From b31f143a21744cb1d2aecac6e404fb82a786b3ed Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:01:02 -0400 Subject: [PATCH 001/228] chore: release main (#18541) --- .release-please-bulk-manifest.json | 4 ++-- librarian.yaml | 4 ++-- packages/google-cloud-bigquery/CHANGELOG.md | 7 +++++++ .../google-cloud-bigquery/google/cloud/bigquery/version.py | 2 +- packages/sqlalchemy-bigquery/CHANGELOG.md | 7 +++++++ .../sqlalchemy-bigquery/sqlalchemy_bigquery/version.py | 2 +- 6 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.release-please-bulk-manifest.json b/.release-please-bulk-manifest.json index 59a31f69be75..08913f4c7cd7 100644 --- a/.release-please-bulk-manifest.json +++ b/.release-please-bulk-manifest.json @@ -53,7 +53,7 @@ "packages/google-cloud-beyondcorp-clientgateways": "0.8.1", "packages/google-cloud-biglake": "0.5.2", "packages/google-cloud-biglake-hive": "0.3.5", - "packages/google-cloud-bigquery": "3.46.0", + "packages/google-cloud-bigquery": "3.46.1", "packages/google-cloud-bigquery-analyticshub": "0.9.1", "packages/google-cloud-bigquery-biglake": "0.8.1", "packages/google-cloud-bigquery-connection": "1.23.0", @@ -277,5 +277,5 @@ "packages/grpc-google-iam-v1": "0.14.5", "packages/pandas-gbq": "0.35.2", "packages/proto-plus": "1.29.0", - "packages/sqlalchemy-bigquery": "1.17.2" + "packages/sqlalchemy-bigquery": "1.17.3" } \ No newline at end of file diff --git a/librarian.yaml b/librarian.yaml index f8e502f5b72c..c3196643dd04 100644 --- a/librarian.yaml +++ b/librarian.yaml @@ -570,7 +570,7 @@ libraries: python: default_version: v1beta - name: google-cloud-bigquery - version: 3.46.0 + version: 3.46.1 python: library_type: GAPIC_COMBO metadata_name_override: bigquery @@ -2672,7 +2672,7 @@ libraries: python: library_type: CORE - name: sqlalchemy-bigquery - version: 1.17.2 + version: 1.17.3 python: library_type: INTEGRATION - name: sqlalchemy-spanner diff --git a/packages/google-cloud-bigquery/CHANGELOG.md b/packages/google-cloud-bigquery/CHANGELOG.md index 1d7feb91df79..71079e69c0f1 100644 --- a/packages/google-cloud-bigquery/CHANGELOG.md +++ b/packages/google-cloud-bigquery/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [3.46.1](https://github.com/googleapis/google-cloud-python/compare/google-cloud-bigquery-v3.46.0...google-cloud-bigquery-v3.46.1) (2026-10-02) + + +### Bug Fixes + +* **bigquery:** account for range element type, rounding mode, and foreign type in SchemaField equality ([#18540](https://github.com/googleapis/google-cloud-python/issues/18540)) ([053dbc9](https://github.com/googleapis/google-cloud-python/commit/053dbc9872ad3f26e10c311ff801b1a85faf1c32)) + ## [3.46.0](https://github.com/googleapis/google-cloud-python/compare/google-cloud-bigquery-v3.45.2...google-cloud-bigquery-v3.46.0) (2026-10-01) diff --git a/packages/google-cloud-bigquery/google/cloud/bigquery/version.py b/packages/google-cloud-bigquery/google/cloud/bigquery/version.py index eef118c03fcb..ad46e1c634c6 100644 --- a/packages/google-cloud-bigquery/google/cloud/bigquery/version.py +++ b/packages/google-cloud-bigquery/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.46.0" +__version__ = "3.46.1" diff --git a/packages/sqlalchemy-bigquery/CHANGELOG.md b/packages/sqlalchemy-bigquery/CHANGELOG.md index 77a42c6139cf..0a954f3a213e 100644 --- a/packages/sqlalchemy-bigquery/CHANGELOG.md +++ b/packages/sqlalchemy-bigquery/CHANGELOG.md @@ -8,6 +8,13 @@ Older versions of this project were distributed as [pybigquery][0]. [1]: https://pypi.org/project/sqlalchemy-bigquery/#history +## [1.17.3](https://github.com/googleapis/google-cloud-python/compare/sqlalchemy-bigquery-v1.17.2...sqlalchemy-bigquery-v1.17.3) (2026-10-02) + + +### Performance Improvements + +* **sqlalchemy-bigquery:** parallelize compliance tests with worker-partitioned datasets and pytest-xdist ([#18534](https://github.com/googleapis/google-cloud-python/issues/18534)) ([855bbde](https://github.com/googleapis/google-cloud-python/commit/855bbde93ea4a5556d2843a6ea6d2b8ee118ba9a)) + ## [1.17.2](https://github.com/googleapis/google-cloud-python/compare/sqlalchemy-bigquery-v1.17.1...sqlalchemy-bigquery-v1.17.2) (2026-08-06) diff --git a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/version.py b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/version.py index faed5f3236fe..45a1be5f3683 100644 --- a/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/version.py +++ b/packages/sqlalchemy-bigquery/sqlalchemy_bigquery/version.py @@ -17,4 +17,4 @@ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -__version__ = "1.17.2" +__version__ = "1.17.3" From d8ae8bf913d94bb1507550c4f40b5344e188dc3b Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Fri, 2 Oct 2026 07:00:24 +0000 Subject: [PATCH 002/228] feat(storage): zero-copy BidiReadObjectResponse parsing for async reads The generated BidiReadObject stub parses every response with proto-plus over upb, which copies each data chunk twice on the event-loop thread before it reaches the caller's buffer. Add a wire-format parser for plain data responses that exposes `checksummed_data.content` as a memoryview of the received message, and let `_AsyncReadObjectStream` use a stub built with it (wrapped like the generated method: error mapping, no default retry or timeout, same client-info header). Responses carrying object metadata, or anything that does not parse cleanly, are handed to the generated parser, so callers keep seeing the real message or its usual DecodeError. CRC32C verification in the read path is unchanged: the memoryview flows through google_crc32c.value(), the user buffer write and the rolling checksum. Because google-crc32c releases up to 1.9.0 only accept bytes there, the fast path is used only when the installed google-crc32c can checksum a memoryview; GOOGLE_CLOUD_STORAGE_FAST_BIDI_READ=0 forces the generated parser as well. --- .../cloud/storage/asyncio/_fast_bidi_read.py | 325 +++++++++++ .../asyncio/async_read_object_stream.py | 10 +- .../asyncio/test_async_read_object_stream.py | 53 ++ .../tests/unit/asyncio/test_fast_bidi_read.py | 516 ++++++++++++++++++ 4 files changed, 901 insertions(+), 3 deletions(-) create mode 100644 packages/google-cloud-storage/google/cloud/storage/asyncio/_fast_bidi_read.py create mode 100644 packages/google-cloud-storage/tests/unit/asyncio/test_fast_bidi_read.py diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/_fast_bidi_read.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/_fast_bidi_read.py new file mode 100644 index 000000000000..9be3f0a61796 --- /dev/null +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/_fast_bidi_read.py @@ -0,0 +1,325 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Zero-copy deserializer for ``BidiReadObjectResponse`` messages. + +The generated gRPC stub parses every response with proto-plus over upb, which +copies each data chunk twice (into the upb arena, then into a Python ``bytes`` +for ``checksummed_data.content``) on the event-loop thread. Data responses are +structurally tiny apart from the chunk itself, so this module walks the wire +format directly and exposes ``content`` as a ``memoryview`` of the raw message. +Responses carrying object metadata (the first message of a stream opened +without a read handle) or anything unexpected fall back to the generated +parser, so callers never see a difference other than the type of ``content``. + +:func:`wrapped_rpc` builds the matching ``BidiReadObject`` callable for a +transport; :class:`~google.cloud.storage.asyncio.async_read_object_stream._AsyncReadObjectStream` +prefers it over the generated one whenever :func:`is_supported` holds. +""" + +import os +import weakref + +import google_crc32c +from google.api_core import gapic_v1 + +from google.cloud import _storage_v2 +from google.cloud._storage_v2.services.storage.transports.base import ( + DEFAULT_CLIENT_INFO, +) +from google.cloud._storage_v2.services.storage.transports.grpc_asyncio import ( + StorageGrpcAsyncIOTransport, +) + +_ENV_VAR = "GOOGLE_CLOUD_STORAGE_FAST_BIDI_READ" +_DISABLED_VALUES = ("0", "false", "no", "off") + +_WT_VARINT, _WT_I64, _WT_LEN, _WT_I32 = 0, 1, 2, 5 + + +class _ReadRange: + __slots__ = ("read_offset", "read_length", "read_id") + + def __init__(self): + self.read_offset = 0 + self.read_length = 0 + self.read_id = 0 + + +class _ChecksummedData: + __slots__ = ("content", "crc32c", "_has_crc32c") + + def __init__(self): + self.content = memoryview(b"") + self.crc32c = 0 + self._has_crc32c = False + + def HasField(self, name): + if name == "crc32c": + return self._has_crc32c + raise ValueError(f"Unknown field {name!r}") + + +class _ObjectRangeData: + __slots__ = ( + "checksummed_data", + "read_range", + "range_end", + "_has_checksummed_data", + "_has_read_range", + ) + + def __init__(self): + self.checksummed_data = _ChecksummedData() + self.read_range = _ReadRange() + self.range_end = False + self._has_checksummed_data = False + self._has_read_range = False + + def HasField(self, name): + if name == "checksummed_data": + return self._has_checksummed_data + if name == "read_range": + return self._has_read_range + raise ValueError(f"Unknown field {name!r}") + + +class FastBidiReadObjectResponse: + """Duck-typed stand-in for a ``BidiReadObjectResponse`` without metadata. + + Only the attributes the async read path uses are provided: + ``object_data_ranges``, ``read_handle``, ``metadata`` (always ``None``), + ``HasField`` and ``_pb`` (itself, so code that unwraps proto-plus messages + keeps working). + """ + + __slots__ = ("object_data_ranges", "read_handle", "metadata") + + def __init__(self, ranges, read_handle): + self.object_data_ranges = ranges + self.read_handle = read_handle + self.metadata = None + + @property + def _pb(self): + return self + + def HasField(self, name): + if name == "read_handle": + return self.read_handle is not None + if name == "metadata": + return False + raise ValueError(f"Unknown field {name!r}") + + +class _Malformed(ValueError): + """Raised when the wire bytes do not look like a well-formed message.""" + + +def _varint(buf, i, end): + """Decode the varint at ``buf[i]``; it must end at or before ``end``.""" + shift = 0 + result = 0 + while True: + b = buf[i] + i += 1 + result |= (b & 0x7F) << shift + if b < 0x80: + break + shift += 7 + if shift > 63: + raise _Malformed("varint longer than 10 bytes") + if i > end: + raise _Malformed("varint runs past its enclosing message") + return result, i + + +def _length_prefixed(buf, i, end): + """Return ``(start, stop)`` of the length-delimited field body at ``i``.""" + n, i = _varint(buf, i, end) + stop = i + n + if stop > end: + raise _Malformed("field runs past its enclosing message") + return i, stop + + +def _skip(buf, i, wt, end): + if wt == _WT_VARINT: + return _varint(buf, i, end)[1] + if wt == _WT_LEN: + return _length_prefixed(buf, i, end)[1] + if wt == _WT_I64: + i += 8 + elif wt == _WT_I32: + i += 4 + else: + raise _Malformed(f"unsupported wire type {wt}") + if i > end: + raise _Malformed("field runs past its enclosing message") + return i + + +def _parse_read_range(buf, i, end): + rr = _ReadRange() + while i < end: + tag, i = _varint(buf, i, end) + f, wt = tag >> 3, tag & 7 + if wt == _WT_VARINT: + v, i = _varint(buf, i, end) + if f == 1: + rr.read_offset = v + elif f == 2: + rr.read_length = v + elif f == 3: + rr.read_id = v + else: + i = _skip(buf, i, wt, end) + return rr + + +def _parse_checksummed(buf, mv, i, end): + cd = _ChecksummedData() + while i < end: + tag, i = _varint(buf, i, end) + f, wt = tag >> 3, tag & 7 + if f == 1 and wt == _WT_LEN: + start, i = _length_prefixed(buf, i, end) + cd.content = mv[start:i] + elif f == 2 and wt == _WT_I32: + if i + 4 > end: + raise _Malformed("crc32c runs past its enclosing message") + cd.crc32c = int.from_bytes(buf[i : i + 4], "little") + cd._has_crc32c = True + i += 4 + else: + i = _skip(buf, i, wt, end) + return cd + + +def _parse_range_data(buf, mv, i, end): + rd = _ObjectRangeData() + while i < end: + tag, i = _varint(buf, i, end) + f, wt = tag >> 3, tag & 7 + if wt == _WT_LEN: + start, i = _length_prefixed(buf, i, end) + if f == 1: + rd.checksummed_data = _parse_checksummed(buf, mv, start, i) + rd._has_checksummed_data = True + elif f == 2: + rd.read_range = _parse_read_range(buf, start, i) + rd._has_read_range = True + elif f == 3 and wt == _WT_VARINT: + v, i = _varint(buf, i, end) + rd.range_end = bool(v) + else: + i = _skip(buf, i, wt, end) + return rd + + +def _parse_handle(buf, i, end): + handle = b"" + while i < end: + tag, i = _varint(buf, i, end) + f, wt = tag >> 3, tag & 7 + if f == 1 and wt == _WT_LEN: + start, i = _length_prefixed(buf, i, end) + handle = bytes(buf[start:i]) + else: + i = _skip(buf, i, wt, end) + # A real message: the handle is stored and passed back in the + # BidiReadObjectSpec of the next open, which only accepts the proto type. + return _storage_v2.BidiReadHandle(handle=handle) + + +_generated_deserialize = _storage_v2.BidiReadObjectResponse.deserialize + + +def deserialize(buf): + """Parse a serialized ``BidiReadObjectResponse``; chunk payloads alias ``buf``. + + Anything that is not a plain data response (object metadata present, or + bytes that do not parse cleanly) is handed to the generated parser, which + either produces the real message or raises its usual ``DecodeError``. + """ + try: + mv = memoryview(buf) + n = len(buf) + i = 0 + ranges = [] + handle = None + while i < n: + tag, i = _varint(buf, i, n) + f, wt = tag >> 3, tag & 7 + if wt == _WT_LEN: + start, end = _length_prefixed(buf, i, n) + if f == 6: + ranges.append(_parse_range_data(buf, mv, start, end)) + elif f == 7: + handle = _parse_handle(buf, start, end) + elif f == 4: + return _generated_deserialize(buf) + i = end + else: + i = _skip(buf, i, wt, n) + return FastBidiReadObjectResponse(ranges, handle) + except Exception: + return _generated_deserialize(buf) + + +def is_supported(): + """Whether the zero-copy parser may be used in this process. + + The parser hands chunk payloads to the read path as ``memoryview`` objects + and checksum verification feeds them to ``google_crc32c``; releases of + google-crc32c that only accept ``bytes`` there would make every verified + read fail, so the generated parser is kept in that case. Setting + ``GOOGLE_CLOUD_STORAGE_FAST_BIDI_READ=0`` forces the generated parser too. + """ + if os.environ.get(_ENV_VAR, "").strip().lower() in _DISABLED_VALUES: + return False + try: + google_crc32c.value(memoryview(b"\0")) + except Exception: + return False + return True + + +_rpcs = weakref.WeakKeyDictionary() + + +def wrapped_rpc(transport): + """Return the zero-copy ``BidiReadObject`` callable for ``transport``. + + The callable is the generated transport's stub with :func:`deserialize` + swapped in, wrapped exactly like ``transport._wrapped_methods`` entries + (error mapping, no default retry or timeout, the client-info header). + Returns ``None`` when the fast path is unsupported or ``transport`` is not + the generated grpc_asyncio transport, in which case callers should use the + generated wrapped method. + """ + if not isinstance(transport, StorageGrpcAsyncIOTransport) or not is_supported(): + return None + rpc = _rpcs.get(transport) + if rpc is None: + stub = transport._logged_channel.stream_stream( + "/google.storage.v2.Storage/BidiReadObject", + request_serializer=_storage_v2.BidiReadObjectRequest.serialize, + response_deserializer=deserialize, + ) + rpc = gapic_v1.method_async.wrap_method( + stub, default_timeout=None, client_info=DEFAULT_CLIENT_INFO + ) + _rpcs[transport] = rpc + return rpc diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py index d3c0780673c7..b536d3b3cf17 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py @@ -19,6 +19,7 @@ from google.api_core.bidi_async import AsyncBidiRpc from google.cloud import _storage_v2 +from google.cloud.storage.asyncio import _fast_bidi_read from google.cloud.storage.asyncio.async_abstract_object_stream import ( _AsyncAbstractObjectStream, ) @@ -76,9 +77,12 @@ def __init__( self._full_bucket_name = f"projects/_/buckets/{self.bucket_name}" - self.rpc = self.client._client._transport._wrapped_methods[ - self.client._client._transport.bidi_read_object - ] + transport = self.client._client._transport + # Zero-copy BidiReadObject responses when google-crc32c can verify + # memoryview payloads; otherwise the generated (proto-plus) parser. + self.rpc = _fast_bidi_read.wrapped_rpc(transport) + if self.rpc is None: + self.rpc = transport._wrapped_methods[transport.bidi_read_object] self.metadata = (("x-goog-request-params", f"bucket={self._full_bucket_name}"),) self.socket_like_rpc: Optional[AsyncBidiRpc] = None self._is_stream_open: bool = False diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_read_object_stream.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_read_object_stream.py index 393fa7b2b709..88d9a0974125 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_read_object_stream.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_read_object_stream.py @@ -113,6 +113,59 @@ def test_init_with_bucket_object_generation(mock_client, mock_async_bidi_rpc): assert read_obj_stream.rpc == rpc_sentinel +@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc") +@mock.patch( + "google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client" +) +@mock.patch( + "google.cloud.storage.asyncio.async_read_object_stream._fast_bidi_read.wrapped_rpc" +) +def test_init_prefers_zero_copy_rpc(mock_wrapped_rpc, mock_client, mock_async_bidi_rpc): + # Arrange + mock_wrapped_rpc.return_value = mock.sentinel.zero_copy_rpc + mock_client._client._transport._wrapped_methods = {} + + # Act + read_obj_stream = _AsyncReadObjectStream( + client=mock_client, + bucket_name=_TEST_BUCKET_NAME, + object_name=_TEST_OBJECT_NAME, + ) + + # Assert + assert read_obj_stream.rpc is mock.sentinel.zero_copy_rpc + mock_wrapped_rpc.assert_called_once_with(mock_client._client._transport) + + +@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc") +@mock.patch( + "google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client" +) +@mock.patch( + "google.cloud.storage.asyncio.async_read_object_stream._fast_bidi_read.wrapped_rpc" +) +def test_init_falls_back_to_generated_rpc( + mock_wrapped_rpc, mock_client, mock_async_bidi_rpc +): + # Arrange + mock_wrapped_rpc.return_value = None + rpc_sentinel = mock.sentinel.A + mock_client._client._transport.bidi_read_object = "bidi_read_object_rpc" + mock_client._client._transport._wrapped_methods = { + "bidi_read_object_rpc": rpc_sentinel, + } + + # Act + read_obj_stream = _AsyncReadObjectStream( + client=mock_client, + bucket_name=_TEST_BUCKET_NAME, + object_name=_TEST_OBJECT_NAME, + ) + + # Assert + assert read_obj_stream.rpc is rpc_sentinel + + @mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc") @mock.patch( "google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client" diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_fast_bidi_read.py b/packages/google-cloud-storage/tests/unit/asyncio/test_fast_bidi_read.py new file mode 100644 index 000000000000..677572cc5690 --- /dev/null +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_fast_bidi_read.py @@ -0,0 +1,516 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the zero-copy ``BidiReadObjectResponse`` deserializer.""" + +import logging + +import google_crc32c +import grpc +import pytest +from google.auth.credentials import AnonymousCredentials +from google.protobuf.message import DecodeError + +from google.cloud import _storage_v2 +from google.cloud._storage_v2.services.storage.transports.grpc_asyncio import ( + StorageGrpcAsyncIOTransport, +) +from google.cloud.storage.asyncio import _fast_bidi_read +from google.cloud.storage.asyncio.async_read_object_stream import ( + _AsyncReadObjectStream, +) + +PAYLOAD = bytes(range(256)) * 8 + +# Field numbers from google/storage/v2/storage.proto. +_RESPONSE_METADATA, _RESPONSE_RANGES, _RESPONSE_HANDLE = 4, 6, 7 +_RANGE_CHECKSUMMED, _RANGE_READ_RANGE, _RANGE_END = 1, 2, 3 +_CHECKSUMMED_CONTENT, _CHECKSUMMED_CRC32C = 1, 2 +_READ_RANGE_OFFSET, _READ_RANGE_LENGTH, _READ_RANGE_ID = 1, 2, 3 +_HANDLE_HANDLE = 1 +_UNKNOWN = 15 + +# --- minimal wire-format encoder ----------------------------------------------- + + +def _varint(value): + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if not value: + out.append(byte) + return bytes(out) + out.append(byte | 0x80) + + +def _tag(number, wire_type): + return _varint(number << 3 | wire_type) + + +def _varint_field(number, value): + return _tag(number, 0) + _varint(value) + + +def _i64_field(number, value): + return _tag(number, 1) + value.to_bytes(8, "little") + + +def _len_field(number, payload): + return _tag(number, 2) + _varint(len(payload)) + payload + + +def _i32_field(number, value): + return _tag(number, 5) + value.to_bytes(4, "little") + + +def _read_range(offset=4096, length=len(PAYLOAD), read_id=7, extra=b""): + return ( + _varint_field(_READ_RANGE_OFFSET, offset) + + _varint_field(_READ_RANGE_LENGTH, length) + + _varint_field(_READ_RANGE_ID, read_id) + + extra + ) + + +def _checksummed(content=PAYLOAD, crc32c=None, extra=b""): + body = _len_field(_CHECKSUMMED_CONTENT, content) + if crc32c is not None: + body += _i32_field(_CHECKSUMMED_CRC32C, crc32c) + return body + extra + + +def _range_data(checksummed=None, read_range=None, range_end=True, extra=b""): + body = b"" + if checksummed is not None: + body += _len_field(_RANGE_CHECKSUMMED, checksummed) + if read_range is not None: + body += _len_field(_RANGE_READ_RANGE, read_range) + if range_end: + body += _varint_field(_RANGE_END, 1) + return body + extra + + +def _response(*ranges, handle=None, extra=b""): + body = b"".join(_len_field(_RESPONSE_RANGES, r) for r in ranges) + if handle is not None: + body += _len_field(_RESPONSE_HANDLE, handle) + return body + extra + + +def _data_response(content=PAYLOAD, crc32c=None, handle=None): + return _response( + _range_data(_checksummed(content, crc32c), _read_range(length=len(content))), + handle=None if handle is None else _len_field(_HANDLE_HANDLE, handle), + ) + + +def _assert_same_as_generated(buf): + """Parse ``buf`` with both parsers and check the fast one matches.""" + expected = _storage_v2.BidiReadObjectResponse.deserialize(buf) + actual = _fast_bidi_read.deserialize(buf) + + assert isinstance(actual, _fast_bidi_read.FastBidiReadObjectResponse) + assert actual.metadata is None + assert not actual.HasField("metadata") + assert actual.HasField("read_handle") == expected._pb.HasField("read_handle") + if actual.HasField("read_handle"): + assert isinstance(actual.read_handle, _storage_v2.BidiReadHandle) + assert actual.read_handle.handle == expected.read_handle.handle + else: + assert actual.read_handle is None + + assert len(actual.object_data_ranges) == len(expected.object_data_ranges) + for got, want in zip(actual.object_data_ranges, expected.object_data_ranges): + assert got.HasField("read_range") == want._pb.HasField("read_range") + assert got.HasField("checksummed_data") == want._pb.HasField("checksummed_data") + assert got.read_range.read_offset == want.read_range.read_offset + assert got.read_range.read_length == want.read_range.read_length + assert got.read_range.read_id == want.read_range.read_id + assert got.range_end == want.range_end + assert isinstance(got.checksummed_data.content, memoryview) + assert bytes(got.checksummed_data.content) == want.checksummed_data.content + assert got.checksummed_data.HasField( + "crc32c" + ) == want.checksummed_data._pb.HasField("crc32c") + assert got.checksummed_data.crc32c == want.checksummed_data.crc32c + return actual + + +# --- deserialize --------------------------------------------------------------- + + +def test_deserialize_data_response_round_trip(): + crc = google_crc32c.value(PAYLOAD) + response = _assert_same_as_generated(_data_response(crc32c=crc)) + + [range_data] = response.object_data_ranges + assert range_data.read_range.read_offset == 4096 + assert range_data.read_range.read_length == len(PAYLOAD) + assert range_data.read_range.read_id == 7 + assert range_data.range_end is True + assert bytes(range_data.checksummed_data.content) == PAYLOAD + assert range_data.checksummed_data.crc32c == crc + + +def test_deserialize_matches_proto_plus_serialization(): + message = _storage_v2.BidiReadObjectResponse( + object_data_ranges=[ + _storage_v2.ObjectRangeData( + checksummed_data=_storage_v2.ChecksummedData( + content=PAYLOAD, crc32c=google_crc32c.value(PAYLOAD) + ), + read_range=_storage_v2.ReadRange( + read_offset=4096, read_length=len(PAYLOAD), read_id=7 + ), + range_end=True, + ) + ], + read_handle=_storage_v2.BidiReadHandle(handle=b"opaque"), + ) + _assert_same_as_generated(_storage_v2.BidiReadObjectResponse.serialize(message)) + + +def test_deserialize_content_is_a_view_of_the_wire_buffer(): + buf = _data_response() + content = _fast_bidi_read.deserialize(buf).object_data_ranges[0] + assert content.checksummed_data.content.obj is buf + + +def test_deserialize_without_crc32c_reports_no_field(): + response = _assert_same_as_generated(_data_response()) + assert response.object_data_ranges[0].checksummed_data.HasField("crc32c") is False + + +def test_deserialize_read_handle_is_a_bidi_read_handle(): + response = _assert_same_as_generated(_data_response(handle=b"opaque")) + + assert response.read_handle.handle == b"opaque" + # The SDK feeds a captured handle straight back into the next open. + spec = _storage_v2.BidiReadObjectSpec(read_handle=response.read_handle) + assert spec.read_handle.handle == b"opaque" + + +def test_deserialize_multiple_ranges(): + buf = _response( + _range_data(_checksummed(b"first"), _read_range(0, 5, 1), range_end=False), + _range_data(_checksummed(b"second"), _read_range(5, 6, 2)), + ) + response = _assert_same_as_generated(buf) + assert [bytes(r.checksummed_data.content) for r in response.object_data_ranges] == [ + b"first", + b"second", + ] + + +def test_deserialize_empty_message(): + response = _assert_same_as_generated(b"") + assert response.object_data_ranges == [] + assert response.read_handle is None + + +def test_deserialize_range_without_read_range_exposes_proto_defaults(): + # _StreamMultiplexer reads ``data_range.read_range.read_id`` without a + # HasField check, so the absent sub-message must look like an empty one. + response = _assert_same_as_generated(_response(_range_data(_checksummed()))) + [range_data] = response.object_data_ranges + assert range_data.HasField("read_range") is False + assert range_data.read_range.read_id == 0 + + +def test_deserialize_range_without_checksummed_data_exposes_proto_defaults(): + response = _assert_same_as_generated( + _response(_range_data(read_range=_read_range())) + ) + [range_data] = response.object_data_ranges + assert range_data.HasField("checksummed_data") is False + assert bytes(range_data.checksummed_data.content) == b"" + + +def test_deserialize_skips_unknown_top_level_fields(): + buf = _data_response() + ( + _varint_field(_UNKNOWN, 1) + + _i64_field(_UNKNOWN, 2) + + _i32_field(_UNKNOWN, 3) + + _len_field(_UNKNOWN, b"ignored") + ) + _assert_same_as_generated(buf) + + +def test_deserialize_skips_unknown_nested_fields(): + unknown = ( + _varint_field(_UNKNOWN, 1) + + _i64_field(_UNKNOWN, 2) + + _i32_field(_UNKNOWN, 3) + + _len_field(_UNKNOWN, b"ignored") + ) + buf = _response( + _range_data( + _checksummed(extra=unknown), + _read_range(extra=unknown), + extra=unknown, + ), + handle=_len_field(_HANDLE_HANDLE, b"opaque") + unknown, + ) + _assert_same_as_generated(buf) + + +def test_deserialize_metadata_response_uses_generated_parser(): + metadata = _storage_v2.Object(name="o", bucket="projects/_/buckets/b", size=10) + buf = _storage_v2.BidiReadObjectResponse.serialize( + _storage_v2.BidiReadObjectResponse(metadata=metadata) + ) + response = _fast_bidi_read.deserialize(buf) + assert isinstance(response, _storage_v2.BidiReadObjectResponse) + assert response.metadata.size == 10 + + +def test_deserialize_unknown_group_uses_generated_parser(): + # Start-group/end-group wire types are valid protobuf that the fast path + # does not understand; the generated parser keeps them as unknown fields. + buf = _data_response() + _tag(_UNKNOWN, 3) + _tag(_UNKNOWN, 4) + response = _fast_bidi_read.deserialize(buf) + assert isinstance(response, _storage_v2.BidiReadObjectResponse) + assert response.object_data_ranges[0].checksummed_data.content == PAYLOAD + + +@pytest.mark.parametrize( + "buf", + [ + pytest.param( + # object_data_ranges (6 bytes) whose checksummed_data claims 8 bytes, + # followed by a well-formed read_handle. + bytes([0x32, 0x06, 0x0A, 0x08]) + b"abcd" + bytes([0x3A, 0x02, 0x0A, 0x00]), + id="inner_length_past_enclosing_message", + ), + pytest.param( + bytes([0x32]) + b"\x80" * 10 + b"\x00", id="varint_longer_than_10_bytes" + ), + pytest.param(bytes([0x32, 0x10, 0x0A, 0x00]), id="truncated_message"), + pytest.param( + _response(_range_data(_checksummed()))[:-3], id="truncated_tag_varint" + ), + pytest.param( + _response(_range_data(_checksummed(extra=_tag(_CHECKSUMMED_CRC32C, 5)))), + id="crc32c_past_enclosing_message", + ), + pytest.param( + _response(_range_data(read_range=_tag(_READ_RANGE_OFFSET, 0) + b"\x80")), + id="read_range_varint_runs_past_enclosing_message", + ), + pytest.param( + _response(_range_data(range_end=False, extra=_tag(_RANGE_END, 0) + b"\x80")) + + _varint_field(_UNKNOWN, 1), + id="range_end_varint_runs_past_enclosing_message", + ), + pytest.param( + _data_response() + _tag(_UNKNOWN, 1) + b"\x00" * 3, + id="truncated_fixed64", + ), + pytest.param( + _data_response() + _tag(_UNKNOWN, 5) + b"\x00", + id="truncated_fixed32", + ), + ], +) +def test_deserialize_malformed_input_raises_like_generated_parser(buf): + with pytest.raises(DecodeError): + _storage_v2.BidiReadObjectResponse.deserialize(buf) + with pytest.raises(DecodeError): + _fast_bidi_read.deserialize(buf) + + +def test_has_field_rejects_unknown_names(): + response = _fast_bidi_read.deserialize(_data_response(crc32c=1)) + [range_data] = response.object_data_ranges + for message in (response, range_data, range_data.checksummed_data): + with pytest.raises(ValueError): + message.HasField("no_such_field") + + +def test_fast_response_pb_is_itself(): + # ReadsResumptionStrategy unwraps ``response._pb`` before calling HasField. + response = _fast_bidi_read.deserialize(_data_response()) + assert response._pb is response + + +# --- is_supported -------------------------------------------------------------- + + +def test_is_supported_requires_crc32c_to_accept_memoryview(monkeypatch): + probed_with = [] + + def bytes_only(data): + probed_with.append(type(data)) + raise TypeError("argument 1 must be read-only bytes-like object") + + monkeypatch.delenv(_fast_bidi_read._ENV_VAR, raising=False) + monkeypatch.setattr(google_crc32c, "value", bytes_only) + assert _fast_bidi_read.is_supported() is False + assert probed_with == [memoryview] + + +def test_is_supported_when_crc32c_accepts_memoryview(monkeypatch): + monkeypatch.delenv(_fast_bidi_read._ENV_VAR, raising=False) + monkeypatch.setattr(google_crc32c, "value", lambda data: len(data)) + assert _fast_bidi_read.is_supported() is True + + +@pytest.mark.parametrize("value", ["0", "false", "No", " off "]) +def test_is_supported_env_kill_switch(monkeypatch, value): + monkeypatch.setattr(google_crc32c, "value", lambda data: len(data)) + monkeypatch.setenv(_fast_bidi_read._ENV_VAR, value) + assert _fast_bidi_read.is_supported() is False + + +# --- wrapped_rpc --------------------------------------------------------------- + + +def _transport(target): + return StorageGrpcAsyncIOTransport( + channel=grpc.aio.insecure_channel(target), credentials=AnonymousCredentials() + ) + + +def test_wrapped_rpc_ignores_transports_it_does_not_know(monkeypatch): + monkeypatch.setattr(_fast_bidi_read, "is_supported", lambda: True) + assert _fast_bidi_read.wrapped_rpc(object()) is None + + +@pytest.mark.asyncio +async def test_wrapped_rpc_returns_none_when_unsupported(monkeypatch): + monkeypatch.setattr(_fast_bidi_read, "is_supported", lambda: False) + transport = _transport("localhost:1") + try: + assert _fast_bidi_read.wrapped_rpc(transport) is None + finally: + await transport.close() + + +@pytest.mark.asyncio +async def test_wrapped_rpc_is_built_once_per_transport(monkeypatch): + monkeypatch.setattr(_fast_bidi_read, "is_supported", lambda: True) + transport = _transport("localhost:1") + other = _transport("localhost:1") + try: + rpc = _fast_bidi_read.wrapped_rpc(transport) + assert rpc is not None + assert rpc is not transport._wrapped_methods[transport.bidi_read_object] + assert _fast_bidi_read.wrapped_rpc(transport) is rpc + assert _fast_bidi_read.wrapped_rpc(other) is not rpc + finally: + await transport.close() + await other.close() + + +# --- end to end through _AsyncReadObjectStream --------------------------------- + + +class _FakeStorageServer: + """In-process gRPC server that answers BidiReadObject with canned bytes.""" + + def __init__(self, responses): + self._responses = list(responses) + self.requests = [] + self.invocation_metadata = None + self._server = grpc.aio.server() + handler = grpc.method_handlers_generic_handler( + "google.storage.v2.Storage", + {"BidiReadObject": grpc.stream_stream_rpc_method_handler(self._handle)}, + ) + self._server.add_generic_rpc_handlers((handler,)) + self._port = self._server.add_insecure_port("localhost:0") + + @property + def target(self): + return f"localhost:{self._port}" + + async def _handle(self, request_iterator, context): + self.invocation_metadata = dict(context.invocation_metadata()) + async for request in request_iterator: + self.requests.append(_storage_v2.BidiReadObjectRequest.deserialize(request)) + yield self._responses.pop(0) + + async def __aenter__(self): + await self._server.start() + return self + + async def __aexit__(self, *exc_info): + await self._server.stop(None) + + +@pytest.fixture +def keep_google_logger_propagation(monkeypatch): + # Building a real GAPIC client runs api-core's client_logging + # initialization, which switches off propagation on the "google" logger + # and would hide log records from ``caplog`` in later test modules. + logger = logging.getLogger("google") + monkeypatch.setattr(logger, "propagate", logger.propagate) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("supported", [True, False]) +async def test_read_object_stream_end_to_end( + monkeypatch, keep_google_logger_propagation, supported +): + monkeypatch.setattr(_fast_bidi_read, "is_supported", lambda: supported) + crc = google_crc32c.value(PAYLOAD) + first = _storage_v2.BidiReadObjectResponse.serialize( + _storage_v2.BidiReadObjectResponse( + metadata=_storage_v2.Object( + name="o", bucket="projects/_/buckets/b", generation=5, size=2048 + ), + read_handle=_storage_v2.BidiReadHandle(handle=b"opaque"), + ) + ) + second = _data_response(crc32c=crc) + + async with _FakeStorageServer([first, second]) as server: + transport = _transport(server.target) + client = _storage_v2.StorageAsyncClient(transport=transport) + stream = _AsyncReadObjectStream(client=client, bucket_name="b", object_name="o") + try: + await stream.open() + assert stream.generation_number == 5 + assert stream.persisted_size == 2048 + assert stream.read_handle.handle == b"opaque" + + await stream.send( + _storage_v2.BidiReadObjectRequest( + read_ranges=[ + _storage_v2.ReadRange( + read_offset=4096, read_length=len(PAYLOAD), read_id=7 + ) + ] + ) + ) + response = await stream.recv() + await stream.close() + finally: + await transport.close() + + [range_data] = response.object_data_ranges + content = range_data.checksummed_data.content + assert isinstance(content, memoryview if supported else bytes) + assert bytes(content) == PAYLOAD + assert range_data.checksummed_data.crc32c == crc + assert range_data.read_range.read_id == 7 + assert isinstance(response, _fast_bidi_read.FastBidiReadObjectResponse) is supported + + assert server.requests[0].read_object_spec.object_ == "o" + assert server.requests[1].read_ranges[0].read_id == 7 + assert server.invocation_metadata["x-goog-request-params"] == ( + "bucket=projects/_/buckets/b" + ) + assert "gapic/" in server.invocation_metadata["x-goog-api-client"] From ff6c2f66bdd059d5ebe86f431da516822f033433 Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Fri, 2 Oct 2026 10:24:51 +0000 Subject: [PATCH 003/228] fix(storage): match the generated parser on int64 sign, split sub-messages and invalid tags The zero-copy BidiReadObjectResponse parser is meant to be indistinguishable from the generated one apart from the type of ``content``; a comparison against upb found a few places where it was not: - ReadRange.read_offset/read_length/read_id are int64, so a ten-byte two's-complement varint must come back negative, and bits above 64 in an over-long varint are dropped rather than kept. - A singular embedded message that occurs twice in its parent is merged; the sub-message parsers now fill in the existing object, which also removes two throw-away allocations per ObjectRangeData. - Field number 0, tags above 32 bits and tags encoded in more than five bytes are malformed and now reach the generated parser's DecodeError. Tests cover each case against the generated parser. --- .../cloud/storage/asyncio/_fast_bidi_read.py | 81 ++++++++++------ .../tests/unit/asyncio/test_fast_bidi_read.py | 94 +++++++++++++++++++ 2 files changed, 146 insertions(+), 29 deletions(-) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/_fast_bidi_read.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/_fast_bidi_read.py index 9be3f0a61796..8c7102ae3c4c 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/_fast_bidi_read.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/_fast_bidi_read.py @@ -127,8 +127,15 @@ class _Malformed(ValueError): """Raised when the wire bytes do not look like a well-formed message.""" +_UINT64_MASK = (1 << 64) - 1 +_UINT32_MAX = (1 << 32) - 1 + + def _varint(buf, i, end): - """Decode the varint at ``buf[i]``; it must end at or before ``end``.""" + """Decode the varint at ``buf[i]``; it must end at or before ``end``. + + Like upb, accept up to ten bytes and keep only the low 64 bits. + """ shift = 0 result = 0 while True: @@ -142,7 +149,25 @@ def _varint(buf, i, end): raise _Malformed("varint longer than 10 bytes") if i > end: raise _Malformed("varint runs past its enclosing message") - return result, i + return result & _UINT64_MASK, i + + +def _int64(value): + """Reinterpret the low 64 bits of a varint as a signed integer.""" + return value - (1 << 64) if value >> 63 else value + + +def _tag(buf, i, end): + """Decode a field tag at ``buf[i]`` into ``(field_number, wire_type, i)``. + + upb rejects field number 0, tags above 32 bits and tags encoded in more + than five bytes. + """ + start = i + tag, i = _varint(buf, i, end) + if tag < 8 or tag > _UINT32_MAX or i - start > 5: + raise _Malformed("invalid field tag") + return tag >> 3, tag & 7, i def _length_prefixed(buf, i, end): @@ -170,29 +195,29 @@ def _skip(buf, i, wt, end): return i -def _parse_read_range(buf, i, end): - rr = _ReadRange() +# The sub-message parsers fill in an existing object so that a singular +# embedded message occurring more than once is merged, as the generated +# parser does, instead of replacing what the earlier occurrence set. + + +def _parse_read_range(rr, buf, i, end): while i < end: - tag, i = _varint(buf, i, end) - f, wt = tag >> 3, tag & 7 + f, wt, i = _tag(buf, i, end) if wt == _WT_VARINT: v, i = _varint(buf, i, end) if f == 1: - rr.read_offset = v + rr.read_offset = _int64(v) elif f == 2: - rr.read_length = v + rr.read_length = _int64(v) elif f == 3: - rr.read_id = v + rr.read_id = _int64(v) else: i = _skip(buf, i, wt, end) - return rr -def _parse_checksummed(buf, mv, i, end): - cd = _ChecksummedData() +def _parse_checksummed(cd, buf, mv, i, end): while i < end: - tag, i = _varint(buf, i, end) - f, wt = tag >> 3, tag & 7 + f, wt, i = _tag(buf, i, end) if f == 1 and wt == _WT_LEN: start, i = _length_prefixed(buf, i, end) cd.content = mv[start:i] @@ -204,21 +229,19 @@ def _parse_checksummed(buf, mv, i, end): i += 4 else: i = _skip(buf, i, wt, end) - return cd def _parse_range_data(buf, mv, i, end): rd = _ObjectRangeData() while i < end: - tag, i = _varint(buf, i, end) - f, wt = tag >> 3, tag & 7 + f, wt, i = _tag(buf, i, end) if wt == _WT_LEN: start, i = _length_prefixed(buf, i, end) if f == 1: - rd.checksummed_data = _parse_checksummed(buf, mv, start, i) + _parse_checksummed(rd.checksummed_data, buf, mv, start, i) rd._has_checksummed_data = True elif f == 2: - rd.read_range = _parse_read_range(buf, start, i) + _parse_read_range(rd.read_range, buf, start, i) rd._has_read_range = True elif f == 3 and wt == _WT_VARINT: v, i = _varint(buf, i, end) @@ -228,19 +251,15 @@ def _parse_range_data(buf, mv, i, end): return rd -def _parse_handle(buf, i, end): - handle = b"" +def _parse_handle(handle, buf, i, end): while i < end: - tag, i = _varint(buf, i, end) - f, wt = tag >> 3, tag & 7 + f, wt, i = _tag(buf, i, end) if f == 1 and wt == _WT_LEN: start, i = _length_prefixed(buf, i, end) handle = bytes(buf[start:i]) else: i = _skip(buf, i, wt, end) - # A real message: the handle is stored and passed back in the - # BidiReadObjectSpec of the next open, which only accepts the proto type. - return _storage_v2.BidiReadHandle(handle=handle) + return handle _generated_deserialize = _storage_v2.BidiReadObjectResponse.deserialize @@ -260,19 +279,23 @@ def deserialize(buf): ranges = [] handle = None while i < n: - tag, i = _varint(buf, i, n) - f, wt = tag >> 3, tag & 7 + f, wt, i = _tag(buf, i, n) if wt == _WT_LEN: start, end = _length_prefixed(buf, i, n) if f == 6: ranges.append(_parse_range_data(buf, mv, start, end)) elif f == 7: - handle = _parse_handle(buf, start, end) + handle = _parse_handle(handle or b"", buf, start, end) elif f == 4: return _generated_deserialize(buf) i = end else: i = _skip(buf, i, wt, n) + if handle is not None: + # A real message: the handle is stored and passed back in the + # BidiReadObjectSpec of the next open, which only accepts the + # proto type. + handle = _storage_v2.BidiReadHandle(handle=handle) return FastBidiReadObjectResponse(ranges, handle) except Exception: return _generated_deserialize(buf) diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_fast_bidi_read.py b/packages/google-cloud-storage/tests/unit/asyncio/test_fast_bidi_read.py index 677572cc5690..0f8ceb1dbe8b 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_fast_bidi_read.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_fast_bidi_read.py @@ -266,6 +266,70 @@ def test_deserialize_skips_unknown_nested_fields(): _assert_same_as_generated(buf) +@pytest.mark.parametrize( + "offset, length, read_id", + [ + pytest.param(-1, -1, -1, id="minus_one"), + pytest.param(-5, 0, -(1 << 55), id="negative"), + pytest.param(-(1 << 63), (1 << 63) - 1, 1, id="int64_extremes"), + ], +) +def test_deserialize_signed_int64_read_range(offset, length, read_id): + # read_offset, read_length and read_id are int64; negative values travel + # as ten-byte two's-complement varints. + message = _storage_v2.BidiReadObjectResponse( + object_data_ranges=[ + _storage_v2.ObjectRangeData( + read_range=_storage_v2.ReadRange( + read_offset=offset, read_length=length, read_id=read_id + ) + ) + ] + ) + response = _assert_same_as_generated( + _storage_v2.BidiReadObjectResponse.serialize(message) + ) + read_range = response.object_data_ranges[0].read_range + assert (read_range.read_offset, read_range.read_length, read_range.read_id) == ( + offset, + length, + read_id, + ) + + +def test_deserialize_over_long_varint_keeps_low_64_bits(): + # A ten-byte varint may carry up to 70 bits; the generated parser keeps + # the low 64 and interprets them as the field's type. + over_long = b"\xff" * 9 + b"\x7f" + read_range = _tag(_READ_RANGE_ID, 0) + over_long + response = _assert_same_as_generated(_response(_range_data(read_range=read_range))) + assert response.object_data_ranges[0].read_range.read_id == -1 + + +def test_deserialize_merges_split_sub_messages(): + # A singular embedded message that occurs twice is merged, so the second + # occurrence must not reset fields set by the first. + read_range = _len_field( + _RANGE_READ_RANGE, _varint_field(_READ_RANGE_OFFSET, 4096) + ) + _len_field(_RANGE_READ_RANGE, _varint_field(_READ_RANGE_ID, 7)) + checksummed = _len_field( + _RANGE_CHECKSUMMED, _len_field(_CHECKSUMMED_CONTENT, PAYLOAD) + ) + _len_field(_RANGE_CHECKSUMMED, _i32_field(_CHECKSUMMED_CRC32C, 1234)) + buf = _response( + _range_data(extra=read_range + checksummed), + handle=_len_field(_HANDLE_HANDLE, b"opaque"), + ) + _len_field(_RESPONSE_HANDLE, b"") + + response = _assert_same_as_generated(buf) + + [range_data] = response.object_data_ranges + assert range_data.read_range.read_offset == 4096 + assert range_data.read_range.read_id == 7 + assert bytes(range_data.checksummed_data.content) == PAYLOAD + assert range_data.checksummed_data.crc32c == 1234 + assert response.read_handle.handle == b"opaque" + + def test_deserialize_metadata_response_uses_generated_parser(): metadata = _storage_v2.Object(name="o", bucket="projects/_/buckets/b", size=10) buf = _storage_v2.BidiReadObjectResponse.serialize( @@ -322,6 +386,36 @@ def test_deserialize_unknown_group_uses_generated_parser(): _data_response() + _tag(_UNKNOWN, 5) + b"\x00", id="truncated_fixed32", ), + pytest.param(b"\x00\x00", id="field_number_zero"), + pytest.param( + _response(_range_data(extra=b"\x00\x00")), + id="field_number_zero_in_range_data", + ), + pytest.param( + _response(_range_data(_checksummed(extra=b"\x00\x00"))), + id="field_number_zero_in_checksummed_data", + ), + pytest.param( + _response(_range_data(read_range=_read_range(extra=b"\x00\x00"))), + id="field_number_zero_in_read_range", + ), + pytest.param( + _response(handle=b"\x00\x00"), + id="field_number_zero_in_read_handle", + ), + pytest.param( + # Wire type 6 ensures the pure-Python protobuf backend also raises + # DecodeError after _tag() rejects the >32-bit tag value. + _tag(1 << 29, 6) + b"\x01" + _data_response(), + id="field_number_too_large", + ), + pytest.param( + # Field 6 / wire type 6 padded to a six-byte varint: _tag() rejects + # i - start > 5 before inspecting the wire type, and both upb and + # the pure-Python backend raise DecodeError on the fallback call. + b"\xb6\x80\x80\x80\x80\x00" + _varint(2) + _varint_field(_RANGE_END, 1), + id="tag_longer_than_five_bytes", + ), ], ) def test_deserialize_malformed_input_raises_like_generated_parser(buf): From d5b5b5d49b6884cc01064fa643c813ac06ecadbb Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 2 Oct 2026 08:48:33 -0400 Subject: [PATCH 004/228] chore: tune renovate limits and ignore non-python lockfiles (#18545) --- renovate.json | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/renovate.json b/renovate.json index eebb15682e1f..dbd88ea1b567 100644 --- a/renovate.json +++ b/renovate.json @@ -1,12 +1,16 @@ { "extends": [ - "config:recommended", - "group:all", - "schedule:weekly" + "config:recommended" ], + "prConcurrentLimit": 5, + "prHourlyLimit": 2, "ignorePaths": [ ".github/workflows/unittest.yml", "**/pyproject.toml", - "**/setup.py" + "**/setup.py", + "**/package.json", + "**/package-lock.json", + "**/pnpm-lock.yaml", + "**/yarn.lock" ] } From 325df14f14c28640a48ad25bac023303606695f7 Mon Sep 17 00:00:00 2001 From: Chandra Shekhar Sirimala Date: Fri, 2 Oct 2026 22:53:11 +0530 Subject: [PATCH 005/228] chore(storage): pass storage_class and finalize_on_close in async grpc write microbenchmarks (#18511) ## Description Passes `storage_class="RAPID"` (when `RUN_RCU_SYSTEM_TESTS=True`) and `finalize_on_close=True` to `AsyncAppendableObjectWriter` in `upload_chunks_using_grpc_async` in microbenchmarks writes. --- .../google/cloud/storage/_helpers.py | 20 +++++++ .../microbenchmarks/writes/test_writes.py | 9 ++- .../tests/system/test_rapid.py | 8 +-- .../tests/unit/test__helpers.py | 55 +++++++++++++++++++ 4 files changed, 86 insertions(+), 6 deletions(-) diff --git a/packages/google-cloud-storage/google/cloud/storage/_helpers.py b/packages/google-cloud-storage/google/cloud/storage/_helpers.py index 4e9d435347d8..9bea6c40845e 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_helpers.py +++ b/packages/google-cloud-storage/google/cloud/storage/_helpers.py @@ -788,3 +788,23 @@ def generate_random_56_bit_integer(): random_bytes = secrets.token_bytes(7) # Convert bytes to an integer return int.from_bytes(random_bytes, "big") + + +def _parse_bool_env(name, default=False): + """Parses a boolean environment variable. + + Raises ValueError if the environment variable is set to an invalid, empty, + or unrecognized boolean value. + """ + env_val = os.getenv(name) + if env_val is None: + return default + cleaned = env_val.strip().lower() + if not cleaned: + raise ValueError(f"{name} cannot be empty or whitespace-only") + if cleaned in ("true", "1", "yes", "on"): + return True + elif cleaned in ("false", "0", "no", "off"): + return False + else: + raise ValueError(f"Invalid value for {name}: {env_val}") diff --git a/packages/google-cloud-storage/tests/perf/microbenchmarks/writes/test_writes.py b/packages/google-cloud-storage/tests/perf/microbenchmarks/writes/test_writes.py index f8b3b3fde80c..1264829d0504 100644 --- a/packages/google-cloud-storage/tests/perf/microbenchmarks/writes/test_writes.py +++ b/packages/google-cloud-storage/tests/perf/microbenchmarks/writes/test_writes.py @@ -32,6 +32,7 @@ import pytest import tests.perf.microbenchmarks.writes.config as config +from google.cloud.storage._helpers import _parse_bool_env from google.cloud.storage.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, ) @@ -45,6 +46,7 @@ # Get write parameters all_params = config.get_write_params() +RCU_SYSTEM_TESTS = _parse_bool_env("RUN_RCU_SYSTEM_TESTS", default=False) async def create_client(): @@ -67,7 +69,10 @@ async def upload_chunks_using_grpc_async(client, filename, other_params): start_time = time.monotonic_ns() writer = AsyncAppendableObjectWriter( - client=client, bucket_name=other_params.bucket_name, object_name=filename + client=client, + bucket_name=other_params.bucket_name, + object_name=filename, + storage_class="RAPID" if RCU_SYSTEM_TESTS else None, ) await writer.open() @@ -80,7 +85,7 @@ async def upload_chunks_using_grpc_async(client, filename, other_params): data = os.urandom(bytes_to_upload) await writer.append(data) uploaded_bytes += bytes_to_upload - await writer.close() + await writer.close(finalize_on_close=True) # print('writer flush count', writer._flush_count) diff --git a/packages/google-cloud-storage/tests/system/test_rapid.py b/packages/google-cloud-storage/tests/system/test_rapid.py index 67ec9f6806c0..52a3c1fba464 100644 --- a/packages/google-cloud-storage/tests/system/test_rapid.py +++ b/packages/google-cloud-storage/tests/system/test_rapid.py @@ -25,8 +25,8 @@ from google.api_core.client_options import ClientOptions from google.api_core.exceptions import FailedPrecondition, NotFound, OutOfRange -# current library imports from google.cloud import kms +from google.cloud.storage._helpers import _parse_bool_env from google.cloud.storage.asyncio.async_appendable_object_writer import ( _DEFAULT_FLUSH_INTERVAL_BYTES, AsyncAppendableObjectWriter, @@ -44,8 +44,8 @@ # Run system test for either Rapid or RCU. But not both => XOR pytestmark = pytest.mark.skipif( not ( - (os.getenv("RUN_RAPID_SYSTEM_TESTS") == "True") - ^ (os.getenv("RUN_RCU_SYSTEM_TESTS") == "True") + _parse_bool_env("RUN_RAPID_SYSTEM_TESTS", default=False) + ^ _parse_bool_env("RUN_RCU_SYSTEM_TESTS", default=False) ), reason="Any one of Rapid or RCU system tests need to be explicitly enabled. This helps scheduling tests in Kokoro and Cloud Build.", ) @@ -57,7 +57,7 @@ _CROSS_REGION_BUCKET = os.getenv("CROSS_REGION_BUCKET") _BYTES_TO_UPLOAD = b"dummy_bytes_to_write_read_and_delete_appendable_object" -RCU_SYSTEM_TESTS = os.getenv("RUN_RCU_SYSTEM_TESTS") == "True" +RCU_SYSTEM_TESTS = _parse_bool_env("RUN_RCU_SYSTEM_TESTS", default=False) _RCU_BUCKET = os.getenv("RCU_BUCKET") _BUCKET_UNDER_TEST = _RCU_BUCKET if RCU_SYSTEM_TESTS else _RAPID_BUCKET diff --git a/packages/google-cloud-storage/tests/unit/test__helpers.py b/packages/google-cloud-storage/tests/unit/test__helpers.py index fa061881c3a3..c7b036d2c655 100644 --- a/packages/google-cloud-storage/tests/unit/test__helpers.py +++ b/packages/google-cloud-storage/tests/unit/test__helpers.py @@ -15,6 +15,7 @@ import unittest import mock +import pytest from google.cloud.storage.retry import ( DEFAULT_RETRY, @@ -720,6 +721,60 @@ def test_hostname_and_scheme(self): self.assertEqual(self._call_fut(host=HOST, scheme=SCHEME), EXPECTED_URL) +class Test__parse_bool_env: + @staticmethod + def _call_fut(*args, **kwargs): + from google.cloud.storage._helpers import _parse_bool_env + + return _parse_bool_env(*args, **kwargs) + + def test_not_set_default_false(self): + with mock.patch("os.getenv", return_value=None): + assert not self._call_fut("TEST_VAR") + + def test_not_set_default_true(self): + with mock.patch("os.getenv", return_value=None): + assert self._call_fut("TEST_VAR", default=True) + + @pytest.mark.parametrize( + "val", + ("true", "True", "TRUE", "1", "yes", "YES", "on", "ON", " true "), + ) + def test_truthy_values(self, val): + with mock.patch("os.getenv", return_value=val): + assert self._call_fut("TEST_VAR") + + @pytest.mark.parametrize( + "val", + ( + "false", + "False", + "FALSE", + "0", + "no", + "NO", + "off", + "OFF", + " false ", + ), + ) + def test_falsy_values(self, val): + with mock.patch("os.getenv", return_value=val): + assert not self._call_fut("TEST_VAR") + + @pytest.mark.parametrize("val", ("", " ", "\t\n")) + def test_empty_or_whitespace_raises(self, val): + with mock.patch("os.getenv", return_value=val): + with pytest.raises(ValueError, match="cannot be empty or whitespace-only"): + self._call_fut("TEST_VAR") + + @pytest.mark.parametrize("val", ("invalid", "2", "foo", "bar")) + def test_invalid_value_raises(self, val): + with mock.patch("os.getenv", return_value=val): + with pytest.raises(ValueError, match="Invalid value for TEST_VAR"): + self._call_fut("TEST_VAR") + + class _MD5Hash(object): def __init__(self, digest_val): self.digest_val = digest_val From 97bc619323b524f8e686bb27c43bd7275f41c239 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 2 Oct 2026 10:42:08 -0700 Subject: [PATCH 006/228] feat(api_core): resumable upload supports request_body in constructor (#18543) Previously, Resumable Uploads would only accept the request_body as part of the `upload` method. Now, it can also be passed in the constructor, to serve as a default value --- .../api_core/resumable_transfer/upload.py | 17 ++- .../resumable_transfer/upload_async.py | 13 ++- .../asyncio/test_resumable_transfer_async.py | 92 +++++++++++++++ .../tests/unit/test_resumable_transfer.py | 109 ++++++++++++++++++ 4 files changed, 228 insertions(+), 3 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index 2cc3b1230929..02d6c0024a8e 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -105,6 +105,7 @@ def __init__( response_type: Optional[Any] = None, start_retry: Optional[google.api_core.retry.Retry] = None, start_timeout: float = DEFAULT_START_TIMEOUT, + request_body: Optional[Union[str, bytes]] = None, ) -> None: """Initializes a ResumableUploadSession. @@ -132,6 +133,9 @@ def __init__( ``UnseekableStreamError``) are never retried. start_timeout: Timeout in seconds for the start request. Defaults to ``60.0`` seconds. + request_body: Optional default metadata payload sent with the start request. + Used by ``upload()`` and ``iter_upload()`` whenever they are called + without an explicit ``request_body``. When ``None``, an empty payload is sent. """ self._config = config or ResumableUploadConfig() self._transport = transport @@ -139,6 +143,9 @@ def __init__( self._response_type = response_type self._start_retry = start_retry self._start_timeout = start_timeout + self._request_body: Union[str, bytes] = ( + request_body if request_body is not None else "" + ) self._response: Optional[Any] = None self._state = upload_state._ProtocolState( upload_url=upload_url, @@ -810,7 +817,7 @@ def attempt_stream() -> Generator[UploadProgress, None, None]: def upload( self, stream: Union[BinaryIO, bytes, Iterable[bytes]], - request_body: Union[str, bytes] = "", + request_body: Optional[Union[str, bytes]] = None, size: Optional[int] = None, transport: Optional[requests.Session] = None, content_type: Optional[str] = None, @@ -822,6 +829,8 @@ def upload( Args: stream: Data payload to upload (file-like stream, bytes, or iterable of bytes). request_body: Initial metadata payload sent with the start request. + When ``None``, the ``request_body`` supplied to the session constructor + (if any) is used. size: Total stream size in bytes, if known. transport: Optional requests session. content_type: Optional MIME type of the stream payload. @@ -859,7 +868,7 @@ def upload( def iter_upload( self, stream: Union[BinaryIO, bytes, Iterable[bytes]], - request_body: Union[str, bytes] = "", + request_body: Optional[Union[str, bytes]] = None, size: Optional[int] = None, transport: Optional[requests.Session] = None, content_type: Optional[str] = None, @@ -871,6 +880,8 @@ def iter_upload( Args: stream: Data payload to upload (file-like stream, bytes, or iterable of bytes). request_body: Initial metadata payload sent with the start request. + When ``None``, the ``request_body`` supplied to the session constructor + (if any) is used. size: Total stream size in bytes, if known. transport: Optional requests session. content_type: Optional MIME type of the stream payload. @@ -896,6 +907,8 @@ def iter_upload( sess = self._get_transport(transport) if content_type is not None: self._content_type = content_type + if request_body is None: + request_body = self._request_body self._reset_transfer_state() progress_queue: List[UploadProgress] = [] try: diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index 13cfcb5c6fbd..538d2c050650 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -171,6 +171,7 @@ def __init__( response_type: Optional[Any] = None, start_retry: Optional[google.api_core.retry.AsyncRetry] = None, start_timeout: float = DEFAULT_START_TIMEOUT, + request_body: Optional[Union[str, bytes]] = None, ) -> None: """Initializes an AsyncResumableUploadSession. @@ -198,6 +199,9 @@ def __init__( retried. start_timeout: Timeout in seconds for the start request. Defaults to ``60.0`` seconds. + request_body: Optional default metadata payload sent with the start request. + Used by ``upload()`` whenever it is called without an explicit + ``request_body``. When ``None``, an empty payload is sent. """ self._config = config or ResumableUploadConfig() self._transport = transport @@ -205,6 +209,9 @@ def __init__( self._response_type = response_type self._start_retry = start_retry self._start_timeout = start_timeout + self._request_body: Union[str, bytes] = ( + request_body if request_body is not None else "" + ) self._response: Optional[Any] = None self._state = upload_state._ProtocolState( upload_url=upload_url, @@ -885,7 +892,7 @@ async def attempt_stream() -> AsyncGenerator[UploadProgress, None]: def upload( self, stream: Union[AsyncIterable[bytes], BinaryIO, bytes, Iterable[bytes]], - request_body: Union[str, bytes] = "", + request_body: Optional[Union[str, bytes]] = None, size: Optional[int] = None, transport: Optional[Any] = None, content_type: Optional[str] = None, @@ -897,6 +904,8 @@ def upload( Args: stream: Data payload to upload (async iterable, binary stream, bytes, or iterable). request_body: Initial metadata payload sent with the start request. + When ``None``, the ``request_body`` supplied to the session constructor + (if any) is used. size: Total stream size in bytes, if known. transport: Optional aiohttp client session. content_type: Optional MIME type of the stream payload. @@ -925,6 +934,8 @@ def upload( if content_type is not None: self._content_type = content_type + if request_body is None: + request_body = self._request_body self._reset_transfer_state() progress_queue: List[UploadProgress] = [] diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index b16211795ee6..4b799647946b 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -262,6 +262,98 @@ async def test_async_upload_direct_execution() -> None: assert session.upload_url == "https://upload.example.com/resumable-async" +def _make_single_chunk_async_transport() -> DummyAsyncSession: + """Builds a DummyAsyncSession returning a start response and a final chunk response.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b"", + ) + return DummyAsyncSession([start_resp, chunk_resp]) + + +@pytest.mark.asyncio +async def test_async_upload_uses_session_request_body() -> None: + """Verifies the constructor request_body is sent with the start request by default.""" + async_transport = _make_single_chunk_async_transport() + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=async_transport, + request_body='{"name": "from-init"}', + ) + + await session.upload(stream=b"payload") + + _, _, start_kwargs = async_transport.requests[0] + assert start_kwargs["data"] == b'{"name": "from-init"}' + assert start_kwargs["headers"]["X-Goog-Upload-Command"] == "start" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "override, expected", + [ + ('{"name": "override"}', b'{"name": "override"}'), + (b'{"name": "override-bytes"}', b'{"name": "override-bytes"}'), + ("", b""), + ], +) +async def test_async_upload_request_body_argument_overrides_session_default( + override: Union[str, bytes], expected: bytes +) -> None: + """Verifies an explicit per-call request_body (even empty) overrides the constructor default.""" + async_transport = _make_single_chunk_async_transport() + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=async_transport, + request_body='{"name": "from-init"}', + ) + + await session.upload(stream=b"payload", request_body=override) + + _, _, start_kwargs = async_transport.requests[0] + assert start_kwargs["data"] == expected + + +@pytest.mark.asyncio +async def test_async_upload_request_body_positional_override() -> None: + """Verifies request_body remains the second positional parameter of upload().""" + async_transport = _make_single_chunk_async_transport() + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=async_transport, + request_body='{"name": "from-init"}', + ) + + await session.upload(b"payload", '{"name": "positional"}') + + _, _, start_kwargs = async_transport.requests[0] + assert start_kwargs["data"] == b'{"name": "positional"}' + + +@pytest.mark.asyncio +async def test_async_upload_request_body_defaults_to_empty() -> None: + """Verifies the start request carries an empty payload when no request_body is configured.""" + async_transport = _make_single_chunk_async_transport() + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=async_transport, + ) + + await session.upload(stream=b"payload") + + _, _, start_kwargs = async_transport.requests[0] + assert start_kwargs["data"] == b"" + + @pytest.mark.asyncio async def test_async_upload_multi_chunk_operation_handle() -> None: """Verifies multi-chunk upload dispatching and AsyncUploadOperation property handles.""" diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index d783af5432a4..d207910f0ba9 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -366,6 +366,115 @@ def test_sync_upload_iterative_progress(): assert session.response.size == 8 +def _make_single_chunk_sync_transport() -> mock.Mock: + """Builds a mock requests.Session returning a start response and a final chunk response.""" + session_transport = mock.create_autospec(requests.Session, instance=True) + start_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + }, + ) + chunk_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "final"}, + content=b"", + ) + session_transport.request.side_effect = [start_resp, chunk_resp] + return session_transport + + +def test_sync_upload_uses_session_request_body(): + # A request_body supplied to the constructor is sent with the start + # request when upload() is called without an explicit request_body. + session_transport = _make_single_chunk_sync_transport() + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + transport=session_transport, + request_body='{"name": "from-init"}', + ) + + session.upload(stream=b"payload") + + start_call = session_transport.request.call_args_list[0] + assert start_call.kwargs["data"] == b'{"name": "from-init"}' + assert start_call.kwargs["headers"]["X-Goog-Upload-Command"] == "start" + + +def test_sync_iter_upload_uses_session_request_body(): + session_transport = _make_single_chunk_sync_transport() + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + transport=session_transport, + request_body=b'{"name": "from-init-bytes"}', + ) + + list(session.iter_upload(stream=b"payload")) + + start_call = session_transport.request.call_args_list[0] + assert start_call.kwargs["data"] == b'{"name": "from-init-bytes"}' + + +@pytest.mark.parametrize( + "override, expected", + [ + ('{"name": "override"}', b'{"name": "override"}'), + (b'{"name": "override-bytes"}', b'{"name": "override-bytes"}'), + ("", b""), + ], +) +def test_sync_upload_request_body_argument_overrides_session_default( + override, expected +): + # An explicit per-call request_body (including an empty one) takes + # precedence over the request_body supplied to the constructor. + session_transport = _make_single_chunk_sync_transport() + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + transport=session_transport, + request_body='{"name": "from-init"}', + ) + + session.upload(stream=b"payload", request_body=override) + + start_call = session_transport.request.call_args_list[0] + assert start_call.kwargs["data"] == expected + + +def test_sync_upload_request_body_positional_override(): + # request_body stays the second positional parameter, so callers that + # pass it positionally are not broken by the constructor default. + session_transport = _make_single_chunk_sync_transport() + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + transport=session_transport, + request_body='{"name": "from-init"}', + ) + + session.upload(b"payload", '{"name": "positional"}') + + start_call = session_transport.request.call_args_list[0] + assert start_call.kwargs["data"] == b'{"name": "positional"}' + + +def test_sync_upload_request_body_defaults_to_empty(): + # Without a request_body on the constructor or the call, the start + # request carries an empty payload. + session_transport = _make_single_chunk_sync_transport() + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + transport=session_transport, + ) + + session.upload(stream=b"payload") + + start_call = session_transport.request.call_args_list[0] + assert start_call.kwargs["data"] == b"" + + def test_sync_resume(): session_transport = mock.create_autospec(requests.Session, instance=True) From 379956865248ebc40e5b851ede520056e65abcac Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 2 Oct 2026 11:16:56 -0700 Subject: [PATCH 007/228] feat(auth): Support sync credentials in AsyncAuthorizedSession (#18542) This PR allows AsyncAuthorizedSession to support standard credential types by wrapping them in an adapter --------- Co-authored-by: Anthonios Partheniou --- .../test_operations_rest_client.py | 28 +- .../google/auth/aio/transport/sessions.py | 125 ++++++++- .../tests/transport/aio/test_sessions.py | 261 +++++++++++++++++- 3 files changed, 386 insertions(+), 28 deletions(-) diff --git a/packages/google-api-core/tests/unit/operations_v1/test_operations_rest_client.py b/packages/google-api-core/tests/unit/operations_v1/test_operations_rest_client.py index cdb0722474d6..17d9150e5f2b 100644 --- a/packages/google-api-core/tests/unit/operations_v1/test_operations_rest_client.py +++ b/packages/google-api-core/tests/unit/operations_v1/test_operations_rest_client.py @@ -1241,24 +1241,20 @@ def test_operations_base_transport_with_adc(): ) def test_operations_auth_adc(client_class): # If no credentials are provided, we should use ADC credentials. + is_async = "async" in str(client_class).lower() + if is_async and parse_version_to_tuple(auth_version) < (2, 60, 0): + # Older versions of google-auth do not accept the synchronous credentials + # returned by ADC in `AsyncAuthorizedSession`. + pytest.skip("ADC with the async REST transport requires google-auth >= 2.60.0") + with mock.patch.object(google.auth, "default", autospec=True) as adc: adc.return_value = (ga_credentials.AnonymousCredentials(), None) - - if "async" in str(client_class).lower(): - # TODO(): Add support for adc to async REST transport. - # NOTE: Ideally, the logic for adc shouldn't be called if transport - # is set to async REST. If the user does not configure credentials - # of type `google.auth.aio.credentials.Credentials`, - # we should raise an exception to avoid the adc workflow. - with pytest.raises(google.auth.exceptions.InvalidType): - client_class() - else: - client_class() - adc.assert_called_once_with( - scopes=None, - default_scopes=(), - quota_project_id=None, - ) + client_class() + adc.assert_called_once_with( + scopes=None, + default_scopes=(), + quota_project_id=None, + ) # TODO(https://github.com/googleapis/python-api-core/issues/705): Add diff --git a/packages/google-auth/google/auth/aio/transport/sessions.py b/packages/google-auth/google/auth/aio/transport/sessions.py index d2375604b127..14e950427b5f 100644 --- a/packages/google-auth/google/auth/aio/transport/sessions.py +++ b/packages/google-auth/google/auth/aio/transport/sessions.py @@ -24,8 +24,9 @@ from contextlib import asynccontextmanager from typing import TYPE_CHECKING, Mapping, Optional, Union +import google.auth.credentials import google.auth.transport._mtls_helper -from google.auth import _exponential_backoff, exceptions +from google.auth import _exponential_backoff, _helpers, exceptions from google.auth.aio import transport from google.auth.aio.credentials import Credentials from google.auth.aio.transport import mtls @@ -104,6 +105,106 @@ async def with_timeout(coro): _remaining_time() +class _SyncCredentialsAdapter(Credentials): + """Adapts synchronous credentials to the asynchronous credentials interface. + + :class:`AsyncAuthorizedSession` wraps :class:`google.auth.credentials.Credentials` + (e.g. application default credentials) with this adapter so that they can be + used with an asynchronous transport. Calls are delegated to the wrapped + credentials using a synchronous transport, and blocking calls such as + refreshing the access token run in a worker thread so that the event loop + is not blocked. + + Args: + credentials (google.auth.credentials.Credentials): The synchronous + credentials to adapt. + """ + + def __init__(self, credentials: google.auth.credentials.Credentials): + self._credentials = credentials + # Synchronous credentials cannot use the asynchronous transport of the + # session, so they are called with a synchronous transport instead. + self._sync_request_instance = None + # Synchronous credentials are not safe to refresh concurrently, which + # concurrent requests would otherwise do from multiple worker threads. + # Instead, at most one refresh is in flight and concurrent callers share it. + self._pending_refresh: Optional["asyncio.Task[None]"] = None + + @property + def _sync_request(self): + if self._sync_request_instance is None: + # Imported here because `requests` is an optional dependency of + # google-auth. It is installed alongside `aiohttp` by the `aiohttp` extra. + from google.auth.transport import requests as sync_requests + + self._sync_request_instance = sync_requests.Request() + return self._sync_request_instance + + def close(self): + if ( + self._sync_request_instance is not None + and hasattr(self._sync_request_instance, "session") + and self._sync_request_instance.session is not None + ): + self._sync_request_instance.session.close() + + @property + def token(self): + """Optional[str]: The bearer token that can be used in HTTP headers to make + authenticated requests.""" + return self._credentials.token + + @property + def expiry(self): + """Optional[datetime]: When the token expires and is no longer valid. + If this is None, the token is assumed to never expire.""" + return self._credentials.expiry + + @property + @_helpers.copy_docstring(google.auth.credentials.Credentials) + def valid(self): + return self._credentials.valid + + @property + @_helpers.copy_docstring(google.auth.credentials.Credentials) + def expired(self): + return self._credentials.expired + + async def _refresh_shared(self): + """Refreshes the wrapped credentials, joining a refresh already in flight. + + The refresh is shielded from cancellation: a caller that is cancelled + while waiting (e.g. because of a timeout) stops waiting, but the refresh + completes so that the next caller joins it rather than starting a + second, concurrent refresh. + """ + if self._pending_refresh is None or self._pending_refresh.done(): + self._pending_refresh = asyncio.create_task( + asyncio.to_thread(self._credentials.refresh, self._sync_request) + ) + await asyncio.shield(self._pending_refresh) + + @_helpers.copy_docstring(Credentials) + async def apply(self, headers, token=None): + self._credentials.apply(headers, token=token) + + @_helpers.copy_docstring(Credentials) + async def refresh(self, request): + await self._refresh_shared() + + @_helpers.copy_docstring(Credentials) + async def before_request(self, request, method, url, headers): + if not self._credentials.valid: + await self._refresh_shared() + await asyncio.to_thread( + self._credentials.before_request, + self._sync_request, + method, + url, + headers, + ) + + class AsyncAuthorizedSession: """This is an asynchronous implementation of :class:`google.auth.requests.AuthorizedSession` class. We utilize an instance of a class that implements :class:`google.auth.aio.transport.Request` configured @@ -126,8 +227,9 @@ class AsyncAuthorizedSession: credentials' headers to the request and refreshing credentials as needed. Args: - credentials (google.auth.aio.credentials.Credentials): - The credentials to add to the request. + credentials (Union[google.auth.aio.credentials.Credentials, google.auth.credentials.Credentials]): + The credentials to add to the request. Synchronous credentials + (e.g. application default credentials) are also supported. auth_request (Optional[google.auth.aio.transport.Request]): An instance of a class that implements :class:`~google.auth.aio.transport.Request` used to make requests @@ -139,17 +241,22 @@ class AsyncAuthorizedSession: - google.auth.exceptions.TransportError: If `auth_request` is `None` and the external package `aiohttp` is not installed. - google.auth.exceptions.InvalidType: If the provided credentials are - not of type `google.auth.aio.credentials.Credentials`. + not of type `google.auth.aio.credentials.Credentials` or + `google.auth.credentials.Credentials`. """ def __init__( - self, credentials: Credentials, auth_request: Optional[transport.Request] = None + self, + credentials: Union[Credentials, google.auth.credentials.Credentials], + auth_request: Optional[transport.Request] = None, ): - if not isinstance(credentials, Credentials): + if isinstance(credentials, google.auth.credentials.Credentials): + credentials = _SyncCredentialsAdapter(credentials) + elif not isinstance(credentials, Credentials): raise exceptions.InvalidType( - f"The configured credentials of type {type(credentials)} are invalid and must be of type `google.auth.aio.credentials.Credentials`" + f"The configured credentials of type {type(credentials)} are invalid and must be of type `google.auth.aio.credentials.Credentials` or `google.auth.credentials.Credentials`" ) - self._credentials = credentials + self._credentials: Credentials = credentials _auth_request = auth_request if not _auth_request and AIOHTTP_INSTALLED: _auth_request = AiohttpRequest() @@ -846,6 +953,8 @@ async def close(self) -> None: if inspect.isawaitable(res): await res finally: + if hasattr(self._credentials, "close"): + self._credentials.close() for old_request in self._old_auth_requests: try: if hasattr(old_request, "close"): diff --git a/packages/google-auth/tests/transport/aio/test_sessions.py b/packages/google-auth/tests/transport/aio/test_sessions.py index 69793fe49f5b..0b4ce134fa9c 100644 --- a/packages/google-auth/tests/transport/aio/test_sessions.py +++ b/packages/google-auth/tests/transport/aio/test_sessions.py @@ -13,13 +13,17 @@ # limitations under the License. import asyncio +import http.client as http_client +import threading from typing import AsyncGenerator -from unittest.mock import Mock, patch +from unittest.mock import AsyncMock, Mock, patch import pytest # type: ignore from aioresponses import aioresponses # type: ignore -from google.auth.aio.credentials import AnonymousCredentials +import google.auth.credentials +import google.auth.transport.requests +from google.auth.aio.credentials import AnonymousCredentials, Credentials from google.auth.aio.transport import ( _DEFAULT_TIMEOUT_SECONDS, DEFAULT_MAX_RETRY_ATTEMPTS, @@ -28,7 +32,12 @@ Response, sessions, ) -from google.auth.exceptions import InvalidType, TimeoutError, TransportError +from google.auth.exceptions import ( + InvalidType, + RefreshError, + TimeoutError, + TransportError, +) @pytest.fixture @@ -203,9 +212,77 @@ async def test_constructor_raises_incorrect_credentials_error(self): sessions.AsyncAuthorizedSession(credentials) exc.match( - f"The configured credentials of type {type(credentials)} are invalid and must be of type `google.auth.aio.credentials.Credentials`" + f"The configured credentials of type {type(credentials)} are invalid and must be of type `google.auth.aio.credentials.Credentials` or `google.auth.credentials.Credentials`" ) + @pytest.mark.asyncio + async def test_constructor_with_sync_credentials(self): + sync_credentials = google.auth.credentials.AnonymousCredentials() + authed_session = sessions.AsyncAuthorizedSession( + sync_credentials, auth_request=MockRequest() + ) + + # Synchronous credentials are adapted to the asynchronous credentials interface. + assert isinstance(authed_session._credentials, Credentials) + assert isinstance(authed_session._credentials, sessions._SyncCredentialsAdapter) + assert authed_session._credentials._credentials is sync_credentials + with patch.object( + authed_session._credentials, + "close", + wraps=authed_session._credentials.close, + ) as mock_close: + await authed_session.close() + mock_close.assert_called_once() + + @pytest.mark.asyncio + async def test_request_with_sync_credentials_success(self, mocked_content): + sync_credentials = Mock(spec=google.auth.credentials.Credentials) + mocked_response = MockResponse( + status_code=http_client.OK, + headers={"Content-Type": "application/json"}, + content=mocked_content, + ) + auth_request = MockRequest(mocked_response) + authed_session = sessions.AsyncAuthorizedSession(sync_credentials, auth_request) + + response = await authed_session.request( + "GET", self.TEST_URL, headers={"x-test": "value"} + ) + + assert response.status_code == http_client.OK + assert await response.read() == b"Cavefish have no sight." + # The synchronous credentials are invoked with a synchronous transport rather + # than the asynchronous transport of the session. + sync_credentials.before_request.assert_called_once() + request, method, url, headers = sync_credentials.before_request.call_args.args + assert isinstance(request, google.auth.transport.requests.Request) + assert method == "GET" + assert url == self.TEST_URL + assert headers == {"x-test": "value"} + await authed_session.close() + + @pytest.mark.asyncio + async def test_request_with_sync_credentials_refreshes_on_unauthorized(self): + sync_credentials = Mock(spec=google.auth.credentials.Credentials) + unauthorized_response = MockResponse(status_code=http_client.UNAUTHORIZED) + ok_response = MockResponse(status_code=http_client.OK) + auth_request = AsyncMock(side_effect=[unauthorized_response, ok_response]) + authed_session = sessions.AsyncAuthorizedSession( + sync_credentials, auth_request=auth_request + ) + + response = await authed_session.request("GET", self.TEST_URL) + + assert response is ok_response + assert auth_request.call_count == 2 + assert unauthorized_response._close + sync_credentials.refresh.assert_called_once() + (refresh_request,) = sync_credentials.refresh.call_args.args + assert isinstance(refresh_request, google.auth.transport.requests.Request) + # The same synchronous transport is reused for every call to the credentials. + assert refresh_request is sync_credentials.before_request.call_args.args[0] + await authed_session.close() + @pytest.mark.asyncio async def test_request_default_auth_request_success(self): with aioresponses() as m: @@ -368,3 +445,179 @@ def test_mock_request_clone(): request = MockRequest() cloned = request._clone() assert cloned is request + + +class BlockingRefreshCredentials(google.auth.credentials.Credentials): + """Synchronous credentials whose refresh blocks until released by the test.""" + + def __init__(self): + super().__init__() + self.refresh_calls = 0 + self.in_flight_refreshes = 0 + self.max_in_flight_refreshes = 0 + self.refresh_started = threading.Event() + self.release_refresh = threading.Event() + + def refresh(self, request): + self.refresh_calls += 1 + self.in_flight_refreshes += 1 + self.max_in_flight_refreshes = max( + self.max_in_flight_refreshes, self.in_flight_refreshes + ) + self.refresh_started.set() + self.release_refresh.wait(timeout=5) + self.in_flight_refreshes -= 1 + self.token = "token" + + +class TestSyncCredentialsAdapter(object): + TEST_URL = "http://example.com/" + + @pytest.mark.asyncio + async def test_delegates_to_sync_credentials(self): + sync_credentials = Mock(spec=google.auth.credentials.Credentials) + sync_credentials.token = "sync-token" + sync_credentials.expiry = Mock() + sync_credentials.valid = True + sync_credentials.expired = False + adapter = sessions._SyncCredentialsAdapter(sync_credentials) + headers = {} + + await adapter.before_request(Mock(), "GET", self.TEST_URL, headers) + await adapter.refresh(Mock()) + await adapter.apply(headers, token="token") + + assert adapter.token == "sync-token" + assert adapter.expiry is sync_credentials.expiry + assert adapter.valid is True + assert adapter.expired is False + + # The same synchronous transport is used for every call. + sync_request = sync_credentials.before_request.call_args.args[0] + assert isinstance(sync_request, google.auth.transport.requests.Request) + sync_credentials.before_request.assert_called_once_with( + sync_request, "GET", self.TEST_URL, headers + ) + sync_credentials.refresh.assert_called_once_with(sync_request) + sync_credentials.apply.assert_called_once_with(headers, token="token") + with patch.object(sync_request.session, "close") as mock_close: + adapter.close() + mock_close.assert_called_once() + + @pytest.mark.asyncio + async def test_blocking_calls_run_off_the_event_loop_thread(self): + sync_credentials = Mock(spec=google.auth.credentials.Credentials) + thread_ids = [] + sync_credentials.before_request.side_effect = lambda *args: thread_ids.append( + threading.get_ident() + ) + sync_credentials.refresh.side_effect = lambda *args: thread_ids.append( + threading.get_ident() + ) + adapter = sessions._SyncCredentialsAdapter(sync_credentials) + + await adapter.before_request(Mock(), "GET", self.TEST_URL, {}) + await adapter.refresh(Mock()) + + assert len(thread_ids) == 2 + assert threading.get_ident() not in thread_ids + + @pytest.mark.asyncio + async def test_concurrent_before_request_refreshes_once(self): + sync_credentials = BlockingRefreshCredentials() + adapter = sessions._SyncCredentialsAdapter(sync_credentials) + headers = [{} for _ in range(5)] + + tasks = [ + asyncio.create_task(adapter.before_request(Mock(), "GET", self.TEST_URL, h)) + for h in headers + ] + await asyncio.to_thread(sync_credentials.refresh_started.wait, 5) + # Yield to the event loop so all tasks run until they await the refresh. + await asyncio.sleep(0) + sync_credentials.release_refresh.set() + await asyncio.gather(*tasks) + + assert sync_credentials.refresh_calls == 1 + assert all(h["authorization"] == "Bearer token" for h in headers) + + @pytest.mark.asyncio + async def test_refresh_and_before_request_share_one_refresh(self): + sync_credentials = BlockingRefreshCredentials() + adapter = sessions._SyncCredentialsAdapter(sync_credentials) + headers = {} + + refresh_task = asyncio.create_task(adapter.refresh(Mock())) + await asyncio.to_thread(sync_credentials.refresh_started.wait, 5) + before_request_task = asyncio.create_task( + adapter.before_request(Mock(), "GET", self.TEST_URL, headers) + ) + # Yield to the event loop so before_request_task reaches the refresh. + await asyncio.sleep(0) + sync_credentials.release_refresh.set() + await asyncio.gather(refresh_task, before_request_task) + + assert sync_credentials.max_in_flight_refreshes == 1 + assert sync_credentials.refresh_calls == 1 + assert headers["authorization"] == "Bearer token" + + @pytest.mark.asyncio + async def test_cancelled_caller_does_not_abandon_refresh(self): + sync_credentials = BlockingRefreshCredentials() + adapter = sessions._SyncCredentialsAdapter(sync_credentials) + + cancelled_task = asyncio.create_task( + adapter.before_request(Mock(), "GET", self.TEST_URL, {}) + ) + await asyncio.to_thread(sync_credentials.refresh_started.wait, 5) + cancelled_task.cancel() + with pytest.raises(asyncio.CancelledError): + await cancelled_task + + # The refresh started on behalf of the cancelled request is still running: + # a new request must wait for it rather than start a second refresh. + headers = {} + waiting_task = asyncio.create_task( + adapter.before_request(Mock(), "GET", self.TEST_URL, headers) + ) + await asyncio.sleep(0) + assert not waiting_task.done() + sync_credentials.release_refresh.set() + await waiting_task + + assert sync_credentials.refresh_calls == 1 + assert sync_credentials.max_in_flight_refreshes == 1 + assert headers["authorization"] == "Bearer token" + + @pytest.mark.asyncio + async def test_failed_refresh_is_not_reused(self): + sync_credentials = Mock(spec=google.auth.credentials.Credentials) + sync_credentials.refresh.side_effect = [RefreshError("refresh failed"), None] + adapter = sessions._SyncCredentialsAdapter(sync_credentials) + + with pytest.raises(RefreshError): + await adapter.refresh(Mock()) + await adapter.refresh(Mock()) + + assert sync_credentials.refresh.call_count == 2 + + @pytest.mark.asyncio + async def test_before_request_with_valid_credentials_does_not_wait_for_refresh( + self, + ): + sync_credentials = BlockingRefreshCredentials() + sync_credentials.token = "token" + adapter = sessions._SyncCredentialsAdapter(sync_credentials) + headers = {} + + refresh_task = asyncio.create_task(adapter.refresh(Mock())) + await asyncio.to_thread(sync_credentials.refresh_started.wait, 5) + # Requests that already have a valid token must not wait for the refresh. + await asyncio.wait_for( + adapter.before_request(Mock(), "GET", self.TEST_URL, headers), timeout=5 + ) + assert headers["authorization"] == "Bearer token" + + sync_credentials.release_refresh.set() + await refresh_task + assert sync_credentials.refresh_calls == 1 From 9b96af76254c33b9c3e0ceb390bbaeab2e33bbea Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 2 Oct 2026 13:19:43 -0700 Subject: [PATCH 008/228] feat(api-core): support AsyncAuthorizedSession transport for resumable uploads (#18544) Currently, resumable uploads only support `aiohttp.ClientSession`. Gapic clients use `AsyncAuthorizedSession` instead. This PR refactors the implementation to support both options --------- Co-authored-by: Anthonios Partheniou --- .../resumable_transfer/upload_async.py | 249 ++++++++++++------ .../asyncio/test_resumable_transfer_async.py | 213 ++++++++++++++- 2 files changed, 380 insertions(+), 82 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index 538d2c050650..4edcc0b6727d 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Asynchronous Resumable Upload session and helpers using aiohttp.""" +"""Asynchronous Resumable Upload session and helpers.""" import asyncio import datetime @@ -28,6 +28,7 @@ Awaitable, BinaryIO, Callable, + Dict, Generator, Generic, Iterable, @@ -37,6 +38,7 @@ Tuple, TypeVar, Union, + cast, ) try: @@ -46,6 +48,14 @@ except ImportError: # pragma: NO COVER _HAS_AIOHTTP = False +try: + from google.auth import exceptions as auth_exceptions + from google.auth.aio.transport.sessions import AsyncAuthorizedSession + + _HAS_GOOGLE_AUTH_AIO = True +except ImportError: # pragma: NO COVER + _HAS_GOOGLE_AUTH_AIO = False + import google.api_core.retry from google.api_core import exceptions from google.api_core.resumable_transfer import common, upload_state @@ -60,6 +70,20 @@ _monotonic_clock = time.monotonic +AsyncTransport = Union["aiohttp.ClientSession", "AsyncAuthorizedSession"] +"""Asynchronous HTTP transports accepted by :class:`AsyncResumableUploadSession`. + +``google.auth.aio.transport.sessions.AsyncAuthorizedSession`` is the transport +used by generated asynchronous REST clients. +""" + +# google-auth >= 2.49.0 accepts ``total_attempts`` to cap AsyncAuthorizedSession's +# own status-code retries; older releases forward unknown kwargs to aiohttp. +_HAS_TOTAL_ATTEMPTS = _HAS_GOOGLE_AUTH_AIO and ( + "total_attempts" in inspect.signature(AsyncAuthorizedSession.request).parameters +) + + def _get_buffer_size(stream: object) -> Optional[int]: """Returns buffer size in bytes if stream exposes getbuffer(), else None.""" getbuffer_fn = getattr(stream, "getbuffer", None) @@ -166,7 +190,7 @@ def __init__( self, upload_url: Optional[str] = None, config: Optional[ResumableUploadConfig] = None, - transport: Optional[Any] = None, + transport: Optional[AsyncTransport] = None, content_type: Optional[str] = None, response_type: Optional[Any] = None, start_retry: Optional[google.api_core.retry.AsyncRetry] = None, @@ -180,7 +204,7 @@ def __init__( new upload, or the pre-existing upload session URL when resuming. config: Optional upload configuration parameters. Defaults to ``ResumableUploadConfig()`` when ``None``. - transport: Optional aiohttp.ClientSession. When ``None``, a + transport: Optional :data:`AsyncTransport`. When ``None``, a transport must be provided to ``upload()`` or ``resume()``. content_type: Optional MIME type of the stream payload. When ``None``, no content-type header is sent unless overridden. @@ -283,6 +307,93 @@ def _ensure_aiohttp(self) -> None: "Please install google-api-core[async_rest]." ) + def _get_transport(self, transport: Optional[AsyncTransport]) -> AsyncTransport: + """Returns the transport to use, raising ValueError when none is set.""" + sess = transport or self._transport + if sess is None: + raise ValueError( + "An aiohttp.ClientSession or AsyncAuthorizedSession transport " + "must be provided." + ) + return sess + + async def _send_request( + self, + transport: AsyncTransport, + method: str, + url: str, + payload: bytes, + headers: Mapping[str, str], + timeout: float, + ) -> Tuple[int, Mapping[str, str], bytes]: + """Sends one request over ``transport`` and reads the whole response. + + Args: + transport: The transport to dispatch the request over. + method: HTTP method verb. + url: Request URL. + payload: Request body bytes. + headers: Request headers. + timeout: Total timeout in seconds for this attempt. + + Returns: + Tuple of (status code, response headers, response body bytes). + + Raises: + exceptions.GoogleAPICallError: If the status is not 200 or 201. + asyncio.TimeoutError: If the attempt times out, whichever transport + is used. + """ + if _HAS_GOOGLE_AUTH_AIO and isinstance(transport, AsyncAuthorizedSession): + # google-auth applies its own overall deadline (``max_allowed_time``, + # 180s by default) on top of the per-request ``timeout``; use the + # caller's timeout for both. Where supported, a single attempt also + # disables google-auth's own status-code retries so that this module + # is the only retry layer. + retry_kwargs: Dict[str, Any] = ( + {"total_attempts": 1} if _HAS_TOTAL_ATTEMPTS else {} + ) + try: + response = await transport.request( + method, + url, + data=payload, + headers=headers, + timeout=timeout, + max_allowed_time=timeout, + **retry_kwargs, + ) + try: + status_code = response.status_code + resp_headers: Mapping[str, str] = dict(response.headers) + body = await response.read() + finally: + await response.close() + except auth_exceptions.TimeoutError as exc: + # google-auth raises its own TimeoutError (not an asyncio one, and + # not reliably chained to one); re-raise it as asyncio.TimeoutError + # so stall control sees the same type as with aiohttp.ClientSession. + raise asyncio.TimeoutError(str(exc)) from exc + else: + # mypy cannot narrow the negative of the guarded check above. + session = cast("aiohttp.ClientSession", transport) + async with session.request( + method, + url, + data=payload, + headers=headers, + timeout=aiohttp.ClientTimeout(total=timeout), + ) as resp: + status_code = resp.status + resp_headers = dict(resp.headers) + body = await resp.read() + + if status_code not in (200, 201): + raise exceptions.from_http_status( + status_code, body.decode("utf-8", errors="replace") + ) + return status_code, resp_headers, body + def _enrich_exception(self, exc: BaseException) -> None: """Attaches session diagnostic metadata to an active exception. @@ -373,7 +484,9 @@ def _get_retry_predicate( ``UnseekableStreamError``) always return ``False``. 2. Protocol-recoverable errors during chunk transfer (``RECOVERABLE_STATUS_CODES`` and ``MissingStatusHeaderError``) - and transport errors (``aiohttp.ClientError`` and + and transport errors (``aiohttp.ClientError``, + ``google.auth.exceptions.TransportError``, + ``google.auth.exceptions.ResponseError``, and ``asyncio.TimeoutError``) always return ``True`` so the session can query server state and recover. @@ -396,6 +509,10 @@ def should_retry(exc: Exception) -> bool: _HAS_AIOHTTP and isinstance(exc, aiohttp.ClientError) ): return True + if _HAS_GOOGLE_AUTH_AIO and isinstance( + exc, (auth_exceptions.TransportError, auth_exceptions.ResponseError) + ): + return True if ( custom_predicate is not None and custom_predicate is not google.api_core.retry.if_transient_error @@ -460,7 +577,7 @@ def _get_async_streaming_retry( async def _initiate( self, - transport: Any, + transport: AsyncTransport, request_body: Union[str, bytes] = "", size: Optional[int] = None, progress_queue: Optional[List[UploadProgress]] = None, @@ -469,7 +586,7 @@ async def _initiate( """Initiates the upload session asynchronously. Args: - transport: The aiohttp client session. + transport: The asynchronous transport to dispatch requests over. request_body: Initial metadata payload sent with start request. size: Total stream size in bytes, if known. progress_queue: Optional queue to receive progress event. @@ -490,21 +607,15 @@ async def _initiate( ) async def do_initiate() -> str: - timeout_sec = self._get_start_timeout() - client_timeout = aiohttp.ClientTimeout(total=timeout_sec) - async with transport.request( - method, url, data=payload, headers=headers, timeout=client_timeout - ) as resp: - resp_headers = dict(resp.headers) - body = await resp.read() - if resp.status not in (200, 201): - raise exceptions.from_http_status( - resp.status, body.decode("utf-8", errors="replace") - ) - session_url = self._state.process_start_response( - resp.status, resp_headers - ) - return session_url + status_code, resp_headers, _ = await self._send_request( + transport, + method, + url, + payload, + headers, + timeout=self._get_start_timeout(), + ) + return self._state.process_start_response(status_code, resp_headers) retry_policy = self._get_async_retry() retryable_initiate = retry_policy(do_initiate) @@ -514,7 +625,7 @@ async def do_initiate() -> str: async def _transmit_chunk( self, - transport: Any, + transport: AsyncTransport, reader_fn: Callable[[int], Awaitable[bytes]], size: Optional[int], progress_queue: Optional[List[UploadProgress]] = None, @@ -523,7 +634,7 @@ async def _transmit_chunk( """Transmits the next data chunk asynchronously with stall control. Args: - transport: The aiohttp client session. + transport: The asynchronous transport to dispatch requests over. reader_fn: Async callable returning chunk bytes. size: Total stream size in bytes, if known. progress_queue: Optional queue to receive progress updates. @@ -570,23 +681,16 @@ async def _transmit_chunk( per_attempt_timeout = self._compute_chunk_timeout( data_len, timeout_override=timeout ) - client_timeout = aiohttp.ClientTimeout(total=per_attempt_timeout) t_start = _monotonic_clock() try: - async with transport.request( + status_code, resp_headers, resp_body = await self._send_request( + transport, method, url, - data=payload, - headers=headers, - timeout=client_timeout, - ) as resp: - resp_headers = dict(resp.headers) - resp_body = await resp.read() - if resp.status not in (200, 201): - raise exceptions.from_http_status( - resp.status, resp_body.decode("utf-8", errors="replace") - ) - status_code = resp.status + payload, + headers, + timeout=per_attempt_timeout, + ) t_elapsed = _monotonic_clock() - t_start except Exception as exc: self._enrich_exception(exc) @@ -691,14 +795,14 @@ def _update_stall_control(self, data_len: int, t_elapsed: float) -> None: async def _recover( self, - transport: Any, + transport: AsyncTransport, stream_obj: Optional[object] = None, progress_queue: Optional[List[UploadProgress]] = None, ) -> Tuple[int, Mapping[str, str], bytes]: """Queries server for committed byte offset and adjusts buffer. Args: - transport: The aiohttp client session. + transport: The asynchronous transport to dispatch requests over. stream_obj: Underlying stream object to rewind if seekable. progress_queue: Optional queue to receive progress updates. @@ -710,18 +814,14 @@ async def _recover( exceptions.GoogleAPICallError: If query request fails on the server. """ method, url, headers, payload = self._state.build_query_request() - timeout_sec = self._get_start_timeout() - client_timeout = aiohttp.ClientTimeout(total=timeout_sec) - async with transport.request( - method, url, data=payload, headers=headers, timeout=client_timeout - ) as resp: - resp_headers = dict(resp.headers) - body = await resp.read() - if resp.status not in (200, 201): - raise exceptions.from_http_status( - resp.status, body.decode("utf-8", errors="replace") - ) - status_code = resp.status + status_code, resp_headers, body = await self._send_request( + transport, + method, + url, + payload, + headers, + timeout=self._get_start_timeout(), + ) received = self._state.process_query_response(status_code, resp_headers) self._notify_progress(common.ProgressState.OFFSET_RECEIVED, progress_queue) @@ -767,37 +867,32 @@ async def _recover( chunk_size=self.chunk_size, ) - async def cancel(self, transport: Optional[Any] = None) -> None: + async def cancel(self, transport: Optional[AsyncTransport] = None) -> None: """Cancels the resumable upload session asynchronously. Args: - transport: Optional aiohttp client session. + transport: Optional aiohttp.ClientSession or AsyncAuthorizedSession. Raises: ValueError: If transport is missing. exceptions.GoogleAPICallError: If cancellation request fails on the server. """ self._ensure_aiohttp() - sess = transport or self._transport - if sess is None: - raise ValueError("An aiohttp.ClientSession transport must be provided.") + sess = self._get_transport(transport) method, url, headers, payload = self._state.build_cancel_request() - timeout_sec = self._get_start_timeout() - client_timeout = aiohttp.ClientTimeout(total=timeout_sec) - async with sess.request( - method, url, data=payload, headers=headers, timeout=client_timeout - ) as resp: - resp_headers = dict(resp.headers) - body = await resp.read() - if resp.status not in (200, 201): - raise exceptions.from_http_status( - resp.status, body.decode("utf-8", errors="replace") - ) - self._state.process_cancel_response(resp.status, resp_headers) + status_code, resp_headers, _ = await self._send_request( + sess, + method, + url, + payload, + headers, + timeout=self._get_start_timeout(), + ) + self._state.process_cancel_response(status_code, resp_headers) async def _transmit_all_chunks( self, - transport: Any, + transport: AsyncTransport, reader_fn: Callable[[int], Awaitable[bytes]], computed_size: Optional[int], progress_queue: Optional[List[UploadProgress]] = None, @@ -808,7 +903,7 @@ async def _transmit_all_chunks( """Transmits chunks until completion using a single outer AsyncStreamingRetry coordinator. Args: - transport: The aiohttp client session. + transport: The asynchronous transport to dispatch requests over. reader_fn: Async callable returning chunk bytes. computed_size: Total stream size in bytes, if known. progress_queue: Optional list receiving UploadProgress snapshots. @@ -894,7 +989,7 @@ def upload( stream: Union[AsyncIterable[bytes], BinaryIO, bytes, Iterable[bytes]], request_body: Optional[Union[str, bytes]] = None, size: Optional[int] = None, - transport: Optional[Any] = None, + transport: Optional[AsyncTransport] = None, content_type: Optional[str] = None, retry: Optional[google.api_core.retry.AsyncStreamingRetry] = None, timeout: Optional[float] = None, @@ -907,7 +1002,7 @@ def upload( When ``None``, the ``request_body`` supplied to the session constructor (if any) is used. size: Total stream size in bytes, if known. - transport: Optional aiohttp client session. + transport: Optional aiohttp.ClientSession or AsyncAuthorizedSession. content_type: Optional MIME type of the stream payload. retry: Optional retry configuration (``AsyncStreamingRetry``) for chunk upload requests. Use this to customize exponential backoff timing between chunk retries or to @@ -928,9 +1023,7 @@ def upload( ValueError: If transport is missing. """ self._ensure_aiohttp() - sess = transport or self._transport - if sess is None: - raise ValueError("An aiohttp.ClientSession transport must be provided.") + sess = self._get_transport(transport) if content_type is not None: self._content_type = content_type @@ -974,7 +1067,7 @@ def resume( stream: Union[AsyncIterable[bytes], BinaryIO, bytes, Iterable[bytes]], size: Optional[int] = None, chunk_size: Optional[int] = None, - transport: Optional[Any] = None, + transport: Optional[AsyncTransport] = None, retry: Optional[google.api_core.retry.AsyncStreamingRetry] = None, timeout: Optional[float] = None, ) -> AsyncUploadOperation: @@ -986,7 +1079,7 @@ def resume( Data payload to resume uploading. size (Optional[int]): Total stream size in bytes, if known. chunk_size (Optional[int]): Optional chunk size override in bytes. - transport (Optional[Any]): Optional aiohttp client session. + transport (Optional[AsyncTransport]): Optional transport override. retry (Optional[google.api_core.retry.AsyncStreamingRetry]): Optional retry configuration (``AsyncStreamingRetry``) for chunk upload requests. Use this to customize exponential backoff timing @@ -1009,9 +1102,7 @@ def resume( ValueError: If transport, upload_url, or stream is missing. """ self._ensure_aiohttp() - sess = transport or self._transport - if sess is None: - raise ValueError("An aiohttp.ClientSession transport must be provided.") + sess = self._get_transport(transport) actual_url = upload_url or self.upload_url if not actual_url: raise ValueError("An upload URL must be provided to resume.") diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 4b799647946b..a31946bd96e4 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -14,8 +14,11 @@ """Asynchronous tests for Resumable Upload protocol implementation.""" +# mypy: disable-error-code="arg-type" + import asyncio import datetime +import inspect import io import json from typing import ( @@ -32,6 +35,7 @@ from unittest import mock import pytest +from google.auth import exceptions as auth_exceptions from google.protobuf import empty_pb2 from google.api_core import exceptions @@ -52,6 +56,8 @@ try: import aiohttp # noqa: F401 import google.auth.aio.transport # noqa: F401 + from google.auth.aio import credentials as aio_credentials + from google.auth.aio.transport import sessions as aio_sessions GOOGLE_AUTH_AIO_INSTALLED = True except ImportError: @@ -1417,12 +1423,12 @@ def test_async_transport_missing_errors() -> None: upload_url="https://api.example.com/start", ) with pytest.raises( - ValueError, match="An aiohttp.ClientSession transport must be provided" + ValueError, match="or AsyncAuthorizedSession transport must be provided" ): session.upload(stream=b"data") with pytest.raises( - ValueError, match="An aiohttp.ClientSession transport must be provided" + ValueError, match="or AsyncAuthorizedSession transport must be provided" ): session.resume(upload_url="https://upload.example.com/123", stream=b"data") @@ -1434,7 +1440,7 @@ async def test_async_cancel_missing_transport_and_error() -> None: upload_url="https://upload.example.com/123", ) with pytest.raises( - ValueError, match="An aiohttp.ClientSession transport must be provided" + ValueError, match="or AsyncAuthorizedSession transport must be provided" ): await session.cancel() @@ -2384,3 +2390,204 @@ def test_async_retry_predicate_includes_asyncio_timeout_error() -> None: session = AsyncResumableUploadSession(upload_url="https://api.example.com/start") predicate = session._get_retry_predicate() assert predicate(asyncio.TimeoutError()) is True + + +# ===================================================================== +# 11. google.auth AsyncAuthorizedSession Transport Tests +# ===================================================================== + + +START_HEADERS = { + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/123", +} + + +def _auth_response( + status_code: int = 200, + headers: Optional[Mapping[str, str]] = None, + body: bytes = b"", +) -> mock.Mock: + """Builds a ``google.auth.aio.transport.Response`` double.""" + return mock.Mock( + status_code=status_code, + headers=dict(headers or {}), + read=mock.AsyncMock(return_value=body), + close=mock.AsyncMock(), + ) + + +def _authorized_session( + *outcomes: Union[mock.Mock, BaseException], +) -> Tuple["aio_sessions.AsyncAuthorizedSession", mock.AsyncMock]: + """Builds a real AsyncAuthorizedSession whose HTTP adapter replays ``outcomes``. + + google-auth invokes the adapter positionally as + ``(url, method, body, headers, timeout, **kwargs)``. + """ + auth_request = mock.AsyncMock(side_effect=list(outcomes)) + transport = aio_sessions.AsyncAuthorizedSession( + aio_credentials.AnonymousCredentials(), auth_request=auth_request + ) + return transport, auth_request + + +def _adapter_commands(auth_request: mock.AsyncMock) -> List[str]: + return [c.args[3]["X-Goog-Upload-Command"] for c in auth_request.call_args_list] + + +def _supports_total_attempts() -> bool: + """Returns True if google-auth (>= 2.49.0) accepts ``total_attempts``.""" + params = inspect.signature(aio_sessions.AsyncAuthorizedSession.request).parameters + return "total_attempts" in params + + +@pytest.mark.asyncio +async def test_async_authorized_session_upload() -> None: + """Uploads through a real AsyncAuthorizedSession (the generated REST client transport). + + Unlike aiohttp.ClientSession, its ``request()`` is a coroutine returning a + ``google.auth.aio.transport.Response`` (``status_code``, explicit ``close()``). + """ + responses = [ + _auth_response(200, START_HEADERS), + _auth_response(200, {"X-Goog-Upload-Status": "active"}), + _auth_response( + 200, {"X-Goog-Upload-Status": "final"}, b'{"name": "auth.txt", "size": 6}' + ), + ] + transport, auth_request = _authorized_session(*responses) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(chunk_size=4), + transport=transport, + response_type=DummyResponse, + start_timeout=7.5, + ) + + with mock.patch.object(transport, "request", wraps=transport.request) as spy: + result = await session.upload(stream=b"012345") + + assert isinstance(result, DummyResponse) + assert (result.name, result.size) == ("auth.txt", 6) + assert session.bytes_uploaded == 6 + assert _adapter_commands(auth_request) == ["start", "upload", "upload, finalize"] + assert [c.args[2] for c in auth_request.call_args_list] == [b"", b"0123", b"45"] + assert [c.args[0] for c in auth_request.call_args_list] == [ + "https://api.example.com/start", + "https://upload.example.com/123", + "https://upload.example.com/123", + ] + for response in responses: + response.close.assert_awaited_once() + # api_core owns the timeout: google-auth's wall-clock guard is bounded by the + # same per-attempt timeout, and nothing extra leaks through to the adapter. + assert spy.call_args_list[0].kwargs["timeout"] == 7.5 + expected_total_attempts = 1 if _supports_total_attempts() else None + for call in spy.call_args_list: + assert call.kwargs["max_allowed_time"] == call.kwargs["timeout"] + assert call.kwargs.get("total_attempts") == expected_total_attempts + assert all(c.kwargs == {} for c in auth_request.call_args_list) + + +@pytest.mark.asyncio +async def test_async_authorized_session_server_error_not_retried_internally() -> None: + """Verifies a 5xx reaches api_core's retry layer instead of google-auth's. + + AsyncAuthorizedSession retries retryable status codes itself by default; + api_core passes ``total_attempts=1`` (google-auth >= 2.49.0) so that its + protocol-aware retry and offset recovery is the single retry layer. + """ + if not _supports_total_attempts(): + pytest.skip("google-auth < 2.49.0 retries 5xx responses internally") + response = _auth_response(503, body=b"Service Unavailable") + transport, auth_request = _authorized_session(response) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=transport, + start_retry=google.api_core.retry.AsyncRetry(predicate=lambda exc: False), + ) + + with pytest.raises(exceptions.ServiceUnavailable, match="Service Unavailable"): + await session.upload(stream=b"data") + + assert auth_request.await_count == 1 + response.close.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("on_read", [False, True], ids=["request", "read"]) +async def test_async_authorized_session_timeout_raises_transfer_stalled( + on_read: bool, +) -> None: + """Verifies google.auth.exceptions.TimeoutError feeds stall control. + + It is not a subclass of asyncio.TimeoutError, so it is normalized to one + (keeping the original as the cause) to surface as TransferStalledError. + """ + read_resp = _auth_response(200, {"X-Goog-Upload-Status": "final"}) + read_resp.read.side_effect = auth_exceptions.TimeoutError("read timed out") + chunk_outcome: Any = ( + read_resp if on_read else auth_exceptions.TimeoutError("timed out") + ) + transport, auth_request = _authorized_session( + _auth_response(200, START_HEADERS), chunk_outcome + ) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", transport=transport + ) + no_retry = google.api_core.retry.AsyncStreamingRetry( + predicate=lambda e: False, timeout=0.001 + ) + + with pytest.raises(TransferStalledError) as exc_info: + await session.upload(stream=b"data", retry=no_retry) + + cause = exc_info.value.__cause__ + assert isinstance(cause, asyncio.TimeoutError) + assert isinstance(cause.__cause__, auth_exceptions.TimeoutError) + assert _adapter_commands(auth_request) == ["start", "upload, finalize"] + if on_read: + read_resp.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_authorized_session_transport_error_recovers() -> None: + """Verifies google.auth.exceptions.TransportError triggers offset recovery.""" + transport, auth_request = _authorized_session( + _auth_response(200, START_HEADERS), + auth_exceptions.TransportError("connection reset"), + _auth_response( + 200, {"X-Goog-Upload-Status": "active", "X-Goog-Upload-Size-Received": "0"} + ), + _auth_response(200, {"X-Goog-Upload-Status": "final"}, b'{"name": "f"}'), + ) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=transport, + response_type=DummyResponse, + ) + + result = await session.upload( + stream=b"data", + retry=google.api_core.retry.AsyncStreamingRetry(initial=0.01, maximum=0.01), + ) + + assert isinstance(result, DummyResponse) + assert result.name == "f" + assert _adapter_commands(auth_request) == [ + "start", + "upload, finalize", + "query", + "upload, finalize", + ] + + +def test_async_retry_predicate_includes_google_auth_errors() -> None: + """Verifies google-auth transport errors are retryable, unlike its other errors.""" + session = AsyncResumableUploadSession(upload_url="https://api.example.com/start") + for is_start in (True, False): + predicate = session._get_retry_predicate(is_start=is_start) + assert predicate(auth_exceptions.TransportError()) is True + assert predicate(auth_exceptions.ResponseError()) is True + assert predicate(auth_exceptions.RefreshError()) is False From 41480cb9acfdf19d7d44af69a8e682901f2a2d16 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 08:32:43 -0700 Subject: [PATCH 009/228] chore(main): release google-auth 2.60.0 (#18552) :robot: I have created a release *beep* *boop* --- ## [2.60.0](https://github.com/googleapis/google-cloud-python/compare/google-auth-v2.59.1...google-auth-v2.60.0) (2026-10-02) ### Features * **auth:** Support sync credentials in AsyncAuthorizedSession ([#18542](https://github.com/googleapis/google-cloud-python/issues/18542)) ([3799568](https://github.com/googleapis/google-cloud-python/commit/379956865248ebc40e5b851ede520056e65abcac)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- .release-please-individual-manifest.json | 2 +- librarian.yaml | 2 +- packages/google-auth/CHANGELOG.md | 7 +++++++ packages/google-auth/google/auth/version.py | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.release-please-individual-manifest.json b/.release-please-individual-manifest.json index 66faf125698f..3a2563fc5238 100644 --- a/.release-please-individual-manifest.json +++ b/.release-please-individual-manifest.json @@ -1,5 +1,5 @@ { - "packages/google-auth": "2.59.1", + "packages/google-auth": "2.60.0", "packages/google-cloud-apptopology": "0.1.0", "packages/google-cloud-ftp": "0.1.0", "packages/google-cloud-spanner": "3.71.0", diff --git a/librarian.yaml b/librarian.yaml index c3196643dd04..6b618d3a7b0d 100644 --- a/librarian.yaml +++ b/librarian.yaml @@ -245,7 +245,7 @@ libraries: metadata_name_override: area120tables default_version: v1alpha1 - name: google-auth - version: 2.59.1 + version: 2.60.0 python: library_type: AUTH - name: google-auth-httplib2 diff --git a/packages/google-auth/CHANGELOG.md b/packages/google-auth/CHANGELOG.md index 1f51b3607fa8..7ed2320daf60 100644 --- a/packages/google-auth/CHANGELOG.md +++ b/packages/google-auth/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-auth/#history +## [2.60.0](https://github.com/googleapis/google-cloud-python/compare/google-auth-v2.59.1...google-auth-v2.60.0) (2026-10-02) + + +### Features + +* **auth:** Support sync credentials in AsyncAuthorizedSession ([#18542](https://github.com/googleapis/google-cloud-python/issues/18542)) ([3799568](https://github.com/googleapis/google-cloud-python/commit/379956865248ebc40e5b851ede520056e65abcac)) + ## [2.59.1](https://github.com/googleapis/google-cloud-python/compare/google-auth-v2.59.0...google-auth-v2.59.1) (2026-09-30) diff --git a/packages/google-auth/google/auth/version.py b/packages/google-auth/google/auth/version.py index b613318b89f2..92bcf75ef2f2 100644 --- a/packages/google-auth/google/auth/version.py +++ b/packages/google-auth/google/auth/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.59.1" +__version__ = "2.60.0" From 8792da2d4f67ddcf7370007eb742164754e90b6e Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 09:23:09 -0700 Subject: [PATCH 010/228] chore: release main (#18551) :robot: I have created a release *beep* *boop* ---
google-api-core: 2.41.0 ## [2.41.0](https://github.com/googleapis/google-cloud-python/compare/google-api-core-v2.40.0...google-api-core-v2.41.0) (2026-10-02) ### Features * **api_core:** resumable upload supports request_body in constructor ([#18543](https://github.com/googleapis/google-cloud-python/issues/18543)) ([97bc619](https://github.com/googleapis/google-cloud-python/commit/97bc619323b524f8e686bb27c43bd7275f41c239)) * **api-core:** support AsyncAuthorizedSession transport for resumable uploads ([#18544](https://github.com/googleapis/google-cloud-python/issues/18544)) ([9b96af7](https://github.com/googleapis/google-cloud-python/commit/9b96af76254c33b9c3e0ceb390bbaeab2e33bbea)) * **auth:** Support sync credentials in AsyncAuthorizedSession ([#18542](https://github.com/googleapis/google-cloud-python/issues/18542)) ([3799568](https://github.com/googleapis/google-cloud-python/commit/379956865248ebc40e5b851ede520056e65abcac))
--- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- .release-please-bulk-manifest.json | 2 +- librarian.yaml | 2 +- packages/google-api-core/CHANGELOG.md | 9 +++++++++ packages/google-api-core/google/api_core/version.py | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.release-please-bulk-manifest.json b/.release-please-bulk-manifest.json index 08913f4c7cd7..2f993b820d09 100644 --- a/.release-please-bulk-manifest.json +++ b/.release-please-bulk-manifest.json @@ -10,7 +10,7 @@ "packages/google-ai-generativelanguage": "0.12.1", "packages/google-analytics-admin": "0.30.2", "packages/google-analytics-data": "0.23.3", - "packages/google-api-core": "2.40.0", + "packages/google-api-core": "2.41.0", "packages/google-apps-card": "0.7.1", "packages/google-apps-chat": "0.10.7", "packages/google-apps-events-subscriptions": "0.6.1", diff --git a/librarian.yaml b/librarian.yaml index 6b618d3a7b0d..4eebd494b8bc 100644 --- a/librarian.yaml +++ b/librarian.yaml @@ -152,7 +152,7 @@ libraries: metadata_name_override: analyticsdata default_version: v1beta - name: google-api-core - version: 2.40.0 + version: 2.41.0 python: library_type: CORE - name: google-apps-card diff --git a/packages/google-api-core/CHANGELOG.md b/packages/google-api-core/CHANGELOG.md index 0980fa798000..93b55a680fe9 100644 --- a/packages/google-api-core/CHANGELOG.md +++ b/packages/google-api-core/CHANGELOG.md @@ -4,6 +4,15 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.41.0](https://github.com/googleapis/google-cloud-python/compare/google-api-core-v2.40.0...google-api-core-v2.41.0) (2026-10-02) + + +### Features + +* **api_core:** resumable upload supports request_body in constructor ([#18543](https://github.com/googleapis/google-cloud-python/issues/18543)) ([97bc619](https://github.com/googleapis/google-cloud-python/commit/97bc619323b524f8e686bb27c43bd7275f41c239)) +* **api-core:** support AsyncAuthorizedSession transport for resumable uploads ([#18544](https://github.com/googleapis/google-cloud-python/issues/18544)) ([9b96af7](https://github.com/googleapis/google-cloud-python/commit/9b96af76254c33b9c3e0ceb390bbaeab2e33bbea)) +* **auth:** Support sync credentials in AsyncAuthorizedSession ([#18542](https://github.com/googleapis/google-cloud-python/issues/18542)) ([3799568](https://github.com/googleapis/google-cloud-python/commit/379956865248ebc40e5b851ede520056e65abcac)) + ## [2.40.0](https://github.com/googleapis/google-cloud-python/compare/google-api-core-v2.39.0...google-api-core-v2.40.0) (2026-09-29) diff --git a/packages/google-api-core/google/api_core/version.py b/packages/google-api-core/google/api_core/version.py index b78bb8c271ea..f540629e2b97 100644 --- a/packages/google-api-core/google/api_core/version.py +++ b/packages/google-api-core/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.40.0" +__version__ = "2.41.0" From 3f8d00bf5d95337a97268bcbdd7ef1ee155f7516 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 5 Oct 2026 20:19:09 -0400 Subject: [PATCH 011/228] feat: generate client, transport, and unit test templates for resumable uploads (#18495) - Add client, transport, and unit test template support for resumable upload RPCs. - Resumable upload methods accept an optional `ResumableUploadConfig` on the client and return `ResumableUploadSession` / `AsyncResumableUploadSession` from `google.api_core.resumable_transfer`. Since resumable uploads use HTTP, the gRPC and gRPC-async transports delegate resumable upload calls to the corresponding REST transport. --- .../gapic/generator/generator.py | 27 +- .../gapic-generator/gapic/schema/wrappers.py | 27 + .../%sub/services/%service/_client_macros.j2 | 11 + .../%sub/services/%service/_shared_macros.j2 | 84 ++- .../%sub/services/%service/async_client.py.j2 | 14 + .../%sub/services/%service/client.py.j2 | 15 +- .../%service/transports/__init__.py.j2 | 6 +- .../services/%service/transports/base.py.j2 | 14 + .../services/%service/transports/grpc.py.j2 | 55 +- .../%service/transports/grpc_asyncio.py.j2 | 76 ++- .../services/%service/transports/rest.py.j2 | 37 +- .../%service/transports/rest_asyncio.py.j2 | 52 +- .../%service/transports/rest_base.py.j2 | 2 +- .../gapic/templates/setup.py.j2 | 9 +- .../constraints-3.10-async-rest.txt.j2 | 9 +- .../templates/testing/constraints-3.10.txt.j2 | 4 + .../%name_%version/%sub/test_%service.py.j2 | 525 +++++++++++++++++- .../gapic/%name_%version/%sub/test_macros.j2 | 69 ++- packages/gapic-generator/noxfile.py | 117 +++- packages/gapic-generator/requirements.txt | 12 +- .../redis_v1/services/cloud_redis/client.py | 12 +- .../cloud_redis/transports/__init__.py | 6 +- .../cloud_redis/transports/rest_asyncio.py | 5 +- .../tests/integration/goldens/redis/setup.py | 3 +- .../testing/constraints-3.10-async-rest.txt | 3 +- .../redis_v1/services/cloud_redis/client.py | 12 +- .../cloud_redis/transports/__init__.py | 6 +- .../cloud_redis/transports/rest_asyncio.py | 5 +- .../goldens/redis_selective/setup.py | 3 +- .../testing/constraints-3.10-async-rest.txt | 3 +- .../resumable_upload_service/async_client.py | 13 +- .../resumable_upload_service/client.py | 13 +- .../transports/base.py | 7 +- .../transports/grpc.py | 47 +- .../transports/grpc_asyncio.py | 71 ++- .../transports/rest.py | 163 ++---- .../transports/rest_asyncio.py | 280 ++++++++++ .../transports/rest_base.py | 2 +- ...ppet_metadata_google.showcase.v1beta1.json | 4 +- .../integration/goldens/showcase/setup.py | 6 +- .../testing/constraints-3.10-async-rest.txt | 14 + .../showcase/testing/constraints-3.10.txt | 2 +- .../test_resumable_upload_service.py | 393 ++++++++++++- .../tests/unit/generator/test_generator.py | 74 +++ .../tests/unit/schema/wrappers/test_method.py | 17 +- 45 files changed, 2040 insertions(+), 289 deletions(-) create mode 100755 packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/rest_asyncio.py create mode 100755 packages/gapic-generator/tests/integration/goldens/showcase/testing/constraints-3.10-async-rest.txt diff --git a/packages/gapic-generator/gapic/generator/generator.py b/packages/gapic-generator/gapic/generator/generator.py index 50a0bc649dcd..76bd9ed4ccee 100644 --- a/packages/gapic-generator/gapic/generator/generator.py +++ b/packages/gapic-generator/gapic/generator/generator.py @@ -342,7 +342,9 @@ def _render_template( ) or ( "transport" in template_name - and not self._is_desired_transport(template_name, opts) + and not self._is_desired_transport( + template_name, opts, service=service + ) ) or # TODO(https://github.com/googleapis/gapic-generator-python/issues/2121): Remove this condition when async rest is GA. @@ -358,8 +360,16 @@ def _render_template( and not api_schema.all_library_settings[ api_schema.naming.proto_package ].python_settings.experimental_features.rest_async_io_enabled + and not ( + "grpc" in opts.transport + and service.has_resumable_upload_methods + ) + ) + or ( + "rest_base" in template_name + and "rest" not in opts.transport + and not service.has_resumable_upload_methods ) - or ("rest_base" in template_name and "rest" not in opts.transport) ): continue @@ -386,9 +396,20 @@ def _render_template( ) return answer - def _is_desired_transport(self, template_name: str, opts: Options) -> bool: + def _is_desired_transport( + self, + template_name: str, + opts: Options, + service: Optional[Any] = None, + ) -> bool: """Returns true if template name contains a desired transport""" desired_transports = ["__init__", "base", "README"] + opts.transport + if ( + service is not None + and service.has_resumable_upload_methods + and "rest" not in desired_transports + ): + desired_transports.append("rest") return any(transport in template_name for transport in desired_transports) def _get_file( diff --git a/packages/gapic-generator/gapic/schema/wrappers.py b/packages/gapic-generator/gapic/schema/wrappers.py index a26f40ff4c45..778800b076a5 100644 --- a/packages/gapic-generator/gapic/schema/wrappers.py +++ b/packages/gapic-generator/gapic/schema/wrappers.py @@ -1640,6 +1640,28 @@ def _client_output(self, enable_asyncio: bool): ) ) + # If this method is a resumable upload, return a PythonType instance + # representing the resumable upload session (while self.output remains + # the underlying protobuf response message for final deserialization). + if self.is_resumable_upload: + return PythonType( + meta=metadata.Metadata( + address=metadata.Address( + name=( + "AsyncResumableUploadSession" + if enable_asyncio + else "ResumableUploadSession" + ), + module="resumable_transfer", + package=("google", "api_core"), + collisions=self.input.ident.collisions, + ), + documentation=utils.doc( + "An object representing a resumable upload session." + ), + ), + ) + # Return the usual output. return self.output @@ -1943,6 +1965,11 @@ def _ref_types(self, recursive: bool) -> Sequence[Union[MessageType, EnumType]]: if self.paged_result_field and self.paged_result_field.message: answer.append(self.paged_result_field.message) + # If this method is a resumable upload, client_output is ResumableUploadSession, + # so explicitly include self.output to ensure the underlying response message is imported. + if self.is_resumable_upload: + answer.append(self.output) + # Done; return the answer. return tuple(answer) diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_client_macros.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_client_macros.j2 index 9029ac8eca23..08da6aaa052b 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_client_macros.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_client_macros.j2 @@ -28,6 +28,9 @@ requests: Optional[Iterator[{{ method.input.ident }}]] = None, *, {% endif %} + {% if method.is_resumable_upload %} + config: Optional[ResumableUploadConfig] = None, + {% endif %} retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, {{ shared_macros.client_method_metadata_argument()|indent(8) }} = {{ shared_macros.client_method_metadata_default_value() }}, @@ -65,6 +68,10 @@ The request object iterator.{{ " " }} {{- method.input.meta.doc|rst(width=72, indent=16, nl=False) }} {% endif %} + {% if method.is_resumable_upload %} + config (Optional[google.api_core.resumable_transfer.ResumableUploadConfig]): + Optional configuration for the resumable upload session. + {% endif %} retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -161,6 +168,10 @@ retry=retry, timeout=timeout, metadata=metadata, + {% if method.is_resumable_upload %} + config=config, + start_retry=retry, + {% endif %} ) {% if method.lro %} diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 index e39425bb8117..e6ce18bfb94c 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 @@ -132,13 +132,13 @@ from google.longrunning import operations_pb2 # type: ignore {% endif %}{# import_ns.has_operations_mixin #} {% endmacro %} -{% macro http_options_method(rules) %} +{% macro http_options_method(rules, is_resumable_upload=False, resumable_upload_prefix="resumable/upload") %} @staticmethod def _get_http_options(): http_options: List[Dict[str, str]] = [ {%- for rule in rules %}{ 'method': '{{ rule.method }}', - 'uri': '{{ rule.uri }}', + 'uri': '{% if is_resumable_upload %}/{{ resumable_upload_prefix }}{% endif %}{{ rule.uri }}', {% if rule.body %} 'body': '{{ rule.body }}', {% endif %}{# rule.body #} @@ -263,6 +263,68 @@ def _get_http_options(): {% endmacro %} +{# rest_resumable_upload_call_method_common includes the common code for a rest + resumable upload __call__ method to be re-used for sync and async REST + __call__ implementation. + + Args: + method: The method. + service: The service. + is_async (bool): Used to determine the code path i.e. whether for sync or async call. + rest_numeric_enums (bool): Used to determine whether to encode enums as numbers. #} +{% macro rest_resumable_upload_call_method_common(method, service, is_async=False, rest_numeric_enums=False) %} +{% set service_name = service.name %} +{% set method_name = method.name %} +{% set body_spec = method.http_options[0].body %} +{% set await_prefix = "await " if is_async else "" %} +{% set client_output_ident = method.client_output_async.ident if is_async else method.client_output.ident %} +{% set retry_class = "retries.AsyncRetry" if is_async else "retries.Retry" %} +http_options = _Base{{ service_name }}RestTransport._Base{{ method_name }}._get_http_options() +request, metadata = {{ await_prefix }}self._interceptor.pre_{{ method_name|snake_case }}(request, metadata) +transcoded_request, body, query_params = transcode_request( + http_options, + request, + required_fields_default_values=getattr( + _Base{{ service_name }}RestTransport._Base{{ method_name }}, + "_Base{{ method_name }}__REQUIRED_FIELDS_DEFAULT_VALUES", + None, + ), + rest_numeric_enums={{ rest_numeric_enums }}, +) + +uri = transcoded_request["uri"] +params = rest_helpers.flatten_query_params(query_params, strict=True) +query_string = f"?{urllib.parse.urlencode(params)}" if params else "" +upload_url = f"{self._host}{uri}{query_string}" +headers: Dict[str, Any] = {**dict(metadata), **dict((config.headers or {}) if config else {})} +headers["Content-Type"] = "application/json" +if config is None: + config = resumable_transfer.ResumableUploadConfig(headers=headers) +else: + config = dataclasses.replace(config, headers=headers) + +session_kwargs: Dict[str, Any] = ( + {"start_timeout": timeout} + if isinstance(timeout, (int, float)) + else {} +) +# ``start_retry`` is used instead of ``retry`` because ``_GapicCallable`` +# consumes the ``retry`` argument before invoking the transport callable +# and only forwards extra keyword arguments such as ``start_retry``. +return {{ client_output_ident }}( + upload_url=upload_url, + config=config, + transport=self._session, + response_type={{ method.output.ident }}, + start_retry=start_retry if isinstance(start_retry, {{ retry_class }}) else None, + {% if body_spec %} + request_body=body, + {% endif %} + **session_kwargs, +) +{%- endmacro %} + + {% macro unary_request_interceptor_common(service) %} logging_enabled = CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor(std_logging.DEBUG) if logging_enabled: # pragma: NO COVER @@ -299,11 +361,12 @@ def _get_http_options(): {%- endmacro %} -{% macro prep_wrapped_messages_async_method(api, service) %} +{% macro prep_wrapped_messages_async_method(api, service, is_rest_asyncio=False) %} +{% set rest_async_io_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.rest_async_io_enabled %} def _prep_wrapped_messages(self, client_info): """ Precompute the wrapped methods, overriding the base class method to use async wrappers.""" self._wrapped_methods = { - {% for method in service.methods.values() %} + {% for method in service.methods.values() if not is_rest_asyncio or rest_async_io_enabled or method.is_resumable_upload %} self.{{ method.transport_safe_name|snake_case }}: self._wrap_method( self.{{ method.transport_safe_name|snake_case }}, {% if method.retry %} @@ -329,6 +392,7 @@ def _prep_wrapped_messages(self, client_info): client_info=client_info, ), {% endfor %}{# service.methods.values() #} + {% if not is_rest_asyncio or rest_async_io_enabled %} {% for method_name in api.mixin_api_methods.keys() %} {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2197): Use `transport_safe_name` similar # to what we do for non-mixin methods above. @@ -339,6 +403,7 @@ def _prep_wrapped_messages(self, client_info): client_info=client_info, ), {% endfor %}{# method_name in api.mixin_api_methods.keys() #} + {% endif %} } {% endmacro %} @@ -362,6 +427,7 @@ def _wrap_method(self, func, *args, **kwargs): # synchronous and asynchronous rest transports #} {% macro create_interceptor_class(api, service, method, is_async=False) %} +{% set rest_async_io_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.rest_async_io_enabled %} {% set async_prefix = "async " if is_async else "" %} {% set async_method_name_prefix = "Async" if is_async else "" %} {% set async_docstring = "Asynchronous " if is_async else "" %} @@ -382,12 +448,12 @@ class {{ async_method_name_prefix }}{{ service.name }}RestInterceptor: .. code-block:: python class MyCustom{{ service.name }}Interceptor({{ service.name }}RestInterceptor): - {% for _, method in service.methods|dictsort if not method.client_streaming %} + {% for _, method in service.methods|dictsort if not method.client_streaming and (not is_async or rest_async_io_enabled or method.is_resumable_upload) %} {{ async_prefix }}def pre_{{ method.name|snake_case }}(self, request, metadata): logging.log(f"Received request: {request}") return request, metadata - {% if not method.void %} + {% if not method.void and not method.is_resumable_upload %} {{ async_prefix }}def post_{{ method.name|snake_case }}(self, response): logging.log(f"Received response: {response}") return response @@ -400,7 +466,7 @@ class {{ async_method_name_prefix }}{{ service.name }}RestInterceptor: """ - {% for method in service.methods.values()|sort(attribute="name") if not method.client_streaming and method.http_options %} + {% for method in service.methods.values()|sort(attribute="name") if not method.client_streaming and method.http_options and (not is_async or rest_async_io_enabled or method.is_resumable_upload) %} {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2147): Remove the condition below once async rest transport supports the guarded methods. #} {{ async_prefix }}def pre_{{ method.name|snake_case }}(self, request: {{method.input.ident}}, {{ client_method_metadata_argument() }}) -> Tuple[{{method.input.ident}}, {{ client_method_metadata_type() }}]: """Pre-rpc interceptor for {{ method.name|snake_case }} @@ -410,7 +476,7 @@ class {{ async_method_name_prefix }}{{ service.name }}RestInterceptor: """ return request, metadata - {% if not method.void %} + {% if not method.void and not method.is_resumable_upload %} {% if not method.server_streaming %} {{ async_prefix }}def post_{{ method.name|snake_case }}(self, response: {{method.output.ident}}) -> {{method.output.ident}}: {% else %} @@ -450,6 +516,7 @@ class {{ async_method_name_prefix }}{{ service.name }}RestInterceptor: {% endif %}{# not method.void #} {% endfor %} + {% if not is_async or rest_async_io_enabled %} {% for name, signature in api.mixin_api_signatures.items() %} {{ async_prefix }}def pre_{{ name|snake_case }}( self, request: {{signature.request_type}}, {{ client_method_metadata_argument() }} @@ -473,6 +540,7 @@ class {{ async_method_name_prefix }}{{ service.name }}RestInterceptor: return response {% endfor %} + {% endif %} {% endmacro %} {% macro generate_mixin_call_method(service, api, name, sig, is_async) %} diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 index a21ba6030a3d..8d96ab289388 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 @@ -21,6 +21,9 @@ from {{package_path}} import gapic_version as package_version from google.api_core.client_options import ClientOptions from google.api_core import exceptions as core_exceptions from google.api_core import gapic_v1 +{% if service.has_resumable_upload_methods %} +from google.api_core.resumable_transfer import ResumableUploadConfig +{% endif %} {% if has_auto_populated_fields %} from {{package_path}}._compat import setup_request_id {% endif %} @@ -289,6 +292,9 @@ class {{ service.async_client_name }}: requests: Optional[AsyncIterator[{{ method.input.ident }}]] = None, *, {% endif %} + {% if method.is_resumable_upload %} + config: Optional[ResumableUploadConfig] = None, + {% endif %} retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, {{ shared_macros.client_method_metadata_argument()|indent(8) }} = {{ shared_macros.client_method_metadata_default_value() }}, @@ -324,6 +330,10 @@ class {{ service.async_client_name }}: The request object AsyncIterator.{{ " " }} {{- method.input.meta.doc|rst(width=72, indent=16, nl=False) }} {% endif %} + {% if method.is_resumable_upload %} + config (Optional[google.api_core.resumable_transfer.ResumableUploadConfig]): + Optional configuration for the resumable upload session. + {% endif %} retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -414,6 +424,10 @@ class {{ service.async_client_name }}: retry=retry, timeout=timeout, metadata=metadata, + {% if method.is_resumable_upload %} + config=config, + start_retry=retry, + {% endif %} ) {% if method.lro %} diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 index 01407a160d99..423c8d61a2fa 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 @@ -30,6 +30,9 @@ from google.api_core import exceptions as core_exceptions from google.api_core import extended_operation {% endif %} from google.api_core import gapic_v1 +{% if service.has_resumable_upload_methods %} +from google.api_core.resumable_transfer import ResumableUploadConfig +{% endif %} from {{package_path}}._compat import get_universe_domain, get_api_endpoint, get_default_mtls_endpoint, should_use_client_cert, read_environment_variables {% if has_auto_populated_fields %} from {{package_path}}._compat import setup_request_id @@ -82,14 +85,12 @@ from .transports.grpc_asyncio import {{ service.grpc_asyncio_transport_name }} from .transports.rest import {{ service.name }}RestTransport {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2121): Remove this condition when async rest is GA. #} {% if rest_async_io_enabled %} -ASYNC_REST_EXCEPTION = None try: from .transports.rest_asyncio import Async{{ service.name }}RestTransport - HAS_ASYNC_REST_DEPENDENCIES = True + HAS_ASYNC_REST_DEPENDENCIES = True # pragma: NO COVER {# NOTE: `pragma: NO COVER` is needed since the coverage for presubmits isn't combined. #} -except ImportError as e: # pragma: NO COVER +except ImportError: # pragma: NO COVER HAS_ASYNC_REST_DEPENDENCIES = False - ASYNC_REST_EXCEPTION = e {% endif %}{# if rest_async_io_enabled #} {% endif %} @@ -133,7 +134,9 @@ class {{ service.client_name }}Meta(type): {% if rest_async_io_enabled %} {# NOTE: `pragma: NO COVER` is needed since the coverage for presubmits isn't combined. #} if label == "rest_asyncio" and not HAS_ASYNC_REST_DEPENDENCIES: # pragma: NO COVER - raise ASYNC_REST_EXCEPTION + raise ImportError( + "`rest_asyncio` transport requires the library to be installed with the `async_rest` extra. Install the library with the `async_rest` extra using `pip install {{ api.naming.warehouse_package_name }}[async_rest]`" + ) {% endif %} if label: return cls._transport_registry[label] @@ -494,7 +497,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta): else cast(Callable[..., {{ service.name }}Transport], transport) ) - if "rest_asyncio" in str(transport_init): + if "rest_asyncio" in str(transport_init): # pragma: NO COVER {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2136): Support the following parameters in async rest: #} unsupported_params = { "google.api_core.client_options.ClientOptions.credentials_file": self._client_options.credentials_file, diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/__init__.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/__init__.py.j2 index 9745b08d789c..5bba915d0d2b 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/__init__.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/__init__.py.j2 @@ -19,9 +19,9 @@ from .rest import {{ service.name }}RestInterceptor ASYNC_REST_CLASSES: Tuple[str, ...] try: from .rest_asyncio import Async{{ service.name }}RestTransport - from .rest_asyncio import Async{{ service.name }}RestInterceptor - ASYNC_REST_CLASSES = ('Async{{ service.name }}RestTransport', 'Async{{ service.name }}RestInterceptor') - HAS_REST_ASYNC = True + from .rest_asyncio import Async{{ service.name }}RestInterceptor # pragma: NO COVER + ASYNC_REST_CLASSES = ('Async{{ service.name }}RestTransport', 'Async{{ service.name }}RestInterceptor') # pragma: NO COVER + HAS_REST_ASYNC = True # pragma: NO COVER except ImportError: # pragma: NO COVER ASYNC_REST_CLASSES = () HAS_REST_ASYNC = False diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/base.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/base.py.j2 index f0cf1178da69..a19c8a242463 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/base.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/base.py.j2 @@ -16,6 +16,9 @@ from google.api_core import retry as retries {% if service.has_lro %} from google.api_core import operations_v1 {% endif %} +{% if service.has_resumable_upload_methods %} +from google.api_core import resumable_transfer +{% endif %} from google.auth import credentials as ga_credentials # type: ignore from google.oauth2 import service_account # type: ignore import google.protobuf @@ -139,6 +142,9 @@ class {{ service.name }}Transport(abc.ABC): # Save the credentials. self._credentials = credentials + {% if service.has_resumable_upload_methods %} + self._client_info = client_info + {% endif %} # Save the hostname. Default to port 443 (HTTPS) if none is specified. if ':' not in host: @@ -210,10 +216,18 @@ class {{ service.name }}Transport(abc.ABC): @property def {{ method.transport_safe_name|snake_case }}(self) -> Callable[ [{{ method.input.ident }}], + {% if method.is_resumable_upload %} + Union[ + {{ method.client_output.ident }}, + {{ method.client_output_async.ident }}, + Awaitable[{{ method.client_output_async.ident }}] + ]]: + {% else %} Union[ {{ method.output.ident }}, Awaitable[{{ method.output.ident }}] ]]: + {% endif %} raise NotImplementedError() {% endfor %} diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 index e906c9d9ea71..7edaf533e71e 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 @@ -10,6 +10,10 @@ import pickle import warnings from typing import Callable, Dict, Optional, Sequence, Tuple, Union +{% if service.has_resumable_upload_methods %} +from google.api_core import exceptions as core_exceptions +from google.api_core import resumable_transfer +{% endif %} from google.api_core import grpc_helpers {% if service.has_lro %} from google.api_core import operations_v1 @@ -49,6 +53,9 @@ from google.longrunning import operations_pb2 # type: ignore {% endif %} {% endfilter %} from .base import {{ service.name }}Transport, DEFAULT_CLIENT_INFO +{% if service.has_resumable_upload_methods %} +from .rest import {{ service.name }}RestTransport +{% endif %} try: from google.api_core import client_logging # type: ignore @@ -186,6 +193,15 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport): {% if service.has_lro %} self._operations_client: Optional[operations_v1.OperationsClient] = None {% endif %} + {% if service.has_resumable_upload_methods %} + # Resumable upload RPCs operate over HTTP/REST rather than gRPC. + # Since gRPC is the default transport for GAPIC clients and services + # mix standard gRPC RPCs with resumable upload RPCs on the same client, + # gRPC transports delegate resumable upload calls to an internal REST + # transport instance. + self._rest_transport: Optional[{{ service.name }}RestTransport] = None + self._client_cert_source_for_mtls = client_cert_source_for_mtls + {% endif %} if api_mtls_endpoint: warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) @@ -334,7 +350,11 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport): @property def {{ method.transport_safe_name|snake_case }}(self) -> Callable[ [{{ method.input.ident }}], + {% if method.is_resumable_upload %} + {{ method.client_output.ident }}]: + {% else %} {{ method.output.ident }}]: + {% endif %} r"""Return a callable for the{{ ' ' }} {{- (method.name|snake_case).replace('_',' ')|wrap( width=70, offset=40, indent=8) -}} @@ -344,7 +364,7 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport): Returns: Callable[[~.{{ method.input.name }}], - ~.{{ method.output.name }}]: + ~.{% if method.is_resumable_upload %}{{ method.client_output.name }}{% else %}{{ method.output.name }}{% endif %}]: A function that, when called, will call the underlying RPC on the server. """ @@ -353,11 +373,40 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport): # gRPC handles serialization and deserialization, so we just need # to pass in the functions for each. if '{{ method.transport_safe_name|snake_case }}' not in self._stubs: + {% if method.is_resumable_upload %} + if not self._credentials: + class _ErrorStub: + def __init__(self, error: Exception): + self._error = error + + def __call__(self, *args, **kwargs): + raise self._error + + self._stubs['{{ method.transport_safe_name|snake_case }}'] = _ErrorStub( + core_exceptions.GoogleAPICallError( + "Resumable upload methods operate over REST and cannot be invoked when the transport is initialized with a pre-constructed gRPC channel. Please supply credentials directly instead of a gRPC channel to use resumable upload functionality." + ) + ) + else: + transport = self + class _RestStub: + def __call__(self, *args, **kwargs): + if transport._rest_transport is None: + transport._rest_transport = {{ service.name }}RestTransport( + host=transport._host, + credentials=transport._credentials, + client_info=transport._client_info, + client_cert_source_for_mtls=transport._client_cert_source_for_mtls, + ) + return transport._rest_transport.{{ method.transport_safe_name|snake_case }}(*args, **kwargs) + self._stubs['{{ method.transport_safe_name|snake_case }}'] = _RestStub() + {% else %} self._stubs['{{ method.transport_safe_name|snake_case }}'] = self._logged_channel.{{ method.grpc_stub_type }}( '/{{ '.'.join(method.meta.address.package) }}.{{ service.name }}/{{ method.name }}', request_serializer={{ method.input.ident }}.{% if method.input.ident.python_import.module.endswith('_pb2') %}SerializeToString{% else %}serialize{% endif %}, response_deserializer={{ method.output.ident }}.{% if method.output.ident.python_import.module.endswith('_pb2') %}FromString{% else %}deserialize{% endif %}, ) + {% endif %} return self._stubs['{{ method.transport_safe_name|snake_case }}'] {% endfor %} @@ -445,6 +494,10 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport): def close(self): self._logged_channel.close() + {% if service.has_resumable_upload_methods %} + if self._rest_transport is not None: + self._rest_transport.close() + {% endif %} {% include '%namespace/%name_%version/%sub/services/%service/transports/_mixins.py.j2' %} diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2 index 7b8a885d227c..e597d1b55de4 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2 @@ -10,7 +10,7 @@ import json import pickle import logging as std_logging import warnings -from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union +from typing import {% if service.has_resumable_upload_methods %}Any, {% endif %}Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union from google.api_core import gapic_v1 from google.api_core import grpc_helpers_async @@ -19,6 +19,9 @@ from google.api_core import retry_async as retries {% if service.has_lro %} from google.api_core import operations_v1 {% endif %} +{% if service.has_resumable_upload_methods %} +from google.api_core import resumable_transfer +{% endif %} from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore from google.protobuf.json_format import MessageToJson @@ -54,6 +57,13 @@ from google.longrunning import operations_pb2 # type: ignore {% endfilter %} from .base import {{ service.name }}Transport, DEFAULT_CLIENT_INFO from .grpc import {{ service.name }}GrpcTransport +{% if service.has_resumable_upload_methods %} +try: + from .rest_asyncio import Async{{ service.name }}RestTransport + HAS_ASYNC_REST = True # pragma: NO COVER +except ImportError: # pragma: NO COVER + HAS_ASYNC_REST = False +{% endif %} try: from google.api_core import client_logging # type: ignore @@ -235,6 +245,15 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport): {% if service.has_lro %} self._operations_client: Optional[operations_v1.OperationsAsyncClient] = None {% endif %} + {% if service.has_resumable_upload_methods %} + # Resumable upload RPCs operate over HTTP/REST rather than gRPC. + # Since gRPC is the default transport for GAPIC clients and services + # mix standard gRPC RPCs with resumable upload RPCs on the same client, + # gRPC transports delegate resumable upload calls to an internal REST + # transport instance. + self._rest_transport: Optional[Any] = None + self._client_cert_source_for_mtls = client_cert_source_for_mtls + {% endif %} if api_mtls_endpoint: warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) @@ -339,7 +358,7 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport): @property def {{ method.transport_safe_name|snake_case }}(self) -> Callable[ [{{ method.input.ident }}], - Awaitable[{{ method.output.ident }}]]: + Awaitable[{% if method.is_resumable_upload %}{{ method.client_output_async.ident }}{% else %}{{ method.output.ident }}{% endif %}]]: r"""Return a callable for the{{ ' ' }} {{- (method.name|snake_case).replace('_',' ')|wrap( width=70, offset=40, indent=8) -}} @@ -349,7 +368,7 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport): Returns: Callable[[~.{{ method.input.name }}], - Awaitable[~.{{ method.output.name }}]]: + Awaitable[~.{% if method.is_resumable_upload %}{{ method.client_output_async.name }}{% else %}{{ method.output.name }}{% endif %}]]: A function that, when called, will call the underlying RPC on the server. """ @@ -358,11 +377,55 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport): # gRPC handles serialization and deserialization, so we just need # to pass in the functions for each. if '{{ method.transport_safe_name|snake_case }}' not in self._stubs: + {% if method.is_resumable_upload %} + class _ErrorStub: + def __init__(self, error: Exception): + self._error = error + + def __call__(self, *args, **kwargs): + async def _raise(): + raise self._error + return _raise() + + if not self._credentials: + self._stubs['{{ method.transport_safe_name|snake_case }}'] = _ErrorStub( + core_exceptions.GoogleAPICallError( + "Resumable upload methods operate over REST and cannot be invoked when the transport is initialized with a pre-constructed gRPC channel. Please supply credentials directly instead of a gRPC channel to use resumable upload functionality." + ) + ) + elif self._client_cert_source_for_mtls: + self._stubs['{{ method.transport_safe_name|snake_case }}'] = _ErrorStub( + core_exceptions.AsyncRestUnsupportedParameterError( + "Mutual TLS (client_cert_source_for_mtls) is not currently supported for async resumable upload methods." + ) + ) + elif HAS_ASYNC_REST: + transport = self + class _AsyncRestStub: + def __call__(self, *args, **kwargs): + if transport._rest_transport is None: + transport._rest_transport = Async{{ service.name }}RestTransport( + host=transport._host, + credentials=transport._credentials, + client_info=transport._client_info, + ) + return transport._rest_transport.{{ method.transport_safe_name|snake_case }}(*args, **kwargs) + self._stubs['{{ method.transport_safe_name|snake_case }}'] = _AsyncRestStub() + else: + self._stubs['{{ method.transport_safe_name|snake_case }}'] = _ErrorStub( + NotImplementedError( + "Async REST transport is required for async resumable upload methods. " + "Install the library with the `async_rest` extra using " + "`pip install {{ api.naming.warehouse_package_name }}[async_rest]`." + ) + ) + {% else %} self._stubs['{{ method.transport_safe_name|snake_case }}'] = self._logged_channel.{{ method.grpc_stub_type }}( '/{{ '.'.join(method.meta.address.package) }}.{{ service.name }}/{{ method.name }}', request_serializer={{ method.input.ident }}.{% if method.input.ident.python_import.module.endswith('_pb2') %}SerializeToString{% else %}serialize{% endif %}, response_deserializer={{ method.output.ident }}.{% if method.output.ident.python_import.module.endswith('_pb2') %}FromString{% else %}deserialize{% endif %}, ) + {% endif %} return self._stubs['{{ method.transport_safe_name|snake_case }}'] {% endfor %} @@ -453,8 +516,15 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport): {{ shared_macros.wrap_async_method_macro()|indent(4) }} + {% if service.has_resumable_upload_methods %} + async def close(self): + await self._logged_channel.close() + if self._rest_transport is not None: + await self._rest_transport.close() + {% else %} def close(self): return self._logged_channel.close() + {% endif %} @property def kind(self) -> str: diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 index 1bc499c068ee..3f4f3f4797b3 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 @@ -13,6 +13,10 @@ from google.api_core import retry as retries from google.api_core import rest_helpers from google.api_core import rest_streaming from google.api_core import gapic_v1 +{% if service.has_resumable_upload_methods %} +import urllib.parse +from google.api_core import resumable_transfer +{% endif %} {% set package_path = api.naming.module_namespace|join('.') + "." + api.naming.versioned_module_name %} from {{package_path}}._compat import transcode_request import google.protobuf @@ -213,18 +217,23 @@ class {{service.name}}RestTransport(_Base{{ service.name }}RestTransport): class {{ method.name|make_private }}(_Base{{ service.name }}RestTransport._Base{{method.name}}, {{service.name}}RestStub): def __hash__(self): return hash("{{service.name}}RestTransport.{{method.name}}") - {% if method.http_options and not method.client_streaming %} + {% if method.http_options and not method.client_streaming and not method.is_resumable_upload %} {% set body_spec = method.http_options[0].body %} {{ shared_macros.response_method(body_spec, is_async=False, is_streaming_method=method.server_streaming)|indent(8) }} - {% endif %}{# method.http_options and not method.client_streaming #} + {% endif %}{# method.http_options and not method.client_streaming and not method.is_resumable_upload #} def __call__(self, request: {{method.input.ident}}, *, + {% if method.is_resumable_upload %} + config: Optional[resumable_transfer.ResumableUploadConfig]=None, + start_retry: OptionalRetry=gapic_v1.method.DEFAULT, + {% else %} retry: OptionalRetry=gapic_v1.method.DEFAULT, + {% endif %} timeout: Optional[float]=None, {{ shared_macros.client_method_metadata_argument()|indent(8) }}={{ shared_macros.client_method_metadata_default_value() }}, - ){% if not method.void %} -> {% if not method.server_streaming %}{{method.output.ident}}{% else %}rest_streaming.ResponseIterator{% endif %}{% endif %}: + ){% if not method.void %} -> {% if method.is_resumable_upload %}{{method.client_output.ident}}{% elif not method.server_streaming %}{{method.output.ident}}{% else %}rest_streaming.ResponseIterator{% endif %}{% endif %}: {% if method.http_options and not method.client_streaming %} r"""Call the {{- ' ' -}} {{ (method.name|snake_case).replace('_',' ')|wrap( @@ -235,17 +244,32 @@ class {{service.name}}RestTransport(_Base{{ service.name }}RestTransport): request (~.{{ method.input.ident }}): The request object.{{ ' ' }} {{- method.input.meta.doc|rst(width=72, indent=16, nl=False) }} + {% if method.is_resumable_upload %} + config (Optional[google.api_core.resumable_transfer.ResumableUploadConfig]): + Optional configuration for the resumable upload session. + start_retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried when initiating the resumable upload session. + {% else %} retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. + {% endif %} timeout (float): The timeout for this request. {{ shared_macros.client_method_metadata_argument_doc()|indent(8) }} {% if not method.void %} Returns: + {% if method.is_resumable_upload %} + ~.{{ method.client_output.ident }}: + {{ method.client_output.meta.doc|rst(width=72, indent=16) }} + {% else %} ~.{{ method.output.ident }}: {{ method.output.meta.doc|rst(width=72, indent=16) }} + {% endif %} {% endif %} """ + {% if method.is_resumable_upload %} + {{ shared_macros.rest_resumable_upload_call_method_common(method, service, False, opts.rest_numeric_enums)|indent(12) }} + {% else %} {{ shared_macros.rest_call_method_common(body_spec, method.name, service, False, method.input.ident.is_proto_plus_type, opts.rest_numeric_enums)|indent(8) }} @@ -304,6 +328,7 @@ class {{service.name}}RestTransport(_Base{{ service.name }}RestTransport): return resp {% endif %}{# method.void #} + {% endif %}{# method.is_resumable_upload #} {% else %}{# method.http_options and not method.client_streaming #} raise NotImplementedError( "Method {{ method.name }} is not available over REST transport" @@ -315,7 +340,11 @@ class {{service.name}}RestTransport(_Base{{ service.name }}RestTransport): @property def {{method.transport_safe_name|snake_case}}(self) -> Callable[ [{{method.input.ident}}], - {{method.output.ident}}]: + {% if method.is_resumable_upload %} + {{ method.client_output.ident }}]: + {% else %} + {{ method.output.ident }}]: + {% endif %} # The return type is fine, but mypy isn't sophisticated enough to determine what's going on here. # In C++ this would require a dynamic_cast return self._{{method.name}}(self._session, self._host, self._interceptor) # type: ignore diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_asyncio.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_asyncio.py.j2 index 0f79d6e1ffef..8ff5ece64292 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_asyncio.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_asyncio.py.j2 @@ -17,10 +17,15 @@ try: except ImportError as e: # pragma: NO COVER raise ImportError("`rest_asyncio` transport requires the library to be installed with the `async_rest` extra. Install the library with the `async_rest` extra using `pip install {{ api.naming.warehouse_package_name }}[async_rest]`") from e +from google.auth import credentials as ga_credentials # type: ignore from google.auth.aio import credentials as ga_credentials_async # type: ignore from google.api_core import exceptions as core_exceptions from google.api_core import gapic_v1 +{% if service.has_resumable_upload_methods %} +import urllib.parse +from google.api_core import resumable_transfer +{% endif %} {% if service.has_lro %} from google.api_core import operations_v1 {% endif %}{# service.has_lro #} @@ -107,7 +112,7 @@ class Async{{service.name}}RestTransport(_Base{{ service.name }}RestTransport): def __init__(self, *, host: str{% if service.host %} = '{{ service.host }}'{% endif %}, - credentials: Optional[ga_credentials_async.Credentials] = None, + credentials: Optional[Union[ga_credentials.Credentials, ga_credentials_async.Credentials]] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, url_scheme: str = 'https', interceptor: Optional[Async{{ service.name }}RestInterceptor] = None, @@ -123,7 +128,7 @@ class Async{{service.name}}RestTransport(_Base{{ service.name }}RestTransport): Args: host ({% if service.host %}Optional[str]{% else %}str{% endif %}): {{ ' ' }}The hostname to connect to {% if service.host %}(default: '{{ service.host }}'){% endif %}. - credentials (Optional[google.auth.aio.credentials.Credentials]): The + credentials (Optional[Union[google.auth.credentials.Credentials, google.auth.aio.credentials.Credentials]]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none are specified, the client will attempt to ascertain the @@ -156,33 +161,38 @@ class Async{{service.name}}RestTransport(_Base{{ service.name }}RestTransport): self._interceptor = interceptor or Async{{ service.name }}RestInterceptor() self._wrap_with_kind = True self._prep_wrapped_messages(client_info) - {% if service.has_lro %} + {% if service.has_lro and rest_async_io_enabled %} self._operations_client: Optional[operations_v1.AsyncOperationsRestClient] = None {% endif %} - {{ shared_macros.prep_wrapped_messages_async_method(api, service)|indent(4) }} + {{ shared_macros.prep_wrapped_messages_async_method(api, service, is_rest_asyncio=True)|indent(4) }} {{ shared_macros.wrap_async_method_macro()|indent(4) }} - {% for method in service.methods.values()|sort(attribute="name") %} + {% for method in service.methods.values()|sort(attribute="name") if rest_async_io_enabled or method.is_resumable_upload %} class {{ method.name|make_private }}(_Base{{ service.name }}RestTransport._Base{{method.name}}, Async{{service.name}}RestStub): def __hash__(self): return hash("Async{{service.name}}RestTransport.{{method.name}}") {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2169): Implement client streaming method. #} - {% if method.http_options and not method.client_streaming %} + {% if method.http_options and not method.client_streaming and not method.is_resumable_upload %} {% set body_spec = method.http_options[0].body %} {{ shared_macros.response_method(body_spec, is_async=True, is_streaming_method=None)|indent(8) }} - {% endif %}{# method.http_options and not method.client_streaming and not method.paged_result_field #} + {% endif %}{# method.http_options and not method.client_streaming and not method.is_resumable_upload #} async def __call__(self, request: {{method.input.ident}}, *, + {% if method.is_resumable_upload %} + config: Optional[resumable_transfer.ResumableUploadConfig]=None, + start_retry: OptionalRetry=gapic_v1.method.DEFAULT, + {% else %} retry: OptionalRetry=gapic_v1.method.DEFAULT, + {% endif %} timeout: Optional[float]=None, {{ shared_macros.client_method_metadata_argument()|indent(12) }}={{ shared_macros.client_method_metadata_default_value() }}, {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2169): Update return type for client streaming method. #} - ){% if not method.void %} -> {% if not method.server_streaming %}{{method.output.ident}}{% else %}rest_streaming_async.AsyncResponseIterator{% endif %}{% endif %}: + ){% if not method.void %} -> {% if method.is_resumable_upload %}{{method.client_output_async.ident}}{% elif not method.server_streaming %}{{method.output.ident}}{% else %}rest_streaming_async.AsyncResponseIterator{% endif %}{% endif %}: {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2169): Implement client streaming method. #} {% if method.http_options and not method.client_streaming %} r"""Call the {{- ' ' -}} @@ -194,17 +204,32 @@ class Async{{service.name}}RestTransport(_Base{{ service.name }}RestTransport): request (~.{{ method.input.ident }}): The request object.{{ ' ' }} {{- method.input.meta.doc|rst(width=72, indent=16, nl=False) }} + {% if method.is_resumable_upload %} + config (Optional[google.api_core.resumable_transfer.ResumableUploadConfig]): + Optional configuration for the resumable upload session. + start_retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried when initiating the resumable upload session. + {% else %} retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, should be retried. + {% endif %} timeout (float): The timeout for this request. {{ shared_macros.client_method_metadata_argument_doc()|indent(8) }} {% if not method.void %} Returns: + {% if method.is_resumable_upload %} + ~.{{ method.client_output_async.ident }}: + {{ method.client_output_async.meta.doc|rst(width=72, indent=16) }} + {% else %} ~.{{ method.output.ident }}: {{ method.output.meta.doc|rst(width=72, indent=16) }} + {% endif %} {% endif %} """ + {% if method.is_resumable_upload %} + {{ shared_macros.rest_resumable_upload_call_method_common(method, service, True, opts.rest_numeric_enums)|indent(12) }} + {% else %} {{ shared_macros.rest_call_method_common(body_spec, method.name, service, True, method.input.ident.is_proto_plus_type, opts.rest_numeric_enums)|indent(8) }} @@ -260,6 +285,7 @@ class Async{{service.name}}RestTransport(_Base{{ service.name }}RestTransport): return resp {% endif %}{# method.void #} + {% endif %}{# method.is_resumable_upload #} {% else %} raise NotImplementedError( @@ -269,7 +295,7 @@ class Async{{service.name}}RestTransport(_Base{{ service.name }}RestTransport): {% endfor %} - {% if service.has_lro %} + {% if service.has_lro and rest_async_io_enabled %} @property def operations_client(self) -> AsyncOperationsRestClient: @@ -316,18 +342,24 @@ class Async{{service.name}}RestTransport(_Base{{ service.name }}RestTransport): return self._operations_client {% endif %} - {% for method in service.methods.values()|sort(attribute="name") %} + {% for method in service.methods.values()|sort(attribute="name") if rest_async_io_enabled or method.is_resumable_upload %} {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2154): Remove `type: ignore`. #} @property def {{method.transport_safe_name|snake_case}}(self) -> Callable[ [{{method.input.ident}}], + {% if method.is_resumable_upload %} + {{method.client_output_async.ident}}]: + {% else %} {{method.output.ident}}]: + {% endif %} return self._{{method.name}}(self._session, self._host, self._interceptor) # type: ignore {% endfor %} + {% if rest_async_io_enabled %} {% for name, sig in api.mixin_api_signatures.items() %} {{ shared_macros.generate_mixin_call_method(service, api, name, sig, is_async=True) | indent(4) }} {% endfor %} + {% endif %} @property def kind(self) -> str: diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_base.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_base.py.j2 index 08887c16eb51..005bdceaeff1 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_base.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_base.py.j2 @@ -120,7 +120,7 @@ class _Base{{ service.name }}RestTransport({{service.name}}Transport): {% set method_http_options = method.http_options %} - {{ shared_macros.http_options_method(method_http_options)|indent(8) }} + {{ shared_macros.http_options_method(method_http_options, method.is_resumable_upload, method.resumable_upload_prefix)|indent(8) }} {% endif %}{# method.http_options and not method.client_streaming #} {% endfor %} diff --git a/packages/gapic-generator/gapic/templates/setup.py.j2 b/packages/gapic-generator/gapic/templates/setup.py.j2 index 610fdc35dc3f..8e0ccbcfdb8d 100644 --- a/packages/gapic-generator/gapic/templates/setup.py.j2 +++ b/packages/gapic-generator/gapic/templates/setup.py.j2 @@ -1,5 +1,5 @@ {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2121): Remove the following variable (and the condition later in this file) for async rest transport once support for it is GA. #} -{% set rest_async_io_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.rest_async_io_enabled %} +{% set rest_async_io_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.rest_async_io_enabled or api.has_resumable_upload_methods %} {% set unversioned_package_disabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.unversioned_package_disabled %} {% extends '_base.py.j2' %} {% from '_pypi_packages.j2' import pypi_packages %} @@ -36,7 +36,11 @@ else: release_status = "Development Status :: 5 - Production/Stable" dependencies = [ + {% if api.has_resumable_upload_methods %} + "google-api-core[grpc] >= 2.41.0, <3.0.0", + {% else %} "google-api-core[grpc] >= 2.28.0, <3.0.0", + {% endif %} # Exclude incompatible versions of `google-auth` # See https://github.com/googleapis/google-cloud-python/issues/12364 "google-auth >= 2.14.1, <3.0.0,!=2.24.0,!=2.25.0", @@ -57,7 +61,8 @@ dependencies = [ extras = { {% if rest_async_io_enabled %} "async_rest": [ - "google-auth[aiohttp] >= 2.35.0, <3.0.0" + "google-auth[aiohttp] >= 2.60.0, <3.0.0", + "google-api-core >= 2.41.0, <3.0.0", ], {% endif %} } diff --git a/packages/gapic-generator/gapic/templates/testing/constraints-3.10-async-rest.txt.j2 b/packages/gapic-generator/gapic/templates/testing/constraints-3.10-async-rest.txt.j2 index 04e798f94d9a..54179f0b0cba 100644 --- a/packages/gapic-generator/gapic/templates/testing/constraints-3.10-async-rest.txt.j2 +++ b/packages/gapic-generator/gapic/templates/testing/constraints-3.10-async-rest.txt.j2 @@ -1,4 +1,4 @@ -{% set rest_async_io_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.rest_async_io_enabled %} +{% set rest_async_io_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.rest_async_io_enabled or api.has_resumable_upload_methods %} {% if rest_async_io_enabled %} # -*- coding: utf-8 -*- {% from '_pypi_packages.j2' import pypi_packages %} @@ -8,8 +8,13 @@ # pinning their versions to their lower bounds. # For example, if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0", # then this file should have google-cloud-foo==1.14.0 +{% if api.has_resumable_upload_methods %} +google-api-core==2.41.0 +{% else %} google-api-core==2.28.0 -google-auth==2.35.0 +{% endif %} +google-auth==2.60.0 +aiohttp==3.8.0 grpcio==1.59.0 proto-plus==1.26.1 protobuf==6.33.5 diff --git a/packages/gapic-generator/gapic/templates/testing/constraints-3.10.txt.j2 b/packages/gapic-generator/gapic/templates/testing/constraints-3.10.txt.j2 index f8d078bfabb9..85af16c110fd 100644 --- a/packages/gapic-generator/gapic/templates/testing/constraints-3.10.txt.j2 +++ b/packages/gapic-generator/gapic/templates/testing/constraints-3.10.txt.j2 @@ -5,7 +5,11 @@ # pinning their versions to their lower bounds. # For example, if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0", # then this file should have google-cloud-foo==1.14.0 +{% if api.has_resumable_upload_methods %} +google-api-core==2.41.0 +{% else %} google-api-core==2.28.0 +{% endif %} google-auth==2.14.1 grpcio==1.59.0 proto-plus==1.26.1 diff --git a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 index d6880c73605a..8240485ce167 100644 --- a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 +++ b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 @@ -16,7 +16,7 @@ from unittest.mock import AsyncMock import grpc from grpc.experimental import aio -{% if "rest" in opts.transport %} +{% if "rest" in opts.transport or service.has_resumable_upload_methods %} from collections.abc import Iterable, AsyncIterable from google.protobuf import json_format {% endif %} @@ -27,8 +27,7 @@ from collections.abc import Sequence, Mapping from google.api_core import api_core_version from proto.marshal.rules.dates import DurationRule, TimestampRule from proto.marshal.rules import wrappers -{% if 'rest' in opts.transport %} -{% if rest_async_io_enabled %} +{% if ('rest' in opts.transport and rest_async_io_enabled) or ('grpc' in opts.transport and service.has_resumable_upload_methods) %} try: import aiohttp # type: ignore from google.auth.aio.transport.sessions import AsyncAuthorizedSession @@ -37,7 +36,8 @@ try: {# NOTE: `pragma: NO COVER` is needed since the coverage for presubmits isn't combined. #} except ImportError: # pragma: NO COVER HAS_ASYNC_REST_EXTRA = False -{% endif %}{# if rest_async_io_enabled #} +{% endif %} +{% if 'rest' in opts.transport %} from requests import Response from requests import Request, PreparedRequest from requests.sessions import Session @@ -1522,6 +1522,523 @@ def test_{{ service.name|snake_case }}_grpc_asyncio_transport_channel(): assert transport._ssl_channel_credentials == None +{% if service.has_resumable_upload_methods and 'grpc' in opts.transport %} +{% for method in service.methods.values() if method.is_resumable_upload %} +def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_grpc_channel_without_credentials_error(): + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + transport = transports.{{ service.name }}GrpcTransport( + host="localhost:7469", + channel=channel, + ) + with pytest.raises(core_exceptions.GoogleAPICallError) as exc_info: + transport.{{ method.transport_safe_name|snake_case }}({{ method.input.ident }}()) + assert "operate over REST and cannot be invoked when the transport is initialized with a pre-constructed gRPC channel" in str(exc_info.value) + transport.close() + + +def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_grpc_rest_stub(): + transport = transports.{{ service.name }}GrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + assert transport._rest_transport is None + stub = transport.{{ method.transport_safe_name|snake_case }} + assert transport._rest_transport is None + + result1 = stub({{ method.input.ident }}()) + rest_transport = transport._rest_transport + assert rest_transport is not None + assert isinstance(result1, {{ method.client_output.ident }}) + + result2 = stub({{ method.input.ident }}()) + assert isinstance(result2, {{ method.client_output.ident }}) + assert transport._rest_transport is rest_transport + + with mock.patch.object(rest_transport, "close") as mock_rest_close: + transport.close() + mock_rest_close.assert_called_once() + + +def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_grpc_rest_stub_mtls(): + with mock.patch( + "google.auth.transport.requests.AuthorizedSession.configure_mtls_channel" + ) as mock_configure_mtls_channel: + transport = transports.{{ service.name }}GrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + client_cert_source_for_mtls=client_cert_source_callback, + ) + mock_configure_mtls_channel.assert_not_called() + stub = transport.{{ method.transport_safe_name|snake_case }} + stub({{ method.input.ident }}()) + mock_configure_mtls_channel.assert_called_once_with( + client_cert_source_callback + ) + transport.close() + + +@pytest.mark.asyncio +async def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_grpc_asyncio_mtls_error(): + transport = transports.{{ service.name }}GrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + client_cert_source_for_mtls=client_cert_source_callback, + ) + with pytest.raises(core_exceptions.AsyncRestUnsupportedParameterError) as exc_info: + await transport.{{ method.transport_safe_name|snake_case }}({{ method.input.ident }}()) + assert "Mutual TLS (client_cert_source_for_mtls) is not currently supported for async resumable upload methods." in str(exc_info.value) + await transport.close() + + +@pytest.mark.asyncio +async def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_grpc_asyncio_channel_without_credentials_error(): + channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) + transport = transports.{{ service.name }}GrpcAsyncIOTransport( + host="localhost:7469", + channel=channel, + ) + with pytest.raises(core_exceptions.GoogleAPICallError) as exc_info: + await transport.{{ method.transport_safe_name|snake_case }}({{ method.input.ident }}()) + assert "operate over REST and cannot be invoked when the transport is initialized with a pre-constructed gRPC channel" in str(exc_info.value) + await transport.close() + + +@pytest.mark.asyncio +async def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_grpc_asyncio_rest_stub(): + with mock.patch.object(transports.grpc_asyncio, "HAS_ASYNC_REST", True), mock.patch.object( + transports.grpc_asyncio, + "Async{{ service.name }}RestTransport", + create=True, + ) as mock_rest_transport_cls: + mock_rest_transport = mock.MagicMock() + mock_rest_transport.close = mock.AsyncMock() + expected_session = {{ method.client_output_async.ident }}() + mock_rest_transport.{{ method.transport_safe_name|snake_case }} = mock.AsyncMock( + return_value=expected_session + ) + mock_rest_transport_cls.return_value = mock_rest_transport + + transport = transports.{{ service.name }}GrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + mock_rest_transport_cls.assert_not_called() + + stub = transport.{{ method.transport_safe_name|snake_case }} + result1 = await stub({{ method.input.ident }}()) + result2 = await stub({{ method.input.ident }}()) + + assert result1 is expected_session + assert result2 is expected_session + mock_rest_transport_cls.assert_called_once_with( + host=transport._host, + credentials=transport._credentials, + client_info=transport._client_info, + ) + assert mock_rest_transport.{{ method.transport_safe_name|snake_case }}.call_count == 2 + await transport.close() + mock_rest_transport.close.assert_called_once() + + +@pytest.mark.asyncio +async def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_grpc_asyncio_rest_stub_sync_credentials(): + if not HAS_ASYNC_REST_EXTRA: + pytest.skip() + transport = transports.{{ service.name }}GrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + assert transport._rest_transport is None + stub = transport.{{ method.transport_safe_name|snake_case }} + assert transport._rest_transport is None + + result1 = await stub({{ method.input.ident }}()) + rest_transport = transport._rest_transport + assert rest_transport is not None + assert isinstance(result1, {{ method.client_output_async.ident }}) + + result2 = await stub({{ method.input.ident }}()) + assert isinstance(result2, {{ method.client_output_async.ident }}) + assert transport._rest_transport is rest_transport + + with mock.patch.object( + rest_transport, "close", wraps=rest_transport.close + ) as mock_rest_close: + await transport.close() + mock_rest_close.assert_called_once() + + +@pytest.mark.asyncio +async def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_grpc_asyncio_start_empty_body(): + if not HAS_ASYNC_REST_EXTRA: + pytest.skip() + client = {{ service.async_client_name }}( + credentials=async_anonymous_credentials(), + ) + with mock.patch.object(AsyncAuthorizedSession, "request") as req: + start_response = mock.MagicMock() + start_response.status_code = 200 + start_response.headers = {"x-goog-upload-status": "active", "x-goog-upload-url": "http://localhost/upload"} + start_response.read = mock.AsyncMock(return_value=b"") + start_response.close = mock.AsyncMock(return_value=None) + + expected_result = {{ method.output.ident }}( + {% for field in method.output.fields.values() | rejectattr('message') %} + {% if not field.oneof or field.proto3_optional %} + {{ field.name }}={{ field.mock_value }}, + {% endif %} + {% endfor %} + ) + {% if method.output.ident.is_proto_plus_type %} + json_return_value = {{ method.output.ident }}.to_json(expected_result) + {% else %} + json_return_value = json_format.MessageToJson(expected_result) + {% endif %} + upload_response = mock.MagicMock() + upload_response.status_code = 200 + upload_response.headers = {"x-goog-upload-status": "final"} + upload_response.read = mock.AsyncMock(return_value=json_return_value.encode("UTF-8")) + upload_response.close = mock.AsyncMock(return_value=None) + + req.side_effect = [start_response, upload_response] + retry = retries.AsyncRetry() + response = await client.{{ method.client_method_name|snake_case }}( + {{ method.input.ident }}(), + config=resumable_transfer.ResumableUploadConfig(headers={"x-custom": "val"}), + retry=retry, + timeout=10.0, + ) + assert isinstance(response, {{ method.client_output_async.ident }}) + assert response._start_retry is retry + result = await response.upload(b"test data") + assert isinstance(result, {{ method.output.ident }}) + assert result == expected_result + start_call = req.call_args_list[0] + assert start_call.kwargs["headers"]["Content-Type"] == "application/json" + assert start_call.kwargs["headers"]["x-custom"] == "val" + + +@pytest.mark.asyncio +async def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_grpc_asyncio_body_and_query_params(): + if not HAS_ASYNC_REST_EXTRA: + pytest.skip() + client = {{ service.async_client_name }}( + credentials=async_anonymous_credentials(), + ) + with mock.patch.object( + AsyncAuthorizedSession, "request" + ) as req, mock.patch.object(path_template, "transcode") as transcode: + request = {{ method.input.ident }}( + {% for field in method.input.fields.values() | rejectattr('message') %} + {% if not field.oneof or field.proto3_optional %} + {{ field.name }}={{ field.mock_value }}, + {% endif %} + {% endfor %} + ) + {% if method.input.ident.is_proto_plus_type %} + pb_request = {{ method.input.ident }}.pb(request) + {% else %} + pb_request = request + {% endif %} + transcode.return_value = { + "uri": "v1/sample_method", + "method": "{{ method.http_options[0].method }}", + "body": pb_request, + "query_params": pb_request, + } + + start_response = mock.MagicMock() + start_response.status_code = 200 + start_response.headers = {"x-goog-upload-status": "active", "x-goog-upload-url": "http://localhost/upload"} + start_response.read = mock.AsyncMock(return_value=b"") + start_response.close = mock.AsyncMock(return_value=None) + + expected_result = {{ method.output.ident }}() + {% if method.output.ident.is_proto_plus_type %} + json_return_value = {{ method.output.ident }}.to_json(expected_result) + {% else %} + json_return_value = json_format.MessageToJson(expected_result) + {% endif %} + upload_response = mock.MagicMock() + upload_response.status_code = 200 + upload_response.headers = {"x-goog-upload-status": "final"} + upload_response.read = mock.AsyncMock(return_value=json_return_value.encode("UTF-8")) + upload_response.close = mock.AsyncMock(return_value=None) + + req.side_effect = [start_response, upload_response] + response = await client.{{ method.client_method_name|snake_case }}(request) + result = await response.upload(b"test data") + assert result == expected_result + + start_call = req.call_args_list[0] + assert start_call.kwargs["headers"]["Content-Type"] == "application/json" + {% for field in method.input.fields.values() | rejectattr('message') %} + {% if not field.oneof or field.proto3_optional %} + assert "{{ field.name | camel_case }}=" in start_call.args[1] + {% endif %} + {% endfor %} + {% if method.http_options[0].body %} + expected_body = json_format.MessageToJson( + pb_request, + use_integers_for_enums={{ opts.rest_numeric_enums }}, + ) + assert start_call.kwargs["data"] == expected_body.encode("UTF-8") + {% endif %} + + +@pytest.mark.asyncio +async def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_grpc_asyncio_without_async_rest_error(): + with mock.patch.object(transports.grpc_asyncio, "HAS_ASYNC_REST", False): + transport = transports.{{ service.name }}GrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(NotImplementedError) as exc_info: + await transport.{{ method.transport_safe_name|snake_case }}({{ method.input.ident }}()) + assert ( + "Async REST transport is required for async resumable upload methods. " + "Install the library with the `async_rest` extra using " + "`pip install {{ api.naming.warehouse_package_name }}[async_rest]`." + ) in str(exc_info.value) +{% endfor %} +{% endif %} + +{% if service.has_resumable_upload_methods and 'rest' in opts.transport %} +{% for method in service.methods.values() if method.is_resumable_upload %} +def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_rest_start_empty_body(): + client = {{ service.client_name }}( + credentials=ga_credentials.AnonymousCredentials(), + transport="rest", + ) + with mock.patch.object(type(client.transport._session), "request") as req: + start_response = mock.MagicMock() + start_response.ok = True + start_response.status_code = 200 + start_response.headers = {"x-goog-upload-status": "active", "x-goog-upload-url": "http://localhost/upload"} + start_response.content = b"" + + expected_result = {{ method.output.ident }}( + {% for field in method.output.fields.values() | rejectattr('message') %} + {% if not field.oneof or field.proto3_optional %} + {{ field.name }}={{ field.mock_value }}, + {% endif %} + {% endfor %} + ) + {% if method.output.ident.is_proto_plus_type %} + json_return_value = {{ method.output.ident }}.to_json(expected_result) + {% else %} + json_return_value = json_format.MessageToJson(expected_result) + {% endif %} + upload_response = mock.MagicMock() + upload_response.ok = True + upload_response.status_code = 200 + upload_response.headers = {"x-goog-upload-status": "final"} + upload_response.content = json_return_value.encode("UTF-8") + + req.side_effect = [start_response, upload_response] + retry = retries.Retry() + response = client.{{ method.client_method_name|snake_case }}( + {{ method.input.ident }}(), + config=resumable_transfer.ResumableUploadConfig(headers={"x-custom": "val"}), + retry=retry, + timeout=10.0, + ) + assert isinstance(response, {{ method.client_output.ident }}) + assert response._start_retry is retry + result = response.upload(b"test data") + assert isinstance(result, {{ method.output.ident }}) + assert result == expected_result + start_call = req.call_args_list[0] + assert start_call.kwargs["headers"]["Content-Type"] == "application/json" + assert start_call.kwargs["headers"]["x-custom"] == "val" + + +{% if rest_async_io_enabled %} +@pytest.mark.asyncio +async def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_rest_asyncio_start_empty_body(): + if not HAS_ASYNC_REST_EXTRA: + pytest.skip() + client = {{ service.async_client_name }}( + credentials=async_anonymous_credentials(), + transport="rest_asyncio", + ) + with mock.patch.object(type(client.transport._session), "request") as req: + start_response = mock.MagicMock() + start_response.status_code = 200 + start_response.headers = {"x-goog-upload-status": "active", "x-goog-upload-url": "http://localhost/upload"} + start_response.read = mock.AsyncMock(return_value=b"") + start_response.close = mock.AsyncMock(return_value=None) + + expected_result = {{ method.output.ident }}( + {% for field in method.output.fields.values() | rejectattr('message') %} + {% if not field.oneof or field.proto3_optional %} + {{ field.name }}={{ field.mock_value }}, + {% endif %} + {% endfor %} + ) + {% if method.output.ident.is_proto_plus_type %} + json_return_value = {{ method.output.ident }}.to_json(expected_result) + {% else %} + json_return_value = json_format.MessageToJson(expected_result) + {% endif %} + upload_response = mock.MagicMock() + upload_response.status_code = 200 + upload_response.headers = {"x-goog-upload-status": "final"} + upload_response.read = mock.AsyncMock(return_value=json_return_value.encode("UTF-8")) + upload_response.close = mock.AsyncMock(return_value=None) + + req.side_effect = [start_response, upload_response] + retry = retries.AsyncRetry() + response = await client.{{ method.client_method_name|snake_case }}( + {{ method.input.ident }}(), + config=resumable_transfer.ResumableUploadConfig(headers={"x-custom": "val"}), + retry=retry, + timeout=10.0, + ) + assert isinstance(response, {{ method.client_output_async.ident }}) + assert response._start_retry is retry + result = await response.upload(b"test data") + assert isinstance(result, {{ method.output.ident }}) + assert result == expected_result + start_call = req.call_args_list[0] + assert start_call.kwargs["headers"]["Content-Type"] == "application/json" + assert start_call.kwargs["headers"]["x-custom"] == "val" +{% endif %} + + +def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_rest_body_and_query_params(): + client = {{ service.client_name }}( + credentials=ga_credentials.AnonymousCredentials(), + transport="rest", + ) + with mock.patch.object(type(client.transport._session), "request") as req, mock.patch.object( + path_template, "transcode" + ) as transcode: + request = {{ method.input.ident }}( + {% for field in method.input.fields.values() | rejectattr('message') %} + {% if not field.oneof or field.proto3_optional %} + {{ field.name }}={{ field.mock_value }}, + {% endif %} + {% endfor %} + ) + {% if method.input.ident.is_proto_plus_type %} + pb_request = {{ method.input.ident }}.pb(request) + {% else %} + pb_request = request + {% endif %} + transcode.return_value = { + "uri": "v1/sample_method", + "method": "{{ method.http_options[0].method }}", + "body": pb_request, + "query_params": pb_request, + } + + start_response = mock.MagicMock() + start_response.ok = True + start_response.status_code = 200 + start_response.headers = {"x-goog-upload-status": "active", "x-goog-upload-url": "http://localhost/upload"} + start_response.content = b"" + + expected_result = {{ method.output.ident }}() + {% if method.output.ident.is_proto_plus_type %} + json_return_value = {{ method.output.ident }}.to_json(expected_result) + {% else %} + json_return_value = json_format.MessageToJson(expected_result) + {% endif %} + upload_response = mock.MagicMock() + upload_response.ok = True + upload_response.status_code = 200 + upload_response.headers = {"x-goog-upload-status": "final"} + upload_response.content = json_return_value.encode("UTF-8") + + req.side_effect = [start_response, upload_response] + response = client.{{ method.client_method_name|snake_case }}(request) + result = response.upload(b"test data") + assert result == expected_result + + start_call = req.call_args_list[0] + assert start_call.kwargs["headers"]["Content-Type"] == "application/json" + {% for field in method.input.fields.values() | rejectattr('message') %} + {% if not field.oneof or field.proto3_optional %} + assert "{{ field.name | camel_case }}=" in start_call.args[1] + {% endif %} + {% endfor %} + {% if method.http_options[0].body %} + expected_body = json_format.MessageToJson( + pb_request, + use_integers_for_enums={{ opts.rest_numeric_enums }}, + ) + assert start_call.kwargs["data"] == expected_body.encode("UTF-8") + {% endif %} + + +{% if rest_async_io_enabled %} +@pytest.mark.asyncio +async def test_{{ service.name|snake_case }}_{{ method.name|snake_case }}_rest_asyncio_body_and_query_params(): + if not HAS_ASYNC_REST_EXTRA: + pytest.skip() + client = {{ service.async_client_name }}( + credentials=async_anonymous_credentials(), + transport="rest_asyncio", + ) + with mock.patch.object( + type(client.transport._session), "request" + ) as req, mock.patch.object(path_template, "transcode") as transcode: + request = {{ method.input.ident }}( + {% for field in method.input.fields.values() | rejectattr('message') %} + {% if not field.oneof or field.proto3_optional %} + {{ field.name }}={{ field.mock_value }}, + {% endif %} + {% endfor %} + ) + {% if method.input.ident.is_proto_plus_type %} + pb_request = {{ method.input.ident }}.pb(request) + {% else %} + pb_request = request + {% endif %} + transcode.return_value = { + "uri": "v1/sample_method", + "method": "{{ method.http_options[0].method }}", + "body": pb_request, + "query_params": pb_request, + } + + start_response = mock.MagicMock() + start_response.status_code = 200 + start_response.headers = {"x-goog-upload-status": "active", "x-goog-upload-url": "http://localhost/upload"} + start_response.read = mock.AsyncMock(return_value=b"") + start_response.close = mock.AsyncMock(return_value=None) + + expected_result = {{ method.output.ident }}() + {% if method.output.ident.is_proto_plus_type %} + json_return_value = {{ method.output.ident }}.to_json(expected_result) + {% else %} + json_return_value = json_format.MessageToJson(expected_result) + {% endif %} + upload_response = mock.MagicMock() + upload_response.status_code = 200 + upload_response.headers = {"x-goog-upload-status": "final"} + upload_response.read = mock.AsyncMock(return_value=json_return_value.encode("UTF-8")) + upload_response.close = mock.AsyncMock(return_value=None) + + req.side_effect = [start_response, upload_response] + response = await client.{{ method.client_method_name|snake_case }}(request) + result = await response.upload(b"test data") + assert result == expected_result + + start_call = req.call_args_list[0] + assert start_call.kwargs["headers"]["Content-Type"] == "application/json" + {% for field in method.input.fields.values() | rejectattr('message') %} + {% if not field.oneof or field.proto3_optional %} + assert "{{ field.name | camel_case }}=" in start_call.args[1] + {% endif %} + {% endfor %} + {% if method.http_options[0].body %} + expected_body = json_format.MessageToJson( + pb_request, + use_integers_for_enums={{ opts.rest_numeric_enums }}, + ) + assert start_call.kwargs["data"] == expected_body.encode("UTF-8") + {% endif %} +{% endif %} +{% endfor %} +{% endif %} + + # Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are # removed from grpc/grpc_asyncio transport constructor. @pytest.mark.filterwarnings("ignore::FutureWarning") diff --git a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2 b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2 index 05c60cbdc9c3..986f46f6c060 100644 --- a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2 +++ b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2 @@ -45,6 +45,8 @@ def test_{{ method_name }}(request_type, transport: str = 'grpc'): call.return_value = operations_pb2.Operation(name='operations/spam') {% elif method.server_streaming %} call.return_value = iter([{{ method.output.ident }}()]) + {% elif method.is_resumable_upload %} + call.return_value = {{ method.client_output.ident }}() {% else %} call.return_value = {{ method.output.ident }}( {% for field in method.output.fields.values() | rejectattr('message')%} @@ -92,6 +94,8 @@ def test_{{ method_name }}(request_type, transport: str = 'grpc'): {% elif method.server_streaming %} for message in response: assert isinstance(message, {{ method.output.ident }}) + {% elif method.is_resumable_upload %} + assert isinstance(response, {{ method.client_output.ident }}) {% else %} {% if "next_page_token" in method.output.fields.values()|map(attribute='name', default="") and not method.paged_result_field %} {# Cheeser assertion to force code coverage for bad paginated methods #} @@ -313,6 +317,10 @@ async def test_{{ method_name }}_async(request_type, transport: str = 'grpc_asyn {% elif method.client_streaming and method.server_streaming %} call.return_value = mock.Mock(aio.StreamStreamCall, autospec=True) call.return_value.read = mock.AsyncMock(side_effect=[{{ method.output.ident }}()]) + {% elif method.is_resumable_upload %} + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + {{ method.client_output_async.ident }}() + ) {% else %} call.return_value ={{ '' }} {%- if not method.client_streaming and not method.server_streaming -%} @@ -357,6 +365,8 @@ async def test_{{ method_name }}_async(request_type, transport: str = 'grpc_asyn {% elif method.server_streaming %} message = await response.read() assert isinstance(message, {{ method.output.ident }}) + {% elif method.is_resumable_upload %} + assert isinstance(response, {{ method.client_output_async.ident }}) {% else %} assert isinstance(response, {{ method.client_output_async.ident }}) {% for field in method_output.fields.values() | rejectattr('message') %} @@ -1832,18 +1842,33 @@ def test_initialize_client_w_{{transport_name}}(): # Mock the http request call within the method and fake a BadRequest error. with mock.patch.object({{mocked_session}}, 'request') as req, pytest.raises(core_exceptions.BadRequest): # Wrap the value into a proper Response obj + {% if method.is_resumable_upload %} + response_value = mock.MagicMock() + {% else %} response_value = mock.Mock() + {% endif %} {% if is_async %} response_value.read = mock.AsyncMock(return_value=b'{}') + {% if method.is_resumable_upload %} + response_value.close = mock.AsyncMock(return_value=None) + {% endif %} {% else %} json_return_value = '' response_value.json = mock.Mock(return_value={}) + {% if method.is_resumable_upload %} + response_value.ok = False + {% endif %} {% endif %}{# if is_async #} response_value.status_code = 400 response_value.request = mock.Mock() req.return_value = response_value req.return_value.headers = {"header-1": "value-1", "header-2": "value-2"} + {% if method.is_resumable_upload %} + response = {{ await_prefix }}client.{{ method_name }}(request) + {{ await_prefix }}response.upload(b"") + {% else %} {{ await_prefix }}client.{{ method_name }}(request) + {% endif %} {% endif %}{# if 'grpc' in transport #} {% endmacro %} @@ -2050,7 +2075,11 @@ def test_initialize_client_w_{{transport_name}}(): {% endif %}{# method.void #} # Wrap the value into a proper Response obj + {% if method.is_resumable_upload %} + response_value = mock.MagicMock() + {% else %} response_value = mock.Mock() + {% endif %} response_value.status_code = 200 {% if method.void %} json_return_value = '' @@ -2076,9 +2105,33 @@ def test_initialize_client_w_{{transport_name}}(): response_value.content = json_return_value.encode('UTF-8') {% endif %}{# is_async #} {% endif %}{# method.server_streaming #} + {% if method.is_resumable_upload %} + start_response = mock.MagicMock() + start_response.status_code = 200 + start_response.headers = {"x-goog-upload-status": "active", "x-goog-upload-url": "http://localhost/upload"} + response_value.headers = {"x-goog-upload-status": "final", "header-1": "value-1", "header-2": "value-2"} + {% if is_async %} + start_response.read = mock.AsyncMock(return_value=b"") + start_response.close = mock.AsyncMock(return_value=None) + response_value.close = mock.AsyncMock(return_value=None) + {% else %} + start_response.ok = True + start_response.content = b"" + response_value.ok = True + {% endif %} + req.side_effect = [start_response, response_value] + response = {{ await_prefix }}client.{{ method_name }}(request) + {% if is_async %} + assert isinstance(response, {{ method.client_output_async.ident }}) + {% else %} + assert isinstance(response, {{ method.client_output.ident }}) + {% endif %} + response = {{ await_prefix }}response.upload(b"test data") + {% else %} req.return_value = response_value req.return_value.headers = {"header-1": "value-1", "header-2": "value-2"} response = {{ await_prefix }}client.{{ method_name }}(request) + {% endif %} {% if "next_page_token" in method_output.fields.values()|map(attribute='name', default="") and not method.paged_result_field %} {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2199): The following assert statement is added to force # code coverage for bad paginated methods. Investigate what bad paginated methods are, and follow up on why this assertion is required. @@ -2103,7 +2156,9 @@ def test_initialize_client_w_{{transport_name}}(): {% elif method.lro %} json_return_value = json_format.MessageToJson(return_value) {% else %} - {% if is_async %} + {% if method.is_resumable_upload %} + assert isinstance(response, {{ method.output.ident }}) + {% elif is_async %} assert isinstance(response, {{ method.client_output_async.ident }}) {% else %} assert isinstance(response, {{ method.client_output.ident }}) @@ -2257,13 +2312,13 @@ def test_initialize_client_w_{{transport_name}}(): {% if method.lro %} mock.patch.object(operation.Operation, "_set_result_from_operation"), \ {% endif %} - {% if not method.void %} + {% if not method.void and not method.is_resumable_upload %} mock.patch.object(transports.{{async_method_prefix}}{{ service.name }}RestInterceptor, "post_{{method.name|snake_case}}") as post, \ mock.patch.object(transports.{{async_method_prefix}}{{ service.name }}RestInterceptor, "post_{{method.name|snake_case}}_with_metadata") as post_with_metadata, \ {% endif %} mock.patch.object(transports.{{async_method_prefix}}{{ service.name }}RestInterceptor, "pre_{{ method.name|snake_case }}") as pre: pre.assert_not_called() - {% if not method.void %} + {% if not method.void and not method.is_resumable_upload %} post.assert_not_called() post_with_metadata.assert_not_called() {% endif %} @@ -2282,7 +2337,7 @@ def test_initialize_client_w_{{transport_name}}(): req.return_value = mock.Mock() req.return_value.status_code = 200 req.return_value.headers = {"header-1": "value-1", "header-2": "value-2"} - {% if not method.void %} + {% if not method.void and not method.is_resumable_upload %} return_value = {% if method.output.ident.is_proto_plus_type %}{{ method.output.ident }}.to_json({{ method.output.ident }}()){% else %}json_format.MessageToJson({{ method.output.ident }}()){% endif %} {% if method.server_streaming %} @@ -2298,7 +2353,7 @@ def test_initialize_client_w_{{transport_name}}(): req.return_value.content = return_value {% endif %}{# end is_async #} {% endif %}{# end method.server_streaming #} - {% endif %}{# end not method.void #} + {% endif %}{# end not method.void and not method.is_resumable_upload #} request = {{ method.input.ident }}() metadata =[ @@ -2306,7 +2361,7 @@ def test_initialize_client_w_{{transport_name}}(): ("cephalopod", "squid"), ] pre.return_value = request, metadata - {% if not method.void %} + {% if not method.void and not method.is_resumable_upload %} post.return_value = {{ method.output.ident }}() post_with_metadata.return_value = {{ method.output.ident }}(), metadata {% endif %} @@ -2314,7 +2369,7 @@ def test_initialize_client_w_{{transport_name}}(): {{await_prefix}}client.{{ method_name }}(request, metadata=[("key", "val"), ("cephalopod", "squid"),]) pre.assert_called_once() - {% if not method.void %} + {% if not method.void and not method.is_resumable_upload %} post.assert_called_once() post_with_metadata.assert_called_once() {% endif %} diff --git a/packages/gapic-generator/noxfile.py b/packages/gapic-generator/noxfile.py index 2fe717cec79e..b9d1870b9e21 100644 --- a/packages/gapic-generator/noxfile.py +++ b/packages/gapic-generator/noxfile.py @@ -193,10 +193,6 @@ def fragment(session, use_ads_templates=False): ) session.install("-e", ".") - # The specific failure is `Plugin output is unparseable` - if session.python == "3.10": - session.install("google-api-core<2.28") - frag_files = ( [Path(f) for f in session.posargs] if session.posargs else FRAGMENT_FILES ) @@ -246,8 +242,30 @@ def showcase_library( include_service_yaml=True, retry_config=True, rest_async_io_enabled=False, + install_async_rest_extra=False, ): - """Install the generated library into the session for showcase tests.""" + """Install the generated library into the session for showcase tests. + + Args: + session: The nox session object. + templates (str): The template directory to use for code generation. + Defaults to "DEFAULT". + other_opts (typing.Iterable[str]): Additional options passed to + `--python_gapic_opt` during code generation. + include_service_yaml (bool): Whether to download and pass + `showcase_v1beta1.yaml` to the generator. + retry_config (bool): Whether to download and pass + `showcase_grpc_service_config.json` to the generator. + rest_async_io_enabled (bool): Whether to enable the experimental + `rest_async_io_enabled` setting in `showcase_v1beta1.yaml` so + `rest_asyncio` transports are generated for all services. When True, + the `[async_rest]` extra is also installed. + install_async_rest_extra (bool): Whether to install the generated + library with the `[async_rest]` extra and its corresponding + `constraints-{python}-async-rest.txt` file even when + `rest_async_io_enabled` is False in `showcase_v1beta1.yaml` (e.g., + for services with resumable upload methods). + """ session.log("-" * 70) session.log("Note: Showcase must be running for these tests to work.") @@ -260,13 +278,6 @@ def showcase_library( # Install grpcio-tools for protoc session.install("grpcio-tools") - # TODO(https://github.com/googleapis/gapic-generator-python/issues/2473): - # Warnings emitted from google-api-core starting in 2.28 - # appear to cause issues when running protoc. - # The specific failure is `Plugin output is unparseable` - if session.python == "3.10": - session.install("google-api-core<2.28") - # Install a client library for Showcase. with tempfile.TemporaryDirectory() as tmp_dir: # Download the Showcase descriptor. @@ -343,18 +354,30 @@ def showcase_library( "transport=grpc+rest", ) ) + # TODO(https://github.com/googleapis/google-cloud-python/issues/16312): + # Add compliance.proto once this bug is fixed + # We should use `"google/showcase/v1beta1/*.proto",` + protos = ( + "google/showcase/v1beta1/echo.proto", + "google/showcase/v1beta1/identity.proto", + "google/showcase/v1beta1/messaging.proto", + "google/showcase/v1beta1/rest_error.proto", + "google/showcase/v1beta1/sequence.proto", + "google/showcase/v1beta1/testing.proto", + ) + if templates == "DEFAULT": + protos += ("google/showcase/v1beta1/resumable_upload.proto",) cmd_tup = ( "python", "-m", "grpc_tools.protoc", - f"--experimental_allow_proto3_optional", + "--experimental_allow_proto3_optional", f"--descriptor_set_in={tmp_dir}{path.sep}showcase.desc", opts, f"--python_gapic_out={tmp_dir}", - f"google/showcase/v1beta1/echo.proto", - f"google/showcase/v1beta1/identity.proto", - f"google/showcase/v1beta1/messaging.proto", + *protos, ) + session.run( *cmd_tup, external=True, @@ -371,7 +394,8 @@ def showcase_library( f"{tmp_dir}/testing/constraints-{session.python}.txt" ) extras = "" - if rest_async_io_enabled: + if rest_async_io_enabled or install_async_rest_extra: + extras = "[async_rest]" async_rest_constraints_path = str( f"{tmp_dir}/testing/constraints-{session.python}-async-rest.txt" ) @@ -382,8 +406,6 @@ def showcase_library( session.log( f"{async_rest_constraints_path} not found. Using base constraints file" ) - extras = "[async_rest]" - session.install("-e", f"{tmp_dir}{extras}", "-r", constraints_path) else: # The ads templates do not have constraints files. @@ -395,8 +417,10 @@ def showcase_library( @nox.session(python=ALL_PYTHON) +@nox.parametrize("install_async_rest_extra", [False, True]) def showcase( session, + install_async_rest_extra=False, templates="DEFAULT", other_opts: typing.Iterable[str] = (), env: typing.Optional[typing.Dict[str, str]] = {}, @@ -407,7 +431,12 @@ def showcase( (useful for local testing and canary validation). """ - with showcase_library(session, templates=templates, other_opts=other_opts): + with showcase_library( + session, + templates=templates, + other_opts=other_opts, + install_async_rest_extra=install_async_rest_extra, + ): # When opt-in environment variable is set (e.g. in canary CI or local testing), # install the local google-api-core package from source. if os.getenv("INSTALL_LOCAL_CORE") == "true": @@ -535,7 +564,26 @@ def showcase_pqc( ) -def run_showcase_unit_tests(session, fail_under=100, rest_async_io_enabled=False): +def run_showcase_unit_tests( + session, + fail_under=100, + rest_async_io_enabled=False, + install_async_rest_extra=False, +): + """Run the generated Showcase unit test suite with coverage verification. + + Args: + session: The nox session object. + fail_under (int): Minimum required test coverage percentage. + Defaults to 100. + rest_async_io_enabled (bool): Whether `rest_async_io_enabled` was enabled + in `showcase_v1beta1.yaml` during code generation. + install_async_rest_extra (bool): Whether the library was installed with + the `[async_rest]` extra. When both `rest_async_io_enabled` and + `install_async_rest_extra` are False, `**/rest_asyncio.py` is omitted + from coverage since optional `async_rest` dependencies are not + installed. + """ session.install( "coverage", "pytest", @@ -546,6 +594,20 @@ def run_showcase_unit_tests(session, fail_under=100, rest_async_io_enabled=False # Freeze and print python environment package versions session.run("python", "-m", "pip", "freeze") + if ( + not rest_async_io_enabled + and not install_async_rest_extra + and path.exists(".coveragerc") + ): + with open(".coveragerc", "r") as f: + coveragerc = f.read() + if "**/rest_asyncio.py" not in coveragerc: + coveragerc = coveragerc.replace( + "omit =\n", "omit =\n **/rest_asyncio.py\n" + ) + with open(".coveragerc", "w") as f: + f.write(coveragerc) + # Run the tests. session.run( "py.test", @@ -564,15 +626,24 @@ def run_showcase_unit_tests(session, fail_under=100, rest_async_io_enabled=False @nox.session(python=ALL_PYTHON) +@nox.parametrize("install_async_rest_extra", [False, True]) def showcase_unit( session, + install_async_rest_extra=False, templates="DEFAULT", other_opts: typing.Iterable[str] = (), ): """Run the generated unit tests against the Showcase library.""" - with showcase_library(session, templates=templates, other_opts=other_opts) as lib: + with showcase_library( + session, + templates=templates, + other_opts=other_opts, + install_async_rest_extra=install_async_rest_extra, + ) as lib: session.chdir(lib) - run_showcase_unit_tests(session) + run_showcase_unit_tests( + session, install_async_rest_extra=install_async_rest_extra + ) # TODO: `showcase_unit_w_rest_async` nox session runs showcase unit tests with the diff --git a/packages/gapic-generator/requirements.txt b/packages/gapic-generator/requirements.txt index ff61f2e8a44e..38eb25a0b012 100644 --- a/packages/gapic-generator/requirements.txt +++ b/packages/gapic-generator/requirements.txt @@ -611,13 +611,13 @@ frozenlist==1.8.0 \ # via # aiohttp # aiosignal -google-api-core==2.40.0 \ - --hash=sha256:4b9e0a80024c269ae173136d5439f4ed284651d6e5e2773ba5c694f481f9c0f4 \ - --hash=sha256:ebee7d1b138b5362beecec260e6e8988ac97346562c7382ccb6f0ad8435c599f +google-api-core==2.41.0 \ + --hash=sha256:1f3d626b7a8022dda8deef200b845057231ddc7cbf0d999026bdf61ba38d7fdf \ + --hash=sha256:73e89a86baef6680934adeee6fbd0ceaf20c1393ab229b2f9b34efb23b0fdef3 # via -r requirements.in -google-auth==2.59.0 \ - --hash=sha256:43e47b1d3067fb90d77bd9ba21f139bb0577b3f2b9f3e2516fc00f37435efcad \ - --hash=sha256:eb32f44f89f6b577947ebee5887c1db46e6b1a278889ba369a88179643f32240 +google-auth==2.60.0 \ + --hash=sha256:34aa0283ef72c99cd410e35db4417211fa3e6aaa5399183227bbe3b82bd15192 \ + --hash=sha256:e78414cd2e22cc02884768707def024ca32da17695f2c0e6194cdbe1316c4cd4 # via google-api-core googleapis-common-protos==1.75.5 \ --hash=sha256:c7a866fc34ed29a3b10af627a4b9b1dc2433313ca6e959f0ae4feb132047ed72 \ diff --git a/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/services/cloud_redis/client.py b/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/services/cloud_redis/client.py index 7b2e7759cd73..bb62ed511a5c 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/services/cloud_redis/client.py +++ b/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/services/cloud_redis/client.py @@ -62,13 +62,11 @@ from .transports.grpc import CloudRedisGrpcTransport from .transports.grpc_asyncio import CloudRedisGrpcAsyncIOTransport from .transports.rest import CloudRedisRestTransport -ASYNC_REST_EXCEPTION = None try: from .transports.rest_asyncio import AsyncCloudRedisRestTransport - HAS_ASYNC_REST_DEPENDENCIES = True -except ImportError as e: # pragma: NO COVER + HAS_ASYNC_REST_DEPENDENCIES = True # pragma: NO COVER +except ImportError: # pragma: NO COVER HAS_ASYNC_REST_DEPENDENCIES = False - ASYNC_REST_EXCEPTION = e class CloudRedisClientMeta(type): @@ -99,7 +97,9 @@ def get_transport_class(cls, """ # If a specific transport is requested, return that one. if label == "rest_asyncio" and not HAS_ASYNC_REST_DEPENDENCIES: # pragma: NO COVER - raise ASYNC_REST_EXCEPTION + raise ImportError( + "`rest_asyncio` transport requires the library to be installed with the `async_rest` extra. Install the library with the `async_rest` extra using `pip install google-cloud-redis[async_rest]`" + ) if label: return cls._transport_registry[label] @@ -506,7 +506,7 @@ def __init__(self, *, else cast(Callable[..., CloudRedisTransport], transport) ) - if "rest_asyncio" in str(transport_init): + if "rest_asyncio" in str(transport_init): # pragma: NO COVER unsupported_params = { "google.api_core.client_options.ClientOptions.credentials_file": self._client_options.credentials_file, "google.api_core.client_options.ClientOptions.scopes": self._client_options.scopes, diff --git a/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/services/cloud_redis/transports/__init__.py b/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/services/cloud_redis/transports/__init__.py index 4b81d3f86f0b..4f1a5dd5f81f 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/services/cloud_redis/transports/__init__.py +++ b/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/services/cloud_redis/transports/__init__.py @@ -24,9 +24,9 @@ ASYNC_REST_CLASSES: Tuple[str, ...] try: from .rest_asyncio import AsyncCloudRedisRestTransport - from .rest_asyncio import AsyncCloudRedisRestInterceptor - ASYNC_REST_CLASSES = ('AsyncCloudRedisRestTransport', 'AsyncCloudRedisRestInterceptor') - HAS_REST_ASYNC = True + from .rest_asyncio import AsyncCloudRedisRestInterceptor # pragma: NO COVER + ASYNC_REST_CLASSES = ('AsyncCloudRedisRestTransport', 'AsyncCloudRedisRestInterceptor') # pragma: NO COVER + HAS_REST_ASYNC = True # pragma: NO COVER except ImportError: # pragma: NO COVER ASYNC_REST_CLASSES = () HAS_REST_ASYNC = False diff --git a/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/services/cloud_redis/transports/rest_asyncio.py b/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/services/cloud_redis/transports/rest_asyncio.py index 4d629a5a8443..fa3f65ce9e36 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/services/cloud_redis/transports/rest_asyncio.py +++ b/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/services/cloud_redis/transports/rest_asyncio.py @@ -23,6 +23,7 @@ except ImportError as e: # pragma: NO COVER raise ImportError("`rest_asyncio` transport requires the library to be installed with the `async_rest` extra. Install the library with the `async_rest` extra using `pip install google-cloud-redis[async_rest]`") from e +from google.auth import credentials as ga_credentials # type: ignore from google.auth.aio import credentials as ga_credentials_async # type: ignore from google.api_core import exceptions as core_exceptions @@ -770,7 +771,7 @@ class AsyncCloudRedisRestTransport(_BaseCloudRedisRestTransport): def __init__(self, *, host: str = 'redis.googleapis.com', - credentials: Optional[ga_credentials_async.Credentials] = None, + credentials: Optional[Union[ga_credentials.Credentials, ga_credentials_async.Credentials]] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, url_scheme: str = 'https', interceptor: Optional[AsyncCloudRedisRestInterceptor] = None, @@ -784,7 +785,7 @@ def __init__(self, Args: host (Optional[str]): The hostname to connect to (default: 'redis.googleapis.com'). - credentials (Optional[google.auth.aio.credentials.Credentials]): The + credentials (Optional[Union[google.auth.credentials.Credentials, google.auth.aio.credentials.Credentials]]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none are specified, the client will attempt to ascertain the diff --git a/packages/gapic-generator/tests/integration/goldens/redis/setup.py b/packages/gapic-generator/tests/integration/goldens/redis/setup.py index 29dcfeccbc8b..fdadb0f8c9a8 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis/setup.py +++ b/packages/gapic-generator/tests/integration/goldens/redis/setup.py @@ -53,7 +53,8 @@ ] extras = { "async_rest": [ - "google-auth[aiohttp] >= 2.35.0, <3.0.0" + "google-auth[aiohttp] >= 2.60.0, <3.0.0", + "google-api-core >= 2.41.0, <3.0.0", ], } url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-cloud-redis" diff --git a/packages/gapic-generator/tests/integration/goldens/redis/testing/constraints-3.10-async-rest.txt b/packages/gapic-generator/tests/integration/goldens/redis/testing/constraints-3.10-async-rest.txt index c0c1ba32f46f..4a2700a3bf8a 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis/testing/constraints-3.10-async-rest.txt +++ b/packages/gapic-generator/tests/integration/goldens/redis/testing/constraints-3.10-async-rest.txt @@ -6,7 +6,8 @@ # For example, if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0", # then this file should have google-cloud-foo==1.14.0 google-api-core==2.28.0 -google-auth==2.35.0 +google-auth==2.60.0 +aiohttp==3.8.0 grpcio==1.59.0 proto-plus==1.26.1 protobuf==6.33.5 diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/services/cloud_redis/client.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/services/cloud_redis/client.py index 771b0baa9989..22043152a1c6 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/services/cloud_redis/client.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/services/cloud_redis/client.py @@ -62,13 +62,11 @@ from .transports.grpc import CloudRedisGrpcTransport from .transports.grpc_asyncio import CloudRedisGrpcAsyncIOTransport from .transports.rest import CloudRedisRestTransport -ASYNC_REST_EXCEPTION = None try: from .transports.rest_asyncio import AsyncCloudRedisRestTransport - HAS_ASYNC_REST_DEPENDENCIES = True -except ImportError as e: # pragma: NO COVER + HAS_ASYNC_REST_DEPENDENCIES = True # pragma: NO COVER +except ImportError: # pragma: NO COVER HAS_ASYNC_REST_DEPENDENCIES = False - ASYNC_REST_EXCEPTION = e class CloudRedisClientMeta(type): @@ -99,7 +97,9 @@ def get_transport_class(cls, """ # If a specific transport is requested, return that one. if label == "rest_asyncio" and not HAS_ASYNC_REST_DEPENDENCIES: # pragma: NO COVER - raise ASYNC_REST_EXCEPTION + raise ImportError( + "`rest_asyncio` transport requires the library to be installed with the `async_rest` extra. Install the library with the `async_rest` extra using `pip install google-cloud-redis[async_rest]`" + ) if label: return cls._transport_registry[label] @@ -506,7 +506,7 @@ def __init__(self, *, else cast(Callable[..., CloudRedisTransport], transport) ) - if "rest_asyncio" in str(transport_init): + if "rest_asyncio" in str(transport_init): # pragma: NO COVER unsupported_params = { "google.api_core.client_options.ClientOptions.credentials_file": self._client_options.credentials_file, "google.api_core.client_options.ClientOptions.scopes": self._client_options.scopes, diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/services/cloud_redis/transports/__init__.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/services/cloud_redis/transports/__init__.py index 4b81d3f86f0b..4f1a5dd5f81f 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/services/cloud_redis/transports/__init__.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/services/cloud_redis/transports/__init__.py @@ -24,9 +24,9 @@ ASYNC_REST_CLASSES: Tuple[str, ...] try: from .rest_asyncio import AsyncCloudRedisRestTransport - from .rest_asyncio import AsyncCloudRedisRestInterceptor - ASYNC_REST_CLASSES = ('AsyncCloudRedisRestTransport', 'AsyncCloudRedisRestInterceptor') - HAS_REST_ASYNC = True + from .rest_asyncio import AsyncCloudRedisRestInterceptor # pragma: NO COVER + ASYNC_REST_CLASSES = ('AsyncCloudRedisRestTransport', 'AsyncCloudRedisRestInterceptor') # pragma: NO COVER + HAS_REST_ASYNC = True # pragma: NO COVER except ImportError: # pragma: NO COVER ASYNC_REST_CLASSES = () HAS_REST_ASYNC = False diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/services/cloud_redis/transports/rest_asyncio.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/services/cloud_redis/transports/rest_asyncio.py index 960d9639a214..d79be431bd8a 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/services/cloud_redis/transports/rest_asyncio.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/services/cloud_redis/transports/rest_asyncio.py @@ -23,6 +23,7 @@ except ImportError as e: # pragma: NO COVER raise ImportError("`rest_asyncio` transport requires the library to be installed with the `async_rest` extra. Install the library with the `async_rest` extra using `pip install google-cloud-redis[async_rest]`") from e +from google.auth import credentials as ga_credentials # type: ignore from google.auth.aio import credentials as ga_credentials_async # type: ignore from google.api_core import exceptions as core_exceptions @@ -506,7 +507,7 @@ class AsyncCloudRedisRestTransport(_BaseCloudRedisRestTransport): def __init__(self, *, host: str = 'redis.googleapis.com', - credentials: Optional[ga_credentials_async.Credentials] = None, + credentials: Optional[Union[ga_credentials.Credentials, ga_credentials_async.Credentials]] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, url_scheme: str = 'https', interceptor: Optional[AsyncCloudRedisRestInterceptor] = None, @@ -520,7 +521,7 @@ def __init__(self, Args: host (Optional[str]): The hostname to connect to (default: 'redis.googleapis.com'). - credentials (Optional[google.auth.aio.credentials.Credentials]): The + credentials (Optional[Union[google.auth.credentials.Credentials, google.auth.aio.credentials.Credentials]]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none are specified, the client will attempt to ascertain the diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/setup.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/setup.py index 29dcfeccbc8b..fdadb0f8c9a8 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/setup.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/setup.py @@ -53,7 +53,8 @@ ] extras = { "async_rest": [ - "google-auth[aiohttp] >= 2.35.0, <3.0.0" + "google-auth[aiohttp] >= 2.60.0, <3.0.0", + "google-api-core >= 2.41.0, <3.0.0", ], } url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-cloud-redis" diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/testing/constraints-3.10-async-rest.txt b/packages/gapic-generator/tests/integration/goldens/redis_selective/testing/constraints-3.10-async-rest.txt index c0c1ba32f46f..4a2700a3bf8a 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/testing/constraints-3.10-async-rest.txt +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/testing/constraints-3.10-async-rest.txt @@ -6,7 +6,8 @@ # For example, if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0", # then this file should have google-cloud-foo==1.14.0 google-api-core==2.28.0 -google-auth==2.35.0 +google-auth==2.60.0 +aiohttp==3.8.0 grpcio==1.59.0 proto-plus==1.26.1 protobuf==6.33.5 diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/async_client.py b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/async_client.py index 4a5769a6fafc..63b639314785 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/async_client.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/async_client.py @@ -24,6 +24,7 @@ from google.api_core.client_options import ClientOptions from google.api_core import exceptions as core_exceptions from google.api_core import gapic_v1 +from google.api_core.resumable_transfer import ResumableUploadConfig from google.showcase_v1beta1._compat import setup_request_id from google.api_core import retry_async as retries from google.auth import credentials as ga_credentials # type: ignore @@ -36,6 +37,7 @@ except AttributeError: # pragma: NO COVER OptionalRetry = Union[retries.AsyncRetry, object, None] # type: ignore +from google.api_core import resumable_transfer from google.cloud.location import locations_pb2 # type: ignore from google.iam.v1 import iam_policy_pb2 # type: ignore from google.iam.v1 import policy_pb2 # type: ignore @@ -261,10 +263,11 @@ def __init__(self, *, async def upload_media(self, request: Optional[Union[resumable_upload.UploadMediaRequest, dict]] = None, *, + config: Optional[ResumableUploadConfig] = None, retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ) -> resumable_upload.UploadMediaResponse: + ) -> resumable_transfer.AsyncResumableUploadSession: r"""A method with media_upload annotation enabled. .. code-block:: python @@ -295,6 +298,8 @@ async def sample_upload_media(): Args: request (Optional[Union[google.showcase_v1beta1.types.UploadMediaRequest, dict]]): The request object. + config (Optional[google.api_core.resumable_transfer.ResumableUploadConfig]): + Optional configuration for the resumable upload session. retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -304,7 +309,9 @@ async def sample_upload_media(): be of type `bytes`. Returns: - google.showcase_v1beta1.types.UploadMediaResponse: + google.api_core.resumable_transfer.AsyncResumableUploadSession: + An object representing a resumable + upload session. """ # Create or coerce a protobuf request object. @@ -326,6 +333,8 @@ async def sample_upload_media(): retry=retry, timeout=timeout, metadata=metadata, + config=config, + start_retry=retry, ) # Done; return the response. diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/client.py b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/client.py index 20f68773522a..4fedb635bc91 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/client.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/client.py @@ -28,6 +28,7 @@ from google.api_core import client_options as client_options_lib from google.api_core import exceptions as core_exceptions from google.api_core import gapic_v1 +from google.api_core.resumable_transfer import ResumableUploadConfig from google.showcase_v1beta1._compat import get_universe_domain, get_api_endpoint, get_default_mtls_endpoint, should_use_client_cert, read_environment_variables from google.showcase_v1beta1._compat import setup_request_id from google.api_core import retry as retries @@ -51,6 +52,7 @@ _LOGGER = std_logging.getLogger(__name__) +from google.api_core import resumable_transfer from google.cloud.location import locations_pb2 # type: ignore from google.iam.v1 import iam_policy_pb2 # type: ignore from google.iam.v1 import policy_pb2 # type: ignore @@ -495,10 +497,11 @@ def __init__(self, *, def upload_media(self, request: Optional[Union[resumable_upload.UploadMediaRequest, dict]] = None, *, + config: Optional[ResumableUploadConfig] = None, retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ) -> resumable_upload.UploadMediaResponse: + ) -> resumable_transfer.ResumableUploadSession: r"""A method with media_upload annotation enabled. .. code-block:: python @@ -529,6 +532,8 @@ def sample_upload_media(): Args: request (Union[google.showcase_v1beta1.types.UploadMediaRequest, dict]): The request object. + config (Optional[google.api_core.resumable_transfer.ResumableUploadConfig]): + Optional configuration for the resumable upload session. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -538,7 +543,9 @@ def sample_upload_media(): be of type `bytes`. Returns: - google.showcase_v1beta1.types.UploadMediaResponse: + google.api_core.resumable_transfer.ResumableUploadSession: + An object representing a resumable + upload session. """ # Create or coerce a protobuf request object. @@ -560,6 +567,8 @@ def sample_upload_media(): retry=retry, timeout=timeout, metadata=metadata, + config=config, + start_retry=retry, ) # Done; return the response. diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/base.py b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/base.py index 856b01756ab3..0009a9c91cbb 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/base.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/base.py @@ -23,6 +23,7 @@ from google.api_core import exceptions as core_exceptions from google.api_core import gapic_v1 from google.api_core import retry as retries +from google.api_core import resumable_transfer from google.auth import credentials as ga_credentials # type: ignore from google.oauth2 import service_account # type: ignore import google.protobuf @@ -116,6 +117,7 @@ def __init__( # Save the credentials. self._credentials = credentials + self._client_info = client_info # Save the hostname. Default to port 443 (HTTPS) if none is specified. if ':' not in host: @@ -196,8 +198,9 @@ def close(self): def upload_media(self) -> Callable[ [resumable_upload.UploadMediaRequest], Union[ - resumable_upload.UploadMediaResponse, - Awaitable[resumable_upload.UploadMediaResponse] + resumable_transfer.ResumableUploadSession, + resumable_transfer.AsyncResumableUploadSession, + Awaitable[resumable_transfer.AsyncResumableUploadSession] ]]: raise NotImplementedError() diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/grpc.py b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/grpc.py index e5d0324cf558..dd52dbc790da 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/grpc.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/grpc.py @@ -19,6 +19,8 @@ import warnings from typing import Callable, Dict, Optional, Sequence, Tuple, Union +from google.api_core import exceptions as core_exceptions +from google.api_core import resumable_transfer from google.api_core import grpc_helpers from google.api_core import gapic_v1 import google.auth # type: ignore @@ -36,6 +38,7 @@ from google.longrunning import operations_pb2 # type: ignore from google.showcase_v1beta1.types import resumable_upload from .base import ResumableUploadServiceTransport, DEFAULT_CLIENT_INFO +from .rest import ResumableUploadServiceRestTransport try: from google.api_core import client_logging # type: ignore @@ -194,6 +197,13 @@ def __init__(self, *, self._grpc_channel = None self._ssl_channel_credentials = ssl_channel_credentials self._stubs: Dict[str, Callable] = {} + # Resumable upload RPCs operate over HTTP/REST rather than gRPC. + # Since gRPC is the default transport for GAPIC clients and services + # mix standard gRPC RPCs with resumable upload RPCs on the same client, + # gRPC transports delegate resumable upload calls to an internal REST + # transport instance. + self._rest_transport: Optional[ResumableUploadServiceRestTransport] = None + self._client_cert_source_for_mtls = client_cert_source_for_mtls if api_mtls_endpoint: warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) @@ -321,14 +331,14 @@ def grpc_channel(self) -> grpc.Channel: @property def upload_media(self) -> Callable[ [resumable_upload.UploadMediaRequest], - resumable_upload.UploadMediaResponse]: + resumable_transfer.ResumableUploadSession]: r"""Return a callable for the upload media method over gRPC. A method with media_upload annotation enabled. Returns: Callable[[~.UploadMediaRequest], - ~.UploadMediaResponse]: + ~.ResumableUploadSession]: A function that, when called, will call the underlying RPC on the server. """ @@ -337,15 +347,38 @@ def upload_media(self) -> Callable[ # gRPC handles serialization and deserialization, so we just need # to pass in the functions for each. if 'upload_media' not in self._stubs: - self._stubs['upload_media'] = self._logged_channel.unary_unary( - '/google.showcase.v1beta1.ResumableUploadService/UploadMedia', - request_serializer=resumable_upload.UploadMediaRequest.serialize, - response_deserializer=resumable_upload.UploadMediaResponse.deserialize, - ) + if not self._credentials: + class _ErrorStub: + def __init__(self, error: Exception): + self._error = error + + def __call__(self, *args, **kwargs): + raise self._error + + self._stubs['upload_media'] = _ErrorStub( + core_exceptions.GoogleAPICallError( + "Resumable upload methods operate over REST and cannot be invoked when the transport is initialized with a pre-constructed gRPC channel. Please supply credentials directly instead of a gRPC channel to use resumable upload functionality." + ) + ) + else: + transport = self + class _RestStub: + def __call__(self, *args, **kwargs): + if transport._rest_transport is None: + transport._rest_transport = ResumableUploadServiceRestTransport( + host=transport._host, + credentials=transport._credentials, + client_info=transport._client_info, + client_cert_source_for_mtls=transport._client_cert_source_for_mtls, + ) + return transport._rest_transport.upload_media(*args, **kwargs) + self._stubs['upload_media'] = _RestStub() return self._stubs['upload_media'] def close(self): self._logged_channel.close() + if self._rest_transport is not None: + self._rest_transport.close() @property def delete_operation( diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/grpc_asyncio.py b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/grpc_asyncio.py index 7bde6f9e3205..40782c9a3092 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/grpc_asyncio.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/grpc_asyncio.py @@ -18,12 +18,13 @@ import pickle import logging as std_logging import warnings -from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union +from typing import Any, Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union from google.api_core import gapic_v1 from google.api_core import grpc_helpers_async from google.api_core import exceptions as core_exceptions from google.api_core import retry_async as retries +from google.api_core import resumable_transfer from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore from google.protobuf.json_format import MessageToJson @@ -40,6 +41,11 @@ from google.showcase_v1beta1.types import resumable_upload from .base import ResumableUploadServiceTransport, DEFAULT_CLIENT_INFO from .grpc import ResumableUploadServiceGrpcTransport +try: + from .rest_asyncio import AsyncResumableUploadServiceRestTransport + HAS_ASYNC_REST = True # pragma: NO COVER +except ImportError: # pragma: NO COVER + HAS_ASYNC_REST = False try: from google.api_core import client_logging # type: ignore @@ -242,6 +248,13 @@ def __init__(self, *, self._grpc_channel = None self._ssl_channel_credentials = ssl_channel_credentials self._stubs: Dict[str, Callable] = {} + # Resumable upload RPCs operate over HTTP/REST rather than gRPC. + # Since gRPC is the default transport for GAPIC clients and services + # mix standard gRPC RPCs with resumable upload RPCs on the same client, + # gRPC transports delegate resumable upload calls to an internal REST + # transport instance. + self._rest_transport: Optional[Any] = None + self._client_cert_source_for_mtls = client_cert_source_for_mtls if api_mtls_endpoint: warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) @@ -327,14 +340,14 @@ def grpc_channel(self) -> aio.Channel: @property def upload_media(self) -> Callable[ [resumable_upload.UploadMediaRequest], - Awaitable[resumable_upload.UploadMediaResponse]]: + Awaitable[resumable_transfer.AsyncResumableUploadSession]]: r"""Return a callable for the upload media method over gRPC. A method with media_upload annotation enabled. Returns: Callable[[~.UploadMediaRequest], - Awaitable[~.UploadMediaResponse]]: + Awaitable[~.AsyncResumableUploadSession]]: A function that, when called, will call the underlying RPC on the server. """ @@ -343,11 +356,47 @@ def upload_media(self) -> Callable[ # gRPC handles serialization and deserialization, so we just need # to pass in the functions for each. if 'upload_media' not in self._stubs: - self._stubs['upload_media'] = self._logged_channel.unary_unary( - '/google.showcase.v1beta1.ResumableUploadService/UploadMedia', - request_serializer=resumable_upload.UploadMediaRequest.serialize, - response_deserializer=resumable_upload.UploadMediaResponse.deserialize, - ) + class _ErrorStub: + def __init__(self, error: Exception): + self._error = error + + def __call__(self, *args, **kwargs): + async def _raise(): + raise self._error + return _raise() + + if not self._credentials: + self._stubs['upload_media'] = _ErrorStub( + core_exceptions.GoogleAPICallError( + "Resumable upload methods operate over REST and cannot be invoked when the transport is initialized with a pre-constructed gRPC channel. Please supply credentials directly instead of a gRPC channel to use resumable upload functionality." + ) + ) + elif self._client_cert_source_for_mtls: + self._stubs['upload_media'] = _ErrorStub( + core_exceptions.AsyncRestUnsupportedParameterError( + "Mutual TLS (client_cert_source_for_mtls) is not currently supported for async resumable upload methods." + ) + ) + elif HAS_ASYNC_REST: + transport = self + class _AsyncRestStub: + def __call__(self, *args, **kwargs): + if transport._rest_transport is None: + transport._rest_transport = AsyncResumableUploadServiceRestTransport( + host=transport._host, + credentials=transport._credentials, + client_info=transport._client_info, + ) + return transport._rest_transport.upload_media(*args, **kwargs) + self._stubs['upload_media'] = _AsyncRestStub() + else: + self._stubs['upload_media'] = _ErrorStub( + NotImplementedError( + "Async REST transport is required for async resumable upload methods. " + "Install the library with the `async_rest` extra using " + "`pip install google-showcase[async_rest]`." + ) + ) return self._stubs['upload_media'] def _prep_wrapped_messages(self, client_info): @@ -410,8 +459,10 @@ def _wrap_method(self, func, *args, **kwargs): kwargs["kind"] = self.kind return gapic_v1.method_async.wrap_method(func, *args, **kwargs) - def close(self): - return self._logged_channel.close() + async def close(self): + await self._logged_channel.close() + if self._rest_transport is not None: + await self._rest_transport.close() @property def kind(self) -> str: diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/rest.py b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/rest.py index 3e29344fc715..032336df4352 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/rest.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/rest.py @@ -23,6 +23,8 @@ from google.api_core import rest_helpers from google.api_core import rest_streaming from google.api_core import gapic_v1 +import urllib.parse +from google.api_core import resumable_transfer from google.showcase_v1beta1._compat import transcode_request import google.protobuf @@ -85,10 +87,6 @@ def pre_upload_media(self, request, metadata): logging.log(f"Received request: {request}") return request, metadata - def post_upload_media(self, response): - logging.log(f"Received response: {response}") - return response - transport = ResumableUploadServiceRestTransport(interceptor=MyCustomResumableUploadServiceInterceptor()) client = ResumableUploadServiceClient(transport=transport) @@ -102,34 +100,6 @@ def pre_upload_media(self, request: resumable_upload.UploadMediaRequest, metadat """ return request, metadata - def post_upload_media(self, response: resumable_upload.UploadMediaResponse) -> resumable_upload.UploadMediaResponse: - """Post-rpc interceptor for upload_media - - DEPRECATED. Please use the `post_upload_media_with_metadata` - interceptor instead. - - Override in a subclass to read or manipulate the response - after it is returned by the ResumableUploadService server but before - it is returned to user code. This `post_upload_media` interceptor runs - before the `post_upload_media_with_metadata` interceptor. - """ - return response - - def post_upload_media_with_metadata(self, response: resumable_upload.UploadMediaResponse, metadata: Sequence[Tuple[str, Union[str, bytes]]]) -> Tuple[resumable_upload.UploadMediaResponse, Sequence[Tuple[str, Union[str, bytes]]]]: - """Post-rpc interceptor for upload_media - - Override in a subclass to read or manipulate the response or metadata after it - is returned by the ResumableUploadService server but before it is returned to user code. - - We recommend only using this `post_upload_media_with_metadata` - interceptor in new development instead of the `post_upload_media` interceptor. - When both interceptors are used, this `post_upload_media_with_metadata` interceptor runs after the - `post_upload_media` interceptor. The (possibly modified) response returned by - `post_upload_media` will be passed to - `post_upload_media_with_metadata`. - """ - return response, metadata - def pre_list_locations( self, request: locations_pb2.ListLocationsRequest, metadata: Sequence[Tuple[str, Union[str, bytes]]] ) -> Tuple[locations_pb2.ListLocationsRequest, Sequence[Tuple[str, Union[str, bytes]]]]: @@ -416,42 +386,22 @@ class _UploadMedia(_BaseResumableUploadServiceRestTransport._BaseUploadMedia, Re def __hash__(self): return hash("ResumableUploadServiceRestTransport.UploadMedia") - @staticmethod - def _get_response( - host, - metadata, - query_params, - session, - timeout, - transcoded_request, - body=None): - - uri = transcoded_request['uri'] - method = transcoded_request['method'] - headers = dict(metadata) - headers['Content-Type'] = 'application/json' - response = getattr(session, method)( - "{host}{uri}".format(host=host, uri=uri), - timeout=timeout, - headers=headers, - params=rest_helpers.flatten_query_params(query_params, strict=True), - data=body, - ) - return response - def __call__(self, request: resumable_upload.UploadMediaRequest, *, - retry: OptionalRetry=gapic_v1.method.DEFAULT, + config: Optional[resumable_transfer.ResumableUploadConfig]=None, + start_retry: OptionalRetry=gapic_v1.method.DEFAULT, timeout: Optional[float]=None, metadata: Sequence[Tuple[str, Union[str, bytes]]]=(), - ) -> resumable_upload.UploadMediaResponse: + ) -> resumable_transfer.ResumableUploadSession: r"""Call the upload media method over HTTP. Args: request (~.resumable_upload.UploadMediaRequest): The request object. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. + config (Optional[google.api_core.resumable_transfer.ResumableUploadConfig]): + Optional configuration for the resumable upload session. + start_retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried when initiating the resumable upload session. timeout (float): The timeout for this request. metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be sent along with the request as metadata. Normally, each value must be of type `str`, @@ -459,10 +409,11 @@ def __call__(self, be of type `bytes`. Returns: - ~.resumable_upload.UploadMediaResponse: + ~.resumable_transfer.ResumableUploadSession: + An object representing a resumable + upload session. """ - http_options = _BaseResumableUploadServiceRestTransport._BaseUploadMedia._get_http_options() request, metadata = self._interceptor.pre_upload_media(request, metadata) transcoded_request, body, query_params = transcode_request( @@ -476,71 +427,39 @@ def __call__(self, rest_numeric_enums=True, ) - if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor(logging.DEBUG): # pragma: NO COVER - request_url = "{host}{uri}".format(host=self._host, uri=transcoded_request['uri']) - method = transcoded_request['method'] - try: - request_payload = type(request).to_json(request) - except: - request_payload = None - http_request = { - "payload": request_payload, - "requestMethod": method, - "requestUrl": request_url, - "headers": dict(metadata), - } - _LOGGER.debug( - f"Sending request for google.showcase_v1beta1.ResumableUploadServiceClient.UploadMedia", - extra = { - "serviceName": "google.showcase.v1beta1.ResumableUploadService", - "rpcName": "UploadMedia", - "httpRequest": http_request, - "metadata": http_request["headers"], - }, - ) - - # Send the request - response = ResumableUploadServiceRestTransport._UploadMedia._get_response(self._host, metadata, query_params, self._session, timeout, transcoded_request, body) - - # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception - # subclass. - if response.status_code >= 400: - raise core_exceptions.from_http_response(response) - - # Return the response - resp = resumable_upload.UploadMediaResponse() - pb_resp = resumable_upload.UploadMediaResponse.pb(resp) - - json_format.Parse(response.content, pb_resp, ignore_unknown_fields=True) - - resp = self._interceptor.post_upload_media(resp) - response_metadata = [(k, str(v)) for k, v in response.headers.items()] - resp, _ = self._interceptor.post_upload_media_with_metadata(resp, response_metadata) - if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor(logging.DEBUG): # pragma: NO COVER - try: - response_payload = resumable_upload.UploadMediaResponse.to_json(response) - except: - response_payload = None - http_response = { - "payload": response_payload, - "headers": dict(response.headers), - "status": response.status_code, - } - _LOGGER.debug( - "Received response for google.showcase_v1beta1.ResumableUploadServiceClient.upload_media", - extra = { - "serviceName": "google.showcase.v1beta1.ResumableUploadService", - "rpcName": "UploadMedia", - "metadata": http_response["headers"], - "httpResponse": http_response, - }, - ) - return resp + uri = transcoded_request["uri"] + params = rest_helpers.flatten_query_params(query_params, strict=True) + query_string = f"?{urllib.parse.urlencode(params)}" if params else "" + upload_url = f"{self._host}{uri}{query_string}" + headers: Dict[str, Any] = {**dict(metadata), **dict((config.headers or {}) if config else {})} + headers["Content-Type"] = "application/json" + if config is None: + config = resumable_transfer.ResumableUploadConfig(headers=headers) + else: + config = dataclasses.replace(config, headers=headers) + + session_kwargs: Dict[str, Any] = ( + {"start_timeout": timeout} + if isinstance(timeout, (int, float)) + else {} + ) + # ``start_retry`` is used instead of ``retry`` because ``_GapicCallable`` + # consumes the ``retry`` argument before invoking the transport callable + # and only forwards extra keyword arguments such as ``start_retry``. + return resumable_transfer.ResumableUploadSession( + upload_url=upload_url, + config=config, + transport=self._session, + response_type=resumable_upload.UploadMediaResponse, + start_retry=start_retry if isinstance(start_retry, retries.Retry) else None, + request_body=body, + **session_kwargs, + ) @property def upload_media(self) -> Callable[ [resumable_upload.UploadMediaRequest], - resumable_upload.UploadMediaResponse]: + resumable_transfer.ResumableUploadSession]: # The return type is fine, but mypy isn't sophisticated enough to determine what's going on here. # In C++ this would require a dynamic_cast return self._UploadMedia(self._session, self._host, self._interceptor) # type: ignore diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/rest_asyncio.py b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/rest_asyncio.py new file mode 100755 index 000000000000..fa206afac970 --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/rest_asyncio.py @@ -0,0 +1,280 @@ +# -*- coding: utf-8 -*- +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import google.auth +try: + import aiohttp # type: ignore + from google.auth.aio.transport.sessions import AsyncAuthorizedSession # type: ignore + from google.api_core import rest_streaming_async # type: ignore + from google.api_core.operations_v1 import AsyncOperationsRestClient # type: ignore +except ImportError as e: # pragma: NO COVER + raise ImportError("`rest_asyncio` transport requires the library to be installed with the `async_rest` extra. Install the library with the `async_rest` extra using `pip install google-showcase[async_rest]`") from e + +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.aio import credentials as ga_credentials_async # type: ignore + +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +import urllib.parse +from google.api_core import resumable_transfer +from google.iam.v1 import iam_policy_pb2 # type: ignore +from google.iam.v1 import policy_pb2 # type: ignore +from google.cloud.location import locations_pb2 # type: ignore +from google.api_core import retry_async as retries +from google.api_core import rest_helpers +from google.api_core import rest_streaming_async # type: ignore +from google.showcase_v1beta1._compat import transcode_request + +import google.protobuf + +from google.protobuf import json_format +from google.iam.v1 import iam_policy_pb2 # type: ignore +from google.iam.v1 import policy_pb2 # type: ignore +from google.cloud.location import locations_pb2 # type: ignore + +import json # type: ignore +import dataclasses +from typing import Any, Dict, List, Callable, Tuple, Optional, Sequence, Union + + +from google.showcase_v1beta1.types import resumable_upload +from google.longrunning import operations_pb2 # type: ignore + + +from .rest_base import _BaseResumableUploadServiceRestTransport + +from .base import DEFAULT_CLIENT_INFO as BASE_DEFAULT_CLIENT_INFO + + +import logging + +try: + from google.api_core import client_logging # type: ignore + CLIENT_LOGGING_SUPPORTED = True # pragma: NO COVER +except ImportError: # pragma: NO COVER + CLIENT_LOGGING_SUPPORTED = False + +_LOGGER = logging.getLogger(__name__) + +try: + OptionalRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.AsyncRetry, object, None] # type: ignore + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=BASE_DEFAULT_CLIENT_INFO.gapic_version, + grpc_version=None, + rest_version=f"google-auth@{google.auth.__version__}", +) + +DEFAULT_CLIENT_INFO.protobuf_runtime_version = google.protobuf.__version__ + + +class AsyncResumableUploadServiceRestInterceptor: + """Asynchronous Interceptor for ResumableUploadService. + + Interceptors are used to manipulate requests, request metadata, and responses + in arbitrary ways. + Example use cases include: + * Logging + * Verifying requests according to service or custom semantics + * Stripping extraneous information from responses + + These use cases and more can be enabled by injecting an + instance of a custom subclass when constructing the AsyncResumableUploadServiceRestTransport. + + .. code-block:: python + class MyCustomResumableUploadServiceInterceptor(ResumableUploadServiceRestInterceptor): + async def pre_upload_media(self, request, metadata): + logging.log(f"Received request: {request}") + return request, metadata + + transport = AsyncResumableUploadServiceRestTransport(interceptor=MyCustomResumableUploadServiceInterceptor()) + client = async ResumableUploadServiceClient(transport=transport) + + + """ + async def pre_upload_media(self, request: resumable_upload.UploadMediaRequest, metadata: Sequence[Tuple[str, Union[str, bytes]]]) -> Tuple[resumable_upload.UploadMediaRequest, Sequence[Tuple[str, Union[str, bytes]]]]: + """Pre-rpc interceptor for upload_media + + Override in a subclass to manipulate the request or metadata + before they are sent to the ResumableUploadService server. + """ + return request, metadata + + +@dataclasses.dataclass +class AsyncResumableUploadServiceRestStub: + _session: AsyncAuthorizedSession + _host: str + _interceptor: AsyncResumableUploadServiceRestInterceptor + +class AsyncResumableUploadServiceRestTransport(_BaseResumableUploadServiceRestTransport): + """Asynchronous REST backend transport for ResumableUploadService. + + A service showcasing universal resumable upload protocol + support. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends JSON representations of protocol buffers over HTTP/1.1 + """ + def __init__(self, + *, + host: str = 'localhost:7469', + credentials: Optional[Union[ga_credentials.Credentials, ga_credentials_async.Credentials]] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + url_scheme: str = 'https', + interceptor: Optional[AsyncResumableUploadServiceRestInterceptor] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'localhost:7469'). + credentials (Optional[Union[google.auth.credentials.Credentials, google.auth.aio.credentials.Credentials]]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you are developing + your own client library. + url_scheme (str): the protocol scheme for the API endpoint. Normally + "https", but for testing or local servers, + "http" can be specified. + interceptor (Optional[AsyncResumableUploadServiceRestInterceptor]): Interceptor used + to manipulate requests, request metadata, and responses. + """ + # Run the base constructor + super().__init__( + host=host, + credentials=credentials, + client_info=client_info, + always_use_jwt_access=False, + url_scheme=url_scheme, + api_audience=None + ) + self._session = AsyncAuthorizedSession(self._credentials) # type: ignore + self._interceptor = interceptor or AsyncResumableUploadServiceRestInterceptor() + self._wrap_with_kind = True + self._prep_wrapped_messages(client_info) + + def _prep_wrapped_messages(self, client_info): + """ Precompute the wrapped methods, overriding the base class method to use async wrappers.""" + self._wrapped_methods = { + self.upload_media: self._wrap_method( + self.upload_media, + default_timeout=None, + client_info=client_info, + ), + } + + def _wrap_method(self, func, *args, **kwargs): + if self._wrap_with_kind: # pragma: NO COVER + kwargs["kind"] = self.kind + return gapic_v1.method_async.wrap_method(func, *args, **kwargs) + + class _UploadMedia(_BaseResumableUploadServiceRestTransport._BaseUploadMedia, AsyncResumableUploadServiceRestStub): + def __hash__(self): + return hash("AsyncResumableUploadServiceRestTransport.UploadMedia") + + async def __call__(self, + request: resumable_upload.UploadMediaRequest, *, + config: Optional[resumable_transfer.ResumableUploadConfig]=None, + start_retry: OptionalRetry=gapic_v1.method.DEFAULT, + timeout: Optional[float]=None, + metadata: Sequence[Tuple[str, Union[str, bytes]]]=(), + ) -> resumable_transfer.AsyncResumableUploadSession: + r"""Call the upload media method over HTTP. + + Args: + request (~.resumable_upload.UploadMediaRequest): + The request object. + config (Optional[google.api_core.resumable_transfer.ResumableUploadConfig]): + Optional configuration for the resumable upload session. + start_retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried when initiating the resumable upload session. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be + sent along with the request as metadata. Normally, each value must be of type `str`, + but for metadata keys ending with the suffix `-bin`, the corresponding values must + be of type `bytes`. + + Returns: + ~.resumable_transfer.AsyncResumableUploadSession: + An object representing a resumable + upload session. + + """ + http_options = _BaseResumableUploadServiceRestTransport._BaseUploadMedia._get_http_options() + request, metadata = await self._interceptor.pre_upload_media(request, metadata) + transcoded_request, body, query_params = transcode_request( + http_options, + request, + required_fields_default_values=getattr( + _BaseResumableUploadServiceRestTransport._BaseUploadMedia, + "_BaseUploadMedia__REQUIRED_FIELDS_DEFAULT_VALUES", + None, + ), + rest_numeric_enums=True, + ) + + uri = transcoded_request["uri"] + params = rest_helpers.flatten_query_params(query_params, strict=True) + query_string = f"?{urllib.parse.urlencode(params)}" if params else "" + upload_url = f"{self._host}{uri}{query_string}" + headers: Dict[str, Any] = {**dict(metadata), **dict((config.headers or {}) if config else {})} + headers["Content-Type"] = "application/json" + if config is None: + config = resumable_transfer.ResumableUploadConfig(headers=headers) + else: + config = dataclasses.replace(config, headers=headers) + + session_kwargs: Dict[str, Any] = ( + {"start_timeout": timeout} + if isinstance(timeout, (int, float)) + else {} + ) + # ``start_retry`` is used instead of ``retry`` because ``_GapicCallable`` + # consumes the ``retry`` argument before invoking the transport callable + # and only forwards extra keyword arguments such as ``start_retry``. + return resumable_transfer.AsyncResumableUploadSession( + upload_url=upload_url, + config=config, + transport=self._session, + response_type=resumable_upload.UploadMediaResponse, + start_retry=start_retry if isinstance(start_retry, retries.AsyncRetry) else None, + request_body=body, + **session_kwargs, + ) + + @property + def upload_media(self) -> Callable[ + [resumable_upload.UploadMediaRequest], + resumable_transfer.AsyncResumableUploadSession]: + return self._UploadMedia(self._session, self._host, self._interceptor) # type: ignore + + @property + def kind(self) -> str: + return "rest_asyncio" + + async def close(self): + await self._session.close() diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/rest_base.py b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/rest_base.py index a02842ea30de..207171c714db 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/rest_base.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/transports/rest_base.py @@ -97,7 +97,7 @@ def __hash__(self): # pragma: NO COVER def _get_http_options(): http_options: List[Dict[str, str]] = [{ 'method': 'post', - 'uri': '/v1beta1/files:upload', + 'uri': '/resumable/upload/v1beta1/files:upload', 'body': '*', }, ] diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/snippet_metadata_google.showcase.v1beta1.json b/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/snippet_metadata_google.showcase.v1beta1.json index 3d6373f36f36..6720f510f805 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/snippet_metadata_google.showcase.v1beta1.json +++ b/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/snippet_metadata_google.showcase.v1beta1.json @@ -4802,7 +4802,7 @@ "type": "Sequence[Tuple[str, Union[str, bytes]]]" } ], - "resultType": "google.showcase_v1beta1.types.UploadMediaResponse", + "resultType": "google.api_core.resumable_transfer.AsyncResumableUploadSession", "shortName": "upload_media" }, "description": "Sample for UploadMedia", @@ -4878,7 +4878,7 @@ "type": "Sequence[Tuple[str, Union[str, bytes]]]" } ], - "resultType": "google.showcase_v1beta1.types.UploadMediaResponse", + "resultType": "google.api_core.resumable_transfer.ResumableUploadSession", "shortName": "upload_media" }, "description": "Sample for UploadMedia", diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/setup.py b/packages/gapic-generator/tests/integration/goldens/showcase/setup.py index e369d99843c5..263074e55d01 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/setup.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/setup.py @@ -42,7 +42,7 @@ release_status = "Development Status :: 5 - Production/Stable" dependencies = [ - "google-api-core[grpc] >= 2.28.0, <3.0.0", + "google-api-core[grpc] >= 2.41.0, <3.0.0", # Exclude incompatible versions of `google-auth` # See https://github.com/googleapis/google-cloud-python/issues/12364 "google-auth >= 2.14.1, <3.0.0,!=2.24.0,!=2.25.0", @@ -53,6 +53,10 @@ "grpc-google-iam-v1 >= 0.14.2, <1.0.0", ] extras = { + "async_rest": [ + "google-auth[aiohttp] >= 2.60.0, <3.0.0", + "google-api-core >= 2.41.0, <3.0.0", + ], } url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-showcase" diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/testing/constraints-3.10-async-rest.txt b/packages/gapic-generator/tests/integration/goldens/showcase/testing/constraints-3.10-async-rest.txt new file mode 100755 index 000000000000..17cd83a2f715 --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/showcase/testing/constraints-3.10-async-rest.txt @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# This constraints file is used to check that lower bounds +# are correct in setup.py +# List all library dependencies and extras in this file, +# pinning their versions to their lower bounds. +# For example, if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0", +# then this file should have google-cloud-foo==1.14.0 +google-api-core==2.41.0 +google-auth==2.60.0 +aiohttp==3.8.0 +grpcio==1.59.0 +proto-plus==1.26.1 +protobuf==6.33.5 +grpc-google-iam-v1==0.14.2 diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/testing/constraints-3.10.txt b/packages/gapic-generator/tests/integration/goldens/showcase/testing/constraints-3.10.txt index 9c33fedfb264..a75b92057fdb 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/testing/constraints-3.10.txt +++ b/packages/gapic-generator/tests/integration/goldens/showcase/testing/constraints-3.10.txt @@ -4,7 +4,7 @@ # pinning their versions to their lower bounds. # For example, if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0", # then this file should have google-cloud-foo==1.14.0 -google-api-core==2.28.0 +google-api-core==2.41.0 google-auth==2.14.1 grpcio==1.59.0 proto-plus==1.26.1 diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/tests/unit/gapic/showcase_v1beta1/test_resumable_upload_service.py b/packages/gapic-generator/tests/integration/goldens/showcase/tests/unit/gapic/showcase_v1beta1/test_resumable_upload_service.py index dac0fc5053f7..1ff0d43412d7 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/tests/unit/gapic/showcase_v1beta1/test_resumable_upload_service.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/tests/unit/gapic/showcase_v1beta1/test_resumable_upload_service.py @@ -30,6 +30,13 @@ from google.api_core import api_core_version from proto.marshal.rules.dates import DurationRule, TimestampRule from proto.marshal.rules import wrappers +try: + import aiohttp # type: ignore + from google.auth.aio.transport.sessions import AsyncAuthorizedSession + from google.api_core.operations_v1 import AsyncOperationsRestClient + HAS_ASYNC_REST_EXTRA = True +except ImportError: # pragma: NO COVER + HAS_ASYNC_REST_EXTRA = False from requests import Response from requests import Request, PreparedRequest from requests.sessions import Session @@ -47,6 +54,7 @@ from google.api_core import grpc_helpers from google.api_core import grpc_helpers_async from google.api_core import path_template +from google.api_core import resumable_transfer from google.api_core import retry as retries from google.auth import credentials as ga_credentials from google.auth.exceptions import MutualTLSChannelError @@ -829,10 +837,7 @@ def test_upload_media(request_type, transport: str = 'grpc'): type(client.transport.upload_media), '__call__') as call: # Designate an appropriate return value for the call. - call.return_value = resumable_upload.UploadMediaResponse( - name='name_value', - size=443, - ) + call.return_value = resumable_transfer.ResumableUploadSession() response = client.upload_media(request) # Establish that the underlying gRPC stub method was called. @@ -842,9 +847,7 @@ def test_upload_media(request_type, transport: str = 'grpc'): assert args[0] == request # Establish that the response is the type that we expect. - assert isinstance(response, resumable_upload.UploadMediaResponse) - assert response.name == 'name_value' - assert response.size == 443 + assert isinstance(response, resumable_transfer.ResumableUploadSession) def test_upload_media_non_empty_request_with_auto_populated_field(): @@ -961,10 +964,9 @@ async def test_upload_media_async(request_type, transport: str = 'grpc_asyncio') type(client.transport.upload_media), '__call__') as call: # Designate an appropriate return value for the call. - call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(resumable_upload.UploadMediaResponse( - name='name_value', - size=443, - )) + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + resumable_transfer.AsyncResumableUploadSession() + ) response = await client.upload_media(request) # Establish that the underlying gRPC stub method was called. @@ -974,9 +976,7 @@ async def test_upload_media_async(request_type, transport: str = 'grpc_asyncio') assert args[0] == request # Establish that the response is the type that we expect. - assert isinstance(response, resumable_upload.UploadMediaResponse) - assert response.name == 'name_value' - assert response.size == 443 + assert isinstance(response, resumable_transfer.AsyncResumableUploadSession) def test_upload_media_rest_use_cached_wrapped_rpc(): @@ -1198,14 +1198,16 @@ def test_upload_media_rest_bad_request(request_type=resumable_upload.UploadMedia # Mock the http request call within the method and fake a BadRequest error. with mock.patch.object(Session, 'request') as req, pytest.raises(core_exceptions.BadRequest): # Wrap the value into a proper Response obj - response_value = mock.Mock() + response_value = mock.MagicMock() json_return_value = '' response_value.json = mock.Mock(return_value={}) + response_value.ok = False response_value.status_code = 400 response_value.request = mock.Mock() req.return_value = response_value req.return_value.headers = {"header-1": "value-1", "header-2": "value-2"} - client.upload_media(request) + response = client.upload_media(request) + response.upload(b"") @pytest.mark.parametrize("request_type", [ @@ -1231,16 +1233,24 @@ def test_upload_media_rest_call_success(request_type): ) # Wrap the value into a proper Response obj - response_value = mock.Mock() + response_value = mock.MagicMock() response_value.status_code = 200 # Convert return value to protobuf type return_value = resumable_upload.UploadMediaResponse.pb(return_value) json_return_value = json_format.MessageToJson(return_value) response_value.content = json_return_value.encode('UTF-8') - req.return_value = response_value - req.return_value.headers = {"header-1": "value-1", "header-2": "value-2"} + start_response = mock.MagicMock() + start_response.status_code = 200 + start_response.headers = {"x-goog-upload-status": "active", "x-goog-upload-url": "http://localhost/upload"} + response_value.headers = {"x-goog-upload-status": "final", "header-1": "value-1", "header-2": "value-2"} + start_response.ok = True + start_response.content = b"" + response_value.ok = True + req.side_effect = [start_response, response_value] response = client.upload_media(request) + assert isinstance(response, resumable_transfer.ResumableUploadSession) + response = response.upload(b"test data") # Establish that the response is the type that we expect. assert isinstance(response, resumable_upload.UploadMediaResponse) @@ -1258,12 +1268,8 @@ def test_upload_media_rest_interceptors(null_interceptor): with mock.patch.object(type(client.transport._session), "request") as req, \ mock.patch.object(path_template, "transcode") as transcode, \ - mock.patch.object(transports.ResumableUploadServiceRestInterceptor, "post_upload_media") as post, \ - mock.patch.object(transports.ResumableUploadServiceRestInterceptor, "post_upload_media_with_metadata") as post_with_metadata, \ mock.patch.object(transports.ResumableUploadServiceRestInterceptor, "pre_upload_media") as pre: pre.assert_not_called() - post.assert_not_called() - post_with_metadata.assert_not_called() pb_message = resumable_upload.UploadMediaRequest.pb(resumable_upload.UploadMediaRequest()) transcode.return_value = { "method": "post", @@ -1275,8 +1281,6 @@ def test_upload_media_rest_interceptors(null_interceptor): req.return_value = mock.Mock() req.return_value.status_code = 200 req.return_value.headers = {"header-1": "value-1", "header-2": "value-2"} - return_value = resumable_upload.UploadMediaResponse.to_json(resumable_upload.UploadMediaResponse()) - req.return_value.content = return_value request = resumable_upload.UploadMediaRequest() metadata =[ @@ -1284,14 +1288,10 @@ def test_upload_media_rest_interceptors(null_interceptor): ("cephalopod", "squid"), ] pre.return_value = request, metadata - post.return_value = resumable_upload.UploadMediaResponse() - post_with_metadata.return_value = resumable_upload.UploadMediaResponse(), metadata client.upload_media(request, metadata=[("key", "val"), ("cephalopod", "squid"),]) pre.assert_called_once() - post.assert_called_once() - post_with_metadata.assert_called_once() def test_list_locations_rest_bad_request(request_type=locations_pb2.ListLocationsRequest): @@ -2103,6 +2103,341 @@ def test_resumable_upload_service_grpc_asyncio_transport_channel(): assert transport._ssl_channel_credentials == None +def test_resumable_upload_service_upload_media_grpc_channel_without_credentials_error(): + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + transport = transports.ResumableUploadServiceGrpcTransport( + host="localhost:7469", + channel=channel, + ) + with pytest.raises(core_exceptions.GoogleAPICallError) as exc_info: + transport.upload_media(resumable_upload.UploadMediaRequest()) + assert "operate over REST and cannot be invoked when the transport is initialized with a pre-constructed gRPC channel" in str(exc_info.value) + transport.close() + + +def test_resumable_upload_service_upload_media_grpc_rest_stub(): + transport = transports.ResumableUploadServiceGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + assert transport._rest_transport is None + stub = transport.upload_media + assert transport._rest_transport is None + + result1 = stub(resumable_upload.UploadMediaRequest()) + rest_transport = transport._rest_transport + assert rest_transport is not None + assert isinstance(result1, resumable_transfer.ResumableUploadSession) + + result2 = stub(resumable_upload.UploadMediaRequest()) + assert isinstance(result2, resumable_transfer.ResumableUploadSession) + assert transport._rest_transport is rest_transport + + with mock.patch.object(rest_transport, "close") as mock_rest_close: + transport.close() + mock_rest_close.assert_called_once() + + +def test_resumable_upload_service_upload_media_grpc_rest_stub_mtls(): + with mock.patch( + "google.auth.transport.requests.AuthorizedSession.configure_mtls_channel" + ) as mock_configure_mtls_channel: + transport = transports.ResumableUploadServiceGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + client_cert_source_for_mtls=client_cert_source_callback, + ) + mock_configure_mtls_channel.assert_not_called() + stub = transport.upload_media + stub(resumable_upload.UploadMediaRequest()) + mock_configure_mtls_channel.assert_called_once_with( + client_cert_source_callback + ) + transport.close() + + +@pytest.mark.asyncio +async def test_resumable_upload_service_upload_media_grpc_asyncio_mtls_error(): + transport = transports.ResumableUploadServiceGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + client_cert_source_for_mtls=client_cert_source_callback, + ) + with pytest.raises(core_exceptions.AsyncRestUnsupportedParameterError) as exc_info: + await transport.upload_media(resumable_upload.UploadMediaRequest()) + assert "Mutual TLS (client_cert_source_for_mtls) is not currently supported for async resumable upload methods." in str(exc_info.value) + await transport.close() + + +@pytest.mark.asyncio +async def test_resumable_upload_service_upload_media_grpc_asyncio_channel_without_credentials_error(): + channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) + transport = transports.ResumableUploadServiceGrpcAsyncIOTransport( + host="localhost:7469", + channel=channel, + ) + with pytest.raises(core_exceptions.GoogleAPICallError) as exc_info: + await transport.upload_media(resumable_upload.UploadMediaRequest()) + assert "operate over REST and cannot be invoked when the transport is initialized with a pre-constructed gRPC channel" in str(exc_info.value) + await transport.close() + + +@pytest.mark.asyncio +async def test_resumable_upload_service_upload_media_grpc_asyncio_rest_stub(): + with mock.patch.object(transports.grpc_asyncio, "HAS_ASYNC_REST", True), mock.patch.object( + transports.grpc_asyncio, + "AsyncResumableUploadServiceRestTransport", + create=True, + ) as mock_rest_transport_cls: + mock_rest_transport = mock.MagicMock() + mock_rest_transport.close = mock.AsyncMock() + expected_session = resumable_transfer.AsyncResumableUploadSession() + mock_rest_transport.upload_media = mock.AsyncMock( + return_value=expected_session + ) + mock_rest_transport_cls.return_value = mock_rest_transport + + transport = transports.ResumableUploadServiceGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + mock_rest_transport_cls.assert_not_called() + + stub = transport.upload_media + result1 = await stub(resumable_upload.UploadMediaRequest()) + result2 = await stub(resumable_upload.UploadMediaRequest()) + + assert result1 is expected_session + assert result2 is expected_session + mock_rest_transport_cls.assert_called_once_with( + host=transport._host, + credentials=transport._credentials, + client_info=transport._client_info, + ) + assert mock_rest_transport.upload_media.call_count == 2 + await transport.close() + mock_rest_transport.close.assert_called_once() + + +@pytest.mark.asyncio +async def test_resumable_upload_service_upload_media_grpc_asyncio_rest_stub_sync_credentials(): + if not HAS_ASYNC_REST_EXTRA: + pytest.skip() + transport = transports.ResumableUploadServiceGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + assert transport._rest_transport is None + stub = transport.upload_media + assert transport._rest_transport is None + + result1 = await stub(resumable_upload.UploadMediaRequest()) + rest_transport = transport._rest_transport + assert rest_transport is not None + assert isinstance(result1, resumable_transfer.AsyncResumableUploadSession) + + result2 = await stub(resumable_upload.UploadMediaRequest()) + assert isinstance(result2, resumable_transfer.AsyncResumableUploadSession) + assert transport._rest_transport is rest_transport + + with mock.patch.object( + rest_transport, "close", wraps=rest_transport.close + ) as mock_rest_close: + await transport.close() + mock_rest_close.assert_called_once() + + +@pytest.mark.asyncio +async def test_resumable_upload_service_upload_media_grpc_asyncio_start_empty_body(): + if not HAS_ASYNC_REST_EXTRA: + pytest.skip() + client = ResumableUploadServiceAsyncClient( + credentials=async_anonymous_credentials(), + ) + with mock.patch.object(AsyncAuthorizedSession, "request") as req: + start_response = mock.MagicMock() + start_response.status_code = 200 + start_response.headers = {"x-goog-upload-status": "active", "x-goog-upload-url": "http://localhost/upload"} + start_response.read = mock.AsyncMock(return_value=b"") + start_response.close = mock.AsyncMock(return_value=None) + + expected_result = resumable_upload.UploadMediaResponse( + name='name_value', + size=443, + ) + json_return_value = resumable_upload.UploadMediaResponse.to_json(expected_result) + upload_response = mock.MagicMock() + upload_response.status_code = 200 + upload_response.headers = {"x-goog-upload-status": "final"} + upload_response.read = mock.AsyncMock(return_value=json_return_value.encode("UTF-8")) + upload_response.close = mock.AsyncMock(return_value=None) + + req.side_effect = [start_response, upload_response] + retry = retries.AsyncRetry() + response = await client.upload_media( + resumable_upload.UploadMediaRequest(), + config=resumable_transfer.ResumableUploadConfig(headers={"x-custom": "val"}), + retry=retry, + timeout=10.0, + ) + assert isinstance(response, resumable_transfer.AsyncResumableUploadSession) + assert response._start_retry is retry + result = await response.upload(b"test data") + assert isinstance(result, resumable_upload.UploadMediaResponse) + assert result == expected_result + start_call = req.call_args_list[0] + assert start_call.kwargs["headers"]["Content-Type"] == "application/json" + assert start_call.kwargs["headers"]["x-custom"] == "val" + + +@pytest.mark.asyncio +async def test_resumable_upload_service_upload_media_grpc_asyncio_body_and_query_params(): + if not HAS_ASYNC_REST_EXTRA: + pytest.skip() + client = ResumableUploadServiceAsyncClient( + credentials=async_anonymous_credentials(), + ) + with mock.patch.object( + AsyncAuthorizedSession, "request" + ) as req, mock.patch.object(path_template, "transcode") as transcode: + request = resumable_upload.UploadMediaRequest( + name='name_value', + ) + pb_request = resumable_upload.UploadMediaRequest.pb(request) + transcode.return_value = { + "uri": "v1/sample_method", + "method": "post", + "body": pb_request, + "query_params": pb_request, + } + + start_response = mock.MagicMock() + start_response.status_code = 200 + start_response.headers = {"x-goog-upload-status": "active", "x-goog-upload-url": "http://localhost/upload"} + start_response.read = mock.AsyncMock(return_value=b"") + start_response.close = mock.AsyncMock(return_value=None) + + expected_result = resumable_upload.UploadMediaResponse() + json_return_value = resumable_upload.UploadMediaResponse.to_json(expected_result) + upload_response = mock.MagicMock() + upload_response.status_code = 200 + upload_response.headers = {"x-goog-upload-status": "final"} + upload_response.read = mock.AsyncMock(return_value=json_return_value.encode("UTF-8")) + upload_response.close = mock.AsyncMock(return_value=None) + + req.side_effect = [start_response, upload_response] + response = await client.upload_media(request) + result = await response.upload(b"test data") + assert result == expected_result + + start_call = req.call_args_list[0] + assert start_call.kwargs["headers"]["Content-Type"] == "application/json" + assert "name=" in start_call.args[1] + expected_body = json_format.MessageToJson( + pb_request, + use_integers_for_enums=True, + ) + assert start_call.kwargs["data"] == expected_body.encode("UTF-8") + + +@pytest.mark.asyncio +async def test_resumable_upload_service_upload_media_grpc_asyncio_without_async_rest_error(): + with mock.patch.object(transports.grpc_asyncio, "HAS_ASYNC_REST", False): + transport = transports.ResumableUploadServiceGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(NotImplementedError) as exc_info: + await transport.upload_media(resumable_upload.UploadMediaRequest()) + assert ( + "Async REST transport is required for async resumable upload methods. " + "Install the library with the `async_rest` extra using " + "`pip install google-showcase[async_rest]`." + ) in str(exc_info.value) + +def test_resumable_upload_service_upload_media_rest_start_empty_body(): + client = ResumableUploadServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="rest", + ) + with mock.patch.object(type(client.transport._session), "request") as req: + start_response = mock.MagicMock() + start_response.ok = True + start_response.status_code = 200 + start_response.headers = {"x-goog-upload-status": "active", "x-goog-upload-url": "http://localhost/upload"} + start_response.content = b"" + + expected_result = resumable_upload.UploadMediaResponse( + name='name_value', + size=443, + ) + json_return_value = resumable_upload.UploadMediaResponse.to_json(expected_result) + upload_response = mock.MagicMock() + upload_response.ok = True + upload_response.status_code = 200 + upload_response.headers = {"x-goog-upload-status": "final"} + upload_response.content = json_return_value.encode("UTF-8") + + req.side_effect = [start_response, upload_response] + retry = retries.Retry() + response = client.upload_media( + resumable_upload.UploadMediaRequest(), + config=resumable_transfer.ResumableUploadConfig(headers={"x-custom": "val"}), + retry=retry, + timeout=10.0, + ) + assert isinstance(response, resumable_transfer.ResumableUploadSession) + assert response._start_retry is retry + result = response.upload(b"test data") + assert isinstance(result, resumable_upload.UploadMediaResponse) + assert result == expected_result + start_call = req.call_args_list[0] + assert start_call.kwargs["headers"]["Content-Type"] == "application/json" + assert start_call.kwargs["headers"]["x-custom"] == "val" + + +def test_resumable_upload_service_upload_media_rest_body_and_query_params(): + client = ResumableUploadServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="rest", + ) + with mock.patch.object(type(client.transport._session), "request") as req, mock.patch.object( + path_template, "transcode" + ) as transcode: + request = resumable_upload.UploadMediaRequest( + name='name_value', + ) + pb_request = resumable_upload.UploadMediaRequest.pb(request) + transcode.return_value = { + "uri": "v1/sample_method", + "method": "post", + "body": pb_request, + "query_params": pb_request, + } + + start_response = mock.MagicMock() + start_response.ok = True + start_response.status_code = 200 + start_response.headers = {"x-goog-upload-status": "active", "x-goog-upload-url": "http://localhost/upload"} + start_response.content = b"" + + expected_result = resumable_upload.UploadMediaResponse() + json_return_value = resumable_upload.UploadMediaResponse.to_json(expected_result) + upload_response = mock.MagicMock() + upload_response.ok = True + upload_response.status_code = 200 + upload_response.headers = {"x-goog-upload-status": "final"} + upload_response.content = json_return_value.encode("UTF-8") + + req.side_effect = [start_response, upload_response] + response = client.upload_media(request) + result = response.upload(b"test data") + assert result == expected_result + + start_call = req.call_args_list[0] + assert start_call.kwargs["headers"]["Content-Type"] == "application/json" + assert "name=" in start_call.args[1] + expected_body = json_format.MessageToJson( + pb_request, + use_integers_for_enums=True, + ) + assert start_call.kwargs["data"] == expected_body.encode("UTF-8") + + # Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are # removed from grpc/grpc_asyncio transport constructor. @pytest.mark.filterwarnings("ignore::FutureWarning") diff --git a/packages/gapic-generator/tests/unit/generator/test_generator.py b/packages/gapic-generator/tests/unit/generator/test_generator.py index 9d8545c4192f..3ca6af45f9b9 100644 --- a/packages/gapic-generator/tests/unit/generator/test_generator.py +++ b/packages/gapic-generator/tests/unit/generator/test_generator.py @@ -263,6 +263,80 @@ def test_get_response_ignores_unwanted_transports_and_clients(): } +def test_get_response_resumable_upload_generates_async_client_and_rest_asyncio(): + # Verify that REST transport files (rest.py, rest_base.py, rest_asyncio.py) + # are generated when a service has a resumable upload method and grpc + # transport is enabled, even if rest_async_io_enabled is False, while + # async_client.py and rest_asyncio.py are not generated for rest-only + # without rest_async_io_enabled. + generator_obj = make_generator() + with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as list_templates: + list_templates.return_value = [ + "foo/%service/transports/grpc.py.j2", + "foo/%service/transports/grpc_asyncio.py.j2", + "foo/%service/transports/rest.py.j2", + "foo/%service/transports/rest_asyncio.py.j2", + "foo/%service/transports/rest_base.py.j2", + "foo/%service/transports/__init__.py.j2", + "foo/%service/transports/base.py.j2", + "foo/%service/async_client.py.j2", + "foo/%service/client.py.j2", + ] + + with mock.patch.object(jinja2.Environment, "get_template") as get_template: + get_template.return_value = jinja2.Template("Service: {{ service.name }}") + api_schema = make_api( + make_proto( + descriptor_pb2.FileDescriptorProto( + name="resumable.proto", + package="foo.v1", + message_type=[ + descriptor_pb2.DescriptorProto(name="UploadMediaRequest"), + descriptor_pb2.DescriptorProto(name="UploadMediaResponse"), + ], + service=[ + descriptor_pb2.ServiceDescriptorProto( + name="ResumableUploadService", + method=[ + descriptor_pb2.MethodDescriptorProto( + name="UploadMedia", + input_type=".foo.v1.UploadMediaRequest", + output_type=".foo.v1.UploadMediaResponse", + ), + ], + ), + ], + ), + ) + ) + + cgr = generator_obj.get_response( + api_schema=api_schema, opts=Options.build("transport=rest") + ) + assert {i.name for i in cgr.file} == { + "foo/resumable_upload_service/transports/rest.py", + "foo/resumable_upload_service/transports/rest_base.py", + "foo/resumable_upload_service/transports/__init__.py", + "foo/resumable_upload_service/transports/base.py", + "foo/resumable_upload_service/client.py", + } + + cgr_grpc = generator_obj.get_response( + api_schema=api_schema, opts=Options.build("transport=grpc") + ) + assert {i.name for i in cgr_grpc.file} == { + "foo/resumable_upload_service/transports/grpc.py", + "foo/resumable_upload_service/transports/grpc_asyncio.py", + "foo/resumable_upload_service/transports/rest.py", + "foo/resumable_upload_service/transports/rest_asyncio.py", + "foo/resumable_upload_service/transports/rest_base.py", + "foo/resumable_upload_service/transports/__init__.py", + "foo/resumable_upload_service/transports/base.py", + "foo/resumable_upload_service/async_client.py", + "foo/resumable_upload_service/client.py", + } + + def test_get_response_enumerates_services(): generator_obj = make_generator() with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as list_templates: diff --git a/packages/gapic-generator/tests/unit/schema/wrappers/test_method.py b/packages/gapic-generator/tests/unit/schema/wrappers/test_method.py index 63a5adc0fcb4..d905c5696d4d 100644 --- a/packages/gapic-generator/tests/unit/schema/wrappers/test_method.py +++ b/packages/gapic-generator/tests/unit/schema/wrappers/test_method.py @@ -1124,9 +1124,24 @@ def test_method_is_resumable_upload(): # UploadMedia (and CreateYouTubeVideoUpload) are temporarily hardcoded as # resumable upload method names until resumable upload proto annotations # are adopted in the future. - # Verify that UploadMedia is identified as a resumable upload method. + # Verify that UploadMedia is identified as a resumable upload method and + # configures ResumableUploadSession / AsyncResumableUploadSession as the + # client output while retaining the raw output message in ref_types. method_upload = make_method("UploadMedia") assert method_upload.is_resumable_upload + assert method_upload.client_output.ident.name == "ResumableUploadSession" + assert ( + str(method_upload.client_output.ident) + == "resumable_transfer.ResumableUploadSession" + ) + assert ( + method_upload.client_output_async.ident.name == "AsyncResumableUploadSession" + ) + assert ( + str(method_upload.client_output_async.ident) + == "resumable_transfer.AsyncResumableUploadSession" + ) + assert method_upload.output in method_upload.ref_types # Verify that CreateYouTubeVideoUpload is also recognized as a resumable upload method. method_youtube_upload = make_method("CreateYouTubeVideoUpload") From c8926ab89d240239fee0c82be2373c752d3507c0 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 5 Oct 2026 20:26:41 -0400 Subject: [PATCH 012/228] feat(gapic-generator): generate code samples for resumable upload RPCs (#18501) - Add sample generation support for resumable upload RPCs. - Generated sync and async snippets demonstrate creating an upload session with `ResumableUploadConfig`, uploading a stream directly, iterating over `UploadProgress` updates per chunk, and resuming an interrupted upload from a saved session URL. --- .../gapic/samplegen/samplegen.py | 17 +- .../templates/examples/feature_fragments.j2 | 67 ++++++- .../gapic/templates/examples/sample.py.j2 | 6 +- .../resumable_upload_service/async_client.py | 29 ++- .../resumable_upload_service/client.py | 29 ++- ...mable_upload_service_upload_media_async.py | 32 ++- ...umable_upload_service_upload_media_sync.py | 30 ++- ...ppet_metadata_google.showcase.v1beta1.json | 40 ++-- .../tests/unit/samplegen/test_integration.py | 185 ++++++++++++++++++ 9 files changed, 401 insertions(+), 34 deletions(-) diff --git a/packages/gapic-generator/gapic/samplegen/samplegen.py b/packages/gapic-generator/gapic/samplegen/samplegen.py index 6f103225ea04..690117d0d975 100644 --- a/packages/gapic-generator/gapic/samplegen/samplegen.py +++ b/packages/gapic-generator/gapic/samplegen/samplegen.py @@ -1183,14 +1183,20 @@ def _get_sample_imports(sample: Dict, rpc: wrappers.Method) -> List[str]: module_name = sample["module_name"] module_import = f"from {module_namespace} import {module_name}" + imports = [module_import] address = rpc.input.meta.address # This checks if the request message is part of the service proto package. # If not, we should try to include a separate import statement. - if address.proto_package.startswith(address.api_naming.proto_package): - return [module_import] - else: - request_import = str(address.python_import) - return sorted([module_import, request_import]) + if not address.proto_package.startswith(address.api_naming.proto_package): + imports.append(str(address.python_import)) + + if rpc.is_resumable_upload: + imports.append( + "from google.api_core.resumable_transfer import ResumableUploadConfig" + ) + imports.append("import io") + + return sorted(imports) def generate_sample( @@ -1222,6 +1228,7 @@ def generate_sample( calling_form = types.CallingForm.method_default(rpc) sample["is_internal"] = rpc.is_internal + sample["is_resumable_upload"] = rpc.is_resumable_upload v = Validator(rpc, api_schema) # Tweak some small aspects of the sample to set defaults for optional diff --git a/packages/gapic-generator/gapic/templates/examples/feature_fragments.j2 b/packages/gapic-generator/gapic/templates/examples/feature_fragments.j2 index 40786ef5946b..5a995725c6cb 100644 --- a/packages/gapic-generator/gapic/templates/examples/feature_fragments.j2 +++ b/packages/gapic-generator/gapic/templates/examples/feature_fragments.j2 @@ -23,7 +23,11 @@ # It may require modifications to work in your environment. # To install the latest published package dependency, execute the following: +{% if sample.get("is_resumable_upload") and sample.transport in ("grpc-async", "rest-async") %} +# python3 -m pip install {{ sample.package_name }}[async_rest] +{% else %} # python3 -m pip install {{ sample.package_name }} +{% endif %} {% endmacro %} {% macro print_string_formatting(string_list) %} @@ -226,7 +230,7 @@ request=request {% endmacro %} -{% macro render_method_call(sample, calling_form, calling_form_enum, transport) %} +{% macro render_method_call(sample, calling_form, calling_form_enum, transport, include_config=False) %} {# Note: this doesn't deal with enums or unions #} {# LROs return operation objects and paged requests return pager objects #} {% if transport == "grpc-async" and calling_form not in @@ -238,14 +242,71 @@ await{{ " "}} client.{{ render_method_name(sample)|trim }}(requests=request_generator()) {% else %}{# TODO: deal with flattening #} {# TODO: set up client streaming once some questions are answered #} -client.{{ render_method_name(sample)|trim }}({{ render_request_params_unary(sample.request)|trim }}) +client.{{ render_method_name(sample)|trim }}({{ render_request_params_unary(sample.request)|trim }}{% if include_config %}, config=config{% endif %}) {% endif %} {% endmacro %} +{% macro render_resumable_upload_samples(sample, calling_form, calling_form_enum) %} +{% set is_async = sample.transport in ("grpc-async", "rest-async") %} +{% set async_prefix = "async " if is_async else "" %} +{% set await_prefix = "await " if is_async else "" %} +{% set fn_name = sample.rpc|snake_case|trim %} +{% set fn_params = print_input_params(sample.request)|trim %} +{% set method_call_with_config = render_method_call(sample, calling_form, calling_form_enum, sample.transport, include_config=True)|trim %} +{{ async_prefix }}def sample_{{ fn_name }}({{ fn_params }}): + {{ render_client_setup(sample.module_name, sample.client_name)|indent }} + {{ render_request_setup(sample.request, sample.request_module_name, sample.request_type, calling_form, calling_form_enum)|indent }} + # Configure optional transfer settings such as chunk size and stall detection + config = ResumableUploadConfig( + chunk_size=8 * 1024 * 1024, # 8 MB + stall_minimum_rate=64 * 1024, + stall_timeout=120, + ) + + # Make the request + upload_session = {{ method_call_with_config }} + + # Option 1: Upload the entire stream directly and {% if is_async %}await{% else %}return{% endif %} the final response + stream = io.BytesIO(b"Example upload data") + {{ "response = " if sample.response else "" }}{{ await_prefix }}upload_session.upload(stream) + + # Option 2: Alternatively, iterate over the upload to receive progress updates per chunk + {% if is_async %} + # async for progress in upload_session.upload(stream): + {% else %} + # for progress in upload_session.iter_upload(stream): + {% endif %} + # print(f"Uploaded {progress.bytes_uploaded} bytes | State: {progress.state.name}") + # print(f"Session URL: {progress.upload_url}") + {% if sample.response %} + # response = upload_session.response + {% endif %} + + # Option 3: Alternatively, resume an interrupted upload from a saved session URL + # {{ "response = " if sample.response else "" }}{{ await_prefix }}upload_session.resume(upload_url, stream, chunk_size=config.chunk_size) + + # Option 4: Alternatively, resume an interrupted upload while receiving progress updates + {% if is_async %} + # async for progress in upload_session.resume(upload_url, stream, chunk_size=config.chunk_size): + {% else %} + # for progress in upload_session.iter_resume(upload_url, stream, chunk_size=config.chunk_size): + {% endif %} + # print(f"Resumed {progress.bytes_uploaded} bytes | State: {progress.state.name}") + {% if sample.response %} + # response = upload_session.response + + # Handle the response + {% for statement in sample.response %} + {{ dispatch_statement(statement)|trim }} + {% endfor %} + {% endif %} +{% endmacro %} + + {# Setting up the method invocation is the responsibility of the caller: #} {# it's just easier to set up client side streaming and other things from outside this macro. #} -{% macro render_calling_form(method_invocation_text, calling_form, calling_form_enum, transport, response_statements ) %} +{% macro render_calling_form(method_invocation_text, calling_form, calling_form_enum, transport, response_statements, sample=None ) %} # Make the request {% if calling_form in [calling_form_enum.Request, calling_form_enum.RequestStreamingClient] %} {% if response_statements %}response = {% endif %}{{ method_invocation_text|trim }} diff --git a/packages/gapic-generator/gapic/templates/examples/sample.py.j2 b/packages/gapic-generator/gapic/templates/examples/sample.py.j2 index 68713f812b85..7b08a54e47c9 100644 --- a/packages/gapic-generator/gapic/templates/examples/sample.py.j2 +++ b/packages/gapic-generator/gapic/templates/examples/sample.py.j2 @@ -32,12 +32,16 @@ {# also need calling form #} +{% if sample.get("is_resumable_upload") %} +{{ frags.render_resumable_upload_samples(sample, calling_form, calling_form_enum) }} +{% else %} {% if sample.transport == "grpc-async" %}async {% endif %}def sample_{{ sample.rpc|snake_case|trim }}({{ frags.print_input_params(sample.request)|trim }}): {{ frags.render_client_setup(sample.module_name, sample.client_name)|indent }} {{ frags.render_request_setup(sample.request, sample.request_module_name, sample.request_type, calling_form, calling_form_enum)|indent }} {% with method_call = frags.render_method_call(sample, calling_form, calling_form_enum, sample.transport) %} - {{ frags.render_calling_form(method_call, calling_form, calling_form_enum, sample.transport, sample.response)|indent -}} + {{ frags.render_calling_form(method_call, calling_form, calling_form_enum, sample.transport, sample.response, sample)|indent -}} {% endwith %} +{% endif %} # [END {{ sample.id }}] {# TODO: Enable main block (or decide to remove main block from python sample) #} diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/async_client.py b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/async_client.py index 63b639314785..adb806e016af 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/async_client.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/async_client.py @@ -280,6 +280,8 @@ async def upload_media(self, # client as shown in: # https://googleapis.dev/python/google-api-core/latest/client_options.html from google import showcase_v1beta1 + from google.api_core.resumable_transfer import ResumableUploadConfig + import io async def sample_upload_media(): # Create a client @@ -289,8 +291,33 @@ async def sample_upload_media(): request = showcase_v1beta1.UploadMediaRequest( ) + # Configure optional transfer settings such as chunk size and stall detection + config = ResumableUploadConfig( + chunk_size=8 * 1024 * 1024, # 8 MB + stall_minimum_rate=64 * 1024, + stall_timeout=120, + ) + # Make the request - response = await client.upload_media(request=request) + upload_session = await client.upload_media(request=request, config=config) + + # Option 1: Upload the entire stream directly and await the final response + stream = io.BytesIO(b"Example upload data") + response = await upload_session.upload(stream) + + # Option 2: Alternatively, iterate over the upload to receive progress updates per chunk + # async for progress in upload_session.upload(stream): + # print(f"Uploaded {progress.bytes_uploaded} bytes | State: {progress.state.name}") + # print(f"Session URL: {progress.upload_url}") + # response = upload_session.response + + # Option 3: Alternatively, resume an interrupted upload from a saved session URL + # response = await upload_session.resume(upload_url, stream, chunk_size=config.chunk_size) + + # Option 4: Alternatively, resume an interrupted upload while receiving progress updates + # async for progress in upload_session.resume(upload_url, stream, chunk_size=config.chunk_size): + # print(f"Resumed {progress.bytes_uploaded} bytes | State: {progress.state.name}") + # response = upload_session.response # Handle the response print(response) diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/client.py b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/client.py index 4fedb635bc91..1f62d75b9951 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/client.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/google/showcase_v1beta1/services/resumable_upload_service/client.py @@ -514,6 +514,8 @@ def upload_media(self, # client as shown in: # https://googleapis.dev/python/google-api-core/latest/client_options.html from google import showcase_v1beta1 + from google.api_core.resumable_transfer import ResumableUploadConfig + import io def sample_upload_media(): # Create a client @@ -523,8 +525,33 @@ def sample_upload_media(): request = showcase_v1beta1.UploadMediaRequest( ) + # Configure optional transfer settings such as chunk size and stall detection + config = ResumableUploadConfig( + chunk_size=8 * 1024 * 1024, # 8 MB + stall_minimum_rate=64 * 1024, + stall_timeout=120, + ) + # Make the request - response = client.upload_media(request=request) + upload_session = client.upload_media(request=request, config=config) + + # Option 1: Upload the entire stream directly and return the final response + stream = io.BytesIO(b"Example upload data") + response = upload_session.upload(stream) + + # Option 2: Alternatively, iterate over the upload to receive progress updates per chunk + # for progress in upload_session.iter_upload(stream): + # print(f"Uploaded {progress.bytes_uploaded} bytes | State: {progress.state.name}") + # print(f"Session URL: {progress.upload_url}") + # response = upload_session.response + + # Option 3: Alternatively, resume an interrupted upload from a saved session URL + # response = upload_session.resume(upload_url, stream, chunk_size=config.chunk_size) + + # Option 4: Alternatively, resume an interrupted upload while receiving progress updates + # for progress in upload_session.iter_resume(upload_url, stream, chunk_size=config.chunk_size): + # print(f"Resumed {progress.bytes_uploaded} bytes | State: {progress.state.name}") + # response = upload_session.response # Handle the response print(response) diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/localhost_v1beta1_generated_resumable_upload_service_upload_media_async.py b/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/localhost_v1beta1_generated_resumable_upload_service_upload_media_async.py index 39d286ead6d8..2bc21047931c 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/localhost_v1beta1_generated_resumable_upload_service_upload_media_async.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/localhost_v1beta1_generated_resumable_upload_service_upload_media_async.py @@ -20,7 +20,7 @@ # It may require modifications to work in your environment. # To install the latest published package dependency, execute the following: -# python3 -m pip install google-showcase +# python3 -m pip install google-showcase[async_rest] # [START localhost_v1beta1_generated_ResumableUploadService_UploadMedia_async] @@ -32,6 +32,8 @@ # client as shown in: # https://googleapis.dev/python/google-api-core/latest/client_options.html from google import showcase_v1beta1 +from google.api_core.resumable_transfer import ResumableUploadConfig +import io async def sample_upload_media(): @@ -42,10 +44,36 @@ async def sample_upload_media(): request = showcase_v1beta1.UploadMediaRequest( ) + # Configure optional transfer settings such as chunk size and stall detection + config = ResumableUploadConfig( + chunk_size=8 * 1024 * 1024, # 8 MB + stall_minimum_rate=64 * 1024, + stall_timeout=120, + ) + # Make the request - response = await client.upload_media(request=request) + upload_session = await client.upload_media(request=request, config=config) + + # Option 1: Upload the entire stream directly and await the final response + stream = io.BytesIO(b"Example upload data") + response = await upload_session.upload(stream) + + # Option 2: Alternatively, iterate over the upload to receive progress updates per chunk + # async for progress in upload_session.upload(stream): + # print(f"Uploaded {progress.bytes_uploaded} bytes | State: {progress.state.name}") + # print(f"Session URL: {progress.upload_url}") + # response = upload_session.response + + # Option 3: Alternatively, resume an interrupted upload from a saved session URL + # response = await upload_session.resume(upload_url, stream, chunk_size=config.chunk_size) + + # Option 4: Alternatively, resume an interrupted upload while receiving progress updates + # async for progress in upload_session.resume(upload_url, stream, chunk_size=config.chunk_size): + # print(f"Resumed {progress.bytes_uploaded} bytes | State: {progress.state.name}") + # response = upload_session.response # Handle the response print(response) + # [END localhost_v1beta1_generated_ResumableUploadService_UploadMedia_async] diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/localhost_v1beta1_generated_resumable_upload_service_upload_media_sync.py b/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/localhost_v1beta1_generated_resumable_upload_service_upload_media_sync.py index 779d7aac1120..b1165a305eed 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/localhost_v1beta1_generated_resumable_upload_service_upload_media_sync.py +++ b/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/localhost_v1beta1_generated_resumable_upload_service_upload_media_sync.py @@ -32,6 +32,8 @@ # client as shown in: # https://googleapis.dev/python/google-api-core/latest/client_options.html from google import showcase_v1beta1 +from google.api_core.resumable_transfer import ResumableUploadConfig +import io def sample_upload_media(): @@ -42,10 +44,36 @@ def sample_upload_media(): request = showcase_v1beta1.UploadMediaRequest( ) + # Configure optional transfer settings such as chunk size and stall detection + config = ResumableUploadConfig( + chunk_size=8 * 1024 * 1024, # 8 MB + stall_minimum_rate=64 * 1024, + stall_timeout=120, + ) + # Make the request - response = client.upload_media(request=request) + upload_session = client.upload_media(request=request, config=config) + + # Option 1: Upload the entire stream directly and return the final response + stream = io.BytesIO(b"Example upload data") + response = upload_session.upload(stream) + + # Option 2: Alternatively, iterate over the upload to receive progress updates per chunk + # for progress in upload_session.iter_upload(stream): + # print(f"Uploaded {progress.bytes_uploaded} bytes | State: {progress.state.name}") + # print(f"Session URL: {progress.upload_url}") + # response = upload_session.response + + # Option 3: Alternatively, resume an interrupted upload from a saved session URL + # response = upload_session.resume(upload_url, stream, chunk_size=config.chunk_size) + + # Option 4: Alternatively, resume an interrupted upload while receiving progress updates + # for progress in upload_session.iter_resume(upload_url, stream, chunk_size=config.chunk_size): + # print(f"Resumed {progress.bytes_uploaded} bytes | State: {progress.state.name}") + # response = upload_session.response # Handle the response print(response) + # [END localhost_v1beta1_generated_ResumableUploadService_UploadMedia_sync] diff --git a/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/snippet_metadata_google.showcase.v1beta1.json b/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/snippet_metadata_google.showcase.v1beta1.json index 6720f510f805..c64d375bfa25 100755 --- a/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/snippet_metadata_google.showcase.v1beta1.json +++ b/packages/gapic-generator/tests/integration/goldens/showcase/samples/generated_samples/snippet_metadata_google.showcase.v1beta1.json @@ -4812,33 +4812,33 @@ "regionTag": "localhost_v1beta1_generated_ResumableUploadService_UploadMedia_async", "segments": [ { - "end": 50, + "end": 78, "start": 27, "type": "FULL" }, { - "end": 50, + "end": 78, "start": 27, "type": "SHORT" }, { - "end": 40, - "start": 38, + "end": 42, + "start": 40, "type": "CLIENT_INITIALIZATION" }, { - "end": 44, - "start": 41, + "end": 53, + "start": 43, "type": "REQUEST_INITIALIZATION" }, { - "end": 47, - "start": 45, + "end": 74, + "start": 54, "type": "REQUEST_EXECUTION" }, { - "end": 51, - "start": 48, + "end": 79, + "start": 75, "type": "RESPONSE_HANDLING" } ], @@ -4888,33 +4888,33 @@ "regionTag": "localhost_v1beta1_generated_ResumableUploadService_UploadMedia_sync", "segments": [ { - "end": 50, + "end": 78, "start": 27, "type": "FULL" }, { - "end": 50, + "end": 78, "start": 27, "type": "SHORT" }, { - "end": 40, - "start": 38, + "end": 42, + "start": 40, "type": "CLIENT_INITIALIZATION" }, { - "end": 44, - "start": 41, + "end": 53, + "start": 43, "type": "REQUEST_INITIALIZATION" }, { - "end": 47, - "start": 45, + "end": 74, + "start": 54, "type": "REQUEST_EXECUTION" }, { - "end": 51, - "start": 48, + "end": 79, + "start": 75, "type": "RESPONSE_HANDLING" } ], diff --git a/packages/gapic-generator/tests/unit/samplegen/test_integration.py b/packages/gapic-generator/tests/unit/samplegen/test_integration.py index c937f78bddd9..65fa11acf144 100644 --- a/packages/gapic-generator/tests/unit/samplegen/test_integration.py +++ b/packages/gapic-generator/tests/unit/samplegen/test_integration.py @@ -807,3 +807,188 @@ def test_generate_sample_config_partial_config(fs): def test_generate_sample_config_fpaths_no_such_file(fs): with pytest.raises(types.InvalidConfig): list(gapic_utils.generate_all_sample_fpaths("cfgs/sample_config.yaml")) + + +def test_generate_sample_resumable_upload(): + classify_target_field = DummyField( + name="classify_target", + type=DummyMessageTypePB(name="ClassifyTarget"), + message=DummyMessage( + type="CLASSIFY TYPE", + fields={ + "location_annotation": DummyField( + type=DummyMessageTypePB(name="Location"), + message=DummyMessage(type="LOCATION TYPE"), + ), + }, + ), + ident=DummyIdent(sphinx="molluscs_v1.ClassifyTarget"), + ) + + input_type = DummyMessage( + type="REQUEST TYPE", + fields={"classify_target": classify_target_field}, + ident=DummyIdent( + name="molluscs.v1.ClassifyRequest", sphinx="molluscs_v1.classify_request" + ), + ) + + output_type = DummyMessage( + type="RESPONSE TYPE", + fields={ + "classification": DummyField( + type=DummyMessageTypePB(name="Classification"), + ) + }, + ident=DummyIdent(sphinx="molluscs_v1.classification"), + ) + + api_naming = naming.NewNaming(name="MolluscClient", namespace=("molluscs", "v1")) + service = wrappers.Service( + service_pb=namedtuple("service_pb", ["name"])("MolluscService"), + methods={ + "Classify": DummyMethod( + name="Classify", + input=input_type, + output=message_factory("$resp.taxonomy"), + client_output_async=output_type, + client_output=output_type, + flattened_fields={"classify_target": classify_target_field}, + is_resumable_upload=True, + ) + }, + visible_resources={}, + ) + + schema = DummyApiSchema( + services={"animalia.mollusca.v1.Mollusc": service}, + naming=api_naming, + ) + + sample = { + "service": "animalia.mollusca.v1.Mollusc", + "region_tag": "molluscs_generated_molluscs_v1_Mollusc_Classify_sync", + "rpc": "Classify", + "transport": "grpc", + "id": "mollusc_classify_sync", + "description": "Determine the full taxonomy of input mollusc", + "request": [ + { + "field": "classify_target.location_annotation", + "value": "New Zealand", + "input_parameter": "location", + }, + ], + "response": [{"print": ['Mollusc is a "%s"', "$resp.taxonomy"]}], + } + + sample_str, _ = samplegen.generate_sample( + sample, schema, env.get_template("examples/sample.py.j2") + ) + + assert ( + "from google.api_core.resumable_transfer import ResumableUploadConfig" + in sample_str + ) + assert "import io" in sample_str + assert "# Make the request" in sample_str + assert "upload_session = client.classify(request=request, config=config)" in sample_str + assert 'stream = io.BytesIO(b"Example upload data")' in sample_str + assert "response = upload_session.upload(stream)" in sample_str + assert "# for progress in upload_session.iter_upload(stream):" in sample_str + assert "# response = upload_session.resume(upload_url, stream, chunk_size=config.chunk_size)" in sample_str + assert "# for progress in upload_session.iter_resume(upload_url, stream, chunk_size=config.chunk_size):" in sample_str + + +def test_generate_sample_resumable_upload_async(): + classify_target_field = DummyField( + name="classify_target", + type=DummyMessageTypePB(name="ClassifyTarget"), + message=DummyMessage( + type="CLASSIFY TYPE", + fields={ + "location_annotation": DummyField( + type=DummyMessageTypePB(name="Location"), + message=DummyMessage(type="LOCATION TYPE"), + ), + }, + ), + ident=DummyIdent(sphinx="molluscs_v1.ClassifyTarget"), + ) + + input_type = DummyMessage( + type="REQUEST TYPE", + fields={"classify_target": classify_target_field}, + ident=DummyIdent( + name="molluscs.v1.ClassifyRequest", sphinx="molluscs_v1.classify_request" + ), + ) + + output_type = DummyMessage( + type="RESPONSE TYPE", + fields={ + "classification": DummyField( + type=DummyMessageTypePB(name="Classification"), + ) + }, + ident=DummyIdent(sphinx="molluscs_v1.classification"), + ) + + api_naming = naming.NewNaming(name="MolluscClient", namespace=("molluscs", "v1")) + service = wrappers.Service( + service_pb=namedtuple("service_pb", ["name"])("MolluscService"), + methods={ + "Classify": DummyMethod( + name="Classify", + input=input_type, + output=message_factory("$resp.taxonomy"), + client_output_async=output_type, + client_output=output_type, + flattened_fields={"classify_target": classify_target_field}, + is_resumable_upload=True, + ) + }, + visible_resources={}, + ) + + schema = DummyApiSchema( + services={"animalia.mollusca.v1.Mollusc": service}, + naming=api_naming, + ) + + sample = { + "service": "animalia.mollusca.v1.Mollusc", + "region_tag": "molluscs_generated_molluscs_v1_Mollusc_Classify_async", + "rpc": "Classify", + "transport": "grpc-async", + "id": "mollusc_classify_async", + "description": "Determine the full taxonomy of input mollusc", + "request": [ + { + "field": "classify_target.location_annotation", + "value": "New Zealand", + "input_parameter": "location", + }, + ], + "response": [{"print": ['Mollusc is a "%s"', "$resp.taxonomy"]}], + } + + sample_str, _ = samplegen.generate_sample( + sample, schema, env.get_template("examples/sample.py.j2") + ) + + assert ( + "from google.api_core.resumable_transfer import ResumableUploadConfig" + in sample_str + ) + assert "import io" in sample_str + assert "python3 -m pip install molluscs-v1-molluscclient[async_rest]" in sample_str + assert "# Make the request" in sample_str + assert "upload_session = await client.classify(request=request, config=config)" in sample_str + assert 'stream = io.BytesIO(b"Example upload data")' in sample_str + assert "response = await upload_session.upload(stream)" in sample_str + assert "# async for progress in upload_session.upload(stream):" in sample_str + assert "# response = await upload_session.resume(upload_url, stream, chunk_size=config.chunk_size)" in sample_str + assert "# async for progress in upload_session.resume(upload_url, stream, chunk_size=config.chunk_size):" in sample_str + + From 37123e5bfc5cb5941d5c3730f6a47a0d7c3b6012 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 6 Oct 2026 15:18:22 +0100 Subject: [PATCH 013/228] chore(deps): update dependency setuptools to v83 [security] (#18569) --- packages/google-crc32c/scripts/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/google-crc32c/scripts/requirements.txt b/packages/google-crc32c/scripts/requirements.txt index 536fc934a879..ee1d42b09d98 100644 --- a/packages/google-crc32c/scripts/requirements.txt +++ b/packages/google-crc32c/scripts/requirements.txt @@ -39,7 +39,7 @@ pip==26.2 \ --hash=sha256:2d8542afcc84cdd8e846c2b36b2861fad1da376dd98f8e7113e9108a3c331690 \ --hash=sha256:931c303696af6fa3417112103b1cad26890e5a07eccb5b99783700e33f2b8aad # via -r requirements.in -setuptools==82.0.1 \ - --hash=sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9 \ - --hash=sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb +setuptools==83.0.0 \ + --hash=sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef \ + --hash=sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3 # via -r requirements.in From 2d19b0a652cbe79005da2b55b553d2c99fe95da9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Tue, 6 Oct 2026 18:30:21 +0200 Subject: [PATCH 014/228] fix(spanner): correctly wrap async streaming calls in metrics interceptor (#18581) Fix an issue where async streaming RPCs were incorrectly wrapped in _AsyncUnaryResponseWrapper instead of _AsyncStreamingResponseWrapper, preventing metrics from being properly recorded. #### Problem In grpc.aio, streaming call objects (like UnaryStreamCall) implement `__aiter__` (returning an async generator) but do not define `__anext__` directly on the call object itself. Because AsyncMetricsInterceptor checked `hasattr(response, "__anext__")`, all async streaming calls (such as ExecuteStreamingSql and StreamingRead) were misclassified as unary calls and wrapped in _AsyncUnaryResponseWrapper. Because _AsyncUnaryResponseWrapper only records metrics when awaited (`__await__`), and callers iterate over streams chunk-by-chunk rather than awaiting the stream object itself: 1. Frontend metrics (gfe_latencies, afe_latencies, and missing header counts) were never recorded for any async queries or streaming reads. 2. Attempt completion was never recorded when the stream finished. Instead, it was only recorded much later when the wrapper was garbage-collected (`__del__`), resulting in wildly inflated attempt latency measurements or attempts incorrectly reporting as CANCELLED. #### Solution - Explicitly dispatch async interceptor wrappers based on the gRPC call type (is_streaming=True for intercept_unary_stream and intercept_stream_stream) rather than fragile runtime dunder checks. - Realign class inheritance so StreamUnaryCall is handled by _AsyncUnaryResponseWrapper, adding write() and done_writing() delegation. - Support sending real HTTP/2 response headers in the in-memory mock Spanner server via an add_header() helper, removing monkeypatching of internal gRPC classes in integration tests. - Add unit tests for direct `__anext__()` consumption (matching StreamedResultSet), delegation methods, error handling, and sync/async response wrappers. --- .../spanner_v1/metrics/metrics_interceptor.py | 36 +- .../cloud/spanner_v1/testing/mock_spanner.py | 19 + .../mockserver_tests/mock_server_test_base.py | 4 + .../mockserver_tests/test_frontend_metrics.py | 418 ++++++++++++---- .../tests/unit/test_metrics_interceptor.py | 470 +++++++++++++++++- 5 files changed, 843 insertions(+), 104 deletions(-) diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/metrics/metrics_interceptor.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/metrics/metrics_interceptor.py index 1205b09c1840..2aea886d0e6e 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/metrics/metrics_interceptor.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/metrics/metrics_interceptor.py @@ -166,23 +166,27 @@ class AsyncMetricsInterceptor( """Async Interceptor that collects metrics for Cloud Spanner operations.""" async def intercept_unary_unary(self, continuation, client_call_details, request): - return await self._async_intercept(continuation, client_call_details, request) + return await self._async_intercept( + continuation, client_call_details, request, is_streaming=False + ) async def intercept_unary_stream(self, continuation, client_call_details, request): - return await self._async_intercept(continuation, client_call_details, request) + return await self._async_intercept( + continuation, client_call_details, request, is_streaming=True + ) async def intercept_stream_unary( self, continuation, client_call_details, request_iterator ): return await self._async_intercept( - continuation, client_call_details, request_iterator + continuation, client_call_details, request_iterator, is_streaming=False ) async def intercept_stream_stream( self, continuation, client_call_details, request_iterator ): return await self._async_intercept( - continuation, client_call_details, request_iterator + continuation, client_call_details, request_iterator, is_streaming=True ) async def _async_intercept( @@ -190,6 +194,7 @@ async def _async_intercept( continuation: Any, call_details: grpc.ClientCallDetails, request_or_iterator: Any, + is_streaming: bool = False, ) -> Any: # Implementation for async interceptor factory = SpannerMetricsTracerFactory() @@ -216,10 +221,9 @@ async def _async_intercept( tracer.record_attempt_start() response = await continuation(call_details, request_or_iterator) - if hasattr(response, "__anext__"): + if is_streaming: return _AsyncStreamingResponseWrapper(response, tracer) - else: - return _AsyncUnaryResponseWrapper(response, tracer) + return _AsyncUnaryResponseWrapper(response, tracer) class _StreamingResponseWrapper: @@ -252,6 +256,8 @@ def _record_metrics(self): return self._metrics_recorded = True try: + # TODO: Extract and pass the actual gRPC status (e.g. from self._response.code() + # or caught exceptions) to record_attempt_completion instead of defaulting to OK. self._tracer.record_attempt_completion() metadata = [] if hasattr(self._response, "initial_metadata"): @@ -273,7 +279,10 @@ def __getattr__(self, name): return getattr(self._response, name) -class _AsyncUnaryResponseWrapper(grpc.aio.UnaryUnaryCall): +class _AsyncUnaryResponseWrapper( + grpc.aio.UnaryUnaryCall, + grpc.aio.StreamUnaryCall, +): """Wrapper for async unary RPC response to defer metrics recording until awaited.""" def __init__(self, response, tracer): @@ -311,6 +320,12 @@ def trailing_metadata(self, *args, **kwargs): def wait_for_connection(self, *args, **kwargs): return getattr(self._response, "wait_for_connection")(*args, **kwargs) + def write(self, *args, **kwargs): + return getattr(self._response, "write")(*args, **kwargs) + + def done_writing(self, *args, **kwargs): + return getattr(self._response, "done_writing")(*args, **kwargs) + def __await__(self): async def _wait(): try: @@ -325,6 +340,8 @@ async def _record_metrics(self): return self._metrics_recorded = True try: + # TODO: Extract and pass the actual gRPC status (e.g. from self._response.code() + # or caught exceptions) to record_attempt_completion instead of defaulting to OK. self._tracer.record_attempt_completion() metadata = [] if hasattr(self._response, "initial_metadata"): @@ -353,7 +370,6 @@ def __getattr__(self, name): class _AsyncStreamingResponseWrapper( grpc.aio.UnaryStreamCall, - grpc.aio.StreamUnaryCall, grpc.aio.StreamStreamCall, ): """Wrapper for async streaming RPC response iterators to defer metrics recording.""" @@ -430,6 +446,8 @@ async def _record_metrics(self): return self._metrics_recorded = True try: + # TODO: Extract and pass the actual gRPC status (e.g. from self._response.code() + # or caught exceptions) to record_attempt_completion instead of defaulting to OK. self._tracer.record_attempt_completion() metadata = [] if hasattr(self._response, "initial_metadata"): diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/testing/mock_spanner.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/testing/mock_spanner.py index ce8c1a932b26..38e028661c48 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/testing/mock_spanner.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/testing/mock_spanner.py @@ -38,12 +38,24 @@ def __init__(self): self.execute_streaming_sql_results = {} self.partition_results = {} self.errors = {} + self.headers = {} def clear_results(self): self.results = {} self.execute_streaming_sql_results = {} self.partition_results = {} self.errors = {} + self.headers = {} + + def add_header(self, method: str, header: str, value: str): + if method not in self.headers: + self.headers[method] = [] + self.headers[method].append((header, value)) + + def send_initial_metadata(self, method_name: str, context): + headers = self.headers.get(method_name) + if headers: + context.send_initial_metadata(tuple(headers)) def add_result(self, sql: str, result: result_set.ResultSet): self.results[sql.lower().strip()] = result @@ -202,6 +214,7 @@ def DeleteSession(self, request, context): def ExecuteSql(self, request, context): self._requests.append(request) self.mock_spanner.pop_error(context) + self.mock_spanner.send_initial_metadata("ExecuteSql", context) started_transaction = self.__maybe_create_transaction(request) result: result_set.ResultSet = self.mock_spanner.get_result(request.sql) if started_transaction: @@ -212,6 +225,7 @@ def ExecuteSql(self, request, context): def ExecuteStreamingSql(self, request, context): self._requests.append(request) self.mock_spanner.pop_error(context) + self.mock_spanner.send_initial_metadata("ExecuteStreamingSql", context) started_transaction = self.__maybe_create_transaction(request) partials = self.mock_spanner.get_execute_streaming_sql_results( request.sql, started_transaction @@ -222,6 +236,7 @@ def ExecuteStreamingSql(self, request, context): def ExecuteBatchDml(self, request, context): self._requests.append(request) self.mock_spanner.pop_error(context) + self.mock_spanner.send_initial_metadata("ExecuteBatchDml", context) response = spanner.ExecuteBatchDmlResponse() started_transaction = self.__maybe_create_transaction(request) @@ -254,10 +269,14 @@ def ExecuteBatchDml(self, request, context): def Read(self, request, context): self._requests.append(request) + self.mock_spanner.pop_error(context) + self.mock_spanner.send_initial_metadata("Read", context) return result_set.ResultSet() def StreamingRead(self, request, context): self._requests.append(request) + self.mock_spanner.pop_error(context) + self.mock_spanner.send_initial_metadata("StreamingRead", context) for result in [result_set.PartialResultSet(), result_set.PartialResultSet()]: yield result diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/mock_server_test_base.py b/packages/google-cloud-spanner/tests/mockserver_tests/mock_server_test_base.py index 42e66d622dc1..1b4baec79ab5 100644 --- a/packages/google-cloud-spanner/tests/mockserver_tests/mock_server_test_base.py +++ b/packages/google-cloud-spanner/tests/mockserver_tests/mock_server_test_base.py @@ -139,6 +139,10 @@ def add_error(method: str, error: status_pb2.Status): get_spanner_service().mock_spanner.add_error(method, error) +def add_header(method: str, header: str, value: str): + get_spanner_service().mock_spanner.add_header(method, header, value) + + def add_result(sql: str, result: result_set.ResultSet): get_spanner_service().mock_spanner.add_result(sql, result) diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/test_frontend_metrics.py b/packages/google-cloud-spanner/tests/mockserver_tests/test_frontend_metrics.py index 367104260c4d..980ca3d1e380 100644 --- a/packages/google-cloud-spanner/tests/mockserver_tests/test_frontend_metrics.py +++ b/packages/google-cloud-spanner/tests/mockserver_tests/test_frontend_metrics.py @@ -21,15 +21,24 @@ from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader +import google.cloud.spanner_v1._async.client as async_client_mod import google.cloud.spanner_v1.client as client_mod -from google.cloud.spanner_v1 import Client +from google.cloud.spanner_v1 import Client, _helpers +from google.cloud.spanner_v1._async.client import Client as AsyncClient +from google.cloud.spanner_v1._async.pool import FixedSizePool as AsyncFixedSizePool from google.cloud.spanner_v1.metrics.metrics_interceptor import MetricsInterceptor from google.cloud.spanner_v1.metrics.spanner_metrics_tracer_factory import ( SpannerMetricsTracerFactory, ) from google.cloud.spanner_v1.pool import FixedSizePool +from google.cloud.spanner_v1.services.spanner.async_client import SpannerAsyncClient +from google.cloud.spanner_v1.services.spanner.transports.grpc_asyncio import ( + SpannerGrpcAsyncIOTransport, +) from tests.mockserver_tests.mock_server_test_base import ( + AsyncMockServerTestBase, MockServerTestBase, + add_header, add_select1_result, ) @@ -37,118 +46,105 @@ class TestFrontendMetricsIntegration(MockServerTestBase): def setUp(self): super().setUp() + self._orig_disable_builtin_metrics = os.environ.get( + "SPANNER_DISABLE_BUILTIN_METRICS" + ) + self._orig_disable_afe_server_timing = os.environ.get( + "SPANNER_DISABLE_AFE_SERVER_TIMING" + ) + self._orig_enable_afe_server_timing = _helpers.ENABLE_AFE_SERVER_TIMING + os.environ["SPANNER_DISABLE_BUILTIN_METRICS"] = "false" + os.environ["SPANNER_DISABLE_AFE_SERVER_TIMING"] = "false" + _helpers.ENABLE_AFE_SERVER_TIMING = True SpannerMetricsTracerFactory._metrics_tracer_factory = None client_mod._metrics_monitor_initialized = False + async_client_mod._metrics_monitor_initialized = False def tearDown(self): super().tearDown() - os.environ["SPANNER_DISABLE_BUILTIN_METRICS"] = "true" + if self._orig_disable_builtin_metrics is None: + os.environ.pop("SPANNER_DISABLE_BUILTIN_METRICS", None) + else: + os.environ["SPANNER_DISABLE_BUILTIN_METRICS"] = ( + self._orig_disable_builtin_metrics + ) + + if self._orig_disable_afe_server_timing is None: + os.environ.pop("SPANNER_DISABLE_AFE_SERVER_TIMING", None) + else: + os.environ["SPANNER_DISABLE_AFE_SERVER_TIMING"] = ( + self._orig_disable_afe_server_timing + ) + + _helpers.ENABLE_AFE_SERVER_TIMING = self._orig_enable_afe_server_timing SpannerMetricsTracerFactory._metrics_tracer_factory = None client_mod._metrics_monitor_initialized = False + async_client_mod._metrics_monitor_initialized = False def test_gfe_metrics_exported(self): add_select1_result() + add_header( + "ExecuteStreamingSql", + "server-timing", + "gfet4t7; dur=55, afe; dur=23", + ) reader = InMemoryMetricReader() meter_provider = MeterProvider(metric_readers=[reader]) - orig_call = grpc._channel._UnaryStreamMultiCallable.__call__ - orig_initial_metadata = grpc._channel._MultiThreadedRendezvous.initial_metadata - orig_trailing_metadata = ( - grpc._channel._MultiThreadedRendezvous.trailing_metadata - ) - - def custom_initial_metadata(self): - mocked = getattr(self, "_is_execute_streaming_sql_mock", False) - if mocked: - return (("server-timing", "gfet4t7; dur=55, afe; dur=23"),) - return orig_initial_metadata(self) - - def custom_trailing_metadata(self): - mocked = getattr(self, "_is_execute_streaming_sql_mock", False) - if mocked: - return (("server-timing", "gfet4t7; dur=55, afe; dur=23"),) - return orig_trailing_metadata(self) - - def custom_call(self_callable, request, *args, **kwargs): - method = getattr(self_callable, "_method", b"") - method_str = method.decode("utf-8") if isinstance(method, bytes) else method - response = orig_call(self_callable, request, *args, **kwargs) - if "ExecuteStreamingSql" in method_str: - response._is_execute_streaming_sql_mock = True - return response - - try: - with ( - mock.patch( - "google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client.MeterProvider", - return_value=meter_provider, + with ( + mock.patch( + "google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1.client.MeterProvider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1.client._get_spanner_emulator_host", + return_value=None, + ), + ): + client = Client( + project="p", + credentials=AnonymousCredentials(), + client_options=ClientOptions( + api_endpoint="localhost:" + str(MockServerTestBase.port), ), - mock.patch( - "google.cloud.spanner_v1.client._get_spanner_emulator_host", - return_value=None, - ), - mock.patch( - "grpc._channel._UnaryStreamMultiCallable.__call__", - custom_call, - ), - mock.patch( - "grpc._channel._MultiThreadedRendezvous.initial_metadata", - custom_initial_metadata, - ), - mock.patch( - "grpc._channel._MultiThreadedRendezvous.trailing_metadata", - custom_trailing_metadata, - ), - ): - client = Client( - project="p", - credentials=AnonymousCredentials(), - client_options=ClientOptions( - api_endpoint="localhost:" + str(MockServerTestBase.port), - ), - ) - instance = client.instance("test-instance") - database = instance.database( - "test-database", - pool=FixedSizePool(size=10), - enable_interceptors_in_tests=True, - ) - database._interceptors.append(MetricsInterceptor()) - database._spanner_api = ( - None # Force recreation with the new interceptor - ) - - with database.snapshot() as snapshot: - results = snapshot.execute_sql("select 1") - # Consume the streaming results to complete the stream - list(results) + ) + instance = client.instance("test-instance") + database = instance.database( + "test-database", + pool=FixedSizePool(size=10), + enable_interceptors_in_tests=True, + ) + database._interceptors.append(MetricsInterceptor()) + database._spanner_api = None # Force recreation with the new interceptor - metric_data = reader.get_metrics_data() - self.assertIsNotNone(metric_data) - metrics = { - metric.name: metric - for rm in metric_data.resource_metrics - for sm in rm.scope_metrics - for metric in sm.metrics - } + with database.snapshot() as snapshot: + results = snapshot.execute_sql("select 1") + # Consume the streaming results to complete the stream + list(results) - self.assertIn("gfe_latencies", metrics, f"Metrics: {list(metrics.keys())}") - gfe_metric = metrics["gfe_latencies"] - point = next(iter(gfe_metric.data.data_points)) - self.assertEqual(point.sum, 55) + metric_data = reader.get_metrics_data() + self.assertIsNotNone(metric_data) + metrics = { + metric.name: metric + for rm in metric_data.resource_metrics + for sm in rm.scope_metrics + for metric in sm.metrics + } - self.assertIn("afe_latencies", metrics, f"Metrics: {list(metrics.keys())}") - afe_metric = metrics["afe_latencies"] - point = next(iter(afe_metric.data.data_points)) - self.assertEqual(point.sum, 23) + self.assertIn("gfe_latencies", metrics, f"Metrics: {list(metrics.keys())}") + gfe_metric = metrics["gfe_latencies"] + point = next(iter(gfe_metric.data.data_points)) + self.assertEqual(point.sum, 55) - finally: - pass + self.assertIn("afe_latencies", metrics, f"Metrics: {list(metrics.keys())}") + afe_metric = metrics["afe_latencies"] + point = next(iter(afe_metric.data.data_points)) + self.assertEqual(point.sum, 23) def test_gfe_missing_header_count_exported(self): add_select1_result() @@ -220,3 +216,237 @@ def test_gfe_missing_header_count_exported(self): self.assertGreaterEqual(afe_point.value, 1) finally: pass + + +class TestFrontendMetricsAsyncIntegration(AsyncMockServerTestBase): + def setUp(self): + super().setUp() + self._orig_disable_builtin_metrics = os.environ.get( + "SPANNER_DISABLE_BUILTIN_METRICS" + ) + self._orig_disable_afe_server_timing = os.environ.get( + "SPANNER_DISABLE_AFE_SERVER_TIMING" + ) + self._orig_enable_afe_server_timing = _helpers.ENABLE_AFE_SERVER_TIMING + + os.environ["SPANNER_DISABLE_BUILTIN_METRICS"] = "false" + os.environ["SPANNER_DISABLE_AFE_SERVER_TIMING"] = "false" + _helpers.ENABLE_AFE_SERVER_TIMING = True + SpannerMetricsTracerFactory._metrics_tracer_factory = None + client_mod._metrics_monitor_initialized = False + async_client_mod._metrics_monitor_initialized = False + + def tearDown(self): + super().tearDown() + if self._orig_disable_builtin_metrics is None: + os.environ.pop("SPANNER_DISABLE_BUILTIN_METRICS", None) + else: + os.environ["SPANNER_DISABLE_BUILTIN_METRICS"] = ( + self._orig_disable_builtin_metrics + ) + + if self._orig_disable_afe_server_timing is None: + os.environ.pop("SPANNER_DISABLE_AFE_SERVER_TIMING", None) + else: + os.environ["SPANNER_DISABLE_AFE_SERVER_TIMING"] = ( + self._orig_disable_afe_server_timing + ) + + _helpers.ENABLE_AFE_SERVER_TIMING = self._orig_enable_afe_server_timing + SpannerMetricsTracerFactory._metrics_tracer_factory = None + client_mod._metrics_monitor_initialized = False + async_client_mod._metrics_monitor_initialized = False + + async def asyncSetUp(self): + await super().asyncSetUp() + if AsyncMockServerTestBase.spanner_service: + AsyncMockServerTestBase.spanner_service.clear_results() + + async def asyncTearDown(self): + await super().asyncTearDown() + if AsyncMockServerTestBase.spanner_service: + AsyncMockServerTestBase.spanner_service.clear_results() + + async def test_async_gfe_metrics_exported(self): + add_select1_result() + add_header( + "ExecuteStreamingSql", + "server-timing", + "gfet4t7; dur=55, afe; dur=23", + ) + reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[reader]) + + with ( + mock.patch( + "google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1.client.MeterProvider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1._async.client.MeterProvider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1.client._get_spanner_emulator_host", + return_value=None, + ), + mock.patch( + "google.cloud.spanner_v1._async.client._get_spanner_emulator_host", + return_value=None, + ), + ): + client = AsyncClient( + project="p", + credentials=AnonymousCredentials(), + client_options=ClientOptions( + api_endpoint="localhost:" + str(AsyncMockServerTestBase.port), + ), + ) + instance = client.instance("test-instance") + database = await instance.database( + "test-database", + pool=AsyncFixedSizePool(size=10), + ) + async with grpc.aio.insecure_channel( + "localhost:" + str(AsyncMockServerTestBase.port), + ) as channel: + transport = SpannerGrpcAsyncIOTransport( + channel=channel, + metrics_interceptor=MetricsInterceptor(), + ) + database._spanner_api = SpannerAsyncClient( + client_info=client._client_info, + transport=transport, + ) + + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql("select 1") + # Consume the async streaming results to complete the stream + async for _ in results: + pass + + metric_data = reader.get_metrics_data() + self.assertIsNotNone(metric_data) + metrics = { + metric.name: metric + for resource_metric in metric_data.resource_metrics + for scope_metric in resource_metric.scope_metrics + for metric in scope_metric.metrics + } + + self.assertIn("gfe_latencies", metrics, f"Metrics: {list(metrics.keys())}") + gfe_metric = metrics["gfe_latencies"] + gfe_point = next(iter(gfe_metric.data.data_points)) + self.assertEqual(gfe_point.sum, 55) + + self.assertIn("afe_latencies", metrics, f"Metrics: {list(metrics.keys())}") + afe_metric = metrics["afe_latencies"] + afe_point = next(iter(afe_metric.data.data_points)) + self.assertEqual(afe_point.sum, 23) + + self.assertIn("attempt_latencies", metrics, f"Metrics: {list(metrics.keys())}") + attempt_latencies_metric = metrics["attempt_latencies"] + streaming_points = [ + point + for point in attempt_latencies_metric.data.data_points + if point.attributes.get("method") == "Spanner.ExecuteStreamingSql" + ] + self.assertTrue(len(streaming_points) > 0) + self.assertEqual(streaming_points[0].attributes.get("status"), "OK") + + async def test_async_gfe_missing_header_count_exported(self): + add_select1_result() + reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[reader]) + + with ( + mock.patch( + "google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1.client.MeterProvider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1._async.client.MeterProvider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1.client._get_spanner_emulator_host", + return_value=None, + ), + mock.patch( + "google.cloud.spanner_v1._async.client._get_spanner_emulator_host", + return_value=None, + ), + ): + client = AsyncClient( + project="p", + credentials=AnonymousCredentials(), + client_options=ClientOptions( + api_endpoint="localhost:" + str(AsyncMockServerTestBase.port), + ), + ) + instance = client.instance("test-instance") + database = await instance.database( + "test-database", + pool=AsyncFixedSizePool(size=10), + ) + async with grpc.aio.insecure_channel( + "localhost:" + str(AsyncMockServerTestBase.port), + ) as channel: + transport = SpannerGrpcAsyncIOTransport( + channel=channel, + metrics_interceptor=MetricsInterceptor(), + ) + database._spanner_api = SpannerAsyncClient( + client_info=client._client_info, + transport=transport, + ) + + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql("select 1") + async for _ in results: + pass + + metric_data = reader.get_metrics_data() + self.assertIsNotNone(metric_data) + metrics = { + metric.name: metric + for resource_metric in metric_data.resource_metrics + for scope_metric in resource_metric.scope_metrics + for metric in scope_metric.metrics + } + + self.assertIn( + "gfe_connectivity_error_count", + metrics, + f"Metrics: {list(metrics.keys())}", + ) + missing_metric = metrics["gfe_connectivity_error_count"] + streaming_points = [ + point + for point in missing_metric.data.data_points + if point.attributes.get("method") == "Spanner.ExecuteStreamingSql" + ] + self.assertTrue(len(streaming_points) > 0) + self.assertGreaterEqual(streaming_points[0].value, 1) + + self.assertIn( + "afe_connectivity_error_count", + metrics, + f"Metrics: {list(metrics.keys())}", + ) + afe_missing_metric = metrics["afe_connectivity_error_count"] + streaming_points = [ + point + for point in afe_missing_metric.data.data_points + if point.attributes.get("method") == "Spanner.ExecuteStreamingSql" + ] + self.assertTrue(len(streaming_points) > 0) + self.assertGreaterEqual(streaming_points[0].value, 1) diff --git a/packages/google-cloud-spanner/tests/unit/test_metrics_interceptor.py b/packages/google-cloud-spanner/tests/unit/test_metrics_interceptor.py index aa31dc5f1210..a62ed4da43be 100644 --- a/packages/google-cloud-spanner/tests/unit/test_metrics_interceptor.py +++ b/packages/google-cloud-spanner/tests/unit/test_metrics_interceptor.py @@ -16,7 +16,13 @@ import pytest -from google.cloud.spanner_v1.metrics.metrics_interceptor import MetricsInterceptor +from google.cloud.spanner_v1.metrics.metrics_interceptor import ( + AsyncMetricsInterceptor, + MetricsInterceptor, + _AsyncStreamingResponseWrapper, + _AsyncUnaryResponseWrapper, + _StreamingResponseWrapper, +) from google.cloud.spanner_v1.metrics.spanner_metrics_tracer_factory import ( SpannerMetricsTracerFactory, ) @@ -118,3 +124,465 @@ def test_intercept_with_tracer(interceptor, mock_tracer_ctx): mock_tracer_ctx.record_attempt_completion.assert_called_once() mock_tracer_ctx.record_front_end_metrics.assert_called_once() mock_invoked_method.assert_called_once_with("request", call_details) + + +@pytest.fixture +def async_interceptor(): + SpannerMetricsTracerFactory(enabled=True) + return AsyncMetricsInterceptor() + + +@pytest.mark.asyncio +async def test_async_intercept_unary_unary(async_interceptor, mock_tracer_ctx): + class MockUnaryCall: + def initial_metadata(self): + return [("server-timing", "gfet4t7; dur=55")] + + def __await__(self): + async def _coroutine(): + return "unary_result" + + return _coroutine().__await__() + + async def mock_continuation(call_details, request): + return MockUnaryCall() + + call_details = MagicMock( + method="/google.spanner.v1.Spanner/ExecuteSql", + metadata=[ + ( + "google-cloud-resource-prefix", + "projects/my_project/instances/my_instance/databases/my_database", + ) + ], + ) + + wrapped_call = await async_interceptor.intercept_unary_unary( + mock_continuation, call_details, "request_payload" + ) + assert isinstance(wrapped_call, _AsyncUnaryResponseWrapper) + result = await wrapped_call + assert result == "unary_result" + mock_tracer_ctx.record_attempt_start.assert_called_once() + mock_tracer_ctx.record_attempt_completion.assert_called_once() + mock_tracer_ctx.record_front_end_metrics.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_intercept_stream_unary(async_interceptor, mock_tracer_ctx): + class MockStreamUnaryCall: + def initial_metadata(self): + return [] + + def __await__(self): + async def _coroutine(): + return "stream_unary_result" + + return _coroutine().__await__() + + async def mock_continuation(call_details, request_iterator): + return MockStreamUnaryCall() + + call_details = MagicMock( + method="/google.spanner.v1.Spanner/Commit", + metadata=[], + ) + + wrapped_call = await async_interceptor.intercept_stream_unary( + mock_continuation, call_details, ["req1"] + ) + assert isinstance(wrapped_call, _AsyncUnaryResponseWrapper) + result = await wrapped_call + assert result == "stream_unary_result" + mock_tracer_ctx.record_attempt_start.assert_called_once() + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_intercept_unary_stream(async_interceptor, mock_tracer_ctx): + # Simulates grpc.aio.UnaryStreamCall: defines __aiter__ returning an async generator, + # but does NOT define __anext__ on the call object itself. + class MockUnaryStreamCall: + def initial_metadata(self): + return [("server-timing", "afe; dur=25")] + + def __aiter__(self): + async def _generator(): + yield "chunk_1" + yield "chunk_2" + + return _generator() + + call = MockUnaryStreamCall() + assert hasattr(call, "__aiter__") + assert not hasattr(call, "__anext__") + + async def mock_continuation(call_details, request): + return call + + call_details = MagicMock( + method="/google.spanner.v1.Spanner/ExecuteStreamingSql", + metadata=[], + ) + + wrapped_stream = await async_interceptor.intercept_unary_stream( + mock_continuation, call_details, "request_payload" + ) + assert isinstance(wrapped_stream, _AsyncStreamingResponseWrapper) + + # Before iteration finishes, attempt completion is not yet recorded + mock_tracer_ctx.record_attempt_start.assert_called_once() + mock_tracer_ctx.record_attempt_completion.assert_not_called() + + items = [] + async for item in wrapped_stream: + items.append(item) + + assert items == ["chunk_1", "chunk_2"] + mock_tracer_ctx.record_attempt_completion.assert_called_once() + mock_tracer_ctx.record_front_end_metrics.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_intercept_stream_stream(async_interceptor, mock_tracer_ctx): + class MockStreamStreamCall: + def initial_metadata(self): + return [] + + def __aiter__(self): + async def _generator(): + yield "stream_stream_chunk" + + return _generator() + + async def mock_continuation(call_details, request_iterator): + return MockStreamStreamCall() + + call_details = MagicMock( + method="/google.spanner.v1.Spanner/StreamingRead", + metadata=[], + ) + + wrapped_stream = await async_interceptor.intercept_stream_stream( + mock_continuation, call_details, ["req1"] + ) + assert isinstance(wrapped_stream, _AsyncStreamingResponseWrapper) + + items = [] + async for item in wrapped_stream: + items.append(item) + + assert items == ["stream_stream_chunk"] + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_intercept_disabled(mock_tracer_ctx): + SpannerMetricsTracerFactory._metrics_tracer_factory = None + SpannerMetricsTracerFactory(enabled=False) + try: + interceptor = AsyncMetricsInterceptor() + + async def mock_continuation(call_details, request): + return "raw_unwrapped_response" + + call_details = MagicMock( + method="/google.spanner.v1.Spanner/ExecuteSql", + metadata=[], + ) + + result = await interceptor.intercept_unary_unary( + mock_continuation, call_details, "request_payload" + ) + assert result == "raw_unwrapped_response" + mock_tracer_ctx.record_attempt_start.assert_not_called() + finally: + SpannerMetricsTracerFactory._metrics_tracer_factory = None + + +@pytest.mark.asyncio +async def test_async_streaming_response_wrapper_exception(mock_tracer_ctx): + class FaultyStreamCall: + def initial_metadata(self): + return [] + + def __aiter__(self): + async def _generator(): + yield "first_chunk" + raise RuntimeError("stream_failed") + + return _generator() + + wrapper = _AsyncStreamingResponseWrapper(FaultyStreamCall(), mock_tracer_ctx) + + with pytest.raises(RuntimeError, match="stream_failed"): + async for _ in wrapper: + pass + + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_streaming_response_wrapper_delegation(mock_tracer_ctx): + mock_response = MagicMock() + mock_response.cancel.return_value = True + mock_response.cancelled.return_value = False + mock_response.code.return_value = 0 + mock_response.details.return_value = "details" + mock_response.done.return_value = True + mock_response.initial_metadata.return_value = [] + mock_response.time_remaining.return_value = 10.0 + mock_response.trailing_metadata.return_value = [] + mock_response.wait_for_connection.return_value = None + mock_response.read.return_value = "chunk" + mock_response.write.return_value = None + mock_response.done_writing.return_value = None + + wrapper = _AsyncStreamingResponseWrapper(mock_response, mock_tracer_ctx) + + assert wrapper.cancel() is True + mock_response.cancel.assert_called_once() + assert wrapper.cancelled() is False + assert wrapper.code() == 0 + assert wrapper.details() == "details" + assert wrapper.done() is True + assert wrapper.initial_metadata() == [] + assert wrapper.time_remaining() == 10.0 + assert wrapper.trailing_metadata() == [] + assert wrapper.wait_for_connection() is None + assert wrapper.read() == "chunk" + assert wrapper.write("msg") is None + mock_response.write.assert_called_once_with("msg") + assert wrapper.done_writing() is None + mock_response.done_writing.assert_called_once() + mock_response.custom_attribute = "custom" + assert wrapper.custom_attribute == "custom" + + callback = MagicMock() + wrapper.add_done_callback(callback) + mock_response.add_done_callback.assert_called_once_with(callback) + + +@pytest.mark.asyncio +async def test_async_unary_response_wrapper_exception(mock_tracer_ctx): + class FaultyUnaryCall: + def initial_metadata(self): + return [] + + def __await__(self): + async def _coroutine(): + raise RuntimeError("unary_failed") + + return _coroutine().__await__() + + wrapper = _AsyncUnaryResponseWrapper(FaultyUnaryCall(), mock_tracer_ctx) + + with pytest.raises(RuntimeError, match="unary_failed"): + await wrapper + + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_unary_response_wrapper_delegation(mock_tracer_ctx): + mock_response = MagicMock() + mock_response.cancel.return_value = True + mock_response.cancelled.return_value = False + mock_response.code.return_value = 0 + mock_response.details.return_value = "details" + mock_response.done.return_value = True + mock_response.initial_metadata.return_value = [] + mock_response.time_remaining.return_value = 10.0 + mock_response.trailing_metadata.return_value = [] + mock_response.wait_for_connection.return_value = None + mock_response.write.return_value = None + mock_response.done_writing.return_value = None + mock_response.custom_attribute = "custom" + + wrapper = _AsyncUnaryResponseWrapper(mock_response, mock_tracer_ctx) + + assert wrapper.cancel() is True + mock_response.cancel.assert_called_once() + assert wrapper.cancelled() is False + assert wrapper.code() == 0 + assert wrapper.details() == "details" + assert wrapper.done() is True + assert wrapper.initial_metadata() == [] + assert wrapper.time_remaining() == 10.0 + assert wrapper.trailing_metadata() == [] + assert wrapper.wait_for_connection() is None + assert wrapper.write("msg") is None + mock_response.write.assert_called_once_with("msg") + assert wrapper.done_writing() is None + mock_response.done_writing.assert_called_once() + assert wrapper.custom_attribute == "custom" + + callback = MagicMock() + wrapper.add_done_callback(callback) + mock_response.add_done_callback.assert_called_once_with(callback) + + +def test_async_unary_response_wrapper_del_unrecorded(mock_tracer_ctx): + mock_response = MagicMock() + wrapper = _AsyncUnaryResponseWrapper(mock_response, mock_tracer_ctx) + wrapper.__del__() + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + # Second call should not record again + wrapper.__del__() + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_streaming_response_wrapper_anext_direct(mock_tracer_ctx): + class MockCall: + def initial_metadata(self): + return [] + + def __aiter__(self): + async def _generator(): + yield "val1" + yield "val2" + + return _generator() + + wrapper = _AsyncStreamingResponseWrapper(MockCall(), mock_tracer_ctx) + # Direct __anext__ without calling __aiter__ first (matching StreamedResultSet._consume_next) + assert await wrapper.__anext__() == "val1" + assert await wrapper.__anext__() == "val2" + with pytest.raises(StopAsyncIteration): + await wrapper.__anext__() + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_streaming_response_wrapper_anext_without_aiter(mock_tracer_ctx): + class MockCallWithoutAiter: + def initial_metadata(self): + return [] + + async def __anext__(self): + raise StopAsyncIteration + + wrapper = _AsyncStreamingResponseWrapper(MockCallWithoutAiter(), mock_tracer_ctx) + with pytest.raises(StopAsyncIteration): + await wrapper.__anext__() + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_streaming_response_wrapper_awaitable_metadata(mock_tracer_ctx): + class MockCallAwaitableMetadata: + async def initial_metadata(self): + return [("server-timing", "gfet4t7; dur=42")] + + def __aiter__(self): + async def _generator(): + yield "item" + + return _generator() + + wrapper = _AsyncStreamingResponseWrapper( + MockCallAwaitableMetadata(), mock_tracer_ctx + ) + async for _ in wrapper: + pass + mock_tracer_ctx.record_front_end_metrics.assert_called_once_with( + [("server-timing", "gfet4t7; dur=42")] + ) + + +def test_async_streaming_response_wrapper_del_unrecorded(mock_tracer_ctx): + mock_response = MagicMock() + wrapper = _AsyncStreamingResponseWrapper(mock_response, mock_tracer_ctx) + wrapper.__del__() + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + # Second call should not record again + wrapper.__del__() + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_streaming_response_wrapper_with_streamed_result_set( + mock_tracer_ctx, +): + from google.cloud.spanner_v1._async.streamed import StreamedResultSet + from google.cloud.spanner_v1._helpers import _make_value_pb + from google.cloud.spanner_v1.types import ( + PartialResultSet, + ResultSetMetadata, + StructType, + Type, + TypeCode, + ) + + metadata = ResultSetMetadata( + row_type=StructType( + fields=[StructType.Field(name="col", type_=Type(code=TypeCode.STRING))] + ) + ) + partial_1 = PartialResultSet(metadata=metadata) + partial_1.values.append(_make_value_pb("val1")) + partial_2 = PartialResultSet() + partial_2.values.append(_make_value_pb("val2")) + + class MockAsyncStreamCall: + def initial_metadata(self): + return [("server-timing", "gfet4t7; dur=50")] + + def __aiter__(self): + async def _gen(): + yield partial_1 + yield partial_2 + + return _gen() + + call = MockAsyncStreamCall() + wrapper = _AsyncStreamingResponseWrapper(call, mock_tracer_ctx) + + streamed = StreamedResultSet(wrapper) + rows = [] + async for row in streamed: + rows.append(row) + + assert rows == [["val1"], ["val2"]] + mock_tracer_ctx.record_attempt_completion.assert_called_once() + mock_tracer_ctx.record_front_end_metrics.assert_called_once() + + +def test_sync_streaming_response_wrapper(mock_tracer_ctx): + mock_response = iter(["item1", "item2"]) + wrapper = _StreamingResponseWrapper(mock_response, mock_tracer_ctx) + + items = list(wrapper) + assert items == ["item1", "item2"] + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + +def test_sync_streaming_response_wrapper_direct_next(mock_tracer_ctx): + mock_response = ["item1"] + wrapper = _StreamingResponseWrapper(mock_response, mock_tracer_ctx) + + assert next(wrapper) == "item1" + with pytest.raises(StopIteration): + next(wrapper) + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + +def test_sync_streaming_response_wrapper_exception(mock_tracer_ctx): + def faulty_generator(): + yield "item1" + raise RuntimeError("stream_failed") + + wrapper = _StreamingResponseWrapper(faulty_generator(), mock_tracer_ctx) + with pytest.raises(RuntimeError, match="stream_failed"): + list(wrapper) + mock_tracer_ctx.record_attempt_completion.assert_called_once() + + +def test_sync_streaming_response_wrapper_del_unrecorded(mock_tracer_ctx): + mock_response = MagicMock() + wrapper = _StreamingResponseWrapper(mock_response, mock_tracer_ctx) + wrapper.__del__() + mock_tracer_ctx.record_attempt_completion.assert_called_once() From 57bae0ed42afe48bf054b7c6bf68a510df8cbf98 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 6 Oct 2026 18:00:41 +0100 Subject: [PATCH 015/228] chore(deps): update dependency pytest to v9 [security] (#18568) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [pytest](https://redirect.github.com/pytest-dev/pytest) ([changelog](https://docs.pytest.org/en/stable/changelog.html)) | `==8.4.2` → `==9.0.3` | ![age](https://developer.mend.io/api/mc/badges/age/pypi/pytest/9.0.3?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/pytest/8.4.2/9.0.3?slim=true) | | [pytest](https://redirect.github.com/pytest-dev/pytest) ([changelog](https://docs.pytest.org/en/stable/changelog.html)) | `==8.3.5` → `==9.0.3` | ![age](https://developer.mend.io/api/mc/badges/age/pypi/pytest/9.0.3?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/pytest/8.3.5/9.0.3?slim=true) | | [pytest](https://redirect.github.com/pytest-dev/pytest) ([changelog](https://docs.pytest.org/en/stable/changelog.html)) | `==7.4.3` → `==9.0.3` | ![age](https://developer.mend.io/api/mc/badges/age/pypi/pytest/9.0.3?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/pytest/7.4.3/9.0.3?slim=true) | --- ### pytest has vulnerable tmpdir handling [CVE-2025-71176](https://nvd.nist.gov/vuln/detail/CVE-2025-71176) / [GHSA-6w46-j5rx-g56g](https://redirect.github.com/advisories/GHSA-6w46-j5rx-g56g)
More information #### Details pytest through 9.0.2 on UNIX relies on directories with the `/tmp/pytest-of-{user}` name pattern, which allows local users to cause a denial of service or possibly gain privileges. #### Severity - CVSS Score: 6.8 / 10 (Medium) - Vector String: `CVSS:3.1/AV:L/AC:L/PR:N/UI:N/S:C/C:L/I:L/A:L` #### References - [https://nvd.nist.gov/vuln/detail/CVE-2025-71176](https://nvd.nist.gov/vuln/detail/CVE-2025-71176) - [https://github.com/pytest-dev/pytest/issues/13669](https://redirect.github.com/pytest-dev/pytest/issues/13669) - [https://www.openwall.com/lists/oss-security/2026/01/21/5](https://www.openwall.com/lists/oss-security/2026/01/21/5) - [https://github.com/pytest-dev/pytest/pull/14343](https://redirect.github.com/pytest-dev/pytest/pull/14343) - [https://github.com/pytest-dev/pytest/commit/95d8423bd24992deea5b9df32555fa1741679e2c](https://redirect.github.com/pytest-dev/pytest/commit/95d8423bd24992deea5b9df32555fa1741679e2c) - [https://github.com/pytest-dev/pytest/releases/tag/9.0.3](https://redirect.github.com/pytest-dev/pytest/releases/tag/9.0.3) - [https://github.com/advisories/GHSA-6w46-j5rx-g56g](https://redirect.github.com/advisories/GHSA-6w46-j5rx-g56g) This data is provided by the [GitHub Advisory Database](https://redirect.github.com/advisories/GHSA-6w46-j5rx-g56g) ([CC-BY 4.0](https://redirect.github.com/github/advisory-database/blob/main/LICENSE.md)).
--- ### Release Notes
pytest-dev/pytest (pytest) ### [`v9.0.3`](https://redirect.github.com/pytest-dev/pytest/releases/tag/9.0.3) [Compare Source](https://redirect.github.com/pytest-dev/pytest/compare/9.0.2...9.0.3) ### pytest 9.0.3 (2026-04-07) #### Bug fixes - [#​12444](https://redirect.github.com/pytest-dev/pytest/issues/12444): Fixed `pytest.approx` which now correctly takes into account `~collections.abc.Mapping` keys order to compare them. - [#​13634](https://redirect.github.com/pytest-dev/pytest/issues/13634): Blocking a `conftest.py` file using the `-p no:` option is now explicitly disallowed. Previously this resulted in an internal assertion failure during plugin loading. Pytest now raises a clear `UsageError` explaining that conftest files are not plugins and cannot be disabled via `-p`. - [#​13734](https://redirect.github.com/pytest-dev/pytest/issues/13734): Fixed crash when a test raises an exceptiongroup with `__tracebackhide__ = True`. - [#​14195](https://redirect.github.com/pytest-dev/pytest/issues/14195): Fixed an issue where non-string messages passed to unittest.TestCase.subTest() were not printed. - [#​14343](https://redirect.github.com/pytest-dev/pytest/issues/14343): Fixed use of insecure temporary directory (CVE-2025-71176). #### Improved documentation - [#​13388](https://redirect.github.com/pytest-dev/pytest/issues/13388): Clarified documentation for `-p` vs `PYTEST_PLUGINS` plugin loading and fixed an incorrect `-p` example. - [#​13731](https://redirect.github.com/pytest-dev/pytest/issues/13731): Clarified that capture fixtures (e.g. `capsys` and `capfd`) take precedence over the `-s` / `--capture=no` command-line options in `Accessing captured output from a test function `. - [#​14088](https://redirect.github.com/pytest-dev/pytest/issues/14088): Clarified that the default `pytest_collection` hook sets `session.items` before it calls `pytest_collection_finish`, not after. - [#​14255](https://redirect.github.com/pytest-dev/pytest/issues/14255): TOML integer log levels must be quoted: Updating reference documentation. #### Contributor-facing changes - [#​12689](https://redirect.github.com/pytest-dev/pytest/issues/12689): The test reports are now published to Codecov from GitHub Actions. The test statistics is visible [on the web interface](https://app.codecov.io/gh/pytest-dev/pytest/tests). \-- by `aleguy02` ### [`v9.0.2`](https://redirect.github.com/pytest-dev/pytest/releases/tag/9.0.2) [Compare Source](https://redirect.github.com/pytest-dev/pytest/compare/9.0.1...9.0.2) ### pytest 9.0.2 (2025-12-06) #### Bug fixes - [#​13896](https://redirect.github.com/pytest-dev/pytest/issues/13896): The terminal progress feature added in pytest 9.0.0 has been disabled by default, except on Windows, due to compatibility issues with some terminal emulators. You may enable it again by passing `-p terminalprogress`. We may enable it by default again once compatibility improves in the future. Additionally, when the environment variable `TERM` is `dumb`, the escape codes are no longer emitted, even if the plugin is enabled. - [#​13904](https://redirect.github.com/pytest-dev/pytest/issues/13904): Fixed the TOML type of the `tmp_path_retention_count` settings in the API reference from number to string. - [#​13946](https://redirect.github.com/pytest-dev/pytest/issues/13946): The private `config.inicfg` attribute was changed in a breaking manner in pytest 9.0.0. Due to its usage in the ecosystem, it is now restored to working order using a compatibility shim. It will be deprecated in pytest 9.1 and removed in pytest 10. - [#​13965](https://redirect.github.com/pytest-dev/pytest/issues/13965): Fixed quadratic-time behavior when handling `unittest` subtests in Python 3.10. #### Improved documentation - [#​4492](https://redirect.github.com/pytest-dev/pytest/issues/4492): The API Reference now contains cross-reference-able documentation of `pytest's command-line flags `. ### [`v9.0.1`](https://redirect.github.com/pytest-dev/pytest/releases/tag/9.0.1) [Compare Source](https://redirect.github.com/pytest-dev/pytest/compare/9.0.0...9.0.1) ### pytest 9.0.1 (2025-11-12) #### Bug fixes - [#​13895](https://redirect.github.com/pytest-dev/pytest/issues/13895): Restore support for skipping tests via `raise unittest.SkipTest`. - [#​13896](https://redirect.github.com/pytest-dev/pytest/issues/13896): The terminal progress plugin added in pytest 9.0 is now automatically disabled when iTerm2 is detected, it generated desktop notifications instead of the desired functionality. - [#​13904](https://redirect.github.com/pytest-dev/pytest/issues/13904): Fixed the TOML type of the verbosity settings in the API reference from number to string. - [#​13910](https://redirect.github.com/pytest-dev/pytest/issues/13910): Fixed UserWarning: Do not expect file\_or\_dir on some earlier Python 3.12 and 3.13 point versions. #### Packaging updates and notes for downstreams - [#​13933](https://redirect.github.com/pytest-dev/pytest/issues/13933): The tox configuration has been adjusted to make sure the desired version string can be passed into its `package_env` through the `SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST` environment variable as a part of the release process -- by `webknjaz`. #### Contributor-facing changes - [#​13891](https://redirect.github.com/pytest-dev/pytest/issues/13891), [#​13942](https://redirect.github.com/pytest-dev/pytest/issues/13942): The CI/CD part of the release automation is now capable of creating GitHub Releases without having a Git checkout on disk -- by `bluetech` and `webknjaz`. - [#​13933](https://redirect.github.com/pytest-dev/pytest/issues/13933): The tox configuration has been adjusted to make sure the desired version string can be passed into its `package_env` through the `SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST` environment variable as a part of the release process -- by `webknjaz`. ### [`v9.0.0`](https://redirect.github.com/pytest-dev/pytest/releases/tag/9.0.0) [Compare Source](https://redirect.github.com/pytest-dev/pytest/compare/8.4.2...9.0.0) ### pytest 9.0.0 (2025-11-05) #### New features - [#​1367](https://redirect.github.com/pytest-dev/pytest/issues/1367): **Support for subtests** has been added. `subtests ` are an alternative to parametrization, useful in situations where the parametrization values are not all known at collection time. Example: ```python def contains_docstring(p: Path) -> bool: """Return True if the given Python file contains a top-level docstring.""" ... def test_py_files_contain_docstring(subtests: pytest.Subtests) -> None: for path in Path.cwd().glob("*.py"): with subtests.test(path=str(path)): assert contains_docstring(path) ``` Each assert failure or error is caught by the context manager and reported individually, giving a clear picture of all files that are missing a docstring. In addition, `unittest.TestCase.subTest` is now also supported. This feature was originally implemented as a separate plugin in [pytest-subtests](https://redirect.github.com/pytest-dev/pytest-subtests), but since then has been merged into the core. > \[!NOTE] > This feature is experimental and will likely evolve in future releases. By that we mean that we might change how subtests are reported on failure, but the functionality and how to use it are stable. - [#​13743](https://redirect.github.com/pytest-dev/pytest/issues/13743): Added support for **native TOML configuration files**. While pytest, since version 6, supports configuration in `pyproject.toml` files under `[tool.pytest.ini_options]`, it does so in an "INI compatibility mode", where all configuration values are treated as strings or list of strings. Now, pytest supports the native TOML data model. In `pyproject.toml`, the native TOML configuration is under the `[tool.pytest]` table. ```toml # pyproject.toml [tool.pytest] minversion = "9.0" addopts = ["-ra", "-q"] testpaths = [ "tests", "integration", ] ``` The `[tool.pytest.ini_options]` table remains supported, but both tables cannot be used at the same time. If you prefer to use a separate configuration file, or don't use `pyproject.toml`, you can use `pytest.toml` or `.pytest.toml`: ```toml # pytest.toml or .pytest.toml [pytest] minversion = "9.0" addopts = ["-ra", "-q"] testpaths = [ "tests", "integration", ] ``` The documentation now (sometimes) shows configuration snippets in both TOML and INI formats, in a tabbed interface. See `config file formats` for full details. - [#​13823](https://redirect.github.com/pytest-dev/pytest/issues/13823): Added a **"strict mode"** enabled by the `strict` configuration option. When set to `true`, the `strict` option currently enables - `strict_config` - `strict_markers` - `strict_parametrization_ids` - `strict_xfail` The individual strictness options can be explicitly set to override the global `strict` setting. The previously-deprecated `--strict` command-line flag now enables strict mode. If pytest adds new strictness options in the future, they will also be enabled in strict mode. Therefore, you should only enable strict mode if you use a pinned/locked version of pytest, or if you want to proactively adopt new strictness options as they are added. See `strict mode` for more details. - [#​13737](https://redirect.github.com/pytest-dev/pytest/issues/13737): Added the `strict_parametrization_ids` configuration option. When set, pytest emits an error if it detects non-unique parameter set IDs, rather than automatically making the IDs unique by adding 0, 1, ... to them. This can be particularly useful for catching unintended duplicates. - [#​13072](https://redirect.github.com/pytest-dev/pytest/issues/13072): Added support for displaying test session **progress in the terminal tab** using the [OSC 9;4;](https://conemu.github.io/en/AnsiEscapeCodes.html#ConEmu_specific_OSC) ANSI sequence. When pytest runs in a supported terminal emulator like ConEmu, Gnome Terminal, Ptyxis, Windows Terminal, Kitty or Ghostty, you'll see the progress in the terminal tab or window, allowing you to monitor pytest's progress at a glance. This feature is automatically enabled when running in a TTY. It is implemented as an internal plugin. If needed, it can be disabled as follows: - On a user level, using `-p no:terminalprogress` on the command line or via an environment variable `PYTEST_ADDOPTS='-p no:terminalprogress'`. - On a project configuration level, using `addopts = "-p no:terminalprogress"`. - [#​478](https://redirect.github.com/pytest-dev/pytest/issues/478): Support PEP420 (implicit namespace packages) as --pyargs target when `consider_namespace_packages` is true in the config. Previously, this option only impacted package imports, now it also impacts tests discovery. - [#​13678](https://redirect.github.com/pytest-dev/pytest/issues/13678): Added a new `faulthandler_exit_on_timeout` configuration option set to "false" by default to let faulthandler interrupt the pytest process after a timeout in case of deadlock. Previously, a faulthandler timeout would only dump the traceback of all threads to stderr, but would not interrupt the pytest process. \-- by `ogrisel`. - [#​13829](https://redirect.github.com/pytest-dev/pytest/issues/13829): Added support for configuration option aliases via the `aliases` parameter in `Parser.addini() `. Plugins can now register alternative names for configuration options, allowing for more flexibility in configuration naming and supporting backward compatibility when renaming options. The canonical name always takes precedence if both the canonical name and an alias are specified in the configuration file. #### Improvements in existing functionality - [#​13330](https://redirect.github.com/pytest-dev/pytest/issues/13330): Having pytest configuration spread over more than one file (for example having both a `pytest.ini` file and `pyproject.toml` with a `[tool.pytest.ini_options]` table) will now print a warning to make it clearer to the user that only one of them is actually used. \-- by `sgaist` - [#​13574](https://redirect.github.com/pytest-dev/pytest/issues/13574): The single argument `--version` no longer loads the entire plugin infrastructure, making it faster and more reliable when displaying only the pytest version. Passing `--version` twice (e.g., `pytest --version --version`) retains the original behavior, showing both the pytest version and plugin information. > \[!NOTE] > Since `--version` is now processed early, it only takes effect when passed directly via the command line. It will not work if set through other mechanisms, such as `PYTEST_ADDOPTS` or `addopts`. - [#​13823](https://redirect.github.com/pytest-dev/pytest/issues/13823): Added `strict_xfail` as an alias to the `xfail_strict` option, `strict_config` as an alias to the `--strict-config` flag, and `strict_markers` as an alias to the `--strict-markers` flag. This makes all strictness options consistently have configuration options with the prefix `strict_`. - [#​13700](https://redirect.github.com/pytest-dev/pytest/issues/13700): --junitxml no longer prints the generated xml file summary at the end of the pytest session when --quiet is given. - [#​13732](https://redirect.github.com/pytest-dev/pytest/issues/13732): Previously, when filtering warnings, pytest would fail if the filter referenced a class that could not be imported. Now, this only outputs a message indicating the problem. - [#​13859](https://redirect.github.com/pytest-dev/pytest/issues/13859): Clarify the error message for pytest.raises() when a regex match fails. - [#​13861](https://redirect.github.com/pytest-dev/pytest/issues/13861): Better sentence structure in a test's expected error message. Previously, the error message would be "expected exception must be \, but got \". Now, it is "Expected \, but got \". #### Removals and backward incompatible breaking changes - [#​12083](https://redirect.github.com/pytest-dev/pytest/issues/12083): Fixed a bug where an invocation such as pytest a/ a/b would cause only tests from a/b to run, and not other tests under a/. The fix entails a few breaking changes to how such overlapping arguments and duplicates are handled: 1. pytest a/b a/ or pytest a/ a/b are equivalent to pytest a; if an argument overlaps another arguments, only the prefix remains. 2. pytest x.py x.py is equivalent to pytest x.py; previously such an invocation was taken as an explicit request to run the tests from the file twice. If you rely on these behaviors, consider using `--keep-duplicates `, which retains its existing behavior (including the bug). - [#​13719](https://redirect.github.com/pytest-dev/pytest/issues/13719): Support for Python 3.9 is dropped following its end of life. - [#​13766](https://redirect.github.com/pytest-dev/pytest/issues/13766): Previously, pytest would assume it was running in a CI/CD environment if either of the environment variables $CI or $BUILD\_NUMBER was defined; now, CI mode is only activated if at least one of those variables is defined and set to a *non-empty* value. - [#​13779](https://redirect.github.com/pytest-dev/pytest/issues/13779): **PytestRemovedIn9Warning deprecation warnings are now errors by default.** Following our plan to remove deprecated features with as little disruption as possible, all warnings of type `PytestRemovedIn9Warning` now generate errors instead of warning messages by default. **The affected features will be effectively removed in pytest 9.1**, so please consult the `deprecations` section in the docs for directions on how to update existing code. In the pytest `9.0.X` series, it is possible to change the errors back into warnings as a stopgap measure by adding this to your `pytest.ini` file: ```ini [pytest] filterwarnings = ignore::pytest.PytestRemovedIn9Warning ``` But this will stop working when pytest `9.1` is released. **If you have concerns** about the removal of a specific feature, please add a comment to `13779`. #### Deprecations (removal in next major release) - [#​13807](https://redirect.github.com/pytest-dev/pytest/issues/13807): `monkeypatch.syspath_prepend() ` now issues a deprecation warning when the prepended path contains legacy namespace packages (those using `pkg_resources.declare_namespace()`). Users should migrate to native namespace packages (`420`). See `monkeypatch-fixup-namespace-packages` for details. #### Bug fixes - [#​13445](https://redirect.github.com/pytest-dev/pytest/issues/13445): Made the type annotations of `pytest.skip` and friends more spec-complaint to have them work across more type checkers. - [#​13537](https://redirect.github.com/pytest-dev/pytest/issues/13537): Fixed a bug in which `ExceptionGroup` with only `Skipped` exceptions in teardown was not handled correctly and showed as error. - [#​13598](https://redirect.github.com/pytest-dev/pytest/issues/13598): Fixed possible collection confusion on Windows when short paths and symlinks are involved. - [#​13716](https://redirect.github.com/pytest-dev/pytest/issues/13716): Fixed a bug where a nonsensical invocation like `pytest x.py[a]` (a file cannot be parametrized) was silently treated as `pytest x.py`. This is now a usage error. - [#​13722](https://redirect.github.com/pytest-dev/pytest/issues/13722): Fixed a misleading assertion failure message when using `pytest.approx` on mappings with differing lengths. - [#​13773](https://redirect.github.com/pytest-dev/pytest/issues/13773): Fixed the static fixture closure calculation to properly consider transitive dependencies requested by overridden fixtures. - [#​13816](https://redirect.github.com/pytest-dev/pytest/issues/13816): Fixed `pytest.approx` which now returns a clearer error message when comparing mappings with different keys. - [#​13849](https://redirect.github.com/pytest-dev/pytest/issues/13849): Hidden `.pytest.ini` files are now picked up as the config file even if empty. This was an inconsistency with non-hidden `pytest.ini`. - [#​13865](https://redirect.github.com/pytest-dev/pytest/issues/13865): Fixed --show-capture with --tb=line. - [#​13522](https://redirect.github.com/pytest-dev/pytest/issues/13522): Fixed `pytester` in subprocess mode ignored all :attr\`pytester.plugins \\` except the first. Fixed `pytester` in subprocess mode silently ignored non-str `pytester.plugins `. Now it errors instead. If you are affected by this, specify the plugin by name, or switch the affected tests to use `pytester.runpytest_inprocess ` explicitly instead. #### Packaging updates and notes for downstreams - [#​13791](https://redirect.github.com/pytest-dev/pytest/issues/13791): Minimum requirements on `iniconfig` and `packaging` were bumped to `1.0.1` and `22.0.0`, respectively. #### Contributor-facing changes - [#​12244](https://redirect.github.com/pytest-dev/pytest/issues/12244): Fixed self-test failures when TERM=dumb. - [#​12474](https://redirect.github.com/pytest-dev/pytest/issues/12474): Added scheduled GitHub Action Workflow to run Sphinx linkchecks in repo documentation. - [#​13621](https://redirect.github.com/pytest-dev/pytest/issues/13621): pytest's own testsuite now handles the `lsof` command hanging (e.g. due to unreachable network filesystems), with the affected selftests being skipped after 10 seconds. - [#​13638](https://redirect.github.com/pytest-dev/pytest/issues/13638): Fixed deprecated `gh pr new` command in `scripts/prepare-release-pr.py`. The script now uses `gh pr create` which is compatible with GitHub CLI v2.0+. - [#​13695](https://redirect.github.com/pytest-dev/pytest/issues/13695): Flush stdout and stderr in Pytester.run to avoid truncated outputs in test\_faulthandler.py::test\_timeout on CI -- by `ogrisel`. - [#​13771](https://redirect.github.com/pytest-dev/pytest/issues/13771): Skip test\_do\_not\_collect\_symlink\_siblings on Windows environments without symlink support to avoid false negatives. - [#​13841](https://redirect.github.com/pytest-dev/pytest/issues/13841): `tox>=4` is now required when contributing to pytest. - [#​13625](https://redirect.github.com/pytest-dev/pytest/issues/13625): Added missing docstrings to `pytest_addoption()`, `pytest_configure()`, and `cacheshow()` functions in `cacheprovider.py`. #### Miscellaneous internal changes - [#​13830](https://redirect.github.com/pytest-dev/pytest/issues/13830): Configuration overrides (`-o`/`--override-ini`) are now processed during startup rather than during `config.getini() `.
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - At any time (no schedule defined) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about these updates again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/googleapis/google-cloud-python). --------- Co-authored-by: Chalmer Lowe --- .../samples/pyarrow/requirements-test.txt | 2 -- .../samples/quickstart/requirements-test.txt | 2 -- .../samples/snippets/requirements-test.txt | 2 -- .../samples/to_dataframe/requirements-test.txt | 2 -- .../samples/geography/requirements-test.txt | 2 +- .../samples/notebooks/requirements-test.txt | 2 +- .../samples/snippets/requirements-test.txt | 2 +- 7 files changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/google-cloud-bigquery-storage/samples/pyarrow/requirements-test.txt b/packages/google-cloud-bigquery-storage/samples/pyarrow/requirements-test.txt index 7752197e3f19..aa857744dee6 100644 --- a/packages/google-cloud-bigquery-storage/samples/pyarrow/requirements-test.txt +++ b/packages/google-cloud-bigquery-storage/samples/pyarrow/requirements-test.txt @@ -1,3 +1 @@ -pytest==7.4.3; python_version == '3.7' -pytest==8.3.5; python_version == '3.8' pytest==9.0.3; python_version >= '3.9' diff --git a/packages/google-cloud-bigquery-storage/samples/quickstart/requirements-test.txt b/packages/google-cloud-bigquery-storage/samples/quickstart/requirements-test.txt index 7752197e3f19..aa857744dee6 100644 --- a/packages/google-cloud-bigquery-storage/samples/quickstart/requirements-test.txt +++ b/packages/google-cloud-bigquery-storage/samples/quickstart/requirements-test.txt @@ -1,3 +1 @@ -pytest==7.4.3; python_version == '3.7' -pytest==8.3.5; python_version == '3.8' pytest==9.0.3; python_version >= '3.9' diff --git a/packages/google-cloud-bigquery-storage/samples/snippets/requirements-test.txt b/packages/google-cloud-bigquery-storage/samples/snippets/requirements-test.txt index 6fc1a97a9e93..69983dbe3624 100644 --- a/packages/google-cloud-bigquery-storage/samples/snippets/requirements-test.txt +++ b/packages/google-cloud-bigquery-storage/samples/snippets/requirements-test.txt @@ -1,4 +1,2 @@ google-cloud-testutils==1.8.0 -pytest==7.4.3; python_version == '3.7' -pytest==8.3.5; python_version == '3.8' pytest==9.0.3; python_version >= '3.9' diff --git a/packages/google-cloud-bigquery-storage/samples/to_dataframe/requirements-test.txt b/packages/google-cloud-bigquery-storage/samples/to_dataframe/requirements-test.txt index 7752197e3f19..aa857744dee6 100644 --- a/packages/google-cloud-bigquery-storage/samples/to_dataframe/requirements-test.txt +++ b/packages/google-cloud-bigquery-storage/samples/to_dataframe/requirements-test.txt @@ -1,3 +1 @@ -pytest==7.4.3; python_version == '3.7' -pytest==8.3.5; python_version == '3.8' pytest==9.0.3; python_version >= '3.9' diff --git a/packages/google-cloud-bigquery/samples/geography/requirements-test.txt b/packages/google-cloud-bigquery/samples/geography/requirements-test.txt index 6fb9ba31062d..81deba00e2e1 100644 --- a/packages/google-cloud-bigquery/samples/geography/requirements-test.txt +++ b/packages/google-cloud-bigquery/samples/geography/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==8.4.2 +pytest==9.0.3 mock==5.2.0 pytest-xdist==3.8.0 diff --git a/packages/google-cloud-bigquery/samples/notebooks/requirements-test.txt b/packages/google-cloud-bigquery/samples/notebooks/requirements-test.txt index 31b836790f7c..73b9b335aa9f 100644 --- a/packages/google-cloud-bigquery/samples/notebooks/requirements-test.txt +++ b/packages/google-cloud-bigquery/samples/notebooks/requirements-test.txt @@ -1,4 +1,4 @@ google-cloud-testutils==1.6.4 -pytest==8.4.2 +pytest==9.0.3 mock==5.2.0 pytest-xdist==3.8.0 diff --git a/packages/google-cloud-bigquery/samples/snippets/requirements-test.txt b/packages/google-cloud-bigquery/samples/snippets/requirements-test.txt index 901f1df1a9d0..d86bc6c49dce 100644 --- a/packages/google-cloud-bigquery/samples/snippets/requirements-test.txt +++ b/packages/google-cloud-bigquery/samples/snippets/requirements-test.txt @@ -1,5 +1,5 @@ # samples/snippets should be runnable with no "extras" google-cloud-testutils==1.6.4 -pytest==8.4.2 +pytest==9.0.3 mock==5.2.0 pytest-xdist==3.8.0 From 2bfdcf46ce32a9aa6b586c689f4b7a114b55d26e Mon Sep 17 00:00:00 2001 From: Noah Dietz Date: Tue, 6 Oct 2026 13:05:55 -0700 Subject: [PATCH 016/228] chore: update librarian version to v0.47.0 (#18585) --- librarian.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/librarian.yaml b/librarian.yaml index 4eebd494b8bc..4d2695ab82de 100644 --- a/librarian.yaml +++ b/librarian.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. language: python -version: v0.46.0 +version: v0.47.0 repo: googleapis/google-cloud-python sources: googleapis: From 734cebe910332adef089979e5648f3036cba71e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Wed, 7 Oct 2026 08:24:42 +0200 Subject: [PATCH 017/228] test(spanner): speed up unit test suite and eliminate idle retry sleeps (#18582) Speed up the unit test suite by ~55% (reducing full suite runtime from ~66s to ~29s) by eliminating wall-clock sleep delays, avoiding live gRPC transport initialization, and reducing cryptographic CPU burn in protocol tests. Key changes: - Intercept retry backoff sleeps: Mock localized `time.sleep` and `asyncio.sleep` in transaction, snapshot, and batch commit retry tests. Each test now verifies retry counts and delay values without idling in real time. - Deterministic concurrency synchronization: Replace arbitrary timing sleeps in session manager and metrics tests with explicit `asyncio.Event` and `threading.Barrier` coordination, eliminating CI flakiness under CPU contention. - Eliminate teardown polling: Remove `_assert_true_with_timeout` and sleep polling from `test_database_session_manager.py` by relying on event-driven rotation callbacks and blocking thread joins. - Isolate GAPIC admin transports: Use autospec mock transports in `test_client.py` and `test_instance.py` instead of default credentials, eliminating live gRPC channel creation and 60-second network timeout risks. - Lower Omni Argon2id test parameters: Use minimal valid test parameters (1 iteration, 8 KB RAM) in protocol flow and state machine tests, reducing key derivation time by ~1,100x while leaving RFC golden vector tests untouched. - Clean up legacy mocks: Convert legacy global `mock.patch("time.sleep")` calls in `test_session.py` to localized module paths adhering to repository mock hygiene rules. --- .../tests/unit/_async/test_session.py | 30 ++++++-- .../tests/unit/_async/test_snapshot.py | 7 +- .../unit/_async/test_transaction_extra.py | 7 +- .../tests/unit/omni/test_login_client.py | 12 +-- .../tests/unit/omni/test_opaque.py | 36 ++++----- .../tests/unit/test_batch.py | 17 ++++- .../tests/unit/test_client.py | 39 ++++++---- .../unit/test_database_session_manager.py | 76 +++++++++---------- .../tests/unit/test_instance.py | 48 +++++++----- .../tests/unit/test_metrics_concurrency.py | 11 ++- .../tests/unit/test_session.py | 36 +++++---- .../tests/unit/test_snapshot.py | 12 ++- 12 files changed, 203 insertions(+), 128 deletions(-) diff --git a/packages/google-cloud-spanner/tests/unit/_async/test_session.py b/packages/google-cloud-spanner/tests/unit/_async/test_session.py index 98758b6b904b..b20dda7e2435 100644 --- a/packages/google-cloud-spanner/tests/unit/_async/test_session.py +++ b/packages/google-cloud-spanner/tests/unit/_async/test_session.py @@ -1219,9 +1219,13 @@ async def unit_of_work(transaction): pass await session.create() - await session.run_in_transaction(unit_of_work) + with mock.patch( + "google.cloud.spanner_v1._async._helpers.asyncio.sleep" + ) as sleep_mock: + await session.run_in_transaction(unit_of_work) self.assertEqual(begin_transaction.call_count, 2) + sleep_mock.assert_called_once() begin_transaction.assert_called_with( request=BeginTransactionRequest( @@ -1261,10 +1265,14 @@ async def unit_of_work(transaction): pass await session.create() - await session.run_in_transaction(unit_of_work) + with mock.patch( + "google.cloud.spanner_v1._async._helpers.asyncio.sleep" + ) as sleep_mock: + await session.run_in_transaction(unit_of_work) # Verify retried BeginTransaction API call. self.assertEqual(begin_transaction.call_count, 2) + sleep_mock.assert_called_once() begin_transaction.assert_called_with( request=BeginTransactionRequest( @@ -1308,10 +1316,14 @@ async def unit_of_work(transaction): pass await session.create() - await session.run_in_transaction(unit_of_work) + with mock.patch( + "google.cloud.spanner_v1._async._helpers.asyncio.sleep" + ) as sleep_mock: + await session.run_in_transaction(unit_of_work) # Verify retried BeginTransaction API call. self.assertEqual(begin_transaction.call_count, 2) + sleep_mock.assert_called_once() begin_transaction.assert_called_with( request=BeginTransactionRequest( @@ -1812,7 +1824,7 @@ def _time(): return _results[0] return 1.0 - with mock.patch("time.time", _time): + with mock.patch("google.cloud.spanner_v1._async._helpers.time.time", _time): with mock.patch( "google.cloud.spanner_v1._async._helpers.asyncio.sleep", new_callable=mock.AsyncMock, @@ -1893,7 +1905,7 @@ def _time(): return 1.0 with ( - mock.patch("time.time", _time), + mock.patch("google.cloud.spanner_v1._async._helpers.time.time", _time), mock.patch( "google.cloud.spanner_v1._helpers.random.random", return_value=0 ), @@ -2817,11 +2829,15 @@ def _time_func(): return 3 # check if current time > deadline - with mock.patch("time.time", _time_func): + with mock.patch( + "google.cloud.spanner_v1._async._helpers.time.time", _time_func + ): with pytest.raises(Exception): _delay_until_retry(exc_mock, 2, 1, default_retry_delay=0) - with mock.patch("time.time", _time_func): + with mock.patch( + "google.cloud.spanner_v1._async._helpers.time.time", _time_func + ): with mock.patch( "google.cloud.spanner_v1._helpers._get_retry_delay" ) as get_retry_delay_mock: diff --git a/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py b/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py index da90f929f615..39926148d9de 100644 --- a/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py +++ b/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py @@ -1367,9 +1367,14 @@ async def test_begin_transaction_retry(self): TransactionPB(id=TXN_ID), ] - tid = await snapshot._begin_transaction() + with mock.patch( + "google.cloud.spanner_v1._async._helpers.asyncio.sleep" + ) as sleep_mock: + tid = await snapshot._begin_transaction() + self.assertEqual(tid, TXN_ID) self.assertEqual(api.begin_transaction.call_count, 2) + sleep_mock.assert_called_once_with(2) async def test_update_for_transaction_pb_w_precommit_token(self): from google.cloud.spanner_v1.types import MultiplexedSessionPrecommitToken diff --git a/packages/google-cloud-spanner/tests/unit/_async/test_transaction_extra.py b/packages/google-cloud-spanner/tests/unit/_async/test_transaction_extra.py index 4e21378cb6b2..69ce9c4d2a68 100644 --- a/packages/google-cloud-spanner/tests/unit/_async/test_transaction_extra.py +++ b/packages/google-cloud-spanner/tests/unit/_async/test_transaction_extra.py @@ -135,8 +135,13 @@ async def test_commit_retry_and_precommit_token(self): final_resp, ] - await txn.commit() + with mock.patch( + "google.cloud.spanner_v1._async._helpers.asyncio.sleep" + ) as sleep_mock: + await txn.commit() + self.assertEqual(self.db.spanner_api.commit.call_count, 3) + sleep_mock.assert_called_once_with(2) async def test_execute_update_request_options_dict(self): # coverage for line 503 diff --git a/packages/google-cloud-spanner/tests/unit/omni/test_login_client.py b/packages/google-cloud-spanner/tests/unit/omni/test_login_client.py index 3d080cd67859..fe33bfea5efc 100644 --- a/packages/google-cloud-spanner/tests/unit/omni/test_login_client.py +++ b/packages/google-cloud-spanner/tests/unit/omni/test_login_client.py @@ -41,9 +41,9 @@ def test_login_successful_flow(self): params = authentication_pb2.HashParameters( argon2_id_parameters=authentication_pb2.HashParameters.Argon2IdParameters( - iteration_count=3, - memory_usage=64 * 1024, - parallelism=4, + iteration_count=1, + memory_usage=8, + parallelism=1, hash_size=32, ) ) @@ -248,9 +248,9 @@ def test_login_missing_access_token_in_final_response(self): password = "test_password" params = authentication_pb2.HashParameters( argon2_id_parameters=authentication_pb2.HashParameters.Argon2IdParameters( - iteration_count=3, - memory_usage=64 * 1024, - parallelism=4, + iteration_count=1, + memory_usage=8, + parallelism=1, hash_size=32, ) ) diff --git a/packages/google-cloud-spanner/tests/unit/omni/test_opaque.py b/packages/google-cloud-spanner/tests/unit/omni/test_opaque.py index e2816ff0c22a..17e93eb90721 100644 --- a/packages/google-cloud-spanner/tests/unit/omni/test_opaque.py +++ b/packages/google-cloud-spanner/tests/unit/omni/test_opaque.py @@ -486,9 +486,9 @@ def test_oprf_evaluate(self): def test_authenticator_validation(self): valid_params = authentication_pb2.HashParameters( argon2_id_parameters=authentication_pb2.HashParameters.Argon2IdParameters( - iteration_count=3, - memory_usage=64 * 1024, - parallelism=4, + iteration_count=1, + memory_usage=8, + parallelism=1, hash_size=32, ) ) @@ -512,9 +512,9 @@ def test_authenticator_validation(self): def test_authenticator_state_errors(self): params = authentication_pb2.HashParameters( argon2_id_parameters=authentication_pb2.HashParameters.Argon2IdParameters( - iteration_count=3, - memory_usage=64 * 1024, - parallelism=4, + iteration_count=1, + memory_usage=8, + parallelism=1, hash_size=32, ) ) @@ -541,9 +541,9 @@ def test_authenticator_state_errors(self): def test_user_authenticator_clear_and_del(self): params = authentication_pb2.HashParameters( argon2_id_parameters=authentication_pb2.HashParameters.Argon2IdParameters( - iteration_count=3, - memory_usage=64 * 1024, - parallelism=4, + iteration_count=1, + memory_usage=8, + parallelism=1, hash_size=32, ) ) @@ -584,9 +584,9 @@ def test_full_opaque_handshake_simulation(self): params = authentication_pb2.HashParameters( argon2_id_parameters=authentication_pb2.HashParameters.Argon2IdParameters( - iteration_count=3, - memory_usage=64 * 1024, - parallelism=4, + iteration_count=1, + memory_usage=8, + parallelism=1, hash_size=32, ) ) @@ -708,9 +708,9 @@ def test_full_opaque_handshake_simulation(self): def test_final_request_invalid_masked_response_length(self): params = authentication_pb2.HashParameters( argon2_id_parameters=authentication_pb2.HashParameters.Argon2IdParameters( - iteration_count=3, - memory_usage=64 * 1024, - parallelism=4, + iteration_count=1, + memory_usage=8, + parallelism=1, hash_size=32, ) ) @@ -964,9 +964,9 @@ def test_final_request_edge_cases(self): params = authentication_pb2.HashParameters( argon2_id_parameters=authentication_pb2.HashParameters.Argon2IdParameters( - iteration_count=3, - memory_usage=64 * 1024, - parallelism=4, + iteration_count=1, + memory_usage=8, + parallelism=1, hash_size=32, ) ) diff --git a/packages/google-cloud-spanner/tests/unit/test_batch.py b/packages/google-cloud-spanner/tests/unit/test_batch.py index 933c47bfa92d..247bdb58ab6c 100644 --- a/packages/google-cloud-spanner/tests/unit/test_batch.py +++ b/packages/google-cloud-spanner/tests/unit/test_batch.py @@ -345,12 +345,25 @@ def test_aborted_exception_on_commit_with_retries(self, mock_region): batch.insert(TABLE_NAME, COLUMNS, VALUES) # Assertion: Ensure that calling batch.commit() raises Aborted - with self.assertRaises(Aborted) as context: - batch.commit(timeout_secs=1.0, default_retry_delay=0) + delay_call_count = 0 + + def fake_delay(exc, *args, **kwargs): + nonlocal delay_call_count + delay_call_count += 1 + if delay_call_count >= 2: + raise exc + + with mock.patch( + "google.cloud.spanner_v1._helpers._delay_until_retry", + side_effect=fake_delay, + ): + with self.assertRaises(Aborted) as context: + batch.commit(timeout_secs=1.0, default_retry_delay=0) # Verify exception includes request_id attribute self.assertIn("409 Transaction was aborted", str(context.exception)) self.assertTrue(hasattr(context.exception, "request_id")) + self.assertEqual(delay_call_count, 2) self.assertGreater( api.commit.call_count, 1, "commit should be called more than once" ) diff --git a/packages/google-cloud-spanner/tests/unit/test_client.py b/packages/google-cloud-spanner/tests/unit/test_client.py index ac279e031230..241b4efa02c3 100644 --- a/packages/google-cloud-spanner/tests/unit/test_client.py +++ b/packages/google-cloud-spanner/tests/unit/test_client.py @@ -59,6 +59,17 @@ def _get_target_class(self): def _make_one(self, *args, **kwargs): return self._get_target_class()(*args, **kwargs) + @staticmethod + def _make_instance_admin_api(): + from google.cloud.spanner_admin_instance_v1 import InstanceAdminClient + from google.cloud.spanner_admin_instance_v1.services.instance_admin.transports.base import ( + InstanceAdminTransport, + ) + + mock_transport = mock.create_autospec(InstanceAdminTransport, instance=True) + mock_transport._wrapped_methods = {} + return InstanceAdminClient(transport=mock_transport) + def _constructor_test_helper( self, expected_scopes, @@ -627,15 +638,14 @@ def test_project_name_property(self): def test_list_instance_configs(self): from google.cloud.spanner_admin_instance_v1 import ( - InstanceAdminClient, - ListInstanceConfigsRequest, - ListInstanceConfigsResponse, + InstanceConfig as InstanceConfigPB, ) from google.cloud.spanner_admin_instance_v1 import ( - InstanceConfig as InstanceConfigPB, + ListInstanceConfigsRequest, + ListInstanceConfigsResponse, ) - api = InstanceAdminClient(credentials=AnonymousCredentials()) + api = self._make_instance_admin_api() credentials = build_scoped_credentials() client = self._make_one(project=self.PROJECT, credentials=credentials) client._instance_admin_api = api @@ -676,16 +686,15 @@ def test_list_instance_configs(self): def test_list_instance_configs_w_options(self): from google.cloud.spanner_admin_instance_v1 import ( - InstanceAdminClient, - ListInstanceConfigsRequest, - ListInstanceConfigsResponse, + InstanceConfig as InstanceConfigPB, ) from google.cloud.spanner_admin_instance_v1 import ( - InstanceConfig as InstanceConfigPB, + ListInstanceConfigsRequest, + ListInstanceConfigsResponse, ) credentials = build_scoped_credentials() - api = InstanceAdminClient(credentials=credentials) + api = self._make_instance_admin_api() client = self._make_one(project=self.PROJECT, credentials=credentials) client._instance_admin_api = api @@ -754,15 +763,16 @@ def test_instance_factory_explicit(self): self.assertIs(instance._client, client) def test_list_instances(self): - from google.cloud.spanner_admin_instance_v1 import Instance as InstancePB from google.cloud.spanner_admin_instance_v1 import ( - InstanceAdminClient, + Instance as InstancePB, + ) + from google.cloud.spanner_admin_instance_v1 import ( ListInstancesRequest, ListInstancesResponse, ) + api = self._make_instance_admin_api() credentials = build_scoped_credentials() - api = InstanceAdminClient(credentials=credentials) client = self._make_one(project=self.PROJECT, credentials=credentials) client._instance_admin_api = api @@ -806,13 +816,12 @@ def test_list_instances(self): def test_list_instances_w_options(self): from google.cloud.spanner_admin_instance_v1 import ( - InstanceAdminClient, ListInstancesRequest, ListInstancesResponse, ) + api = self._make_instance_admin_api() credentials = build_scoped_credentials() - api = InstanceAdminClient(credentials=credentials) client = self._make_one(project=self.PROJECT, credentials=credentials) client._instance_admin_api = api diff --git a/packages/google-cloud-spanner/tests/unit/test_database_session_manager.py b/packages/google-cloud-spanner/tests/unit/test_database_session_manager.py index 47fa96e5a468..f7b1fd50b88a 100644 --- a/packages/google-cloud-spanner/tests/unit/test_database_session_manager.py +++ b/packages/google-cloud-spanner/tests/unit/test_database_session_manager.py @@ -11,14 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import threading from datetime import timedelta from os import environ -from time import sleep, time -from typing import Callable from unittest import TestCase from google.api_core.exceptions import BadRequest, FailedPrecondition -from mock import MagicMock, Mock, patch +from mock import DEFAULT, MagicMock, Mock, patch from google.cloud.spanner_v1.database_sessions_manager import ( DatabaseSessionsManager, @@ -27,12 +26,6 @@ from tests._builders import build_database -# Shorten polling and refresh intervals for testing. -@patch.multiple( - DatabaseSessionsManager, - _MAINTENANCE_THREAD_POLLING_INTERVAL=timedelta(seconds=1), - _MAINTENANCE_THREAD_REFRESH_INTERVAL=timedelta(seconds=2), -) class TestDatabaseSessionManager(TestCase): @classmethod def setUpClass(cls): @@ -64,7 +57,8 @@ def tearDown(self): if thread and thread.is_alive(): manager._multiplexed_session_terminate_event.set() - self._assert_true_with_timeout(lambda: not thread.is_alive()) + thread.join(timeout=10) + self.assertFalse(thread.is_alive()) def test_read_only_pooled(self): manager = self._manager @@ -180,19 +174,31 @@ def test_read_write_multiplexed(self): # Verify create_session was called. manager._database.spanner_api.create_session.assert_called_once() + @patch.multiple( + DatabaseSessionsManager, + _MAINTENANCE_THREAD_POLLING_INTERVAL=timedelta(milliseconds=5), + _MAINTENANCE_THREAD_REFRESH_INTERVAL=timedelta(milliseconds=10), + ) def test_multiplexed_maintenance(self): manager = self._manager self._enable_multiplexed_sessions() + rotated = threading.Event() + + def on_create_session(*args, **kwargs): + if manager._database.spanner_api.create_session.call_count > 1: + rotated.set() + return DEFAULT + + manager._database.spanner_api.create_session.side_effect = on_create_session + # Maintenance thread is started. session_1 = manager.get_session(TransactionType.READ_ONLY) self.assertTrue(session_1.is_multiplexed) self.assertTrue(manager._multiplexed_session_thread.is_alive()) - # Wait for maintenance thread to execute. - self._assert_true_with_timeout( - lambda: manager._database.spanner_api.create_session.call_count > 1 - ) + # Wait for maintenance thread to execute without polling. + self.assertTrue(rotated.wait(timeout=10.0)) # Verify that maintenance thread created new multiplexed session. session_2 = manager.get_session(TransactionType.READ_ONLY) @@ -222,24 +228,32 @@ def test_concurrent_get_multiplexed_session_no_deadlock(self): # Mock maintenance thread creation to avoid spawning background tasks manager._build_maintenance_thread = Mock(return_value=Mock()) - # Mock _build_multiplexed_session to include a suspension point - async def slow_build(): - await asyncio.sleep(0.5) - return Mock() - - manager._build_multiplexed_session = slow_build - # Enable multiplexed sessions in environment for verification environ[DatabaseSessionsManager._ENV_VAR_MULTIPLEXED] = "true" async def run_concurrent(): + entered_slow_build = asyncio.Event() + release_slow_build = asyncio.Event() + + # Mock _build_multiplexed_session to include a suspension point + async def slow_build(): + entered_slow_build.set() + await release_slow_build.wait() + return Mock() + + manager._build_multiplexed_session = slow_build + # Trigger Coroutine 1 task1 = asyncio.create_task(manager._get_multiplexed_session()) - await asyncio.sleep(0.1) # Allow Coroutine 1 to acquire lock and suspend + # Wait until Coroutine 1 has acquired the lock and suspended + await entered_slow_build.wait() - # Trigger Coroutine 2 - this would previously block the main thread + # Trigger Coroutine 2 while Coroutine 1 holds the lock task2 = asyncio.create_task(manager._get_multiplexed_session()) + # Release Coroutine 1 to complete + release_slow_build.set() + await asyncio.gather(task1, task2) try: @@ -645,22 +659,6 @@ def test_rotate_multiplexed_session_build_failure(self): self.assertIs(manager._multiplexed_session, current_session) current_session.delete.assert_not_called() - def _assert_true_with_timeout(self, condition: Callable) -> None: - """Asserts that the given condition is met within a timeout period. - - :type condition: Callable - :param condition: A callable that returns a boolean indicating whether the condition is met. - """ - - sleep_seconds = 0.1 - timeout_seconds = 10 - - start_time = time() - while not condition() and time() - start_time < timeout_seconds: - sleep(sleep_seconds) - - self.assertTrue(condition()) - @staticmethod def _disable_multiplexed_sessions() -> None: """Sets environment variables to disable multiplexed sessions for all transactions types.""" diff --git a/packages/google-cloud-spanner/tests/unit/test_instance.py b/packages/google-cloud-spanner/tests/unit/test_instance.py index b089777cc853..9f52d70b1518 100644 --- a/packages/google-cloud-spanner/tests/unit/test_instance.py +++ b/packages/google-cloud-spanner/tests/unit/test_instance.py @@ -15,7 +15,6 @@ import unittest import mock -from google.auth.credentials import AnonymousCredentials from google.cloud.spanner_v1 import DefaultTransactionOptions @@ -52,6 +51,17 @@ def _getTargetClass(self): def _make_one(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) + @staticmethod + def _make_database_admin_api(): + from google.cloud.spanner_admin_database_v1 import DatabaseAdminClient + from google.cloud.spanner_admin_database_v1.services.database_admin.transports.base import ( + DatabaseAdminTransport, + ) + + mock_transport = mock.create_autospec(DatabaseAdminTransport, instance=True) + mock_transport._wrapped_methods = {} + return DatabaseAdminClient(transport=mock_transport) + def test_constructor_defaults(self): from google.cloud.spanner_v1.instance import DEFAULT_NODE_COUNT @@ -586,14 +596,15 @@ def test_database_factory_explicit(self): self.assertIs(database._proto_descriptors, proto_descriptors) def test_list_databases(self): - from google.cloud.spanner_admin_database_v1 import Database as DatabasePB from google.cloud.spanner_admin_database_v1 import ( - DatabaseAdminClient, + Database as DatabasePB, + ) + from google.cloud.spanner_admin_database_v1 import ( ListDatabasesRequest, ListDatabasesResponse, ) - api = DatabaseAdminClient(credentials=AnonymousCredentials()) + api = self._make_database_admin_api() client = _Client(self.PROJECT) client.database_admin_api = api instance = self._make_one(self.INSTANCE_ID, client) @@ -629,12 +640,11 @@ def test_list_databases(self): def test_list_databases_w_options(self): from google.cloud.spanner_admin_database_v1 import ( - DatabaseAdminClient, ListDatabasesRequest, ListDatabasesResponse, ) - api = DatabaseAdminClient(credentials=AnonymousCredentials()) + api = self._make_database_admin_api() client = _Client(self.PROJECT) client.database_admin_api = api instance = self._make_one(self.INSTANCE_ID, client) @@ -711,14 +721,15 @@ def test_backup_factory_explicit(self): self.assertEqual(backup._encryption_config, encryption_config) def test_list_backups_defaults(self): - from google.cloud.spanner_admin_database_v1 import Backup as BackupPB from google.cloud.spanner_admin_database_v1 import ( - DatabaseAdminClient, + Backup as BackupPB, + ) + from google.cloud.spanner_admin_database_v1 import ( ListBackupsRequest, ListBackupsResponse, ) - api = DatabaseAdminClient(credentials=AnonymousCredentials()) + api = self._make_database_admin_api() client = _Client(self.PROJECT) client.database_admin_api = api instance = self._make_one(self.INSTANCE_ID, client) @@ -752,14 +763,15 @@ def test_list_backups_defaults(self): ) def test_list_backups_w_options(self): - from google.cloud.spanner_admin_database_v1 import Backup as BackupPB from google.cloud.spanner_admin_database_v1 import ( - DatabaseAdminClient, + Backup as BackupPB, + ) + from google.cloud.spanner_admin_database_v1 import ( ListBackupsRequest, ListBackupsResponse, ) - api = DatabaseAdminClient(credentials=AnonymousCredentials()) + api = self._make_database_admin_api() client = _Client(self.PROJECT) client.database_admin_api = api instance = self._make_one(self.INSTANCE_ID, client) @@ -801,12 +813,11 @@ def test_list_backup_operations_defaults(self): from google.cloud.spanner_admin_database_v1 import ( CreateBackupMetadata, - DatabaseAdminClient, ListBackupOperationsRequest, ListBackupOperationsResponse, ) - api = DatabaseAdminClient(credentials=AnonymousCredentials()) + api = self._make_database_admin_api() client = _Client(self.PROJECT) client.database_admin_api = api instance = self._make_one(self.INSTANCE_ID, client) @@ -849,12 +860,11 @@ def test_list_backup_operations_w_options(self): from google.cloud.spanner_admin_database_v1 import ( CreateBackupMetadata, - DatabaseAdminClient, ListBackupOperationsRequest, ListBackupOperationsResponse, ) - api = DatabaseAdminClient(credentials=AnonymousCredentials()) + api = self._make_database_admin_api() client = _Client(self.PROJECT) client.database_admin_api = api instance = self._make_one(self.INSTANCE_ID, client) @@ -899,13 +909,12 @@ def test_list_database_operations_defaults(self): from google.cloud.spanner_admin_database_v1 import ( CreateDatabaseMetadata, - DatabaseAdminClient, ListDatabaseOperationsRequest, ListDatabaseOperationsResponse, OptimizeRestoredDatabaseMetadata, ) - api = DatabaseAdminClient(credentials=AnonymousCredentials()) + api = self._make_database_admin_api() client = _Client(self.PROJECT) client.database_admin_api = api instance = self._make_one(self.INSTANCE_ID, client) @@ -955,7 +964,6 @@ def test_list_database_operations_w_options(self): from google.protobuf.any_pb2 import Any from google.cloud.spanner_admin_database_v1 import ( - DatabaseAdminClient, ListDatabaseOperationsRequest, ListDatabaseOperationsResponse, RestoreDatabaseMetadata, @@ -963,7 +971,7 @@ def test_list_database_operations_w_options(self): UpdateDatabaseDdlMetadata, ) - api = DatabaseAdminClient(credentials=AnonymousCredentials()) + api = self._make_database_admin_api() client = _Client(self.PROJECT) client.database_admin_api = api instance = self._make_one(self.INSTANCE_ID, client) diff --git a/packages/google-cloud-spanner/tests/unit/test_metrics_concurrency.py b/packages/google-cloud-spanner/tests/unit/test_metrics_concurrency.py index 83d88b0ab023..1ad20eef2740 100644 --- a/packages/google-cloud-spanner/tests/unit/test_metrics_concurrency.py +++ b/packages/google-cloud-spanner/tests/unit/test_metrics_concurrency.py @@ -13,7 +13,6 @@ # limitations under the License. import threading -import time import unittest from google.cloud.spanner_v1.metrics.metrics_capture import MetricsCapture @@ -34,6 +33,7 @@ def test_concurrent_tracers(self): factory.enabled = True errors = [] + barrier = threading.Barrier(10) def worker(idx): try: @@ -43,14 +43,15 @@ def worker(idx): tracer = SpannerMetricsTracerFactory.get_current_tracer() if tracer is None: errors.append(f"Thread {idx}: Tracer is None inside Capture") + barrier.abort() return # Set a unique attribute for this thread project_name = f"project-{idx}" tracer.set_project(project_name) - # Simulate some work - time.sleep(0.01) + # Synchronize all threads so they are all concurrently inside MetricsCapture + barrier.wait(timeout=10.0) # Verify verify we still have OUR tracer current_tracer = SpannerMetricsTracerFactory.get_current_tracer() @@ -67,7 +68,10 @@ def worker(idx): if interceptor_tracer is not tracer: errors.append(f"Thread {idx}: Interceptor tracer mismatch") + except threading.BrokenBarrierError: + pass except Exception as e: + barrier.abort() errors.append(f"Thread {idx}: Exception {e}") threads = [] @@ -79,6 +83,7 @@ def worker(idx): for t in threads: t.join() + self.assertFalse(barrier.broken, "Barrier timed out or was broken") self.assertEqual(errors, [], f"Concurrency errors found: {errors}") def test_context_var_cleanup(self): diff --git a/packages/google-cloud-spanner/tests/unit/test_session.py b/packages/google-cloud-spanner/tests/unit/test_session.py index 17704cb59b2f..0bb064d20a9b 100644 --- a/packages/google-cloud-spanner/tests/unit/test_session.py +++ b/packages/google-cloud-spanner/tests/unit/test_session.py @@ -1151,9 +1151,11 @@ def unit_of_work(transaction): list(transaction.read(TABLE_NAME, COLUMNS, KEYSET)) session.create() - session.run_in_transaction(unit_of_work) + with mock.patch("google.cloud.spanner_v1._helpers.time.sleep") as sleep_mock: + session.run_in_transaction(unit_of_work) self.assertEqual(begin_transaction.call_count, 2) + sleep_mock.assert_called_once() begin_transaction.assert_called_with( request=BeginTransactionRequest( @@ -1191,10 +1193,12 @@ def unit_of_work(transaction): list(transaction.read(TABLE_NAME, COLUMNS, KEYSET)) session.create() - session.run_in_transaction(unit_of_work) + with mock.patch("google.cloud.spanner_v1._helpers.time.sleep") as sleep_mock: + session.run_in_transaction(unit_of_work) # Verify retried BeginTransaction API call. self.assertEqual(begin_transaction.call_count, 2) + sleep_mock.assert_called_once() begin_transaction.assert_called_with( request=BeginTransactionRequest( @@ -1236,10 +1240,12 @@ def unit_of_work(transaction): list(transaction.read(TABLE_NAME, COLUMNS, KEYSET)) session.create() - session.run_in_transaction(unit_of_work) + with mock.patch("google.cloud.spanner_v1._helpers.time.sleep") as sleep_mock: + session.run_in_transaction(unit_of_work) # Verify retried BeginTransaction API call. self.assertEqual(begin_transaction.call_count, 2) + sleep_mock.assert_called_once() begin_transaction.assert_called_with( request=BeginTransactionRequest( @@ -1524,7 +1530,7 @@ def unit_of_work(txn, *args, **kw): called_with.append((txn, args, kw)) txn.insert(TABLE_NAME, COLUMNS, VALUES) - with mock.patch("time.sleep") as sleep_mock: + with mock.patch("google.cloud.spanner_v1._helpers.time.sleep") as sleep_mock: session.run_in_transaction(unit_of_work, "abc", some_arg="def") sleep_mock.assert_called_once_with(RETRY_SECONDS + RETRY_NANOS / 1.0e9) @@ -1639,7 +1645,7 @@ def unit_of_work(txn, *args, **kw): raise _make_rpc_error(Aborted, trailing_metadata) txn.insert(TABLE_NAME, COLUMNS, VALUES) - with mock.patch("time.sleep") as sleep_mock: + with mock.patch("google.cloud.spanner_v1._helpers.time.sleep") as sleep_mock: session.run_in_transaction(unit_of_work) sleep_mock.assert_called_once_with(RETRY_SECONDS + RETRY_NANOS / 1.0e9) @@ -1726,8 +1732,10 @@ def _time(): return _results[0] return 1.0 - with mock.patch("time.time", _time): - with mock.patch("time.sleep") as sleep_mock: + with mock.patch("google.cloud.spanner_v1._helpers.time.time", _time): + with mock.patch( + "google.cloud.spanner_v1._helpers.time.sleep" + ) as sleep_mock: # Exception has request_id attribute added with self.assertRaises(Aborted) as context: session.run_in_transaction(unit_of_work, "abc", timeout_secs=1) @@ -1803,11 +1811,11 @@ def _time(): return 1.0 with ( - mock.patch("time.time", _time), + mock.patch("google.cloud.spanner_v1._helpers.time.time", _time), mock.patch( "google.cloud.spanner_v1._helpers.random.random", return_value=0 ), - mock.patch("time.sleep") as sleep_mock, + mock.patch("google.cloud.spanner_v1._helpers.time.sleep") as sleep_mock, ): # Exception has request_id attribute added with self.assertRaises(Aborted) as context: @@ -2219,7 +2227,7 @@ def unit_of_work(txn, *args, **kw): called_with.append((txn, args, kw)) txn.insert(TABLE_NAME, COLUMNS, VALUES) - with mock.patch("time.sleep") as sleep_mock: + with mock.patch("google.cloud.spanner_v1._helpers.time.sleep") as sleep_mock: session.run_in_transaction( unit_of_work, "abc", @@ -2669,15 +2677,17 @@ def _time_func(): return 3 # check if current time > deadline - with mock.patch("time.time", _time_func): + with mock.patch("google.cloud.spanner_v1._helpers.time.time", _time_func): with self.assertRaises(Exception): _delay_until_retry(exc_mock, 2, 1, default_retry_delay=0) - with mock.patch("time.time", _time_func): + with mock.patch("google.cloud.spanner_v1._helpers.time.time", _time_func): with mock.patch( "google.cloud.spanner_v1._helpers._get_retry_delay" ) as get_retry_delay_mock: - with mock.patch("time.sleep") as sleep_mock: + with mock.patch( + "google.cloud.spanner_v1._helpers.time.sleep" + ) as sleep_mock: get_retry_delay_mock.return_value = None _delay_until_retry(exc_mock, 6, 1) diff --git a/packages/google-cloud-spanner/tests/unit/test_snapshot.py b/packages/google-cloud-spanner/tests/unit/test_snapshot.py index bd17f9cca606..bfc52ee4461d 100644 --- a/packages/google-cloud-spanner/tests/unit/test_snapshot.py +++ b/packages/google-cloud-spanner/tests/unit/test_snapshot.py @@ -1084,11 +1084,12 @@ def test_begin_precommit_token(self, mock_region): self._execute_begin(derived) + @mock.patch("google.cloud.spanner_v1._helpers.time.sleep") @mock.patch( "google.cloud.spanner_v1._opentelemetry_tracing._get_cloud_region", return_value="global", ) - def test_begin_retry_for_internal_server_error(self, mock_region): + def test_begin_retry_for_internal_server_error(self, mock_region, mock_sleep): derived = _build_snapshot_derived(multi_use=True) begin_transaction = derived._session._database.spanner_api.begin_transaction @@ -1098,6 +1099,7 @@ def test_begin_retry_for_internal_server_error(self, mock_region): ] self._execute_begin(derived, attempts=2) + mock_sleep.assert_called_once_with(2) expected_statuses = [ ( @@ -1108,11 +1110,12 @@ def test_begin_retry_for_internal_server_error(self, mock_region): actual_statuses = self.finished_spans_events_statuses() self.assertEqual(expected_statuses, actual_statuses) + @mock.patch("google.cloud.spanner_v1._helpers.time.sleep") @mock.patch( "google.cloud.spanner_v1._opentelemetry_tracing._get_cloud_region", return_value="global", ) - def test_begin_retry_for_aborted(self, mock_region): + def test_begin_retry_for_aborted(self, mock_region, mock_sleep): derived = _build_snapshot_derived(multi_use=True) begin_transaction = derived._session._database.spanner_api.begin_transaction @@ -1122,6 +1125,7 @@ def test_begin_retry_for_aborted(self, mock_region): ] self._execute_begin(derived, attempts=2) + mock_sleep.assert_called_once_with(2) expected_statuses = [ ( @@ -2057,7 +2061,8 @@ def test_partition_read_other_error(self, mock_region): ), ) - def test_partition_read_w_retry(self): + @mock.patch("google.cloud.spanner_v1._helpers.time.sleep") + def test_partition_read_w_retry(self, mock_sleep): from google.cloud.spanner_v1 import Partition, PartitionResponse, Transaction from google.cloud.spanner_v1.keyset import KeySet @@ -2087,6 +2092,7 @@ def test_partition_read_w_retry(self): list(derived.partition_read(TABLE_NAME, COLUMNS, keyset)) self.assertEqual(api.partition_read.call_count, 2) + mock_sleep.assert_called_once_with(2) @mock.patch( "google.cloud.spanner_v1._opentelemetry_tracing._get_cloud_region", From 873ef9980dee9cd518733dec164f4ae42d25afbe Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 7 Oct 2026 03:54:15 -0400 Subject: [PATCH 018/228] fix(sqlalchemy-spanner): resolve compliance and migration failures with Alembic 1.20 and SQLAlchemy 2.1 (#18570) --- .../sqlalchemy_spanner/sqlalchemy_spanner.py | 133 ++++++++++-------- packages/sqlalchemy-spanner/noxfile.py | 52 +++---- packages/sqlalchemy-spanner/setup.py | 4 +- .../sqlalchemy-spanner/tests/test_suite_20.py | 26 +++- .../tests/unit/test_alembic.py | 36 ++++- 5 files changed, 163 insertions(+), 88 deletions(-) diff --git a/packages/sqlalchemy-spanner/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py b/packages/sqlalchemy-spanner/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py index ee8e72eb5665..6c6e1fbfec0c 100644 --- a/packages/sqlalchemy-spanner/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py +++ b/packages/sqlalchemy-spanner/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py @@ -15,14 +15,6 @@ import re import sqlalchemy -from alembic.ddl.base import ( - ColumnNullable, - ColumnType, - alter_column, - alter_table, - format_server_default, - format_type, -) from google.api_core.client_options import ClientOptions from google.auth.credentials import AnonymousCredentials from google.cloud.spanner_v1 import Client, TransactionOptions @@ -51,6 +43,26 @@ from google.cloud.sqlalchemy_spanner import version as sqlalchemy_spanner_version from google.cloud.sqlalchemy_spanner._opentelemetry_tracing import trace_call +# Defensively decouple the Alembic import so the Spanner dialect does not +# hard-depend on Alembic at runtime. A database dialect does not inherently +# require a schema migration tool, and consumers using only SQLAlchemy Core +# or ORM (or managing DDL outside of Alembic) can still import and use the +# dialect even if Alembic is omitted or unavailable in the environment +# (see #18584). +try: + from alembic.ddl.base import ( + ColumnNullable, + ColumnType, + alter_column, + alter_table, + format_server_default, + format_type, + ) + + HAS_ALEMBIC_INSTALLED = True +except ImportError: + HAS_ALEMBIC_INSTALLED = False + USING_SQLACLCHEMY_20 = False if sqlalchemy.__version__.split(".")[0] == "2": USING_SQLACLCHEMY_20 = True @@ -1882,55 +1894,60 @@ def do_execute_no_params(self, cursor, statement, context=None): cursor.execute(statement) -# Alembic ALTER operation override -@compiles(ColumnNullable, "spanner+spanner") -def visit_column_nullable( - element: "ColumnNullable", compiler: "SpannerDDLCompiler", **kw -) -> str: - return _format_alter_column( - compiler, - element.table_name, - element.schema, - element.column_name, - element.existing_type, - element.nullable, - element.existing_server_default, - ) - - -# Alembic ALTER operation override -@compiles(ColumnType, "spanner+spanner") -def visit_column_type( - element: "ColumnType", compiler: "SpannerDDLCompiler", **kw -) -> str: - return _format_alter_column( - compiler, - element.table_name, - element.schema, - element.column_name, - element.type_, - element.existing_nullable, - element.existing_server_default, - ) +# Cloud Spanner requires ALTER TABLE ... ALTER COLUMN statements to specify the +# complete column definition (type, nullability, and default expression), whereas +# Alembic's default DDL compiler emits partial clauses (e.g., only SET NOT NULL +# or TYPE). Because the @compiles decorators reference Alembic's ColumnNullable +# and ColumnType classes at module import time, we only register these overrides +# when Alembic is available in the environment. +if HAS_ALEMBIC_INSTALLED: + # Alembic ALTER operation override + @compiles(ColumnNullable, "spanner+spanner") + def visit_column_nullable( + element: "ColumnNullable", compiler: "SpannerDDLCompiler", **kw + ) -> str: + return _format_alter_column( + compiler, + element.table_name, + element.schema, + element.column_name, + element.existing_type, + element.nullable, + element.existing_server_default, + ) + # Alembic ALTER operation override + @compiles(ColumnType, "spanner+spanner") + def visit_column_type( + element: "ColumnType", compiler: "SpannerDDLCompiler", **kw + ) -> str: + return _format_alter_column( + compiler, + element.table_name, + element.schema, + element.column_name, + element.type_, + element.existing_nullable, + element.existing_server_default, + ) -def _format_alter_column( - compiler, table_name, schema, column_name, type_, nullable, server_default -): - # Older versions of SQLAlchemy pass in a boolean to indicate whether there - # is an existing DEFAULT constraint, instead of the actual DEFAULT constraint - # expression. In those cases, we do not want to explicitly include the DEFAULT - # constraint in the expression that is generated here. - if isinstance(server_default, bool): - server_default = None - return "%s %s %s%s%s" % ( - alter_table(compiler, table_name, schema), - alter_column(compiler, column_name), - format_type(compiler, type_), - "" if nullable else " NOT NULL", - ( - "" - if server_default is None - else f" DEFAULT {format_server_default(compiler, server_default)}" - ), - ) + def _format_alter_column( + compiler, table_name, schema, column_name, type_, nullable, server_default + ): + # Older versions of SQLAlchemy pass in a boolean to indicate whether there + # is an existing DEFAULT constraint, instead of the actual DEFAULT constraint + # expression. In those cases, we do not want to explicitly include the DEFAULT + # constraint in the expression that is generated here. + if isinstance(server_default, bool): + server_default = None + return "%s %s %s%s%s" % ( + alter_table(compiler, table_name, schema), + alter_column(compiler, column_name), + format_type(compiler, type_), + "" if nullable else " NOT NULL", + ( + "" + if server_default is None + else f" DEFAULT {format_server_default(compiler, server_default)}" + ), + ) diff --git a/packages/sqlalchemy-spanner/noxfile.py b/packages/sqlalchemy-spanner/noxfile.py index 56884d95296d..c05f20d74300 100644 --- a/packages/sqlalchemy-spanner/noxfile.py +++ b/packages/sqlalchemy-spanner/noxfile.py @@ -120,6 +120,7 @@ class = StreamHandler SQLALCHEMY_14_DEPENDENCIES = [ "sqlalchemy>=1.4,<2.0", + "alembic<1.20.0", ] SQLALCHEMY_20_DEPENDENCIES = [ @@ -139,7 +140,6 @@ class = StreamHandler "compliance_test_14", "compliance_test_20", "migration_test", - "_migration_test", "mockserver", ] @@ -206,13 +206,7 @@ def compliance_test_14(session): try: session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES) - session.install(".[tracing]") - session.run( - "pip", - "install", - *SQLALCHEMY_14_DEPENDENCIES, - "--force-reinstall", - ) + session.install(".[tracing]", *SQLALCHEMY_14_DEPENDENCIES) session.run( "python", "create_test_database.py", @@ -335,20 +329,27 @@ def mockserver(session): ) -@nox.session(python=SYSTEM_COMPLIANCE_MIGRATION_TEST_PYTHON_VERSIONS[0]) -def migration_test(session): - """Test migrations with SQLAlchemy v1.4 and Alembic""" - session.run( - "pip", - "install", - *SQLALCHEMY_14_DEPENDENCIES, - "--force-reinstall", - ) - _migration_test(session) +@nox.session +@nox.parametrize( + ("python", "extra_dependencies"), + [ + ( + SYSTEM_COMPLIANCE_MIGRATION_TEST_PYTHON_VERSIONS[0], + SQLALCHEMY_14_DEPENDENCIES, + ), + ( + DEFAULT_PYTHON_VERSION_FOR_SQLALCHEMY_20, + SQLALCHEMY_20_DEPENDENCIES, + ), + ], + ids=["14", "20"], +) +def migration_test(session, extra_dependencies): + """Test migrations with SQLAlchemy and Alembic.""" + _run_migration_test(session, extra_dependencies=extra_dependencies) -@nox.session(python=SYSTEM_COMPLIANCE_MIGRATION_TEST_PYTHON_VERSIONS[-1]) -def _migration_test(session): +def _run_migration_test(session, extra_dependencies=()): """Migrate with SQLAlchemy and Alembic and check the result.""" import glob import os @@ -356,8 +357,7 @@ def _migration_test(session): config_file = f"test_migration_{session.python}_{uuid.uuid4().hex[:6]}.cfg" - session.install(*MIGRATION_TEST_DEPENDENCIES) - session.install(".") + session.install(*MIGRATION_TEST_DEPENDENCIES, *extra_dependencies, ".") try: session.run( @@ -482,9 +482,13 @@ def system(session, test_type): elif test_type == "compliance_20": return compliance_test_20(session) elif test_type == "migration_14": - return migration_test(session) + return _run_migration_test( + session, extra_dependencies=SQLALCHEMY_14_DEPENDENCIES + ) elif test_type == "migration_20": - return _migration_test(session) + return _run_migration_test( + session, extra_dependencies=SQLALCHEMY_20_DEPENDENCIES + ) config_file = f"test_{test_type}_{session.python}_{uuid.uuid4().hex[:6]}.cfg" diff --git a/packages/sqlalchemy-spanner/setup.py b/packages/sqlalchemy-spanner/setup.py index 91ae4b47d9dc..2968c746907a 100644 --- a/packages/sqlalchemy-spanner/setup.py +++ b/packages/sqlalchemy-spanner/setup.py @@ -22,9 +22,9 @@ name = "sqlalchemy-spanner" description = "SQLAlchemy dialect integrated into Cloud Spanner database" dependencies = [ - "sqlalchemy>=1.1.13", + "sqlalchemy>=1.1.13,<3.0.0", "google-cloud-spanner>=3.55.0", - "alembic", + "alembic>=1.0.0,<2.0.0", ] extras = { "tracing": [ diff --git a/packages/sqlalchemy-spanner/tests/test_suite_20.py b/packages/sqlalchemy-spanner/tests/test_suite_20.py index 6c975004a8c4..2142fef94358 100644 --- a/packages/sqlalchemy-spanner/tests/test_suite_20.py +++ b/packages/sqlalchemy-spanner/tests/test_suite_20.py @@ -78,7 +78,6 @@ LongNameBlowoutTest as _LongNameBlowoutTest, ) from sqlalchemy.testing.suite.test_ddl import TableDDLTest as _TableDDLTest -from sqlalchemy.testing.suite.test_deprecations import * # noqa: F401, F403 from sqlalchemy.testing.suite.test_dialect import * # noqa: F401, F403 from sqlalchemy.testing.suite.test_dialect import ( DifficultParametersTest as _DifficultParametersTest, @@ -225,6 +224,15 @@ get_project, ) +try: + # SQLAlchemy 2.1+ removed test_deprecations from sqlalchemy.testing.suite. + # Guard this import so the test suite remains compatible with both 2.0.x and 2.1+. + # Tell flake8 to ignore F401 (unused import) and F403 (wildcard import) since + # pytest discovers the imported SQLAlchemy compliance test classes at module scope. + from sqlalchemy.testing.suite.test_deprecations import * # noqa: F401, F403 +except ModuleNotFoundError: + pass + config.test_schema = "" @@ -721,6 +729,7 @@ def test_get_columns(self, connection, use_views, use_schema): [ types.Integer, types.Numeric, + types.Float, types.DateTime, types.Date, types.Time, @@ -2916,6 +2925,10 @@ def test_has_table_nonexistent_schema(self): def test_has_table_schema(self): pass + @pytest.mark.skip("Not supported by Cloud Spanner") + def test_has_multi_table_schema(self): + pass + @pytest.mark.skip("Not supported by Cloud Spanner") def test_has_table_cache(self): pass @@ -3066,19 +3079,26 @@ def test_eval_none_flag_orm(self): pass @pytest.mark.skip( - "Spanner JSON_VALUE() always returns STRING," + "Spanner JSON_VALUE() always returns STRING, " "thus, this test case can't be executed." ) def test_index_typed_comparison(self): pass @pytest.mark.skip( - "Spanner JSON_VALUE() always returns STRING," + "Spanner JSON_VALUE() always returns STRING, " "thus, this test case can't be executed." ) def test_path_typed_comparison(self): pass + @pytest.mark.skip( + "Spanner JSON_VALUE() always returns STRING, " + "thus, this test case can't be executed." + ) + def test_index_cross_casts(self): + pass + @pytest.mark.skip("Custom JSON de-/serializers are not supported.") def test_round_trip_custom_json(self): pass diff --git a/packages/sqlalchemy-spanner/tests/unit/test_alembic.py b/packages/sqlalchemy-spanner/tests/unit/test_alembic.py index 298326497daf..7aa40fce09f5 100644 --- a/packages/sqlalchemy-spanner/tests/unit/test_alembic.py +++ b/packages/sqlalchemy-spanner/tests/unit/test_alembic.py @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import importlib.util +import sys +from unittest import mock + from alembic.ddl import base as ddl_base -from sqlalchemy import String, TextClause +from sqlalchemy import String, TextClause, event from sqlalchemy.testing import eq_ from sqlalchemy.testing.plugin.plugin_base import fixtures @@ -96,3 +100,33 @@ def test_visit_column_type_with_default(self): "ALTER COLUMN col " "STRING(256) NOT NULL DEFAULT (GENERATE_UUID())", ) + + def test_dialect_import_without_alembic(self): + """Verify that sqlalchemy_spanner imports cleanly when Alembic is unavailable. + + Why we test this way instead of calling importlib.reload(sqlalchemy_spanner): + 1. Setting sys.modules["alembic.ddl.base"] = None via mock.patch.dict causes + Python to raise ModuleNotFoundError (a subclass of ImportError) when + sqlalchemy_spanner attempts to import from alembic.ddl.base. + 2. Calling importlib.reload(sqlalchemy_spanner) would re-execute the module + in-place inside the existing sqlalchemy_spanner.__dict__. That has two + undesirable side effects: + - Functions defined during the initial import (like visit_column_nullable) + remain in sqlalchemy_spanner.__dict__ even if skipped on reload. + - Classes (like SpannerIdentifierPreparer) are recreated with new class + identities in sqlalchemy_spanner.__dict__, which breaks other test + modules (such as test_dialect.py) that already imported SpannerDialect + before the reload and rely on super(SpannerIdentifierPreparer, self). + 3. Creating a fresh module object via importlib.util.module_from_spec and + executing the loader into that isolated namespace tests a clean import + without mutating the shared sqlalchemy_spanner module in sys.modules. + """ + with mock.patch.dict(sys.modules, {"alembic.ddl.base": None}): + module = importlib.util.module_from_spec(sqlalchemy_spanner.__spec__) + sqlalchemy_spanner.__spec__.loader.exec_module(module) + # Executing the module registers module.reset_connection on the shared + # SQLAlchemy Pool class via @listens_for(Pool, "reset"); remove that + # temporary listener so global Pool state stays clean for other tests. + event.remove(sqlalchemy_spanner.Pool, "reset", module.reset_connection) + assert not module.HAS_ALEMBIC_INSTALLED + assert not hasattr(module, "visit_column_nullable") From 9f5fd7bdf0707a17c363482b094ccd70a4358389 Mon Sep 17 00:00:00 2001 From: Negar <99771966+nbayati@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:57:13 -0700 Subject: [PATCH 019/228] feat(auth): support bound access and ID tokens for GKE (#18594) This PR enables bound access tokens, bound ID tokens, and default mTLS certificate discovery for Agent Identity workloads on GKE/CR, and adds per-request control for ID token binding: * Fall back to the well-known GKE workload credential bundle (`/var/run/secrets/workload-spiffe-credentials/x509.credential-bundle.private-key.pem`) in `_agent_identity_utils.get_agent_identity_certificate_path()`, `_mtls_helper._get_workload_cert_and_key()`, `_mtls_helper.check_use_client_cert()`, and `mtls.has_default_client_cert_source()` when no explicit or default certificate config file is present. * Add an optional `bind_id_token` parameter (`True`, `False`, or `None`) to `compute_engine.IDTokenCredentials`, `fetch_id_token_credentials()`, and `fetch_id_token()` (sync and async) so callers can explicitly opt in or out of binding metadata server ID tokens per request. * Scope default bound ID tokens to GKE by keeping ID tokens unbound when `GOOGLE_API_CERTIFICATE_CONFIG` is set (unless `bind_id_token=True`), while continuing to bind access tokens by default on both Cloud Run and GKE. * Add and update unit tests for GKE credential bundle fallback, `bind_id_token` options, and the `GOOGLE_API_CERTIFICATE_CONFIG` default gate. ---- Note: 1. GKE mounts a single combined PEM file (`x509.credential-bundle.private-key.pem`) at `/var/run/secrets/workload-spiffe-credentials/` without setting `GOOGLE_API_CERTIFICATE_CONFIG`. With this fallback, GKE Agent Identity workloads automatically use bound access tokens and bound ID tokens by default when the file is present. 2. On Cloud Run (where `GOOGLE_API_CERTIFICATE_CONFIG` is set), default ID token requests (`bind_id_token=None`) temporarily remain unbound so existing callers do not receive bound ID tokens. Callers on Cloud Run can pass `bind_id_token=True` to request bound ID tokens now, and a follow-up PR will remove this temporary gate to enable bound ID tokens by default on Cloud Run as well. design: go/sdk-mds-bound-token b/545779266 --- .../google/auth/_agent_identity_utils.py | 158 ++++-- .../google/auth/aio/transport/sessions.py | 16 +- .../google/auth/compute_engine/_metadata.py | 81 ++- .../google/auth/compute_engine/credentials.py | 123 ++-- .../google/auth/environment_vars.py | 14 +- .../google-auth/google/auth/identity_pool.py | 9 +- .../google/auth/transport/_mtls_helper.py | 139 ++++- .../google-auth/google/auth/transport/mtls.py | 19 +- .../google/oauth2/_id_token_async.py | 23 +- .../google-auth/google/oauth2/id_token.py | 42 +- .../tests/compute_engine/test__metadata.py | 133 +++-- .../tests/compute_engine/test_credentials.py | 417 ++++++++++++-- packages/google-auth/tests/conftest.py | 32 ++ .../google-auth/tests/oauth2/test_id_token.py | 57 +- .../tests/test_agent_identity_utils.py | 536 +++++++++++++++--- .../google-auth/tests/test_identity_pool.py | 43 +- .../tests/transport/aio/test_sessions_mtls.py | 26 +- .../tests/transport/test__mtls_helper.py | 343 ++++++++++- .../google-auth/tests/transport/test_mtls.py | 94 ++- .../tests/transport/test_requests.py | 31 +- .../tests/transport/test_urllib3.py | 29 +- .../tests_async/oauth2/test_id_token.py | 29 +- 22 files changed, 1966 insertions(+), 428 deletions(-) diff --git a/packages/google-auth/google/auth/_agent_identity_utils.py b/packages/google-auth/google/auth/_agent_identity_utils.py index 0fa6a92f57ff..ed4264293443 100644 --- a/packages/google-auth/google/auth/_agent_identity_utils.py +++ b/packages/google-auth/google/auth/_agent_identity_utils.py @@ -16,6 +16,7 @@ import base64 import hashlib +import json import os import re import stat @@ -73,13 +74,32 @@ def _is_certificate_file_ready(path): return False +def _is_in_well_known_dir(path): + """Checks if the given path is inside the well-known Agent Identity directory.""" + if not path: + return False + well_known_dir = os.path.dirname(_WELL_KNOWN_CERT_PATH) + try: + real_path = os.path.realpath(path) + real_well_known_dir = os.path.realpath(well_known_dir) + return ( + os.path.commonpath([real_well_known_dir, real_path]) == real_well_known_dir + ) + except ValueError: + return False + + def get_agent_identity_certificate_path(): - """Gets the agent certificate path from the certificate config file. + """Gets the agent certificate path from the certificate config file or GKE bundle. The path to the certificate config file is read from the GOOGLE_API_CERTIFICATE_CONFIG environment variable. This function can optionally trigger polling to handle cases where the environment variable is set before the files are available on the filesystem. + When GOOGLE_API_CERTIFICATE_CONFIG and + CLOUDSDK_CONTEXT_AWARE_CERTIFICATE_CONFIG_FILE_PATH are unset and no + implicit certificate_config.json is present, it falls back to checking + the GKE workload credential bundle path without polling. Returns: Optional[str]: The path to the agent's certificate file, or None if unavailable. @@ -88,9 +108,18 @@ def get_agent_identity_certificate_path(): google.auth.exceptions.RefreshError: If the certificate config file or the certificate file cannot be found after retries. """ + from google.auth.transport import _mtls_helper + cert_config_path = os.environ.get(environment_vars.GOOGLE_API_CERTIFICATE_CONFIG) if not cert_config_path: + if ( + not _mtls_helper._has_explicit_cert_config_env() + and _mtls_helper._should_use_gke_credential_bundle( + _mtls_helper._get_cert_config_path() + ) + ): + return _mtls_helper._GKE_CREDENTIAL_BUNDLE_PATH return None # We trigger polling only if the config path points to the well-known directory. @@ -98,16 +127,7 @@ def get_agent_identity_certificate_path(): # config file and the certificate file may experience a brief startup latency. # For all other paths, we return early to avoid introducing unnecessary startup # delays. - well_known_dir = os.path.dirname(_WELL_KNOWN_CERT_PATH) - try: - abs_cert_path = os.path.abspath(cert_config_path) - abs_well_known_dir = os.path.abspath(well_known_dir) - should_poll = ( - os.path.commonpath([abs_well_known_dir, abs_cert_path]) - == abs_well_known_dir - ) - except ValueError: - should_poll = False + should_poll = _is_in_well_known_dir(cert_config_path) return _get_cert_path_with_optional_polling(cert_config_path, should_poll) @@ -141,9 +161,9 @@ def _get_cert_path_with_optional_polling(cert_config_path, should_poll): if _is_certificate_file_ready(cert_path): return cert_path - # The config was parsed, but the cert file is not ready yet - if not should_poll: - # If polling is disabled, return early. + # The config was parsed, but the cert file is not ready yet. + # Only poll if both the config path and cert path are in the well-known directory. + if not (should_poll and _is_in_well_known_dir(cert_path)): return None if not has_logged_cert_warning: @@ -182,7 +202,7 @@ def _get_cert_path_with_optional_polling(cert_config_path, should_poll): raise exceptions.RefreshError( "Certificate config or certificate file not found after multiple retries. " f"Token binding protection is failing. You can turn off this protection by setting " - f"{environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES} to false " + f"{environment_vars.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN} to false " "to fall back to unbound tokens." ) @@ -203,8 +223,6 @@ def _parse_cert_path_from_config(cert_config_path): KeyError: If the certificate config file does not contain the expected structure. """ - import json - with open(cert_config_path, "r", encoding="utf-8") as f: cert_config = json.load(f) @@ -221,64 +239,105 @@ def _parse_cert_path_from_config(cert_config_path): return workload_config["cert_path"] -def get_and_parse_agent_identity_certificate(): +def _is_bound_token_opted_out(): + """Returns True only if bound tokens are explicitly disabled via env vars.""" + val = os.environ.get( + environment_vars.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "" + ).strip() + if not val: + # Fall back to the deprecated env var for backward compatibility + val = os.environ.get( + environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, + "true", + ).strip() + return val.lower() == "false" + + +def get_agent_identity_certificate_and_bytes(): """Gets and parses the agent identity certificate if not opted out. Checks if the user has opted out of certificate-bound tokens. If not, it gets the certificate path, reads the file, and parses it. Returns: - The parsed certificate object if found and not opted out, otherwise None. + Tuple[Optional[cryptography.x509.Certificate], Optional[bytes]]: A tuple + of (parsed certificate object, certificate bytes) if found and not + opted out, otherwise (None, None). """ + from google.auth.transport import _mtls_helper + # If the user has opted out of cert bound tokens, there is no need to # look up the certificate. - is_opted_out = ( - os.environ.get( - environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, - "true", - ).lower() - == "false" - ) - if is_opted_out: - return None + if _is_bound_token_opted_out(): + return None, None # Respect explicit opt-out of mTLS / client certs - from google.auth.transport import _mtls_helper - env_override = _mtls_helper._check_use_client_cert_env() if env_override is False: - return None + return None, None cert_path = get_agent_identity_certificate_path() if not cert_path: - return None + return None, None try: with open(cert_path, "rb") as cert_file: - cert_bytes = cert_file.read() - except PermissionError as e: + raw_bytes = cert_file.read() + except OSError as e: warnings.warn( f"Failed to read agent identity certificate file at {cert_path}: {e}. " "Token binding protection cannot be enabled. Falling back to unbound tokens." ) - return None + return None, None - return parse_certificate(cert_bytes) + cert_blocks = _mtls_helper._CERT_REGEX.findall(raw_bytes) + if not cert_blocks: + warnings.warn( + f"No PEM certificate blocks found in {cert_path}. " + "Token binding protection cannot be enabled. Falling back to unbound tokens." + ) + return None, None + + cert_bytes = b"\n".join(block.strip() for block in cert_blocks) + b"\n" + try: + return parse_certificate(raw_bytes), cert_bytes + except (ValueError, ImportError) as e: + warnings.warn( + f"Failed to parse agent identity certificate at {cert_path}: {e}. " + "Token binding protection cannot be enabled. Falling back to unbound tokens." + ) + return None, None def parse_certificate(cert_bytes): - """Parses a PEM-encoded certificate. + """Parses a PEM-encoded certificate or certificate chain and returns the leaf certificate. Args: cert_bytes (bytes): The PEM-encoded certificate bytes. Returns: - cryptography.x509.Certificate: The parsed certificate object. + cryptography.x509.Certificate: The leaf (first) parsed certificate object. + + Raises: + ValueError: If no certificates are found or any certificate in the chain + is malformed. + ImportError: If the cryptography library is not installed. """ + from google.auth.transport import _mtls_helper + + if not cert_bytes: + raise ValueError("Certificate bytes cannot be empty or None.") + try: from cryptography import x509 - return x509.load_pem_x509_certificate(cert_bytes) + cert_blocks = _mtls_helper._CERT_REGEX.findall(cert_bytes) + if not cert_blocks: + return x509.load_pem_x509_certificate(cert_bytes) + if _mtls_helper._has_unmatched_pem_markers(cert_bytes, cert_blocks): + raise ValueError("Malformed or truncated PEM certificate chain.") + certs = [x509.load_pem_x509_certificate(block) for block in cert_blocks] + return certs[0] except ImportError as e: raise ImportError(CRYPTOGRAPHY_NOT_FOUND_ERROR) from e @@ -350,8 +409,9 @@ def calculate_certificate_fingerprint(cert): def should_request_bound_token(cert): """Determines if a bound token should be requested. - This is based on the GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES - environment variable and whether the certificate is an agent identity cert. + This is based on the GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN env var + (falls back to the deprecated GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES + if unset or empty) and whether the certificate is an agent identity cert. Args: cert (cryptography.x509.Certificate): The parsed certificate object. @@ -359,25 +419,17 @@ def should_request_bound_token(cert): Returns: bool: True if a bound token should be requested, False otherwise. """ - is_agent_cert = _is_agent_identity_certificate(cert) - is_opted_in = ( - os.environ.get( - environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, - "true", - ).lower() - == "true" - ) - if not (is_agent_cert and is_opted_in): + from google.auth.transport import _mtls_helper + + if _is_bound_token_opted_out(): return False # Respect explicit opt-out of mTLS / client certs - from google.auth.transport import _mtls_helper - env_override = _mtls_helper._check_use_client_cert_env() if env_override is False: return False - return True + return _is_agent_identity_certificate(cert) def get_cached_cert_fingerprint(cached_cert): diff --git a/packages/google-auth/google/auth/aio/transport/sessions.py b/packages/google-auth/google/auth/aio/transport/sessions.py index 14e950427b5f..980c86f5ea19 100644 --- a/packages/google-auth/google/auth/aio/transport/sessions.py +++ b/packages/google-auth/google/auth/aio/transport/sessions.py @@ -19,7 +19,6 @@ import inspect import logging import time -import urllib.parse import warnings from contextlib import asynccontextmanager from typing import TYPE_CHECKING, Mapping, Optional, Union @@ -44,11 +43,6 @@ ClientTimeout = None _LOGGER = logging.getLogger(__name__) -_MTLS_URL_PREFIXES = [ - "mtls.googleapis.com", - "mtls.sandbox.googleapis.com", - "p.googleapis.com", -] # Tracks the internal aiohttp installation and usage try: @@ -482,13 +476,9 @@ async def request( async def _recover_auth_state(): is_mtls_endpoint = False if self._is_mtls: - hostname = urllib.parse.urlsplit(url).hostname - if hostname: - is_mtls_endpoint = any( - hostname == prefix - or hostname.endswith("." + prefix) - for prefix in _MTLS_URL_PREFIXES - ) + is_mtls_endpoint = ( + google.auth.transport._mtls_helper.is_mtls_endpoint(url) + ) # Snapshot the stale certificate state BEFORE acquiring the lock. # This represents the cert that caused the 401 rejection. if is_mtls_endpoint: diff --git a/packages/google-auth/google/auth/compute_engine/_metadata.py b/packages/google-auth/google/auth/compute_engine/_metadata.py index 58c509dfa768..7404cf2896e1 100644 --- a/packages/google-auth/google/auth/compute_engine/_metadata.py +++ b/packages/google-auth/google/auth/compute_engine/_metadata.py @@ -27,7 +27,14 @@ import requests -from google.auth import _helpers, environment_vars, exceptions, metrics, transport +from google.auth import ( + _agent_identity_utils, + _helpers, + environment_vars, + exceptions, + metrics, + transport, +) from google.auth._exponential_backoff import ExponentialBackoff from google.auth.compute_engine import _mtls @@ -250,6 +257,8 @@ def get( headers=None, return_none_for_not_found_error=False, timeout=_METADATA_DEFAULT_TIMEOUT, + method="GET", + body=None, ): """Fetch a resource from the metadata server. @@ -271,6 +280,8 @@ def get( return_none_for_not_found_error (Optional[bool]): If True, returns None for 404 error instead of throwing an exception. timeout (int): How long to wait, in seconds for the metadata server to respond. + method (str): The HTTP method to use for the request. Defaults to "GET". + body (Optional[bytes]): The HTTP request body payload to send. Defaults to None. Returns: Union[Mapping, str]: If the metadata server returns JSON, a mapping of @@ -283,8 +294,12 @@ def get( google.auth.exceptions.MutualTLSChannelError: if using mtls and the environment configuration is invalid for mTLS (for example, the metadata host has been overridden in strict mTLS mode). + ValueError: if a request body is specified with the GET method. """ + if body is not None and method.upper() == "GET": + raise ValueError("Request body cannot be specified with GET method.") + use_mtls = _mtls.should_use_mds_mtls() # Prepare the request object for mTLS if needed. # This will create a new request object with the mTLS session. @@ -314,9 +329,15 @@ def get( last_exception = None for attempt in backoff: try: - response = request( - url=url, method="GET", headers=headers_to_use, timeout=timeout - ) + kwargs = { + "url": url, + "method": method, + "headers": headers_to_use, + "timeout": timeout, + } + if body is not None: + kwargs["body"] = body + response = request(**kwargs) if response.status in transport.DEFAULT_RETRYABLE_STATUS_CODES: _LOGGER.warning( "Compute Engine Metadata server unavailable on " @@ -460,6 +481,40 @@ def get_service_account_info(request, service_account="default"): return get(request, path, params={"recursive": "true"}) +def _build_token_request_options(metrics_header_value, bind_token=True): + """Returns (method, body, headers) for a metadata server token request. + + Defaults to a standard GET request with the x-goog-api-client metrics header. + If ``bind_token`` is ``True``, upgrades to a POST request with a JSON + certificate_chain body and Content-Type header if an Agent Identity + certificate is present and bound tokens are enabled. + + Args: + metrics_header_value (str): Value for the x-goog-api-client header. + bind_token (bool): Whether to bind the token to the Agent Identity + certificate. Defaults to ``True``. If ``False``, returns a ``GET`` + request without reading the certificate. If ``True``, requests a + bound token via ``POST`` when a valid Agent Identity certificate is + present and token binding is enabled. + + Returns: + Tuple[str, Optional[bytes], Mapping[str, str]]: A tuple of + (HTTP method, request body bytes, request headers). + """ + headers = {metrics.API_CLIENT_HEADER: metrics_header_value} + if not bind_token: + return "GET", None, headers + + cert, cert_bytes = _agent_identity_utils.get_agent_identity_certificate_and_bytes() + if cert and _agent_identity_utils.should_request_bound_token(cert): + headers["Content-Type"] = "application/json" + body = json.dumps({"certificate_chain": cert_bytes.decode("utf-8")}).encode( + "utf-8" + ) + return "POST", body, headers + return "GET", None, headers + + def get_service_account_token(request, service_account="default", scopes=None): """Get the OAuth 2.0 access token for a service account. @@ -478,26 +533,20 @@ def get_service_account_token(request, service_account="default", scopes=None): google.auth.exceptions.TransportError: if an error occurred while retrieving metadata. """ - from google.auth import _agent_identity_utils - params = {} if scopes: if not isinstance(scopes, str): scopes = ",".join(scopes) params["scopes"] = scopes - cert = _agent_identity_utils.get_and_parse_agent_identity_certificate() - if cert: - if _agent_identity_utils.should_request_bound_token(cert): - fingerprint = _agent_identity_utils.calculate_certificate_fingerprint(cert) - params["bindCertificateFingerprint"] = fingerprint - - metrics_header = { - metrics.API_CLIENT_HEADER: metrics.token_request_access_token_mds() - } + method, body, headers = _build_token_request_options( + metrics.token_request_access_token_mds() + ) path = "instance/service-accounts/{0}/token".format(service_account) - token_json = get(request, path, params=params, headers=metrics_header) + token_json = get( + request, path, params=params, headers=headers, method=method, body=body + ) token_expiry = _helpers.utcnow() + datetime.timedelta( seconds=token_json["expires_in"] ) diff --git a/packages/google-auth/google/auth/compute_engine/credentials.py b/packages/google-auth/google/auth/compute_engine/credentials.py index d0d24b41f4e4..4a71b36f9308 100644 --- a/packages/google-auth/google/auth/compute_engine/credentials.py +++ b/packages/google-auth/google/auth/compute_engine/credentials.py @@ -21,12 +21,14 @@ import datetime import logging +import os from typing import TYPE_CHECKING, Optional from google.auth import ( _helpers, _regional_access_boundary_utils, credentials, + environment_vars, exceptions, iam, jwt, @@ -340,6 +342,7 @@ def __init__( signer=None, use_metadata_identity_endpoint=False, quota_project_id=None, + bind_id_token=None, ): """ Args: @@ -364,18 +367,34 @@ def __init__( otherwise ValueError will be raised. quota_project_id (Optional[str]): The project ID used for quota and billing. + bind_id_token (Optional[bool]): Controls whether to request a + certificate-bound ID token. Can only be set when + ``use_metadata_identity_endpoint`` is ``True``. + If ``True``, requests a bound token whenever a valid Agent + Identity certificate is available and token binding is not + disabled via ``GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN``, falling + back to an unbound token otherwise (or raising + :class:`~google.auth.exceptions.RefreshError` if a configured + certificate is not found after retries). If ``False``, always + requests an unbound token. If ``None`` (default), token binding + is determined automatically by the library. Set ``True`` or + ``False`` explicitly if your application requires a specific + behavior. Raises: ValueError: If ``use_metadata_identity_endpoint`` is set to True, and one of ``token_uri``, ``additional_claims``, ``service_account_email``, - ``signer`` arguments is set. + ``signer`` arguments is set, or if + ``use_metadata_identity_endpoint`` is False and + ``bind_id_token`` is not None. """ super(IDTokenCredentials, self).__init__() self._quota_project_id = quota_project_id self._use_metadata_identity_endpoint = use_metadata_identity_endpoint self._target_audience = target_audience + self._bind_id_token = bind_id_token if use_metadata_identity_endpoint: if token_uri or additional_claims or service_account_email or signer: @@ -387,8 +406,13 @@ def __init__( self._token_uri = None self._additional_claims = None self._signer = None + elif bind_id_token is not None: + raise ValueError( + "If use_metadata_identity_endpoint is False, bind_id_token " + "must not be set" + ) - if service_account_email is None: + if service_account_email is None and request is not None: sa_info = _metadata.get_service_account_info(request) self._service_account_email = sa_info["email"] else: @@ -409,79 +433,60 @@ def __init__( else: self._additional_claims = {} - def with_target_audience(self, target_audience): - """Create a copy of these credentials with the specified target - audience. - Args: - target_audience (str): The intended audience for these credentials, - used when requesting the ID Token. - Returns: - google.auth.service_account.IDTokenCredentials: A new credentials - instance. - """ + def _make_copy(self): # since the signer is already instantiated, # the request is not needed if self._use_metadata_identity_endpoint: - return self.__class__( + cred = self.__class__( None, - target_audience=target_audience, + target_audience=self._target_audience, use_metadata_identity_endpoint=True, quota_project_id=self._quota_project_id, ) + cred._service_account_email = self._service_account_email + cred._bind_id_token = getattr(self, "_bind_id_token", None) + return cred else: return self.__class__( None, service_account_email=self._service_account_email, token_uri=self._token_uri, - target_audience=target_audience, + target_audience=self._target_audience, additional_claims=self._additional_claims.copy(), signer=self.signer, use_metadata_identity_endpoint=False, quota_project_id=self._quota_project_id, ) + def with_target_audience(self, target_audience): + """Create a copy of these credentials with the specified target + audience. + Args: + target_audience (str): The intended audience for these credentials, + used when requesting the ID Token. + Returns: + google.auth.service_account.IDTokenCredentials: A new credentials + instance. + """ + cred = self._make_copy() + cred._target_audience = target_audience + return cred + @_helpers.copy_docstring(credentials.CredentialsWithQuotaProject) def with_quota_project(self, quota_project_id): - # since the signer is already instantiated, - # the request is not needed - if self._use_metadata_identity_endpoint: - return self.__class__( - None, - target_audience=self._target_audience, - use_metadata_identity_endpoint=True, - quota_project_id=quota_project_id, - ) - else: - return self.__class__( - None, - service_account_email=self._service_account_email, - token_uri=self._token_uri, - target_audience=self._target_audience, - additional_claims=self._additional_claims.copy(), - signer=self.signer, - use_metadata_identity_endpoint=False, - quota_project_id=quota_project_id, - ) + cred = self._make_copy() + cred._quota_project_id = quota_project_id + return cred @_helpers.copy_docstring(credentials.CredentialsWithTokenUri) def with_token_uri(self, token_uri): - # since the signer is already instantiated, - # the request is not needed if self._use_metadata_identity_endpoint: raise ValueError( "If use_metadata_identity_endpoint is set, token_uri must not be set" ) - else: - return self.__class__( - None, - service_account_email=self._service_account_email, - token_uri=token_uri, - target_audience=self._target_audience, - additional_claims=self._additional_claims.copy(), - signer=self.signer, - use_metadata_identity_endpoint=False, - quota_project_id=self.quota_project_id, - ) + cred = self._make_copy() + cred._token_uri = token_uri + return cred def _make_authorization_grant_assertion(self): """Create the OAuth 2.0 assertion. @@ -530,11 +535,25 @@ def _call_metadata_identity_endpoint(self, request): try: path = "instance/service-accounts/default/identity" params = {"audience": self._target_audience, "format": "full"} - metrics_header = { - metrics.API_CLIENT_HEADER: metrics.token_request_id_token_mds() - } + bind_token = getattr(self, "_bind_id_token", None) + if bind_token is None: + # Temporary gate: keep default ID tokens unbound when + # GOOGLE_API_CERTIFICATE_CONFIG is set (Cloud Run). + bind_token = not os.environ.get( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG + ) + method, body, headers = _metadata._build_token_request_options( + metrics.token_request_id_token_mds(), + bind_token=bind_token, + ) + id_token = _metadata.get( - request, path, params=params, headers=metrics_header + request, + path, + params=params, + headers=headers, + method=method, + body=body, ) except exceptions.TransportError as caught_exc: new_exc = exceptions.RefreshError(caught_exc) diff --git a/packages/google-auth/google/auth/environment_vars.py b/packages/google-auth/google/auth/environment_vars.py index 54550945f994..e8f3fb6e0670 100644 --- a/packages/google-auth/google/auth/environment_vars.py +++ b/packages/google-auth/google/auth/environment_vars.py @@ -128,10 +128,22 @@ """Environment variable defining the location of Google API certificate config file. This variable is the fallback of GOOGLE_API_CERTIFICATE_CONFIG.""" +GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN = "GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN" +"""Environment variable controlling whether to enable runtime bound tokens. + +Defaults to enabled; only a case-insensitive ``"false"`` disables it. When set +to a non-empty value, this variable takes precedence over +:data:`GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES`. +""" + GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES = ( "GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES" ) -"""Environment variable to prevent agent token sharing for GCP services.""" +"""Environment variable to prevent agent token sharing for GCP services. + +.. deprecated:: v2.59.0 + Use :data:`GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN` instead. +""" GOOGLE_API_USE_MTLS_ENDPOINT = "GOOGLE_API_USE_MTLS_ENDPOINT" """Environment variable controlling whether to use mTLS endpoint or not.""" diff --git a/packages/google-auth/google/auth/identity_pool.py b/packages/google-auth/google/auth/identity_pool.py index 41b312e4f841..6607bb3aecd1 100644 --- a/packages/google-auth/google/auth/identity_pool.py +++ b/packages/google-auth/google/auth/identity_pool.py @@ -404,9 +404,10 @@ def _get_mtls_cert_and_key_paths(self): 'The credential is not configured to use mtls requests. The credential should include a "certificate" section in the credential source.' ) else: - return _mtls_helper._get_workload_cert_and_key_paths( + cert_path, key_path, _ = _mtls_helper._get_workload_cert_and_key_paths( self._certificate_config_location ) + return cert_path, key_path def _get_cert_bytes(self): cert_path, _ = self._get_mtls_cert_and_key_paths() @@ -572,12 +573,12 @@ def refresh(self, request): if self._credential_source_certificate is not None: try: cert_bytes = self._get_cert_bytes() - except (exceptions.ClientCertError, OSError) as e: + cert = _agent_identity_utils.parse_certificate(cert_bytes) + except (exceptions.ClientCertError, OSError, ValueError) as e: raise exceptions.RefreshError( - "Failed to retrieve certificate bytes for external" + "Failed to retrieve or parse certificate for external" " account credentials" ) from e - cert = _agent_identity_utils.parse_certificate(cert_bytes) if _agent_identity_utils.should_request_bound_token(cert): cert_fingerprint = ( _agent_identity_utils.calculate_certificate_fingerprint(cert) diff --git a/packages/google-auth/google/auth/transport/_mtls_helper.py b/packages/google-auth/google/auth/transport/_mtls_helper.py index 469932be058c..65d54e3b762f 100644 --- a/packages/google-auth/google/auth/transport/_mtls_helper.py +++ b/packages/google-auth/google/auth/transport/_mtls_helper.py @@ -32,9 +32,10 @@ # Default gcloud config path, to be used with path.expanduser for cross-platform compatibility. CERTIFICATE_CONFIGURATION_DEFAULT_PATH = "~/.config/gcloud/certificate_config.json" +_GKE_CREDENTIAL_BUNDLE_PATH = "/var/run/secrets/workload-spiffe-credentials/x509.credential-bundle.private-key.pem" _CERT_PROVIDER_COMMAND = "cert_provider_command" _CERT_REGEX = re.compile( - b"-----BEGIN CERTIFICATE-----.+-----END CERTIFICATE-----\r?\n?", re.DOTALL + b"-----BEGIN CERTIFICATE-----.+?-----END CERTIFICATE-----\r?\n?", re.DOTALL ) # support various format of key files, e.g. @@ -43,7 +44,7 @@ # "-----BEGIN RSA PRIVATE KEY-----..." # "-----BEGIN ENCRYPTED PRIVATE KEY-----" _KEY_REGEX = re.compile( - b"-----BEGIN [A-Z ]*PRIVATE KEY-----.+-----END [A-Z ]*PRIVATE KEY-----\r?\n?", + b"-----BEGIN [A-Z ]*PRIVATE KEY-----.+?-----END [A-Z ]*PRIVATE KEY-----\r?\n?", re.DOTALL, ) @@ -367,12 +368,37 @@ def _load_json_file(path): return json_data +def _has_explicit_cert_config_env(): + """Returns True if an explicit certificate config environment variable is set.""" + return bool( + environ.get(environment_vars.GOOGLE_API_CERTIFICATE_CONFIG) + or environ.get( + environment_vars.CLOUDSDK_CONTEXT_AWARE_CERTIFICATE_CONFIG_FILE_PATH + ) + ) + + +def _should_use_gke_credential_bundle(config_file_path=None): + """Returns True if no certificate config is present and the GKE credential bundle file is present.""" + if config_file_path is not None or _has_explicit_cert_config_env(): + return False + try: + return _agent_identity_utils._is_certificate_file_ready( + _GKE_CREDENTIAL_BUNDLE_PATH + ) + except PermissionError: + return False + + def _get_workload_cert_and_key( certificate_config_path=None, include_context_aware=True ): """Read the workload identity cert and key files specified in the certificate config provided. If no config path is provided, check the environment variable: "GOOGLE_API_CERTIFICATE_CONFIG" - first, then the well known gcloud location: "~/.config/gcloud/certificate_config.json". + first, then the well known gcloud location: "~/.config/gcloud/certificate_config.json", + and finally fall back to the GKE workload credential bundle at + "/var/run/secrets/workload-spiffe-credentials/x509.credential-bundle.private-key.pem" + when no explicit or implicit certificate_config.json is present. Args: certificate_config_path (string): The certificate config path. If no path is provided, @@ -389,11 +415,15 @@ def _get_workload_cert_and_key( the certificate or key information. """ - cert_path, key_path = _get_workload_cert_and_key_paths( + cert_path, key_path, config_file_path = _get_workload_cert_and_key_paths( certificate_config_path, include_context_aware ) if cert_path is None and key_path is None: + if certificate_config_path is None and _should_use_gke_credential_bundle( + config_file_path + ): + return _read_credential_bundle_file(_GKE_CREDENTIAL_BUNDLE_PATH) return None, None return _read_cert_and_key_files(cert_path, key_path) @@ -453,10 +483,10 @@ def _get_cert_config_path(certificate_config_path=None, include_context_aware=Tr return certificate_config_path -def _get_workload_cert_and_key_paths(config_path, include_context_aware=True): +def _get_workload_cert_and_key_paths(config_path=None, include_context_aware=True): absolute_path = _get_cert_config_path(config_path, include_context_aware) if absolute_path is None: - return None, None + return None, None, None data = _load_json_file(absolute_path) @@ -503,13 +533,15 @@ def _get_workload_cert_and_key_paths(config_path, include_context_aware=True): pass if not isinstance(cert_configs, dict) or "workload" not in cert_configs: - return None, None + return None, None, absolute_path workload = cert_configs["workload"] if ( not isinstance(workload, dict) - or "cert_path" not in workload - or "key_path" not in workload + or not isinstance(workload.get("cert_path"), str) + or not workload.get("cert_path") + or not isinstance(workload.get("key_path"), str) + or not workload.get("key_path") ): raise exceptions.ClientCertError( 'Workload certificate configuration is missing "cert_path" or "key_path" in {}'.format( @@ -519,7 +551,20 @@ def _get_workload_cert_and_key_paths(config_path, include_context_aware=True): cert_path = workload["cert_path"] key_path = workload["key_path"] - return cert_path, key_path + return cert_path, key_path, absolute_path + + +def _read_credential_bundle_file(bundle_path): + """Reads a combined PEM credential bundle containing certificate(s) and a private key.""" + # Read the bundle once so a certificate rotation on disk cannot pair an + # old certificate chain with a new private key. + with open(bundle_path, "rb") as bundle_file: + bundle_data = bundle_file.read() + + return ( + _extract_cert_chain(bundle_data, bundle_path), + _extract_private_key(bundle_data, bundle_path), + ) def _read_cert_and_key_files(cert_path, key_path): @@ -529,24 +574,32 @@ def _read_cert_and_key_files(cert_path, key_path): return cert_data, key_data -def _read_cert_file(cert_path): - with open(cert_path, "rb") as cert_file: - cert_data = cert_file.read() +def _join_cert_chain(cert_match): + return b"".join( + m if m.endswith(b"\n") or i == len(cert_match) - 1 else m + b"\n" + for i, m in enumerate(cert_match) + ) + +def _has_unmatched_pem_markers(pem_bytes, cert_blocks): + """Checks that every BEGIN/END CERTIFICATE marker belongs to a complete cert block.""" + return len(cert_blocks) != pem_bytes.count(b"-----BEGIN CERTIFICATE-----") or len( + cert_blocks + ) != pem_bytes.count(b"-----END CERTIFICATE-----") + + +def _extract_cert_chain(cert_data, cert_path): cert_match = re.findall(_CERT_REGEX, cert_data) - if len(cert_match) != 1: + if not cert_match or _has_unmatched_pem_markers(cert_data, cert_match): raise exceptions.ClientCertError( - "Certificate file {} is in an invalid format, a single PEM formatted certificate is expected".format( + "Certificate file {} is in an invalid format, at least one PEM formatted certificate is expected".format( cert_path ) ) - return cert_match[0] - + return _join_cert_chain(cert_match) -def _read_key_file(key_path): - with open(key_path, "rb") as key_file: - key_data = key_file.read() +def _extract_private_key(key_data, key_path): key_match = re.findall(_KEY_REGEX, key_data) if len(key_match) != 1: raise exceptions.ClientCertError( @@ -558,6 +611,20 @@ def _read_key_file(key_path): return key_match[0] +def _read_cert_file(cert_path): + with open(cert_path, "rb") as cert_file: + cert_data = cert_file.read() + + return _extract_cert_chain(cert_data, cert_path) + + +def _read_key_file(key_path): + with open(key_path, "rb") as key_file: + key_data = key_file.read() + + return _extract_private_key(key_data, key_path) + + def _run_cert_provider_command(command, expect_encrypted_key=False): """Run the provided command, and return client side mTLS cert, key and passphrase. @@ -591,8 +658,9 @@ def _run_cert_provider_command(command, expect_encrypted_key=False): # Extract certificate (chain), key and passphrase. cert_match = re.findall(_CERT_REGEX, stdout) - if len(cert_match) != 1: + if not cert_match or _has_unmatched_pem_markers(stdout, cert_match): raise exceptions.ClientCertError("Client SSL certificate is missing or invalid") + cert_chain = _join_cert_chain(cert_match) key_match = re.findall(_KEY_REGEX, stdout) if len(key_match) != 1: raise exceptions.ClientCertError("Client SSL key is missing or invalid") @@ -603,13 +671,13 @@ def _run_cert_provider_command(command, expect_encrypted_key=False): raise exceptions.ClientCertError("Passphrase is missing or invalid") if b"ENCRYPTED" not in key_match[0]: raise exceptions.ClientCertError("Encrypted private key is expected") - return cert_match[0], key_match[0], passphrase_match[0].strip() + return cert_chain, key_match[0], passphrase_match[0].strip() if b"ENCRYPTED" in key_match[0]: raise exceptions.ClientCertError("Encrypted private key is not expected") if len(passphrase_match) > 0: raise exceptions.ClientCertError("Passphrase is not expected") - return cert_match[0], key_match[0], None + return cert_chain, key_match[0], None def get_client_ssl_credentials( @@ -768,7 +836,9 @@ def check_use_client_cert(): as True (auto-enabled) if a workload config file exists (pointed at by GOOGLE_API_CERTIFICATE_CONFIG or CLOUDSDK_CONTEXT_AWARE_CERTIFICATE_CONFIG_FILE_PATH, or the default path like ~/.config/gcloud/certificate_config.json) - containing a "workload" section. + containing a "workload" section, or if no certificate config file is + configured/present and the GKE credential bundle exists at + /var/run/secrets/workload-spiffe-credentials/x509.credential-bundle.private-key.pem. Otherwise, it returns False. Returns: @@ -806,22 +876,26 @@ def check_use_client_cert(): "mTLS auto-enablement failed: Certificate configuration file at %s is missing the required ['cert_configs']['workload'] section.", cert_path, ) - return False + return False + + return _should_use_gke_credential_bundle(cert_path) def check_parameters_for_unauthorized_response(cached_cert): """Returns the cached and current cert fingerprint for reconfiguring mTLS. Args: - cached_cert(bytes): The cached client certificate. + cached_cert (Optional[bytes]): The cached client certificate. Returns: - bytes: The client callback cert bytes. - bytes: The client callback key bytes. - str: The base64-encoded SHA256 cached fingerprint. - str: The base64-encoded SHA256 current cert fingerprint. + Tuple[Optional[bytes], Optional[bytes], Optional[str], Optional[str]]: + The client callback cert bytes, client callback key bytes, + base64-encoded SHA256 cached fingerprint, and base64-encoded SHA256 + current cert fingerprint. """ call_cert_bytes, call_key_bytes = call_client_cert_callback() + if not call_cert_bytes: + return None, None, None, None cert_obj = _agent_identity_utils.parse_certificate(call_cert_bytes) current_cert_fingerprint = _agent_identity_utils.calculate_certificate_fingerprint( cert_obj @@ -847,16 +921,19 @@ def call_client_cert_callback(): ".mtls.googleapis.com", ".mtls.sandbox.googleapis.com", ".p.googleapis.com", + ".mtls.run.app", ) _MTLS_EXACT_HOSTS = ( "mtls.googleapis.com", "mtls.sandbox.googleapis.com", "p.googleapis.com", + "mtls.run.app", ) def is_mtls_endpoint(url: Optional[Union[str, bytes, object]]) -> bool: - """Checks if the given URL corresponds to an mTLS or Private Service Connect (PSC) endpoint. + """Checks if the given URL corresponds to an mTLS (Google APIs or Cloud Run) + or Private Service Connect (PSC) endpoint. Args: url (Optional[Union[str, bytes, object]]): The request URL. diff --git a/packages/google-auth/google/auth/transport/mtls.py b/packages/google-auth/google/auth/transport/mtls.py index 54b675b3a1fa..db6d47daf688 100644 --- a/packages/google-auth/google/auth/transport/mtls.py +++ b/packages/google-auth/google/auth/transport/mtls.py @@ -42,10 +42,21 @@ def has_default_client_cert_source(include_context_aware=True): Returns: bool: indicating if the default client cert source exists. """ - cert_path = _mtls_helper._get_cert_config_path( - include_context_aware=include_context_aware - ) - if cert_path is not None: + try: + ( + cert_path, + key_path, + config_file_path, + ) = _mtls_helper._get_workload_cert_and_key_paths( + None, include_context_aware=include_context_aware + ) + except (exceptions.ClientCertError, OSError): + # Config exists, but is malformed or unreadable. Let caller surface error. + return True + + if cert_path is not None and key_path is not None: + return True + if _mtls_helper._should_use_gke_credential_bundle(config_file_path): return True if ( include_context_aware diff --git a/packages/google-auth/google/oauth2/_id_token_async.py b/packages/google-auth/google/oauth2/_id_token_async.py index 38d79ef5d612..024a305c0808 100644 --- a/packages/google-auth/google/oauth2/_id_token_async.py +++ b/packages/google-auth/google/oauth2/_id_token_async.py @@ -194,7 +194,7 @@ async def verify_firebase_token( ) -async def fetch_id_token(request, audience): +async def fetch_id_token(request, audience, bind_id_token=None): """Fetch the ID Token from the current environment. This function acquires ID token from the environment in the following order. @@ -223,6 +223,19 @@ async def fetch_id_token(request, audience): request (google.auth.transport.aiohttp_requests.Request): A callable used to make HTTP requests. audience (str): The audience that this ID token is intended for. + bind_id_token (Optional[bool]): Controls whether to request a + certificate-bound ID token from the metadata server identity + endpoint. If ``True``, requests a bound token whenever a valid + Agent Identity certificate is available and token binding is not + disabled via ``GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN``, falling back + to an unbound token otherwise (or raising + :class:`~google.auth.exceptions.RefreshError` if a configured + certificate is not found after retries). If ``False``, always + requests an unbound token. If ``None`` (default), token binding is + determined automatically by the library. Set ``True`` or ``False`` + explicitly if your application requires a specific behavior. Has no + effect when credentials are loaded from + ``GOOGLE_APPLICATION_CREDENTIALS``. Returns: str: The ID token. @@ -231,6 +244,9 @@ async def fetch_id_token(request, audience): ~google.auth.exceptions.DefaultCredentialsError: If metadata server doesn't exist and no valid service account credentials are found. + ~google.auth.exceptions.RefreshError: + If an error occurred while fetching the ID token or if a required + certificate file is not found after retries. """ # 1. Try to get credentials from the GOOGLE_APPLICATION_CREDENTIALS environment # variable. @@ -273,7 +289,10 @@ async def fetch_id_token(request, audience): request_new = requests.Request() if _metadata.ping(request_new): credentials = compute_engine.IDTokenCredentials( - request_new, audience, use_metadata_identity_endpoint=True + request_new, + audience, + use_metadata_identity_endpoint=True, + bind_id_token=bind_id_token, ) credentials.refresh(request_new) return credentials.token diff --git a/packages/google-auth/google/oauth2/id_token.py b/packages/google-auth/google/oauth2/id_token.py index 1cab9440a61a..c73d11b50f94 100644 --- a/packages/google-auth/google/oauth2/id_token.py +++ b/packages/google-auth/google/oauth2/id_token.py @@ -236,7 +236,7 @@ def verify_firebase_token(id_token, request, audience=None, clock_skew_in_second ) -def fetch_id_token_credentials(audience, request=None): +def fetch_id_token_credentials(audience, request=None, bind_id_token=None): """Create the ID Token credentials from the current environment. This function acquires ID token from the environment in the following order. @@ -272,6 +272,19 @@ def fetch_id_token_credentials(audience, request=None): audience (str): The audience that this ID token is intended for. request (Optional[google.auth.transport.Request]): A callable used to make HTTP requests. A request object will be created if not provided. + bind_id_token (Optional[bool]): Controls whether to request a + certificate-bound ID token from the metadata server identity + endpoint. If ``True``, requests a bound token whenever a valid + Agent Identity certificate is available and token binding is not + disabled via ``GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN``, falling back + to an unbound token otherwise (or raising + :class:`~google.auth.exceptions.RefreshError` if a configured + certificate is not found after retries). If ``False``, always + requests an unbound token. If ``None`` (default), token binding is + determined automatically by the library. Set ``True`` or ``False`` + explicitly if your application requires a specific behavior. Has no + effect when credentials are loaded from + ``GOOGLE_APPLICATION_CREDENTIALS``. Returns: google.auth.credentials.Credentials: The ID token credentials. @@ -335,7 +348,10 @@ def fetch_id_token_credentials(audience, request=None): if _metadata.ping(request): return compute_engine.IDTokenCredentials( - request, audience, use_metadata_identity_endpoint=True + request, + audience, + use_metadata_identity_endpoint=True, + bind_id_token=bind_id_token, ) except (ImportError, exceptions.TransportError): pass @@ -345,7 +361,7 @@ def fetch_id_token_credentials(audience, request=None): ) -def fetch_id_token(request, audience): +def fetch_id_token(request, audience, bind_id_token=None): """Fetch the ID Token from the current environment. This function acquires ID token from the environment in the following order. @@ -374,6 +390,19 @@ def fetch_id_token(request, audience): request (google.auth.transport.Request): A callable used to make HTTP requests. audience (str): The audience that this ID token is intended for. + bind_id_token (Optional[bool]): Controls whether to request a + certificate-bound ID token from the metadata server identity + endpoint. If ``True``, requests a bound token whenever a valid + Agent Identity certificate is available and token binding is not + disabled via ``GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN``, falling back + to an unbound token otherwise (or raising + :class:`~google.auth.exceptions.RefreshError` if a configured + certificate is not found after retries). If ``False``, always + requests an unbound token. If ``None`` (default), token binding is + determined automatically by the library. Set ``True`` or ``False`` + explicitly if your application requires a specific behavior. Has no + effect when credentials are loaded from + ``GOOGLE_APPLICATION_CREDENTIALS``. Returns: str: The ID token. @@ -382,7 +411,12 @@ def fetch_id_token(request, audience): ~google.auth.exceptions.DefaultCredentialsError: If metadata server doesn't exist and no valid service account credentials are found. + ~google.auth.exceptions.RefreshError: + If an error occurred while fetching the ID token or if a required + certificate file is not found after retries. """ - id_token_credentials = fetch_id_token_credentials(audience, request=request) + id_token_credentials = fetch_id_token_credentials( + audience, request=request, bind_id_token=bind_id_token + ) id_token_credentials.refresh(request) return id_token_credentials.token diff --git a/packages/google-auth/tests/compute_engine/test__metadata.py b/packages/google-auth/tests/compute_engine/test__metadata.py index 0b151722af07..af8573a80c66 100644 --- a/packages/google-auth/tests/compute_engine/test__metadata.py +++ b/packages/google-auth/tests/compute_engine/test__metadata.py @@ -484,6 +484,57 @@ def test_get_failure_bad_json(): ) +def test_get_body_with_get_method_raises_value_error(): + request = make_request("{}") + + with pytest.raises( + ValueError, match="Request body cannot be specified with GET method." + ): + _metadata.get(request, PATH, method="GET", body=b"some_body") + + request.assert_not_called() + + +@mock.patch("time.sleep", return_value=None) +def test_get_post_retry_preserves_method_body_and_headers(mock_sleep): + response_503 = mock.create_autospec(transport.Response, instance=True) + response_503.status = http_client.SERVICE_UNAVAILABLE + response_503.data = _helpers.to_bytes("Service Unavailable") + response_503.headers = {} + + response_ok = mock.create_autospec(transport.Response, instance=True) + response_ok.status = http_client.OK + response_ok.data = _helpers.to_bytes( + json.dumps({"access_token": "bound_token", "expires_in": 3600}) + ) + response_ok.headers = {"content-type": "application/json"} + + request = mock.create_autospec(transport.Request) + request.side_effect = [ + response_503, + exceptions.TransportError("transient transport error"), + response_ok, + ] + + expected_body = json.dumps({"certificate_chain": "fake_pem_chain"}).encode("utf-8") + result = _metadata.get( + request, + PATH, + method="POST", + body=expected_body, + headers={"Content-Type": "application/json"}, + ) + + assert result == {"access_token": "bound_token", "expires_in": 3600} + assert request.call_count == 3 + for call_args in request.call_args_list: + _, kwargs = call_args + assert kwargs["method"] == "POST" + assert kwargs["body"] == expected_body + assert kwargs["headers"]["Content-Type"] == "application/json" + assert kwargs["headers"][_metadata._METADATA_FLAVOR_HEADER] == "Google" + + def test_get_project_id(): project = "example-project" request = make_request(project, headers={"content-type": "text/plain"}) @@ -635,18 +686,12 @@ def test_get_universe_domain_other_error(): ) -@mock.patch( - "google.auth._agent_identity_utils.get_and_parse_agent_identity_certificate", - return_value=None, -) @mock.patch( "google.auth.metrics.token_request_access_token_mds", return_value=ACCESS_TOKEN_REQUEST_METRICS_HEADER_VALUE, ) @mock.patch("google.auth._helpers.utcnow", return_value=datetime.datetime.min) -def test_get_service_account_token( - utcnow, mock_metrics_header_value, mock_get_agent_cert -): +def test_get_service_account_token(utcnow, mock_metrics_header_value): ttl = 500 request = make_request( json.dumps({"access_token": "token", "expires_in": ttl}), @@ -668,18 +713,12 @@ def test_get_service_account_token( assert expiry == utcnow() + datetime.timedelta(seconds=ttl) -@mock.patch( - "google.auth._agent_identity_utils.get_and_parse_agent_identity_certificate", - return_value=None, -) @mock.patch( "google.auth.metrics.token_request_access_token_mds", return_value=ACCESS_TOKEN_REQUEST_METRICS_HEADER_VALUE, ) @mock.patch("google.auth._helpers.utcnow", return_value=datetime.datetime.min) -def test_get_service_account_token_with_scopes_list( - utcnow, mock_metrics_header_value, mock_get_agent_cert -): +def test_get_service_account_token_with_scopes_list(utcnow, mock_metrics_header_value): ttl = 500 request = make_request( json.dumps({"access_token": "token", "expires_in": ttl}), @@ -704,17 +743,13 @@ def test_get_service_account_token_with_scopes_list( assert expiry == utcnow() + datetime.timedelta(seconds=ttl) -@mock.patch( - "google.auth._agent_identity_utils.get_and_parse_agent_identity_certificate", - return_value=None, -) @mock.patch( "google.auth.metrics.token_request_access_token_mds", return_value=ACCESS_TOKEN_REQUEST_METRICS_HEADER_VALUE, ) @mock.patch("google.auth._helpers.utcnow", return_value=datetime.datetime.min) def test_get_service_account_token_with_scopes_string( - utcnow, mock_metrics_header_value, mock_get_agent_cert + utcnow, mock_metrics_header_value ): ttl = 500 request = make_request( @@ -740,10 +775,9 @@ def test_get_service_account_token_with_scopes_string( assert expiry == utcnow() + datetime.timedelta(seconds=ttl) -@mock.patch("google.auth._agent_identity_utils.calculate_certificate_fingerprint") @mock.patch("google.auth._agent_identity_utils.should_request_bound_token") @mock.patch( - "google.auth._agent_identity_utils.get_and_parse_agent_identity_certificate" + "google.auth._agent_identity_utils.get_agent_identity_certificate_and_bytes" ) @mock.patch( "google.auth.metrics.token_request_access_token_mds", @@ -753,38 +787,39 @@ def test_get_service_account_token_with_scopes_string( def test_get_service_account_token_with_bound_token( utcnow, mock_metrics_header_value, - mock_get_and_parse, + mock_get_cert_and_bytes, mock_should_request, - mock_calculate_fingerprint, ): # Test the successful path where a certificate is found and a bound token # is requested. mock_cert = mock.sentinel.cert - mock_get_and_parse.return_value = mock_cert + mock_cert_bytes = b"fake_cert_bytes" + mock_get_cert_and_bytes.return_value = (mock_cert, mock_cert_bytes) mock_should_request.return_value = True - mock_calculate_fingerprint.return_value = "fake_fingerprint" token_response = json.dumps({"access_token": "token", "expires_in": 3600}) request = make_request(token_response, headers={"content-type": "application/json"}) _metadata.get_service_account_token(request) - mock_get_and_parse.assert_called_once() + mock_get_cert_and_bytes.assert_called_once() mock_should_request.assert_called_once_with(mock_cert) - mock_calculate_fingerprint.assert_called_once_with(mock_cert) request.assert_called_once() _, kwargs = request.call_args - url = kwargs["url"] - assert "bindCertificateFingerprint=fake_fingerprint" in url + assert kwargs["method"] == "POST" + assert kwargs["body"] == json.dumps( + {"certificate_chain": mock_cert_bytes.decode("utf-8")} + ).encode("utf-8") + assert kwargs["headers"]["Content-Type"] == "application/json" @mock.patch( - "google.auth._agent_identity_utils.get_and_parse_agent_identity_certificate" + "google.auth._agent_identity_utils.get_agent_identity_certificate_and_bytes" ) -def test_get_service_account_token_no_cert(mock_get_and_parse): - # Test that no fingerprint is added when no certificate is found. - mock_get_and_parse.return_value = None +def test_get_service_account_token_no_cert(mock_get_cert_and_bytes): + # Test that a standard GET request without a body is sent when no certificate is found. + mock_get_cert_and_bytes.return_value = (None, None) token_response = json.dumps({"access_token": "token", "expires_in": 3600}) request = make_request(token_response, headers={"content-type": "application/json"}) @@ -792,19 +827,20 @@ def test_get_service_account_token_no_cert(mock_get_and_parse): request.assert_called_once() _, kwargs = request.call_args - url = kwargs["url"] - assert "bindCertificateFingerprint" not in url + assert kwargs["method"] == "GET" + assert "body" not in kwargs + assert "Content-Type" not in kwargs["headers"] @mock.patch("google.auth._agent_identity_utils.should_request_bound_token") @mock.patch( - "google.auth._agent_identity_utils.get_and_parse_agent_identity_certificate" + "google.auth._agent_identity_utils.get_agent_identity_certificate_and_bytes" ) def test_get_service_account_token_should_not_bind( - mock_get_and_parse, mock_should_request + mock_get_cert_and_bytes, mock_should_request ): - # Test that no fingerprint is added when a cert is found but should not be used. - mock_get_and_parse.return_value = mock.sentinel.cert + # Test that a standard GET request without a body is sent when a cert is found but should not be used. + mock_get_cert_and_bytes.return_value = (mock.sentinel.cert, b"fake_cert_bytes") mock_should_request.return_value = False token_response = json.dumps({"access_token": "token", "expires_in": 3600}) request = make_request(token_response, headers={"content-type": "application/json"}) @@ -813,8 +849,23 @@ def test_get_service_account_token_should_not_bind( request.assert_called_once() _, kwargs = request.call_args - url = kwargs["url"] - assert "bindCertificateFingerprint" not in url + assert kwargs["method"] == "GET" + assert "body" not in kwargs + assert "Content-Type" not in kwargs["headers"] + + +@mock.patch( + "google.auth._agent_identity_utils.get_agent_identity_certificate_and_bytes" +) +def test__build_token_request_options_unbound(mock_get_cert_and_bytes): + method, body, headers = _metadata._build_token_request_options( + ACCESS_TOKEN_REQUEST_METRICS_HEADER_VALUE, bind_token=False + ) + + mock_get_cert_and_bytes.assert_not_called() + assert method == "GET" + assert body is None + assert headers == {"x-goog-api-client": ACCESS_TOKEN_REQUEST_METRICS_HEADER_VALUE} def test_get_service_account_info(): diff --git a/packages/google-auth/tests/compute_engine/test_credentials.py b/packages/google-auth/tests/compute_engine/test_credentials.py index 6e30d8807bf3..9490ca3e010a 100644 --- a/packages/google-auth/tests/compute_engine/test_credentials.py +++ b/packages/google-auth/tests/compute_engine/test_credentials.py @@ -13,13 +13,21 @@ # limitations under the License. import base64 import datetime +import json import re from unittest import mock import pytest # type: ignore import responses # type: ignore -from google.auth import _helpers, exceptions, jwt, transport +from google.auth import ( + _helpers, + environment_vars, + exceptions, + jwt, + metrics, + transport, +) from google.auth.compute_engine import credentials from google.auth.transport import requests @@ -471,29 +479,23 @@ def test_regional_access_boundary_disabled_state_transitions( # Subsequent check calls should return False early assert creds._is_regional_access_boundary_lookup_required() is False - @mock.patch("google.auth.compute_engine._metadata.get") - @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") - @mock.patch("google.auth._agent_identity_utils.parse_certificate") @mock.patch( - "google.auth._agent_identity_utils.should_request_bound_token", - return_value=True, + "google.auth._agent_identity_utils.get_agent_identity_certificate_and_bytes" ) @mock.patch( - "google.auth._agent_identity_utils.calculate_certificate_fingerprint", - return_value="fingerprint", + "google.auth._agent_identity_utils.should_request_bound_token", + return_value=True, ) + @mock.patch("google.auth.compute_engine._metadata.get", autospec=True) def test_refresh_with_agent_identity( self, - mock_calculate_fingerprint, - mock_should_request, - mock_parse_certificate, - mock_get_path, mock_metadata_get, - tmpdir, + mock_should_request, + mock_get_cert_and_bytes, ): - cert_path = tmpdir.join("cert.pem") - cert_path.write(b"cert_content") - mock_get_path.return_value = str(cert_path) + mock_cert = mock.sentinel.cert + mock_cert_bytes = b"cert_content" + mock_get_cert_and_bytes.return_value = (mock_cert, mock_cert_bytes) mock_metadata_get.side_effect = [ { @@ -506,33 +508,32 @@ def test_refresh_with_agent_identity( self.credentials.refresh(None) assert self.credentials.token == "token" - mock_parse_certificate.assert_called_once_with(b"cert_content") - mock_should_request.assert_called_once_with(mock_parse_certificate.return_value) + mock_get_cert_and_bytes.assert_called_once() + mock_should_request.assert_called_once_with(mock_cert) kwargs = mock_metadata_get.call_args[1] assert kwargs["params"] == { "scopes": "one,two", - "bindCertificateFingerprint": "fingerprint", } + assert kwargs["method"] == "POST" + assert kwargs["body"] == json.dumps( + {"certificate_chain": mock_cert_bytes.decode("utf-8")} + ).encode("utf-8") + assert kwargs["headers"]["Content-Type"] == "application/json" - @mock.patch("google.auth.compute_engine._metadata.get") - @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") - @mock.patch("google.auth._agent_identity_utils.parse_certificate") + @mock.patch( + "google.auth._agent_identity_utils.get_agent_identity_certificate_and_bytes" + ) @mock.patch( "google.auth._agent_identity_utils.should_request_bound_token", return_value=False, ) + @mock.patch("google.auth.compute_engine._metadata.get", autospec=True) def test_refresh_with_agent_identity_opt_out_or_not_agent( self, - mock_should_request, - mock_parse_certificate, - mock_get_path, mock_metadata_get, - tmpdir, + mock_should_request, + mock_get_cert_and_bytes, ): - cert_path = tmpdir.join("cert.pem") - cert_path.write(b"cert_content") - mock_get_path.return_value = str(cert_path) - mock_metadata_get.side_effect = [ { "email": "service-account@project.iam.gserviceaccount.com", @@ -541,13 +542,44 @@ def test_refresh_with_agent_identity_opt_out_or_not_agent( {"access_token": "token", "expires_in": 500}, ] + mock_cert = mock.sentinel.cert + mock_cert_bytes = b"cert_content" + mock_get_cert_and_bytes.return_value = (mock_cert, mock_cert_bytes) + + self.credentials.refresh(None) + + assert self.credentials.token == "token" + mock_get_cert_and_bytes.assert_called_once() + mock_should_request.assert_called_once_with(mock_cert) + kwargs = mock_metadata_get.call_args[1] + assert kwargs["method"] == "GET" + assert kwargs["body"] is None + assert "Content-Type" not in kwargs["headers"] + + @mock.patch( + "google.auth._agent_identity_utils.get_agent_identity_certificate_and_bytes" + ) + @mock.patch("google.auth.compute_engine._metadata.get", autospec=True) + def test_refresh_without_agent_identity_certificate( + self, + mock_metadata_get, + mock_get_cert_and_bytes, + ): + mock_metadata_get.side_effect = [ + {"email": "service-account@example.com", "scopes": ["one", "two"]}, + {"access_token": "token", "expires_in": 500}, + ] + + mock_get_cert_and_bytes.return_value = (None, None) + self.credentials.refresh(None) assert self.credentials.token == "token" - mock_parse_certificate.assert_called_once_with(b"cert_content") - mock_should_request.assert_called_once_with(mock_parse_certificate.return_value) + mock_get_cert_and_bytes.assert_called_once() kwargs = mock_metadata_get.call_args[1] - assert "bindCertificateFingerprint" not in kwargs.get("params", {}) + assert kwargs["method"] == "GET" + assert kwargs["body"] is None + assert "Content-Type" not in kwargs["headers"] def test_set_blocking_regional_access_boundary_lookup(self): creds = self.credentials @@ -855,6 +887,204 @@ def test_with_target_audience_integration(self): assert self.credentials.token is not None + @pytest.mark.parametrize( + "cert_config_env,bind_id_token", + [(None, None), (None, True), ("/path/to/config.json", True)], + ) + @mock.patch( + "google.auth._agent_identity_utils.get_agent_identity_certificate_and_bytes" + ) + @mock.patch( + "google.auth._agent_identity_utils.should_request_bound_token", + return_value=True, + ) + @mock.patch("google.auth.compute_engine._metadata.get", autospec=True) + def test_refresh_with_agent_identity( + self, + mock_metadata_get, + mock_should_request, + mock_get_cert_and_bytes, + cert_config_env, + bind_id_token, + monkeypatch, + ): + if cert_config_env: + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, cert_config_env + ) + mock_metadata_get.side_effect = [ + {"email": "service-account@example.com", "scopes": ["one", "two"]}, + SAMPLE_ID_TOKEN, + ] + + mock_cert = mock.sentinel.cert + mock_cert_bytes = b"cert_content" + mock_get_cert_and_bytes.return_value = (mock_cert, mock_cert_bytes) + + request = mock.create_autospec(transport.Request, instance=True) + self.credentials = credentials.IDTokenCredentials( + request=request, + target_audience="https://audience.com", + use_metadata_identity_endpoint=True, + bind_id_token=bind_id_token, + ) + + self.credentials.refresh(None) + + assert self.credentials.token == SAMPLE_ID_TOKEN + mock_get_cert_and_bytes.assert_called_once() + mock_should_request.assert_called_once_with(mock_cert) + + kwargs = mock_metadata_get.call_args[1] + assert kwargs["method"] == "POST" + assert kwargs["body"] == json.dumps( + {"certificate_chain": mock_cert_bytes.decode("utf-8")} + ).encode("utf-8") + assert kwargs["headers"]["Content-Type"] == "application/json" + assert ( + kwargs["headers"][metrics.API_CLIENT_HEADER] + == metrics.token_request_id_token_mds() + ) + + @pytest.mark.parametrize( + "cert_config_env,bind_id_token", + [ + (None, False), + ("/path/to/config.json", False), + ("/path/to/config.json", None), + ], + ) + @mock.patch( + "google.auth._agent_identity_utils.get_agent_identity_certificate_and_bytes" + ) + @mock.patch("google.auth.compute_engine._metadata.get", autospec=True) + def test_refresh_with_agent_identity_unbound( + self, + mock_metadata_get, + mock_get_cert_and_bytes, + cert_config_env, + bind_id_token, + monkeypatch, + ): + if cert_config_env: + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, cert_config_env + ) + mock_metadata_get.side_effect = [ + {"email": "service-account@example.com", "scopes": ["one", "two"]}, + SAMPLE_ID_TOKEN, + ] + + request = mock.create_autospec(transport.Request, instance=True) + self.credentials = credentials.IDTokenCredentials( + request=request, + target_audience="https://audience.com", + use_metadata_identity_endpoint=True, + bind_id_token=bind_id_token, + ) + + self.credentials.refresh(None) + + assert self.credentials.token == SAMPLE_ID_TOKEN + mock_get_cert_and_bytes.assert_not_called() + + kwargs = mock_metadata_get.call_args[1] + assert kwargs["method"] == "GET" + assert kwargs["body"] is None + assert "Content-Type" not in kwargs["headers"] + assert ( + kwargs["headers"][metrics.API_CLIENT_HEADER] + == metrics.token_request_id_token_mds() + ) + + @pytest.mark.parametrize("bind_id_token", [None, True]) + @mock.patch( + "google.auth._agent_identity_utils.get_agent_identity_certificate_and_bytes" + ) + @mock.patch( + "google.auth._agent_identity_utils.should_request_bound_token", + return_value=False, + ) + @mock.patch("google.auth.compute_engine._metadata.get", autospec=True) + def test_refresh_with_agent_identity_opt_out_or_not_agent( + self, + mock_metadata_get, + mock_should_request, + mock_get_cert_and_bytes, + bind_id_token, + ): + mock_metadata_get.side_effect = [ + {"email": "service-account@example.com", "scopes": ["one", "two"]}, + SAMPLE_ID_TOKEN, + ] + + mock_cert = mock.sentinel.cert + mock_cert_bytes = b"cert_content" + mock_get_cert_and_bytes.return_value = (mock_cert, mock_cert_bytes) + + request = mock.create_autospec(transport.Request, instance=True) + self.credentials = credentials.IDTokenCredentials( + request=request, + target_audience="https://audience.com", + use_metadata_identity_endpoint=True, + bind_id_token=bind_id_token, + ) + + self.credentials.refresh(None) + + assert self.credentials.token == SAMPLE_ID_TOKEN + mock_get_cert_and_bytes.assert_called_once() + mock_should_request.assert_called_once_with(mock_cert) + + kwargs = mock_metadata_get.call_args[1] + assert kwargs["method"] == "GET" + assert kwargs["body"] is None + assert "Content-Type" not in kwargs["headers"] + assert ( + kwargs["headers"][metrics.API_CLIENT_HEADER] + == metrics.token_request_id_token_mds() + ) + + @pytest.mark.parametrize("bind_id_token", [None, True]) + @mock.patch( + "google.auth._agent_identity_utils.get_agent_identity_certificate_and_bytes" + ) + @mock.patch("google.auth.compute_engine._metadata.get", autospec=True) + def test_refresh_without_agent_identity_certificate( + self, + mock_metadata_get, + mock_get_cert_and_bytes, + bind_id_token, + ): + mock_metadata_get.side_effect = [ + {"email": "service-account@example.com", "scopes": ["one", "two"]}, + SAMPLE_ID_TOKEN, + ] + + mock_get_cert_and_bytes.return_value = (None, None) + + request = mock.create_autospec(transport.Request, instance=True) + self.credentials = credentials.IDTokenCredentials( + request=request, + target_audience="https://audience.com", + use_metadata_identity_endpoint=True, + bind_id_token=bind_id_token, + ) + + self.credentials.refresh(None) + + assert self.credentials.token == SAMPLE_ID_TOKEN + mock_get_cert_and_bytes.assert_called_once() + + kwargs = mock_metadata_get.call_args[1] + assert kwargs["method"] == "GET" + assert kwargs["body"] is None + assert "Content-Type" not in kwargs["headers"] + assert ( + kwargs["headers"][metrics.API_CLIENT_HEADER] + == metrics.token_request_id_token_mds() + ) + @mock.patch( "google.auth._helpers.utcnow", return_value=_helpers.utcfromtimestamp(0), @@ -1220,6 +1450,7 @@ def test_get_id_token_from_metadata( assert cred.token == SAMPLE_ID_TOKEN assert cred.expiry == _helpers.utcfromtimestamp(SAMPLE_ID_TOKEN_EXP) assert cred._use_metadata_identity_endpoint + assert cred._bind_id_token is None assert cred._signer is None assert cred._token_uri is None assert cred._service_account_email == "foo@project.iam.gserviceaccount.com" @@ -1227,43 +1458,120 @@ def test_get_id_token_from_metadata( with pytest.raises(ValueError): cred.sign_bytes(b"bytes") - @mock.patch( - "google.auth.compute_engine._metadata.get_service_account_info", autospec=True - ) - def test_with_target_audience_for_metadata(self, get_service_account_info): - get_service_account_info.return_value = { - "email": "foo@project.iam.gserviceaccount.com" - } + @pytest.mark.parametrize("bind_id_token", [None, True, False]) + @responses.activate + def test_with_target_audience_for_metadata(self, bind_id_token): + responses.add( + responses.GET, + "http://metadata.google.internal/computeMetadata/v1/instance/" + "service-accounts/default/?recursive=true", + status=200, + content_type="application/json", + json={"email": "foo@project.iam.gserviceaccount.com"}, + ) + responses.add( + responses.GET, + "http://metadata.google.internal/computeMetadata/v1/instance/" + "service-accounts/default/identity?audience=new_audience&format=full", + status=200, + content_type="text/plain", + body=SAMPLE_ID_TOKEN, + ) cred = credentials.IDTokenCredentials( - mock.Mock(), "audience", use_metadata_identity_endpoint=True + requests.Request(), + "audience", + use_metadata_identity_endpoint=True, + bind_id_token=bind_id_token, ) cred = cred.with_target_audience("new_audience") + cred.refresh(requests.Request()) assert cred._target_audience == "new_audience" assert cred._use_metadata_identity_endpoint + assert cred._bind_id_token is bind_id_token assert cred._signer is None assert cred._token_uri is None assert cred._service_account_email == "foo@project.iam.gserviceaccount.com" + assert cred.token == SAMPLE_ID_TOKEN.decode("utf-8") - @mock.patch( - "google.auth.compute_engine._metadata.get_service_account_info", autospec=True - ) - def test_id_token_with_quota_project(self, get_service_account_info): - get_service_account_info.return_value = { - "email": "foo@project.iam.gserviceaccount.com" - } + @pytest.mark.parametrize("bind_id_token", [None, True, False]) + @responses.activate + def test_id_token_with_quota_project(self, bind_id_token): + responses.add( + responses.GET, + "http://metadata.google.internal/computeMetadata/v1/instance/" + "service-accounts/default/?recursive=true", + status=200, + content_type="application/json", + json={"email": "foo@project.iam.gserviceaccount.com"}, + ) + responses.add( + responses.GET, + "http://metadata.google.internal/computeMetadata/v1/instance/" + "service-accounts/default/identity?audience=audience&format=full", + status=200, + content_type="text/plain", + body=SAMPLE_ID_TOKEN, + ) cred = credentials.IDTokenCredentials( - mock.Mock(), "audience", use_metadata_identity_endpoint=True + requests.Request(), + "audience", + use_metadata_identity_endpoint=True, + bind_id_token=bind_id_token, ) cred = cred.with_quota_project("project-foo") + cred.refresh(requests.Request()) assert cred._quota_project_id == "project-foo" assert cred._use_metadata_identity_endpoint + assert cred._bind_id_token is bind_id_token assert cred._signer is None assert cred._token_uri is None assert cred._service_account_email == "foo@project.iam.gserviceaccount.com" + assert cred.token == SAMPLE_ID_TOKEN.decode("utf-8") + + @mock.patch( + "google.auth._agent_identity_utils.get_agent_identity_certificate_and_bytes", + return_value=(None, None), + ) + @responses.activate + def test_id_token_missing_bind_id_token_attribute(self, mock_get_agent_cert): + responses.add( + responses.GET, + "http://metadata.google.internal/computeMetadata/v1/instance/" + "service-accounts/default/?recursive=true", + status=200, + content_type="application/json", + json={"email": "foo@project.iam.gserviceaccount.com"}, + ) + responses.add( + responses.GET, + "http://metadata.google.internal/computeMetadata/v1/instance/" + "service-accounts/default/identity?audience=audience&format=full", + status=200, + content_type="text/plain", + body=SAMPLE_ID_TOKEN, + ) + + cred = credentials.IDTokenCredentials( + requests.Request(), + "audience", + use_metadata_identity_endpoint=True, + ) + # Simulate credentials pickled with an older version without _bind_id_token. + del cred._bind_id_token + + cred_with_audience = cred.with_target_audience("new_audience") + assert cred_with_audience._bind_id_token is None + + cred_with_quota = cred.with_quota_project("project-foo") + assert cred_with_quota._bind_id_token is None + + cred.refresh(requests.Request()) + assert cred.token == SAMPLE_ID_TOKEN.decode("utf-8") + mock_get_agent_cert.assert_called_once() @mock.patch( "google.auth.compute_engine._metadata.get_service_account_info", autospec=True @@ -1329,3 +1637,14 @@ def test_get_id_token_from_metadata_constructor(self): use_metadata_identity_endpoint=True, service_account_email="foo@project.iam.gserviceaccount.com", ) + for bind_id_token in (True, False): + with pytest.raises( + ValueError, + match="If use_metadata_identity_endpoint is False, bind_id_token must not be set", + ): + credentials.IDTokenCredentials( + mock.Mock(), + "audience", + use_metadata_identity_endpoint=False, + bind_id_token=bind_id_token, + ) diff --git a/packages/google-auth/tests/conftest.py b/packages/google-auth/tests/conftest.py index fbce2575e929..3be7b4ad1b05 100644 --- a/packages/google-auth/tests/conftest.py +++ b/packages/google-auth/tests/conftest.py @@ -18,6 +18,9 @@ import pytest # type: ignore +from google.auth import environment_vars +from google.auth.transport import _mtls_helper + def pytest_configure(): """Load public certificate and private key.""" @@ -29,6 +32,35 @@ def pytest_configure(): with open(os.path.join(pytest.data_dir, "public_cert.pem"), "rb") as fh: pytest.public_cert_bytes = fh.read() + _mtls_helper._GKE_CREDENTIAL_BUNDLE_PATH = os.path.join( + pytest.data_dir, "nonexistent_gke_credential_bundle.pem" + ) + + +@pytest.fixture(autouse=True) +def clean_cert_config_env(monkeypatch, tmp_path): + monkeypatch.delenv( + environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, + raising=False, + ) + monkeypatch.delenv( + environment_vars.CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE, + raising=False, + ) + monkeypatch.delenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, + raising=False, + ) + monkeypatch.delenv( + environment_vars.CLOUDSDK_CONTEXT_AWARE_CERTIFICATE_CONFIG_FILE_PATH, + raising=False, + ) + monkeypatch.setattr( + _mtls_helper, + "_GKE_CREDENTIAL_BUNDLE_PATH", + str(tmp_path / "gke" / "credential" / "bundle" / "path"), + ) + @pytest.fixture def mock_non_existent_module(monkeypatch): diff --git a/packages/google-auth/tests/oauth2/test_id_token.py b/packages/google-auth/tests/oauth2/test_id_token.py index 456ca58b89f6..07fd2c3f1935 100644 --- a/packages/google-auth/tests/oauth2/test_id_token.py +++ b/packages/google-auth/tests/oauth2/test_id_token.py @@ -285,7 +285,18 @@ def test_fetch_id_token_credentials_optional_request(monkeypatch): mock_request.assert_called() -def test_fetch_id_token_credentials_from_metadata_server(monkeypatch): +@pytest.mark.parametrize( + "call_kwargs,expected_bind_id_token", + [ + ({}, None), + ({"bind_id_token": None}, None), + ({"bind_id_token": True}, True), + ({"bind_id_token": False}, False), + ], +) +def test_fetch_id_token_credentials_from_metadata_server( + monkeypatch, call_kwargs, expected_bind_id_token +): monkeypatch.delenv(environment_vars.CREDENTIALS, raising=False) mock_req = mock.Mock() @@ -294,24 +305,39 @@ def test_fetch_id_token_credentials_from_metadata_server(monkeypatch): with mock.patch( "google.auth.compute_engine.IDTokenCredentials.__init__", return_value=None ) as mock_init: - id_token.fetch_id_token_credentials(ID_TOKEN_AUDIENCE, request=mock_req) + id_token.fetch_id_token_credentials( + ID_TOKEN_AUDIENCE, request=mock_req, **call_kwargs + ) mock_init.assert_called_once_with( - mock_req, ID_TOKEN_AUDIENCE, use_metadata_identity_endpoint=True + mock_req, + ID_TOKEN_AUDIENCE, + use_metadata_identity_endpoint=True, + bind_id_token=expected_bind_id_token, ) -def test_fetch_id_token_credentials_from_explicit_cred_json_file(monkeypatch): +@pytest.mark.parametrize("bind_id_token", [None, True, False]) +def test_fetch_id_token_credentials_from_explicit_cred_json_file( + monkeypatch, bind_id_token +): monkeypatch.setenv(environment_vars.CREDENTIALS, SERVICE_ACCOUNT_FILE) - cred = id_token.fetch_id_token_credentials(ID_TOKEN_AUDIENCE) + cred = id_token.fetch_id_token_credentials( + ID_TOKEN_AUDIENCE, bind_id_token=bind_id_token + ) assert isinstance(cred, service_account.IDTokenCredentials) assert cred._target_audience == ID_TOKEN_AUDIENCE -def test_fetch_id_token_credentials_from_impersonated_cred_json_file(monkeypatch): +@pytest.mark.parametrize("bind_id_token", [None, True, False]) +def test_fetch_id_token_credentials_from_impersonated_cred_json_file( + monkeypatch, bind_id_token +): monkeypatch.setenv(environment_vars.CREDENTIALS, IMPERSONATED_SERVICE_ACCOUNT_FILE) - cred = id_token.fetch_id_token_credentials(ID_TOKEN_AUDIENCE) + cred = id_token.fetch_id_token_credentials( + ID_TOKEN_AUDIENCE, bind_id_token=bind_id_token + ) assert isinstance(cred, impersonated_credentials.IDTokenCredentials) assert cred._target_audience == ID_TOKEN_AUDIENCE @@ -373,7 +399,16 @@ def test_fetch_id_token_credentials_invalid_cred_path(monkeypatch): ) -def test_fetch_id_token(monkeypatch): +@pytest.mark.parametrize( + "call_kwargs,expected_bind_id_token", + [ + ({}, None), + ({"bind_id_token": None}, None), + ({"bind_id_token": True}, True), + ({"bind_id_token": False}, False), + ], +) +def test_fetch_id_token(call_kwargs, expected_bind_id_token): mock_cred = mock.MagicMock() mock_cred.token = "token" @@ -382,7 +417,9 @@ def test_fetch_id_token(monkeypatch): with mock.patch( "google.oauth2.id_token.fetch_id_token_credentials", return_value=mock_cred ) as mock_fetch: - token = id_token.fetch_id_token(mock_req, ID_TOKEN_AUDIENCE) - mock_fetch.assert_called_once_with(ID_TOKEN_AUDIENCE, request=mock_req) + token = id_token.fetch_id_token(mock_req, ID_TOKEN_AUDIENCE, **call_kwargs) + mock_fetch.assert_called_once_with( + ID_TOKEN_AUDIENCE, request=mock_req, bind_id_token=expected_bind_id_token + ) mock_cred.refresh.assert_called_once_with(mock_req) assert token == "token" diff --git a/packages/google-auth/tests/test_agent_identity_utils.py b/packages/google-auth/tests/test_agent_identity_utils.py index 7448dfa7aa0b..9ab375f840c2 100644 --- a/packages/google-auth/tests/test_agent_identity_utils.py +++ b/packages/google-auth/tests/test_agent_identity_utils.py @@ -16,6 +16,7 @@ import hashlib import json import os +import stat import urllib.parse from unittest import mock @@ -23,6 +24,7 @@ from cryptography import x509 from google.auth import _agent_identity_utils, environment_vars, exceptions +from google.auth.transport import _mtls_helper # A mock PEM-encoded certificate without an Agent Identity SPIFFE ID. NON_AGENT_IDENTITY_CERT_BYTES = ( @@ -48,34 +50,202 @@ ) -class TestAgentIdentityUtils: - @pytest.fixture(autouse=True) - def clean_env(self, monkeypatch): - monkeypatch.delenv( - environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, - raising=False, - ) - monkeypatch.delenv( - environment_vars.CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE, - raising=False, - ) +# Synthetic self-signed test certificate (CN=agent-identity-test) with SAN URI +# "spiffe://agents.global.proj-12345.system.id.goog/workload". +# To regenerate safely, use cryptography.x509.CertificateBuilder with a +# throwaway key (discard the private key; never commit it) and a dummy project +# ID in the SPIFFE URI. +AGENT_IDENTITY_CERT_BYTES = ( + b"-----BEGIN CERTIFICATE-----\n" + b"MIIDEjCCAfqgAwIBAgIUKZAXnXnxf8hsn+ojS1N8bN3hXrUwDQYJKoZIhvcNAQEL\n" + b"BQAwHjEcMBoGA1UEAwwTYWdlbnQtaWRlbnRpdHktdGVzdDAeFw0yNDAxMDEwMDAw\n" + b"MDBaFw0zNDAxMDEwMDAwMDBaMB4xHDAaBgNVBAMME2FnZW50LWlkZW50aXR5LXRl\n" + b"c3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC+3eSHkp1oBj7rFehL\n" + b"5VJmBF9KLZ5PXQuZNYrGSpxGJ0Dx1T5ancrl8e66AfAepw9O4zdA+8Afub39PQLh\n" + b"wMTEY3O9Uqetch+2apkwXQ+yYpnorgMwqykY77ptApA8WPHzEOj58FPtyC4UqXJ7\n" + b"YKVpN92lVi1l73XBn6axo/q72KjeEdssR6UMtAd3dbGqY3af/AZNppJWRmCMWs8Z\n" + b"oAuxTH5LqYuxwvCDfYLpmQSbv4IJ/UBjkvjRIlzPo2zHg9PMdf/j6Bg9n3kkFaVH\n" + b"Oep+Zm+DtdT8JvwnG3sQ8Qn/ZCqU0z3DkT//XaElAikLwr/1MFrVHhfrYEWnDvuy\n" + b"9PHxAgMBAAGjSDBGMEQGA1UdEQQ9MDuGOXNwaWZmZTovL2FnZW50cy5nbG9iYWwu\n" + b"cHJvai0xMjM0NS5zeXN0ZW0uaWQuZ29vZy93b3JrbG9hZDANBgkqhkiG9w0BAQsF\n" + b"AAOCAQEAWEyBk7TbetWeQTYEdJwH/pNmiqoCzDYcqCSuNqJhrItHuLmSAlKBGCz6\n" + b"I6ptzY6vT7ARXoW07ivf9Ffl3TMUDLjd5Tkfn1q8JjyM1Ugbfuq7rdF2g9+5h6wg\n" + b"tjeV10LqAimr+fFaNvRiGsMfokuwPyUKYe/9d6x5NhcTTNgMQDG5SWnRe1JqPy94\n" + b"GKilWCyzDl4qzHAU5gc7lZ/6WKbYPwjJDDT4/d3AvNx1O/cQCG7Mz4veDuG2Jqh+\n" + b"FPUqQ4G9RL4zdPuXlbKfSknkmZWld1+adyitai6BzDCG9zkEEVJmLE2/e3XvNC93\n" + b"fa2asspu5y/ViCmPS0J2rzWEk7zI5w==\n" + b"-----END CERTIFICATE-----\n" +) + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch): + monkeypatch.delenv( + environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, + raising=False, + ) + monkeypatch.delenv( + environment_vars.CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE, + raising=False, + ) + monkeypatch.delenv( + environment_vars.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, + raising=False, + ) + monkeypatch.delenv( + environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, + raising=False, + ) + + +class TestAgentIdentityUtils: @mock.patch("cryptography.x509.load_pem_x509_certificate") def test_parse_certificate(self, mock_load_cert): + mock_load_cert.return_value = mock.sentinel.cert result = _agent_identity_utils.parse_certificate(b"cert_bytes") mock_load_cert.assert_called_once_with(b"cert_bytes") - assert result == mock_load_cert.return_value + assert result == mock.sentinel.cert + + @pytest.mark.parametrize("invalid_input", [b"", None]) + def test_parse_certificate_empty_or_none_raises_value_error(self, invalid_input): + with pytest.raises( + ValueError, match="Certificate bytes cannot be empty or None" + ): + _agent_identity_utils.parse_certificate(invalid_input) + + @pytest.mark.parametrize( + "second_cert_block", + [ + # Valid Base64, invalid ASN.1 DER + b"-----BEGIN CERTIFICATE-----\n" + + base64.b64encode(b"not valid asn1 der payload") + + b"\n-----END CERTIFICATE-----\n", + # Corrupted Base64 + b"-----BEGIN CERTIFICATE-----\n!!!not_base64!!!\n-----END CERTIFICATE-----\n", + # Non-UTF-8 bytes + b"-----BEGIN CERTIFICATE-----\n\xff\xfe\xfd\n-----END CERTIFICATE-----\n", + # Truncated block missing END CERTIFICATE + b"-----BEGIN CERTIFICATE-----\nMIIB\n", + # Truncated block missing BEGIN CERTIFICATE + b"MIIB\n-----END CERTIFICATE-----\n", + ], + ) + def test_parse_certificate_full_chain_rejects_malformed_intermediate( + self, second_cert_block + ): + chain_bytes = NON_AGENT_IDENTITY_CERT_BYTES + second_cert_block + with pytest.raises(ValueError): + _agent_identity_utils.parse_certificate(chain_bytes) def test_is_certificate_file_ready_empty_path(self): result = _agent_identity_utils._is_certificate_file_ready("") assert result is False - def test_get_agent_identity_certificate_path_empty_env(self, monkeypatch): - monkeypatch.delenv( - environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, raising=False + def test_is_in_well_known_dir_empty_path(self): + assert _agent_identity_utils._is_in_well_known_dir("") is False + + def test_is_in_well_known_dir_resolves_symlinks(self, tmpdir, monkeypatch): + real_run_dir = tmpdir.mkdir("run") + var_dir = tmpdir.mkdir("var") + symlink_var_run = var_dir.join("run") + os.symlink(str(real_run_dir), str(symlink_var_run)) + + well_known_via_symlink = os.path.join( + str(symlink_var_run), + "secrets", + "workload-spiffe-credentials", + "certificates.pem", ) + monkeypatch.setattr( + "google.auth._agent_identity_utils._WELL_KNOWN_CERT_PATH", + well_known_via_symlink, + ) + + resolved_cert_path = os.path.join( + str(real_run_dir), + "secrets", + "workload-spiffe-credentials", + "certificates.pem", + ) + assert _agent_identity_utils._is_in_well_known_dir(resolved_cert_path) is True + + @mock.patch( + "google.auth.transport._mtls_helper._get_cert_config_path", + return_value=None, + ) + @mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=False, + ) + def test_get_agent_identity_certificate_path_empty_env( + self, mock_is_ready, mock_get_config + ): result = _agent_identity_utils.get_agent_identity_certificate_path() assert result is None + mock_is_ready.assert_called_once_with(_mtls_helper._GKE_CREDENTIAL_BUNDLE_PATH) + + @mock.patch("google.auth._agent_identity_utils.time.sleep") + @mock.patch( + "google.auth.transport._mtls_helper._get_cert_config_path", + return_value=None, + ) + @mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + ) + def test_get_agent_identity_certificate_path_gke_bundle_fallback( + self, mock_is_ready, mock_get_config, mock_sleep + ): + result = _agent_identity_utils.get_agent_identity_certificate_path() + assert result == _mtls_helper._GKE_CREDENTIAL_BUNDLE_PATH + mock_is_ready.assert_called_once_with(_mtls_helper._GKE_CREDENTIAL_BUNDLE_PATH) + mock_sleep.assert_not_called() + + @mock.patch( + "google.auth.transport._mtls_helper._get_cert_config_path", + return_value=None, + ) + @mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + side_effect=PermissionError("Access denied"), + ) + def test_get_agent_identity_certificate_path_gke_bundle_permission_error( + self, mock_is_ready, mock_get_config + ): + result = _agent_identity_utils.get_agent_identity_certificate_path() + assert result is None + mock_is_ready.assert_called_once_with(_mtls_helper._GKE_CREDENTIAL_BUNDLE_PATH) + + @mock.patch( + "google.auth.transport._mtls_helper._get_cert_config_path", + return_value="/home/user/.config/gcloud/certificate_config.json", + ) + @mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + ) + def test_get_agent_identity_certificate_path_no_gke_fallback_when_implicit_config_exists( + self, mock_is_ready, mock_get_config + ): + result = _agent_identity_utils.get_agent_identity_certificate_path() + assert result is None + mock_get_config.assert_called_once() + mock_is_ready.assert_not_called() + + @mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + ) + def test_get_agent_identity_certificate_path_no_gke_fallback_when_context_aware_env_set( + self, mock_is_ready, monkeypatch + ): + monkeypatch.setenv( + environment_vars.CLOUDSDK_CONTEXT_AWARE_CERTIFICATE_CONFIG_FILE_PATH, + "/missing/context_aware.json", + ) + result = _agent_identity_utils.get_agent_identity_certificate_path() + assert result is None + mock_is_ready.assert_not_called() @mock.patch("google.auth._agent_identity_utils.os.path.commonpath") @mock.patch( @@ -137,8 +307,6 @@ def test_is_certificate_file_ready_os_error(self, mock_stat): @mock.patch("google.auth._agent_identity_utils.os.stat") def test_is_certificate_file_ready_not_a_file(self, mock_stat): - import stat - mock_stat.return_value = mock.MagicMock(st_mode=stat.S_IFDIR, st_size=4096) result = _agent_identity_utils._is_certificate_file_ready("/path/to/cert") assert result is False @@ -207,37 +375,73 @@ def test_calculate_certificate_fingerprint(self): assert fingerprint == expected_fingerprint + @pytest.mark.parametrize( + "primary,fallback,expected", + [ + # Default (both unset) -> not opted out + (None, None, False), + # Explicit opt-in / non-false values on primary -> not opted out + ("true", None, False), + ("TRUE", None, False), + ("1", None, False), + ("invalid", None, False), + ("", None, False), + # Explicit opt-out via primary (including surrounding whitespace) -> opted out + ("false", None, True), + ("FALSE", None, True), + (" false ", None, True), + (" FALSE\n", None, True), + # Primary overrides fallback + ("false", "true", True), + ("true", "false", False), + # Empty or whitespace-only string on primary falls back to secondary + ("", " false ", True), + (" ", " false ", True), + # Fallback when primary is unset + (None, "false", True), + (None, "true", False), + ], + ) + def test_is_bound_token_opted_out(self, monkeypatch, primary, fallback, expected): + if primary is not None: + monkeypatch.setenv( + environment_vars.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, + primary, + ) + if fallback is not None: + monkeypatch.setenv( + environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, + fallback, + ) + assert _agent_identity_utils._is_bound_token_opted_out() is expected + @mock.patch("google.auth._agent_identity_utils._is_agent_identity_certificate") def test_should_request_bound_token(self, mock_is_agent, monkeypatch): - # Agent cert, default env var (opt-in) + # Agent cert, opted in mock_is_agent.return_value = True - monkeypatch.delenv( - environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, - raising=False, - ) - assert _agent_identity_utils.should_request_bound_token(mock.sentinel.cert) - - # Agent cert, explicit opt-in monkeypatch.setenv( - environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, + environment_vars.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true", ) assert _agent_identity_utils.should_request_bound_token(mock.sentinel.cert) - # Agent cert, explicit opt-out + # Agent cert, opted out + mock_is_agent.reset_mock() monkeypatch.setenv( - environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, + environment_vars.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "false", ) assert not _agent_identity_utils.should_request_bound_token(mock.sentinel.cert) + mock_is_agent.assert_not_called() - # Non-agent cert, opt-in + # Non-agent cert, opted in mock_is_agent.return_value = False monkeypatch.setenv( - environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, + environment_vars.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true", ) assert not _agent_identity_utils.should_request_bound_token(mock.sentinel.cert) + mock_is_agent.assert_called_once_with(mock.sentinel.cert) @mock.patch("google.auth._agent_identity_utils._is_agent_identity_certificate") def test_should_request_bound_token_explicit_use_client_cert_false( @@ -249,6 +453,7 @@ def test_should_request_bound_token_explicit_use_client_cert_false( "false", ) assert not _agent_identity_utils.should_request_bound_token(mock.sentinel.cert) + mock_is_agent.assert_not_called() @mock.patch("google.auth._agent_identity_utils._is_agent_identity_certificate") def test_should_request_bound_token_explicit_use_client_cert_invalid( @@ -260,6 +465,7 @@ def test_should_request_bound_token_explicit_use_client_cert_invalid( "foo", ) assert not _agent_identity_utils.should_request_bound_token(mock.sentinel.cert) + mock_is_agent.assert_not_called() @mock.patch("google.auth._agent_identity_utils._is_agent_identity_certificate") def test_should_request_bound_token_auto_enablement(self, mock_is_agent): @@ -381,9 +587,8 @@ def test_get_agent_identity_certificate_path_failure( _agent_identity_utils.get_agent_identity_certificate_path() assert "not found after multiple retries" in str(excinfo.value) - assert ( - environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES - in str(excinfo.value) + assert environment_vars.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN in str( + excinfo.value ) assert mock_sleep.call_count == len(_agent_identity_utils._POLLING_INTERVALS) @@ -412,9 +617,8 @@ def test_get_agent_identity_certificate_path_fail_fast_config_missing( mock_sleep.assert_not_called() @mock.patch("time.sleep") - @mock.patch("google.auth._agent_identity_utils.os.path.exists") def test_get_agent_identity_certificate_path_fail_fast_cert_missing( - self, mock_exists, mock_sleep, tmpdir, monkeypatch + self, mock_sleep, tmpdir, monkeypatch ): # Simulate config path outside well-known dir where config is valid but cert is missing. well_known_path = tmpdir.mkdir("well_known_cert").join("certificates.pem") @@ -434,10 +638,36 @@ def test_get_agent_identity_certificate_path_fail_fast_cert_missing( environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_path) ) - def exists_side_effect(path): - return path == str(config_path) + result = _agent_identity_utils.get_agent_identity_certificate_path() - mock_exists.side_effect = exists_side_effect + assert result is None + mock_sleep.assert_not_called() + + @mock.patch("time.sleep") + def test_get_agent_identity_certificate_path_well_known_config_external_missing_cert_no_poll( + self, mock_sleep, tmpdir, monkeypatch + ): + well_known_dir = tmpdir.mkdir("workload-spiffe-credentials") + external_dir = tmpdir.mkdir("external_certs") + monkeypatch.setattr( + "google.auth._agent_identity_utils._WELL_KNOWN_CERT_PATH", + str(well_known_dir.join("certificates.pem")), + ) + config_path = well_known_dir.join("config.json") + config_path.write( + json.dumps( + { + "cert_configs": { + "workload": { + "cert_path": str(external_dir.join("missing_cert.pem")) + } + } + } + ) + ) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_path) + ) result = _agent_identity_utils.get_agent_identity_certificate_path() @@ -445,9 +675,8 @@ def exists_side_effect(path): mock_sleep.assert_not_called() @mock.patch("time.sleep") - @mock.patch("google.auth._agent_identity_utils.os.path.exists") def test_get_agent_identity_certificate_path_cert_not_found( - self, mock_exists, mock_sleep, tmpdir, monkeypatch + self, mock_sleep, tmpdir, monkeypatch ): monkeypatch.setattr( "google.auth._agent_identity_utils._WELL_KNOWN_CERT_PATH", @@ -462,11 +691,6 @@ def test_get_agent_identity_certificate_path_cert_not_found( environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_path) ) - def exists_side_effect(path): - return path == str(config_path) - - mock_exists.side_effect = exists_side_effect - with pytest.raises(exceptions.RefreshError): _agent_identity_utils.get_agent_identity_certificate_path() @@ -589,89 +813,196 @@ def test_get_agent_identity_certificate_path_permission_error_cert_file( mock_sleep.assert_not_called() @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") - def test_get_and_parse_agent_identity_certificate_opted_out( - self, mock_get_path, monkeypatch + def test_get_agent_identity_certificate_and_bytes_success( + self, mock_get_path, tmpdir ): - monkeypatch.setenv( - environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, - "false", + cert_file = tmpdir.join("cert.pem") + cert_file.write_binary(NON_AGENT_IDENTITY_CERT_BYTES) + mock_get_path.return_value = str(cert_file) + + ( + cert, + cert_bytes, + ) = _agent_identity_utils.get_agent_identity_certificate_and_bytes() + + assert isinstance(cert, x509.Certificate) + assert cert_bytes == NON_AGENT_IDENTITY_CERT_BYTES + + @pytest.mark.parametrize("key_index", [0, 1, 2]) + @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") + def test_get_agent_identity_certificate_and_bytes_combined_bundle( + self, mock_get_path, tmpdir, key_index + ): + non_utf8_bag_attrs = b"Bag Attributes\n friendlyName: \xff\xfe\n" + private_key_pem = ( + b"-----BEGIN PRIVATE KEY-----\n" + b"MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC3\n" + b"-----END PRIVATE KEY-----\n" ) - result = _agent_identity_utils.get_and_parse_agent_identity_certificate() - assert result is None - mock_get_path.assert_not_called() + blocks = [ + AGENT_IDENTITY_CERT_BYTES.rstrip(b"\n") + b" \n", + NON_AGENT_IDENTITY_CERT_BYTES, + ] + blocks.insert(key_index, private_key_pem) + cert_file = tmpdir.join("credentialbundle.pem") + cert_file.write_binary(non_utf8_bag_attrs + b"".join(blocks)) + mock_get_path.return_value = str(cert_file) + + ( + cert, + cert_bytes, + ) = _agent_identity_utils.get_agent_identity_certificate_and_bytes() + + expected_certs = AGENT_IDENTITY_CERT_BYTES + NON_AGENT_IDENTITY_CERT_BYTES + assert isinstance(cert, x509.Certificate) + assert _agent_identity_utils._is_agent_identity_certificate(cert) + assert cert_bytes == expected_certs + assert cert_bytes.decode("utf-8") == expected_certs.decode("utf-8") @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") - def test_get_and_parse_agent_identity_certificate_no_path( - self, mock_get_path, monkeypatch + def test_get_agent_identity_certificate_and_bytes_no_cert_blocks( + self, mock_get_path, tmpdir ): - monkeypatch.setenv( - environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, - "true", + cert_file = tmpdir.join("empty_or_key_only.pem") + cert_file.write_binary( + b"-----BEGIN PRIVATE KEY-----\nMIIB\n-----END PRIVATE KEY-----\n" ) - mock_get_path.return_value = None - result = _agent_identity_utils.get_and_parse_agent_identity_certificate() - assert result is None - mock_get_path.assert_called_once() + mock_get_path.return_value = str(cert_file) + + with pytest.warns(UserWarning, match="No PEM certificate blocks found"): + ( + cert, + cert_bytes, + ) = _agent_identity_utils.get_agent_identity_certificate_and_bytes() + + assert cert is None + assert cert_bytes is None - @mock.patch("google.auth._agent_identity_utils.parse_certificate") @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") - def test_get_and_parse_agent_identity_certificate_success( - self, mock_get_path, mock_parse_certificate, monkeypatch + def test_get_agent_identity_certificate_and_bytes_os_error( + self, mock_get_path, tmpdir ): - monkeypatch.setenv( - environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, - "true", + missing_cert_file = tmpdir.join("deleted_during_rotation.pem") + mock_get_path.return_value = str(missing_cert_file) + + with pytest.warns( + UserWarning, match="Failed to read agent identity certificate file" + ): + ( + cert, + cert_bytes, + ) = _agent_identity_utils.get_agent_identity_certificate_and_bytes() + + assert cert is None + assert cert_bytes is None + + @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") + def test_get_agent_identity_certificate_and_bytes_corrupt_cert_value_error( + self, mock_get_path, tmpdir + ): + cert_file = tmpdir.join("corrupt_cert.pem") + cert_file.write_binary( + b"-----BEGIN CERTIFICATE-----\nnot_valid_base64_or_der\n-----END CERTIFICATE-----\n" ) - mock_get_path.return_value = "/fake/cert.pem" - mock_open = mock.mock_open(read_data=b"cert_bytes") + mock_get_path.return_value = str(cert_file) - with mock.patch("builtins.open", mock_open): - result = _agent_identity_utils.get_and_parse_agent_identity_certificate() + with pytest.warns( + UserWarning, match="Failed to parse agent identity certificate" + ): + ( + cert, + cert_bytes, + ) = _agent_identity_utils.get_agent_identity_certificate_and_bytes() - mock_open.assert_called_once_with("/fake/cert.pem", "rb") - mock_parse_certificate.assert_called_once_with(b"cert_bytes") - assert result == mock_parse_certificate.return_value + assert cert is None + assert cert_bytes is None + + @pytest.mark.parametrize( + "corrupt_intermediate", + [ + b"-----BEGIN CERTIFICATE-----\n" + + base64.b64encode(b"invalid_asn1_der_intermediate") + + b"\n-----END CERTIFICATE-----\n", + b"-----BEGIN CERTIFICATE-----\ntruncated_without_end_marker\n", + ], + ) + @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") + def test_get_agent_identity_certificate_and_bytes_corrupt_intermediate_warns( + self, mock_get_path, corrupt_intermediate, tmpdir + ): + cert_file = tmpdir.join("chain_with_corrupt_intermediate.pem") + cert_file.write_binary(AGENT_IDENTITY_CERT_BYTES + corrupt_intermediate) + mock_get_path.return_value = str(cert_file) + + with pytest.warns( + UserWarning, match="Failed to parse agent identity certificate" + ): + ( + cert, + cert_bytes, + ) = _agent_identity_utils.get_agent_identity_certificate_and_bytes() + + assert cert is None + assert cert_bytes is None @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") - def test_get_and_parse_agent_identity_certificate_use_client_cert_false( + def test_get_agent_identity_certificate_and_bytes_opted_out( self, mock_get_path, monkeypatch ): monkeypatch.setenv( - environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, + environment_vars.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "false", ) - result = _agent_identity_utils.get_and_parse_agent_identity_certificate() - assert result is None + ( + cert, + cert_bytes, + ) = _agent_identity_utils.get_agent_identity_certificate_and_bytes() + assert cert is None + assert cert_bytes is None mock_get_path.assert_not_called() @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") - def test_get_and_parse_agent_identity_certificate_use_client_cert_invalid( + def test_get_agent_identity_certificate_and_bytes_no_path(self, mock_get_path): + mock_get_path.return_value = None + ( + cert, + cert_bytes, + ) = _agent_identity_utils.get_agent_identity_certificate_and_bytes() + assert cert is None + assert cert_bytes is None + mock_get_path.assert_called_once() + + @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") + def test_get_agent_identity_certificate_and_bytes_use_client_cert_false( self, mock_get_path, monkeypatch ): monkeypatch.setenv( environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, - "foo", + "false", ) - result = _agent_identity_utils.get_and_parse_agent_identity_certificate() - assert result is None + ( + cert, + cert_bytes, + ) = _agent_identity_utils.get_agent_identity_certificate_and_bytes() + assert cert is None + assert cert_bytes is None mock_get_path.assert_not_called() @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") - def test_get_and_parse_agent_identity_certificate_file_read_error( + def test_get_agent_identity_certificate_and_bytes_use_client_cert_invalid( self, mock_get_path, monkeypatch ): monkeypatch.setenv( - environment_vars.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, - "true", + environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, + "foo", ) - mock_get_path.return_value = "/fake/cert.pem" - mock_open = mock.mock_open() - mock_open.side_effect = PermissionError("Permission denied") - - with mock.patch("builtins.open", mock_open): - result = _agent_identity_utils.get_and_parse_agent_identity_certificate() - - assert result is None + ( + cert, + cert_bytes, + ) = _agent_identity_utils.get_agent_identity_certificate_and_bytes() + assert cert is None + assert cert_bytes is None + mock_get_path.assert_not_called() def test_get_cached_cert_fingerprint_no_cert(self): with pytest.raises(ValueError, match="mTLS connection is not configured."): @@ -709,3 +1040,20 @@ def test_is_agent_identity_certificate_raises_import_error(self): def test_calculate_certificate_fingerprint_raises_import_error(self): with pytest.raises(ImportError, match="The cryptography library is required"): _agent_identity_utils.calculate_certificate_fingerprint(mock.sentinel.cert) + + @mock.patch("google.auth._agent_identity_utils.get_agent_identity_certificate_path") + def test_get_agent_identity_certificate_and_bytes_warns_without_cryptography( + self, mock_get_path, tmpdir + ): + cert_file = tmpdir.join("cert.pem") + cert_file.write_binary(AGENT_IDENTITY_CERT_BYTES) + mock_get_path.return_value = str(cert_file) + + with pytest.warns(UserWarning, match="The cryptography library is required"): + ( + cert, + cert_bytes, + ) = _agent_identity_utils.get_agent_identity_certificate_and_bytes() + + assert cert is None + assert cert_bytes is None diff --git a/packages/google-auth/tests/test_identity_pool.py b/packages/google-auth/tests/test_identity_pool.py index 7d6a7c1b68d8..a6b59ef4552a 100644 --- a/packages/google-auth/tests/test_identity_pool.py +++ b/packages/google-auth/tests/test_identity_pool.py @@ -1005,7 +1005,7 @@ def test_retrieve_subject_token_json_file(self): @mock.patch( "google.auth.transport._mtls_helper._get_workload_cert_and_key_paths", - return_value=(CERT_FILE, KEY_FILE), + return_value=(CERT_FILE, KEY_FILE, None), ) def test_retrieve_subject_token_certificate_default( self, mock_get_workload_cert_and_key_paths @@ -1020,7 +1020,7 @@ def test_retrieve_subject_token_certificate_default( @mock.patch( "google.auth.transport._mtls_helper._get_workload_cert_and_key_paths", - return_value=(CERT_FILE, KEY_FILE), + return_value=(CERT_FILE, KEY_FILE, None), ) def test_retrieve_subject_token_certificate_non_default_path( self, mock_get_workload_cert_and_key_paths @@ -1035,7 +1035,7 @@ def test_retrieve_subject_token_certificate_non_default_path( @mock.patch( "google.auth.transport._mtls_helper._get_workload_cert_and_key_paths", - return_value=(CERT_FILE, KEY_FILE), + return_value=(CERT_FILE, KEY_FILE, None), ) def test_retrieve_subject_token_certificate_trust_chain_with_leaf( self, mock_get_workload_cert_and_key_paths @@ -1049,7 +1049,7 @@ def test_retrieve_subject_token_certificate_trust_chain_with_leaf( @mock.patch( "google.auth.transport._mtls_helper._get_workload_cert_and_key_paths", - return_value=(CERT_FILE, KEY_FILE), + return_value=(CERT_FILE, KEY_FILE, None), ) def test_retrieve_subject_token_certificate_trust_chain_without_leaf( self, mock_get_workload_cert_and_key_paths @@ -1063,7 +1063,7 @@ def test_retrieve_subject_token_certificate_trust_chain_without_leaf( @mock.patch( "google.auth.transport._mtls_helper._get_workload_cert_and_key_paths", - return_value=(CERT_FILE, KEY_FILE), + return_value=(CERT_FILE, KEY_FILE, None), ) def test_retrieve_subject_token_certificate_trust_chain_invalid_order( self, mock_get_workload_cert_and_key_paths @@ -1081,7 +1081,7 @@ def test_retrieve_subject_token_certificate_trust_chain_invalid_order( @mock.patch( "google.auth.transport._mtls_helper._get_workload_cert_and_key_paths", - return_value=(CERT_FILE, KEY_FILE), + return_value=(CERT_FILE, KEY_FILE, None), ) def test_retrieve_subject_token_certificate_trust_chain_file_does_not_exist( self, mock_get_workload_cert_and_key_paths @@ -1102,7 +1102,7 @@ def test_retrieve_subject_token_certificate_trust_chain_file_does_not_exist( @mock.patch( "google.auth.transport._mtls_helper._get_workload_cert_and_key_paths", - return_value=(CERT_FILE, KEY_FILE), + return_value=(CERT_FILE, KEY_FILE, None), ) def test_retrieve_subject_token_certificate_invalid_trust_chain_file( self, mock_get_workload_cert_and_key_paths @@ -1764,7 +1764,7 @@ def test_refresh_success_supplier_without_impersonation_url(self): @mock.patch( "google.auth.transport._mtls_helper._get_workload_cert_and_key_paths", - return_value=("cert", "key"), + return_value=("cert", "key", None), ) def test_get_mtls_certs(self, mock_get_workload_cert_and_key_paths): credentials = self.make_credentials( @@ -1789,7 +1789,7 @@ def test_get_mtls_certs_invalid(self): @mock.patch( "google.auth.transport._mtls_helper._get_workload_cert_and_key_paths", - return_value=(None, None), + return_value=(None, None, None), ) def test_get_cert_bytes_none_raises_error( self, mock_get_workload_cert_and_key_paths @@ -1819,7 +1819,7 @@ def test_refresh_cert_error_raises_refresh_error(self, mock_get_cert_bytes): credentials.refresh(None) assert excinfo.match( - "Failed to retrieve certificate bytes for external account credentials" + "Failed to retrieve or parse certificate for external account credentials" ) @mock.patch.object( @@ -1835,9 +1835,30 @@ def test_refresh_os_error_raises_refresh_error(self, mock_get_cert_bytes): with pytest.raises(exceptions.RefreshError) as excinfo: credentials.refresh(None) - msg = "Failed to retrieve certificate bytes for external" + msg = "Failed to retrieve or parse certificate for external" assert excinfo.match(msg + " account credentials") + @mock.patch( + "google.auth._agent_identity_utils.parse_certificate", + side_effect=ValueError("malformed certificate chain"), + ) + @mock.patch.object( + identity_pool.Credentials, "_get_cert_bytes", return_value=b"bad_cert" + ) + def test_refresh_parse_certificate_value_error_raises_refresh_error( + self, mock_get_cert_bytes, mock_parse_certificate + ): + credentials = self.make_credentials( + credential_source=self.CREDENTIAL_SOURCE_CERTIFICATE.copy() + ) + + with pytest.raises(exceptions.RefreshError) as excinfo: + credentials.refresh(None) + + assert excinfo.match( + "Failed to retrieve or parse certificate for external account credentials" + ) + @mock.patch("google.auth._agent_identity_utils.parse_certificate") @mock.patch( "google.auth._agent_identity_utils.should_request_bound_token", diff --git a/packages/google-auth/tests/transport/aio/test_sessions_mtls.py b/packages/google-auth/tests/transport/aio/test_sessions_mtls.py index b390a8dfe91b..a96a1e5a404e 100644 --- a/packages/google-auth/tests/transport/aio/test_sessions_mtls.py +++ b/packages/google-auth/tests/transport/aio/test_sessions_mtls.py @@ -539,7 +539,14 @@ async def test_psc_endpoint_triggers_cert_rotation(self): await session.close() @pytest.mark.asyncio - async def test_non_mtls_url_bypasses_rotation(self): + @pytest.mark.parametrize( + "non_mtls_url", + [ + "https://pubsub.googleapis.com/test", + "https://my-service-xyz-uc.a.run.app/test", + ], + ) + async def test_non_mtls_url_bypasses_rotation(self, non_mtls_url): """Verifies that standard non-mTLS URLs bypass certificate rotation.""" mock_creds = mock.AsyncMock(spec=credentials.Credentials) mock_creds.before_request = mock.AsyncMock(return_value=None) @@ -570,7 +577,7 @@ async def test_non_mtls_url_bypasses_rotation(self): session, "configure_mtls_channel", new_callable=mock.AsyncMock ) as mock_conf, ): - resp = await session.request("GET", "https://pubsub.googleapis.com/test") + resp = await session.request("GET", non_mtls_url) assert resp == mock_resp_200 mock_check.assert_not_called() @@ -862,7 +869,14 @@ async def slow_refresh(*args, **kwargs): await session.close() @pytest.mark.asyncio - async def test_cert_rotation_with_completed_mtls_init_task(self): + @pytest.mark.parametrize( + "mtls_url", + [ + "https://pubsub.mtls.googleapis.com/test", + "https://my-service-123456.us-central1.mtls.run.app/test", + ], + ) + async def test_cert_rotation_with_completed_mtls_init_task(self, mtls_url): """ Verifies that when _mtls_init_task is already completed, receiving a 401 with rotated certificates properly resets _mtls_init_task and reconfigures mTLS. @@ -904,10 +918,8 @@ async def dummy_completed(): ) as mock_conf, ): mock_check.return_value = (new_cert, new_key, b"old_fp", b"new_fp") - # Must use a hostname matching _MTLS_URL_PREFIXES (e.g. *.mtls.googleapis.com) - resp = await session.request( - "GET", "https://pubsub.mtls.googleapis.com/test" - ) + # Must use a hostname matching _mtls_helper.is_mtls_endpoint + resp = await session.request("GET", mtls_url) assert resp == mock_resp_200 mock_conf.assert_called_once() # Verify the previous completed task was cleared during rotation diff --git a/packages/google-auth/tests/transport/test__mtls_helper.py b/packages/google-auth/tests/transport/test__mtls_helper.py index 675f54b46324..f335aff2712d 100644 --- a/packages/google-auth/tests/transport/test__mtls_helper.py +++ b/packages/google-auth/tests/transport/test__mtls_helper.py @@ -56,7 +56,7 @@ def check_cert_and_key(content, expected_cert, expected_key): success = True cert_match = re.findall(_mtls_helper._CERT_REGEX, content) - success = success and len(cert_match) == 1 and cert_match[0] == expected_cert + success = success and len(cert_match) >= 1 and b"".join(cert_match) == expected_cert key_match = re.findall(_mtls_helper._KEY_REGEX, content) success = success and len(key_match) == 1 and key_match[0] == expected_key @@ -67,32 +67,40 @@ def check_cert_and_key(content, expected_cert, expected_key): class TestCertAndKeyRegex(object): def test_cert_and_key(self): # Test single cert and single key - check_cert_and_key( + assert check_cert_and_key( pytest.public_cert_bytes + pytest.private_key_bytes, pytest.public_cert_bytes, pytest.private_key_bytes, ) - check_cert_and_key( + assert check_cert_and_key( pytest.private_key_bytes + pytest.public_cert_bytes, pytest.public_cert_bytes, pytest.private_key_bytes, ) # Test cert chain and single key - check_cert_and_key( + assert check_cert_and_key( pytest.public_cert_bytes + pytest.public_cert_bytes + pytest.private_key_bytes, pytest.public_cert_bytes + pytest.public_cert_bytes, pytest.private_key_bytes, ) - check_cert_and_key( + assert check_cert_and_key( pytest.private_key_bytes + pytest.public_cert_bytes + pytest.public_cert_bytes, pytest.public_cert_bytes + pytest.public_cert_bytes, pytest.private_key_bytes, ) + # Test interleaved key between certificates in a combined bundle + assert check_cert_and_key( + pytest.public_cert_bytes + + pytest.private_key_bytes + + pytest.public_cert_bytes, + pytest.public_cert_bytes + pytest.public_cert_bytes, + pytest.private_key_bytes, + ) def test_key(self): # Create some fake keys for regex check. @@ -109,13 +117,13 @@ def test_key(self): /fy3ZpsL7WqgsZS7Q+0VRK8gKfqkxg5OYQIDAQAB -----END EC PRIVATE KEY-----""" - check_cert_and_key( + assert check_cert_and_key( pytest.public_cert_bytes + KEY, pytest.public_cert_bytes, KEY ) - check_cert_and_key( + assert check_cert_and_key( pytest.public_cert_bytes + RSA_KEY, pytest.public_cert_bytes, RSA_KEY ) - check_cert_and_key( + assert check_cert_and_key( pytest.public_cert_bytes + EC_KEY, pytest.public_cert_bytes, EC_KEY ) @@ -200,6 +208,21 @@ def test_success_with_cert_chain(self, mock_popen): assert key == ENCRYPTED_EC_PRIVATE_KEY assert passphrase == PASSPHRASE_VALUE + @pytest.mark.parametrize( + "trailing_bytes", + [ + b"-----BEGIN CERTIFICATE-----\nMIIB\n", + b"MIIB\n-----END CERTIFICATE-----\n", + ], + ) + @mock.patch("subprocess.Popen", autospec=True) + def test_truncated_cert_chain_raises_error(self, mock_popen, trailing_bytes): + mock_popen.return_value = self.create_mock_process( + pytest.public_cert_bytes + trailing_bytes + pytest.private_key_bytes, b"" + ) + with pytest.raises(exceptions.ClientCertError): + _mtls_helper._run_cert_provider_command(["command"]) + @mock.patch("subprocess.Popen", autospec=True) def test_missing_cert(self, mock_popen): mock_popen.return_value = self.create_mock_process( @@ -743,6 +766,37 @@ def test_no_key_file( with pytest.raises(exceptions.ClientCertError): _mtls_helper._get_workload_cert_and_key("") + @pytest.mark.parametrize( + "workload", + [ + {"cert_path": None, "key_path": "path/to/key"}, + {"cert_path": "path/to/cert", "key_path": None}, + {"cert_path": None, "key_path": None}, + {"cert_path": "", "key_path": "path/to/key"}, + {"cert_path": "path/to/cert", "key_path": ""}, + {"cert_path": 123, "key_path": "path/to/key"}, + {"cert_path": "path/to/cert", "key_path": 123}, + ], + ) + @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) + @mock.patch( + "google.auth.transport._mtls_helper._get_cert_config_path", autospec=True + ) + def test_invalid_cert_or_key_path_value( + self, + mock_get_cert_config_path, + mock_load_json_file, + workload, + ): + mock_get_cert_config_path.return_value = "/path/to/cert" + mock_load_json_file.return_value = {"cert_configs": {"workload": workload}} + + with pytest.raises( + exceptions.ClientCertError, + match='Workload certificate configuration is missing "cert_path" or "key_path"', + ): + _mtls_helper._get_workload_cert_and_key("") + class TestReadCertAndKeyFile(object): def test_success(self): @@ -779,6 +833,40 @@ def test_invalid_key_file(self): with pytest.raises(exceptions.ClientCertError): _mtls_helper._read_cert_and_key_files(cert_path, key_path) + def test_combined_bundle_with_interleaved_key(self, tmp_path): + bundle_file = tmp_path / "credentialbundle.pem" + bundle_file.write_bytes( + pytest.public_cert_bytes.rstrip(b"\n") + + pytest.private_key_bytes + + pytest.public_cert_bytes + ) + actual_cert, actual_key = _mtls_helper._read_cert_and_key_files( + str(bundle_file), str(bundle_file) + ) + assert actual_cert == pytest.public_cert_bytes + pytest.public_cert_bytes + assert actual_key == pytest.private_key_bytes + + def test_multiple_keys_raises_error(self, tmp_path): + cert_path = os.path.join(pytest.data_dir, "public_cert.pem") + key_file = tmp_path / "two_keys.pem" + key_file.write_bytes(pytest.private_key_bytes + pytest.private_key_bytes) + with pytest.raises(exceptions.ClientCertError): + _mtls_helper._read_cert_and_key_files(cert_path, str(key_file)) + + @pytest.mark.parametrize( + "trailing_bytes", + [ + b"-----BEGIN CERTIFICATE-----\nMIIB\n", + b"MIIB\n-----END CERTIFICATE-----\n", + ], + ) + def test_truncated_multi_cert_raises_error(self, tmp_path, trailing_bytes): + cert_file = tmp_path / "truncated_chain.pem" + cert_file.write_bytes(pytest.public_cert_bytes + trailing_bytes) + key_path = os.path.join(pytest.data_dir, "privatekey.pem") + with pytest.raises(exceptions.ClientCertError): + _mtls_helper._read_cert_and_key_files(str(cert_file), key_path) + class TestGetCertConfigPath(object): def test_success_with_override(self): @@ -1223,6 +1311,21 @@ def test_check_parameters_for_unauthorized_response_without_cached_cert( mock_call_client_cert_callback.assert_called_once() mock_agent_identity_utils.get_cached_cert_fingerprint.assert_not_called() + @mock.patch("google.auth.transport._mtls_helper.call_client_cert_callback") + @mock.patch("google.auth.transport._mtls_helper._agent_identity_utils") + def test_check_parameters_for_unauthorized_response_no_call_cert( + self, mock_agent_identity_utils, mock_call_client_cert_callback + ): + mock_call_client_cert_callback.return_value = (None, None) + + result = _mtls_helper.check_parameters_for_unauthorized_response( + cached_cert=CERT_MOCK_VAL + ) + + assert result == (None, None, None, None) + mock_call_client_cert_callback.assert_called_once() + mock_agent_identity_utils.parse_certificate.assert_not_called() + @mock.patch("google.auth.transport._mtls_helper.get_client_ssl_credentials") def test_call_client_cert_callback(self, mock_get_client_ssl_credentials): mock_get_client_ssl_credentials.return_value = ( @@ -1913,6 +2016,12 @@ class TestIsMtlsEndpoint(object): "https://p.googleapis.com/", "https://p.googleapis.com:443/v1", "https://p.googleapis.com.", + "https://mtls.run.app", + "https://mtls.run.app/", + "https://my-service-123456.us-central1.mtls.run.app", + "https://my-service-123456.us-central1.mtls.run.app/v1/invocations", + "https://tag---my-service-123456.us-central1.mtls.run.app.", + b"https://my-service-123456.us-central1.mtls.run.app", ], ) def test_is_mtls_endpoint_true(self, url): @@ -1926,6 +2035,11 @@ def test_is_mtls_endpoint_true(self, url): "https://storage.googleapis.com:443/b/my-bucket", "https://storage.googleapis.com:443/bucket/mtls.googleapis.com?pageSize=10#frag", "https://storage.googleapis.com/bucket/mtls.googleapis.com", + "https://my-service-xyz-uc.a.run.app", + "https://my-service-123456.us-central1.run.app", + "https://my-service-xyz-uc.a.run.app/mtls.run.app", + "https://fake-mtls.run.app/v1", + "https://fake-mtls.run.app.attacker.com/v1", "https://[2001:db8::1]:443/mtls.googleapis.com", "https://[::1]:8443/mtls.googleapis.com", "https://logging.googleapis.com/v2/entries?filter=mtls.googleapis.com", @@ -1944,6 +2058,7 @@ def test_is_mtls_endpoint_true(self, url): "https://storage.googleapis.com/bucket/mtls.googleapis.com" ), "https://.", + "https://[::1", "", None, 123, @@ -1952,3 +2067,215 @@ def test_is_mtls_endpoint_true(self, url): ) def test_is_mtls_endpoint_false(self, url): assert _mtls_helper.is_mtls_endpoint(url) is False + + +class TestReadCredentialBundleFile(object): + @pytest.mark.parametrize("key_index", [0, 1]) + def test_single_cert_and_key(self, tmpdir, key_index): + bundle_file = tmpdir.join("x509.credential-bundle.private-key.pem") + blocks = [pytest.public_cert_bytes] + blocks.insert(key_index, pytest.private_key_bytes) + bundle_file.write_binary(b"".join(blocks)) + + with mock.patch("builtins.open", wraps=open) as mock_open: + cert_bytes, key_bytes = _mtls_helper._read_credential_bundle_file( + str(bundle_file) + ) + mock_open.assert_called_once_with(str(bundle_file), "rb") + assert cert_bytes == pytest.public_cert_bytes + assert key_bytes == pytest.private_key_bytes + + @pytest.mark.parametrize("key_index", [0, 1, 2]) + def test_multi_cert_chain_and_key(self, tmpdir, key_index): + bundle_file = tmpdir.join("x509.credential-bundle.private-key.pem") + blocks = [ + pytest.public_cert_bytes.rstrip(b"\n") + b" \n", + pytest.public_cert_bytes, + ] + blocks.insert(key_index, pytest.private_key_bytes) + bundle_file.write_binary(b"".join(blocks)) + + cert_bytes, key_bytes = _mtls_helper._read_credential_bundle_file( + str(bundle_file) + ) + assert cert_bytes == pytest.public_cert_bytes + pytest.public_cert_bytes + assert key_bytes == pytest.private_key_bytes + + def test_missing_cert_raises_error(self, tmpdir): + bundle_file = tmpdir.join("key_only.pem") + bundle_file.write_binary(pytest.private_key_bytes) + + with pytest.raises(exceptions.ClientCertError, match="at least one PEM"): + _mtls_helper._read_credential_bundle_file(str(bundle_file)) + + def test_missing_key_raises_error(self, tmpdir): + bundle_file = tmpdir.join("cert_only.pem") + bundle_file.write_binary(pytest.public_cert_bytes) + + with pytest.raises( + exceptions.ClientCertError, match="a single PEM formatted private key" + ): + _mtls_helper._read_credential_bundle_file(str(bundle_file)) + + def test_multiple_keys_raises_error(self, tmpdir): + bundle_file = tmpdir.join("multi_key.pem") + bundle_file.write_binary( + pytest.public_cert_bytes + + pytest.private_key_bytes + + pytest.private_key_bytes + ) + + with pytest.raises( + exceptions.ClientCertError, match="a single PEM formatted private key" + ): + _mtls_helper._read_credential_bundle_file(str(bundle_file)) + + def test_os_error_propagates(self, tmpdir): + missing_bundle = str(tmpdir.join("nonexistent_rotated_symlink.pem")) + with pytest.raises(OSError): + _mtls_helper._read_credential_bundle_file(missing_bundle) + + +class TestGkeCredentialBundleDiscovery(object): + @mock.patch( + "google.auth.transport._mtls_helper._get_cert_config_path", + return_value=None, + autospec=True, + ) + @mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + autospec=True, + ) + @mock.patch( + "google.auth.transport._mtls_helper._read_credential_bundle_file", + autospec=True, + ) + def test_get_workload_cert_and_key_gke_fallback( + self, mock_read_bundle, mock_is_ready, mock_get_config_path + ): + mock_read_bundle.return_value = ( + pytest.public_cert_bytes, + pytest.private_key_bytes, + ) + + cert, key = _mtls_helper._get_workload_cert_and_key() + assert cert == pytest.public_cert_bytes + assert key == pytest.private_key_bytes + mock_is_ready.assert_called_once_with(_mtls_helper._GKE_CREDENTIAL_BUNDLE_PATH) + mock_read_bundle.assert_called_once_with( + _mtls_helper._GKE_CREDENTIAL_BUNDLE_PATH + ) + + @mock.patch( + "google.auth.transport._mtls_helper.path.exists", + return_value=False, + autospec=True, + ) + @mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + autospec=True, + ) + def test_get_workload_cert_and_key_no_gke_fallback_when_explicit_path_set( + self, mock_is_ready, mock_exists + ): + cert, key = _mtls_helper._get_workload_cert_and_key("/missing/config.json") + assert cert is None + assert key is None + mock_exists.assert_called_once_with("/missing/config.json") + mock_is_ready.assert_not_called() + + @mock.patch( + "google.auth.transport._mtls_helper.path.exists", + return_value=False, + autospec=True, + ) + @mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + autospec=True, + ) + def test_get_workload_cert_and_key_no_gke_fallback_when_explicit_env_set( + self, mock_is_ready, mock_exists, monkeypatch + ): + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, "/missing/config.json" + ) + cert, key = _mtls_helper._get_workload_cert_and_key() + assert cert is None + assert key is None + mock_exists.assert_called_once_with("/missing/config.json") + mock_is_ready.assert_not_called() + + @pytest.mark.parametrize("include_context_aware", [True, False]) + @mock.patch( + "google.auth.transport._mtls_helper.path.exists", + return_value=False, + autospec=True, + ) + @mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + autospec=True, + ) + def test_get_workload_cert_and_key_no_gke_fallback_when_context_aware_env_set( + self, mock_is_ready, mock_exists, include_context_aware, monkeypatch + ): + monkeypatch.setenv( + environment_vars.CLOUDSDK_CONTEXT_AWARE_CERTIFICATE_CONFIG_FILE_PATH, + "/missing/context_aware_config.json", + ) + cert, key = _mtls_helper._get_workload_cert_and_key( + include_context_aware=include_context_aware + ) + assert cert is None + assert key is None + assert mock_exists.call_count == 1 + mock_is_ready.assert_not_called() + + @mock.patch( + "google.auth.transport._mtls_helper._get_cert_config_path", + return_value=None, + autospec=True, + ) + @mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + autospec=True, + ) + def test_check_use_client_cert_gke_auto_enabled( + self, mock_is_ready, mock_get_config_path + ): + assert _mtls_helper.check_use_client_cert() is True + mock_is_ready.assert_called_once_with(_mtls_helper._GKE_CREDENTIAL_BUNDLE_PATH) + + @mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + autospec=True, + ) + def test_check_use_client_cert_gke_disabled_by_env( + self, mock_is_ready, monkeypatch + ): + monkeypatch.setenv(environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false") + assert _mtls_helper.check_use_client_cert() is False + mock_is_ready.assert_not_called() + + @mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + autospec=True, + ) + def test_no_gke_fallback_when_implicit_config_has_no_workload( + self, mock_is_ready, tmpdir, monkeypatch + ): + config_file = tmpdir.join("certificate_config.json") + config_file.write('{"cert_configs": {}}') + monkeypatch.setattr( + _mtls_helper._cloud_sdk, "get_config_path", lambda: str(tmpdir) + ) + + assert _mtls_helper._get_workload_cert_and_key() == (None, None) + assert _mtls_helper.check_use_client_cert() is False + mock_is_ready.assert_not_called() diff --git a/packages/google-auth/tests/transport/test_mtls.py b/packages/google-auth/tests/transport/test_mtls.py index 7caf63861751..5899a1b1c73b 100644 --- a/packages/google-auth/tests/transport/test_mtls.py +++ b/packages/google-auth/tests/transport/test_mtls.py @@ -46,7 +46,7 @@ def side_effect(path): # Assert assert result is True - mock_get_cert.assert_called_once_with(include_context_aware=True) + mock_get_cert.assert_called_once_with(None, True) mock_check.assert_any_call(_mtls_helper.CONTEXT_AWARE_METADATA_PATH) @@ -63,13 +63,21 @@ def test_has_default_client_cert_source_without_context_aware( result = mtls.has_default_client_cert_source(include_context_aware=False) assert result is False - mock_get_cert.assert_called_once_with(include_context_aware=False) + mock_get_cert.assert_called_once_with(None, False) mock_check.assert_not_called() +@mock.patch( + "google.auth.transport._mtls_helper._load_json_file", + return_value={ + "cert_configs": {"workload": {"cert_path": "cert.pem", "key_path": "key.pem"}} + }, +) @mock.patch("google.auth.transport._mtls_helper._check_config_path") @mock.patch("google.auth.transport._mtls_helper._get_cert_config_path") -def test_has_default_client_cert_source_falls_back(mock_get_cert, mock_check): +def test_has_default_client_cert_source_falls_back( + mock_get_cert, mock_check, mock_load_json +): """ Tests that it checks X.509 WIF first, and if found, returns True without checking context aware metadata. """ @@ -83,21 +91,28 @@ def test_has_default_client_cert_source_falls_back(mock_get_cert, mock_check): # Assert assert result is True # Verify the sequence of calls - mock_get_cert.assert_called_once_with(include_context_aware=True) + mock_get_cert.assert_called_once_with(None, True) mock_check.assert_not_called() +@mock.patch( + "google.auth.transport._mtls_helper._load_json_file", + return_value={ + "cert_configs": {"workload": {"cert_path": "cert.pem", "key_path": "key.pem"}} + }, + autospec=True, +) @mock.patch("google.auth.transport._mtls_helper._get_cert_config_path", autospec=True) @mock.patch("google.auth.transport._mtls_helper._check_config_path", autospec=True) def test_has_default_client_cert_source_env_var_success( - check_config_path, get_cert_config_path + check_config_path, get_cert_config_path, mock_load_json ): check_config_path.return_value = None get_cert_config_path.return_value = "/absolute/path/to/cert.json" assert mtls.has_default_client_cert_source(True) - get_cert_config_path.assert_called_with(include_context_aware=True) + get_cert_config_path.assert_called_with(None, True) @mock.patch("google.auth.transport._mtls_helper._get_cert_config_path", autospec=True) @@ -476,3 +491,70 @@ def test_get_default_ssl_context_no_default_source(mock_has_default, mock_should result = mtls.get_default_ssl_context() assert result is None + + +@mock.patch( + "google.auth.transport._mtls_helper._get_cert_config_path", + return_value=None, + autospec=True, +) +@mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + autospec=True, +) +def test_has_default_client_cert_source_gke_bundle(mock_is_ready, mock_get_cert): + assert mtls.has_default_client_cert_source(include_context_aware=False) is True + mock_is_ready.assert_called_once_with(_mtls_helper._GKE_CREDENTIAL_BUNDLE_PATH) + + +@mock.patch( + "google.auth.transport._mtls_helper._load_json_file", + return_value={"cert_configs": {}}, + autospec=True, +) +@mock.patch( + "google.auth.transport._mtls_helper._get_cert_config_path", + return_value="/home/user/.config/gcloud/certificate_config.json", + autospec=True, +) +@mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + autospec=True, +) +def test_has_default_client_cert_source_config_without_workload_no_gke_fallback( + mock_is_ready, mock_get_cert, mock_load_json +): + assert mtls.has_default_client_cert_source(include_context_aware=False) is False + mock_is_ready.assert_not_called() + + +@pytest.mark.parametrize( + "side_effect", + [ + exceptions.ClientCertError("Invalid JSON"), + OSError("Permission denied"), + ], +) +@mock.patch( + "google.auth.transport._mtls_helper._load_json_file", + autospec=True, +) +@mock.patch( + "google.auth.transport._mtls_helper._get_cert_config_path", + return_value="/home/user/.config/gcloud/certificate_config.json", + autospec=True, +) +@mock.patch( + "google.auth._agent_identity_utils._is_certificate_file_ready", + return_value=True, + autospec=True, +) +def test_has_default_client_cert_source_malformed_config_no_gke_fallback( + mock_is_ready, mock_get_cert, mock_load_json, side_effect +): + mock_load_json.side_effect = side_effect + + assert mtls.has_default_client_cert_source(include_context_aware=False) is True + mock_is_ready.assert_not_called() diff --git a/packages/google-auth/tests/transport/test_requests.py b/packages/google-auth/tests/transport/test_requests.py index ae102bcb33e5..5ca8788ca42b 100644 --- a/packages/google-auth/tests/transport/test_requests.py +++ b/packages/google-auth/tests/transport/test_requests.py @@ -1280,7 +1280,14 @@ def test_cert_rotation_logic_skipped_on_other_refresh_status_codes(self): # Assert mTLS check logic was SKIPPED (Inner Check was False) assert not mock_helper.check_parameters_for_unauthorized_response.called - def test_cert_rotation_skipped_on_non_mtls_url(self): + @pytest.mark.parametrize( + "non_mtls_url", + [ + "https://storage.googleapis.com/bucket/mtls.googleapis.com", + "https://my-service-xyz-uc.a.run.app/mtls.run.app", + ], + ) + def test_cert_rotation_skipped_on_non_mtls_url(self, non_mtls_url): """ Tests that mTLS cert rotation is skipped on non-mTLS URLs even if mTLS is enabled and an UNAUTHORIZED (401) response is received. @@ -1293,7 +1300,6 @@ def test_cert_rotation_skipped_on_non_mtls_url(self): make_response(status=http_client.OK), ] ) - non_mtls_url = "https://storage.googleapis.com/bucket/mtls.googleapis.com" authed_session = google.auth.transport.requests.AuthorizedSession( credentials, refresh_timeout=60 ) @@ -1313,10 +1319,18 @@ def test_cert_rotation_skipped_on_non_mtls_url(self): # Assert mTLS check logic was SKIPPED assert not mock_check_params.called - def test_cert_rotation_triggered_on_psc_url(self): + @pytest.mark.parametrize( + "mtls_url", + [ + "https://storage.p.googleapis.com/b/my-bucket", + "https://my-service-123456.us-central1.mtls.run.app/v1", + ], + ) + def test_cert_rotation_triggered_on_mtls_url(self, mtls_url): """ - Tests that mTLS cert rotation IS triggered on a Private Service Connect - (PSC) mTLS endpoint when an UNAUTHORIZED (401) response is received. + Tests that mTLS cert rotation IS triggered on Private Service Connect + (PSC) and Cloud Run mTLS endpoints when an UNAUTHORIZED (401) response + is received. """ credentials = mock.Mock(wraps=CredentialsStub()) adapter = AdapterStub( @@ -1325,11 +1339,10 @@ def test_cert_rotation_triggered_on_psc_url(self): make_response(status=http_client.OK), ] ) - psc_url = "https://storage.p.googleapis.com/b/my-bucket" authed_session = google.auth.transport.requests.AuthorizedSession( credentials, refresh_timeout=60 ) - authed_session.mount(psc_url, adapter) + authed_session.mount(mtls_url, adapter) authed_session._is_mtls = True authed_session._cached_cert = b"cached_cert" @@ -1338,9 +1351,9 @@ def test_cert_rotation_triggered_on_psc_url(self): "check_parameters_for_unauthorized_response", return_value=(b"new_cert", b"new_key", "old_fp", "old_fp"), ) as mock_check_params: - authed_session.request("GET", psc_url) + authed_session.request("GET", mtls_url) - # Assert mTLS check logic was called on PSC endpoint + # Assert mTLS check logic was called on PSC / Cloud Run mTLS endpoint mock_check_params.assert_called_once() assert credentials.refresh.called diff --git a/packages/google-auth/tests/transport/test_urllib3.py b/packages/google-auth/tests/transport/test_urllib3.py index b9136a4f9f7c..7ffd50aeee32 100644 --- a/packages/google-auth/tests/transport/test_urllib3.py +++ b/packages/google-auth/tests/transport/test_urllib3.py @@ -657,7 +657,14 @@ def test_cert_rotation_logic_skipped_on_other_refresh_status_codes(self): # Assert mTLS check logic was SKIPPED (Inner Check was False) assert not mock_helper.check_parameters_for_unauthorized_response.called - def test_cert_rotation_skipped_on_non_mtls_url(self): + @pytest.mark.parametrize( + "non_mtls_url", + [ + "https://storage.googleapis.com/bucket/mtls.googleapis.com", + "https://my-service-xyz-uc.a.run.app/mtls.run.app", + ], + ) + def test_cert_rotation_skipped_on_non_mtls_url(self, non_mtls_url): """ Tests that mTLS cert rotation is skipped on non-mTLS URLs even if mTLS is enabled and an UNAUTHORIZED (401) response is received. @@ -669,7 +676,6 @@ def test_cert_rotation_skipped_on_non_mtls_url(self): ResponseStub(status=http_client.OK), ] ) - non_mtls_url = "https://storage.googleapis.com/bucket/mtls.googleapis.com" authed_http = google.auth.transport.urllib3.AuthorizedHttp( credentials, http=http ) @@ -688,10 +694,18 @@ def test_cert_rotation_skipped_on_non_mtls_url(self): # Assert mTLS check logic was SKIPPED assert not mock_check_params.called - def test_cert_rotation_triggered_on_psc_url(self): + @pytest.mark.parametrize( + "mtls_url", + [ + "https://storage.p.googleapis.com/b/my-bucket", + "https://my-service-123456.us-central1.mtls.run.app/v1", + ], + ) + def test_cert_rotation_triggered_on_mtls_url(self, mtls_url): """ - Tests that mTLS cert rotation IS triggered on a Private Service Connect - (PSC) mTLS endpoint when an UNAUTHORIZED (401) response is received. + Tests that mTLS cert rotation IS triggered on Private Service Connect + (PSC) and Cloud Run mTLS endpoints when an UNAUTHORIZED (401) response + is received. """ credentials = mock.Mock(wraps=CredentialsStub()) http = HttpStub( @@ -700,7 +714,6 @@ def test_cert_rotation_triggered_on_psc_url(self): ResponseStub(status=http_client.OK), ] ) - psc_url = "https://storage.p.googleapis.com/b/my-bucket" authed_http = google.auth.transport.urllib3.AuthorizedHttp( credentials, http=http ) @@ -712,9 +725,9 @@ def test_cert_rotation_triggered_on_psc_url(self): "check_parameters_for_unauthorized_response", return_value=(b"new_cert", b"new_key", "old_fp", "old_fp"), ) as mock_check_params: - authed_http.urlopen("GET", psc_url) + authed_http.urlopen("GET", mtls_url) - # Assert mTLS check logic was called on PSC endpoint + # Assert mTLS check logic was called on PSC / Cloud Run mTLS endpoint mock_check_params.assert_called_once() assert credentials.refresh.called diff --git a/packages/google-auth/tests_async/oauth2/test_id_token.py b/packages/google-auth/tests_async/oauth2/test_id_token.py index 22456c6bdace..883a9ee9c5ec 100644 --- a/packages/google-auth/tests_async/oauth2/test_id_token.py +++ b/packages/google-auth/tests_async/oauth2/test_id_token.py @@ -209,11 +209,25 @@ async def test_verify_firebase_token_clock_skew(verify_token): @pytest.mark.asyncio -async def test_fetch_id_token_from_metadata_server(monkeypatch): +@pytest.mark.parametrize( + "call_kwargs,expected_bind_id_token", + [ + ({}, None), + ({"bind_id_token": None}, None), + ({"bind_id_token": True}, True), + ({"bind_id_token": False}, False), + ], +) +async def test_fetch_id_token_from_metadata_server( + monkeypatch, call_kwargs, expected_bind_id_token +): monkeypatch.delenv(environment_vars.CREDENTIALS, raising=False) - def mock_init(self, request, audience, use_metadata_identity_endpoint): + def mock_init( + self, request, audience, use_metadata_identity_endpoint, bind_id_token + ): assert use_metadata_identity_endpoint + assert bind_id_token is expected_bind_id_token self.token = "id_token" with mock.patch("google.auth.compute_engine._metadata.ping", return_value=True): @@ -224,13 +238,16 @@ def mock_init(self, request, audience, use_metadata_identity_endpoint): ): request = mock.AsyncMock() token = await id_token.fetch_id_token( - request, "https://pubsub.googleapis.com" + request, + "https://pubsub.googleapis.com", + **call_kwargs, ) assert token == "id_token" @pytest.mark.asyncio -async def test_fetch_id_token_from_explicit_cred_json_file(monkeypatch): +@pytest.mark.parametrize("bind_id_token", [None, True, False]) +async def test_fetch_id_token_from_explicit_cred_json_file(monkeypatch, bind_id_token): monkeypatch.setenv(environment_vars.CREDENTIALS, test_id_token.SERVICE_ACCOUNT_FILE) async def mock_refresh(self, request): @@ -240,7 +257,9 @@ async def mock_refresh(self, request): _service_account_async.IDTokenCredentials, "refresh", mock_refresh ): request = mock.AsyncMock() - token = await id_token.fetch_id_token(request, "https://pubsub.googleapis.com") + token = await id_token.fetch_id_token( + request, "https://pubsub.googleapis.com", bind_id_token=bind_id_token + ) assert token == "id_token" From 037036717747eb3f16e8b13a71c4c5e77824ba62 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:23:04 -0700 Subject: [PATCH 020/228] chore(main): release google-auth 2.61.0 (#18595) :robot: I have created a release *beep* *boop* --- ## [2.61.0](https://github.com/googleapis/google-cloud-python/compare/google-auth-v2.60.0...google-auth-v2.61.0) (2026-10-07) ### Features * **auth:** support bound access and ID tokens for GKE ([#18594](https://github.com/googleapis/google-cloud-python/issues/18594)) ([9f5fd7b](https://github.com/googleapis/google-cloud-python/commit/9f5fd7bdf0707a17c363482b094ccd70a4358389)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- .release-please-individual-manifest.json | 2 +- librarian.yaml | 2 +- packages/google-auth/CHANGELOG.md | 7 +++++++ packages/google-auth/google/auth/version.py | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.release-please-individual-manifest.json b/.release-please-individual-manifest.json index 3a2563fc5238..e64004459cc8 100644 --- a/.release-please-individual-manifest.json +++ b/.release-please-individual-manifest.json @@ -1,5 +1,5 @@ { - "packages/google-auth": "2.60.0", + "packages/google-auth": "2.61.0", "packages/google-cloud-apptopology": "0.1.0", "packages/google-cloud-ftp": "0.1.0", "packages/google-cloud-spanner": "3.71.0", diff --git a/librarian.yaml b/librarian.yaml index 4d2695ab82de..c014d8d18d6d 100644 --- a/librarian.yaml +++ b/librarian.yaml @@ -245,7 +245,7 @@ libraries: metadata_name_override: area120tables default_version: v1alpha1 - name: google-auth - version: 2.60.0 + version: 2.61.0 python: library_type: AUTH - name: google-auth-httplib2 diff --git a/packages/google-auth/CHANGELOG.md b/packages/google-auth/CHANGELOG.md index 7ed2320daf60..4118e4c54a90 100644 --- a/packages/google-auth/CHANGELOG.md +++ b/packages/google-auth/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-auth/#history +## [2.61.0](https://github.com/googleapis/google-cloud-python/compare/google-auth-v2.60.0...google-auth-v2.61.0) (2026-10-07) + + +### Features + +* **auth:** support bound access and ID tokens for GKE ([#18594](https://github.com/googleapis/google-cloud-python/issues/18594)) ([9f5fd7b](https://github.com/googleapis/google-cloud-python/commit/9f5fd7bdf0707a17c363482b094ccd70a4358389)) + ## [2.60.0](https://github.com/googleapis/google-cloud-python/compare/google-auth-v2.59.1...google-auth-v2.60.0) (2026-10-02) diff --git a/packages/google-auth/google/auth/version.py b/packages/google-auth/google/auth/version.py index 92bcf75ef2f2..52ccde909832 100644 --- a/packages/google-auth/google/auth/version.py +++ b/packages/google-auth/google/auth/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.60.0" +__version__ = "2.61.0" From d0505e8bc7615f2c649d3e25ebc7e3ccca45f22b Mon Sep 17 00:00:00 2001 From: Noah Dietz Date: Wed, 7 Oct 2026 12:29:03 -0700 Subject: [PATCH 021/228] feat: update API sources and regenerate (#18593) --- librarian.yaml | 4 +- .../docs/admanager_v1/creative_service.rst | 10 + .../docs/admanager_v1/forecast_service.rst | 6 + ...line_item_creative_association_service.rst | 10 + .../line_item_template_service.rst | 10 + .../docs/admanager_v1/services_.rst | 4 + .../docs/admanager_v1/user_service.rst | 4 + .../google/ads/admanager/__init__.py | 309 +- .../google/ads/admanager_v1/__init__.py | 326 +- .../ads/admanager_v1/gapic_metadata.json | 226 + .../ad_review_center_ad_service/client.py | 279 +- .../transports/base.py | 38 + .../transports/rest.py | 476 +- .../transports/rest_base.py | 33 + .../audience_segment_service/client.py | 248 + .../transports/base.py | 34 + .../transports/rest.py | 454 + .../transports/rest_base.py | 34 + .../child_publisher_service/client.py | 60 +- .../transports/rest.py | 34 +- .../services/company_service/client.py | 39 +- .../company_service/transports/rest.py | 23 +- .../services/creative_service/__init__.py | 18 + .../services/creative_service/client.py | 1331 ++ .../services/creative_service/pagers.py | 117 + .../creative_service/transports/README.rst | 10 + .../creative_service/transports/__init__.py | 30 + .../creative_service/transports/base.py | 256 + .../creative_service/transports/rest.py | 1367 ++ .../creative_service/transports/rest_base.py | 186 + .../entity_signals_mapping_service/client.py | 8 +- .../transports/rest.py | 8 +- .../services/forecast_service/__init__.py | 18 + .../services/forecast_service/client.py | 1527 ++ .../forecast_service/transports/README.rst | 10 + .../forecast_service/transports/__init__.py | 30 + .../forecast_service/transports/base.py | 242 + .../forecast_service/transports/rest.py | 1152 + .../forecast_service/transports/rest_base.py | 171 + .../__init__.py | 18 + .../client.py | 1084 + .../pagers.py | 131 + .../transports/README.rst | 10 + .../transports/__init__.py | 33 + .../transports/base.py | 234 + .../transports/rest.py | 975 + .../transports/rest_base.py | 157 + .../services/line_item_service/client.py | 1705 ++ .../line_item_service/transports/base.py | 230 + .../line_item_service/transports/rest.py | 3334 ++- .../line_item_service/transports/rest_base.py | 239 + .../line_item_template_service/__init__.py | 18 + .../line_item_template_service/client.py | 993 + .../line_item_template_service/pagers.py | 122 + .../transports/README.rst | 10 + .../transports/__init__.py | 33 + .../transports/base.py | 228 + .../transports/rest.py | 933 + .../transports/rest_base.py | 155 + .../services/native_style_service/client.py | 258 + .../native_style_service/transports/base.py | 34 + .../native_style_service/transports/rest.py | 450 + .../transports/rest_base.py | 34 + .../services/network_service/client.py | 4 +- .../services/partner_service/client.py | 23 +- .../partner_service/transports/rest.py | 17 +- .../services/user_service/client.py | 830 + .../services/user_service/pagers.py | 117 + .../services/user_service/transports/base.py | 112 + .../services/user_service/transports/rest.py | 1539 +- .../user_service/transports/rest_base.py | 118 + .../viewability_provider_service/client.py | 24 +- .../transports/rest.py | 18 +- .../google/ads/admanager_v1/types/__init__.py | 300 + .../types/ad_review_center_ad_enums.py | 41 + .../types/ad_review_center_ad_messages.py | 61 + .../types/ad_review_center_ad_service.py | 247 +- .../ad_rule_fill_order_direction_enum.py | 2 +- .../admanager_v1/types/ad_rule_messages.py | 34 +- ...d_rule_slot_midroll_frequency_type_enum.py | 6 +- .../admanager_v1/types/ad_spot_messages.py | 2 +- .../admanager_v1/types/ad_unit_messages.py | 2 +- .../ads/admanager_v1/types/asset_enums.py | 65 + .../ads/admanager_v1/types/asset_messages.py | 45 + .../types/audience_segment_service.py | 69 + .../types/break_template_messages.py | 10 +- .../types/cdn_security_policy_enum.py | 2 +- .../types/child_publisher_messages.py | 4 +- .../types/child_publisher_service.py | 54 +- .../admanager_v1/types/company_messages.py | 9 +- .../ads/admanager_v1/types/company_service.py | 43 +- .../types/content_bundle_messages.py | 2 +- .../admanager_v1/types/content_messages.py | 20 +- .../ads/admanager_v1/types/creative_asset.py | 169 + .../ads/admanager_v1/types/creative_enums.py | 185 + .../admanager_v1/types/creative_messages.py | 2183 ++ .../types/creative_placeholder.py | 9 +- .../admanager_v1/types/creative_service.py | 220 + .../types/creative_set_messages.py | 2 +- ...hird_party_data_declaration_status_enum.py | 78 + .../types/custom_creative_asset.py | 66 + .../admanager_v1/types/custom_field_value.py | 24 +- .../admanager_v1/types/custom_pacing_curve.py | 9 +- .../types/custom_pacing_goal_unit_enum.py | 3 +- .../types/dai_encoding_profile_messages.py | 3 +- .../types/dai_session_messages.py | 2 +- .../ads/admanager_v1/types/date_range.py | 54 + .../types/entity_signals_mapping_service.py | 4 +- .../admanager_v1/types/forecast_messages.py | 842 + .../admanager_v1/types/forecast_service.py | 239 + .../admanager_v1/types/forecasting_enums.py | 234 + .../google/ads/admanager_v1/types/goal.py | 2 +- .../ads/admanager_v1/types/goal_enums.py | 35 +- .../ads/admanager_v1/types/grp_settings.py | 16 +- .../line_item_creative_association_enums.py | 56 + ...line_item_creative_association_messages.py | 319 + .../line_item_creative_association_service.py | 179 + .../admanager_v1/types/line_item_discount.py | 23 +- .../ads/admanager_v1/types/line_item_enums.py | 31 +- .../admanager_v1/types/line_item_messages.py | 11 +- .../admanager_v1/types/line_item_service.py | 410 + .../types/line_item_template_messages.py | 159 + .../types/line_item_template_service.py | 162 + .../admanager_v1/types/network_messages.py | 3 +- .../admanager_v1/types/partner_messages.py | 9 +- .../ads/admanager_v1/types/partner_service.py | 31 +- .../types/private_auction_deal_messages.py | 4 +- .../admanager_v1/types/report_definition.py | 200 +- .../ads/admanager_v1/types/report_messages.py | 8 + .../types/request_platform_enum.py | 3 +- ...rich_media_studio_child_asset_type_enum.py | 60 + ..._studio_creative_billing_attribute_enum.py | 75 + .../rich_media_studio_creative_format_enum.py | 124 + .../types/rich_media_studio_messages.py | 99 + .../ads/admanager_v1/types/slate_messages.py | 4 +- .../types/targeted_video_bumper_type_enum.py | 7 +- .../third_party_data_declaration_type_enum.py | 2 +- .../types/third_party_measurement_settings.py | 29 +- .../ads/admanager_v1/types/user_service.py | 303 + .../types/vast_redirect_type_enum.py | 57 + .../admanager_v1/types/video_tracking_url.py | 2 +- .../types/viewability_provider_service.py | 32 +- ...ply_ad_review_center_custom_labels_sync.py | 53 + ...tch_ad_review_center_custom_labels_sync.py | 53 + ...ice_batch_update_audience_segments_sync.py | 53 + ...nt_service_update_audience_segment_sync.py | 51 + ...e_service_batch_activate_creatives_sync.py | 54 + ...service_batch_deactivate_creatives_sync.py | 54 + ...ated_creative_service_get_creative_sync.py | 53 + ...ed_creative_service_list_creatives_sync.py | 54 + ..._service_run_availability_forecast_sync.py | 54 + ...cast_service_run_delivery_forecast_sync.py | 57 + ..._forecast_service_run_traffic_data_sync.py | 53 + ...get_line_item_creative_association_sync.py | 53 + ...st_line_item_creative_associations_sync.py | 54 + ..._service_batch_activate_line_items_sync.py | 54 + ...m_service_batch_archive_line_items_sync.py | 54 + ...em_service_batch_create_line_items_sync.py | 60 + ...em_service_batch_delete_line_items_sync.py | 51 + ...tem_service_batch_pause_line_items_sync.py | 54 + ...m_service_batch_release_line_items_sync.py | 54 + ...ch_reserve_and_overbook_line_items_sync.py | 54 + ...m_service_batch_reserve_line_items_sync.py | 54 + ...tch_resume_and_overbook_line_items_sync.py | 54 + ...em_service_batch_resume_line_items_sync.py | 54 + ...service_batch_unarchive_line_items_sync.py | 54 + ...em_service_batch_update_line_items_sync.py | 59 + ...line_item_service_create_line_item_sync.py | 59 + ...line_item_service_update_line_item_sync.py | 58 + ...ate_service_get_line_item_template_sync.py | 53 + ...e_service_list_line_item_templates_sync.py | 54 + ..._style_service_create_native_style_sync.py | 59 + ..._style_service_update_native_style_sync.py | 58 + ..._user_service_batch_activate_users_sync.py | 54 + ...ed_user_service_batch_create_users_sync.py | 57 + ...ser_service_batch_deactivate_users_sync.py | 54 + ...ed_user_service_batch_update_users_sync.py | 53 + ...generated_user_service_create_user_sync.py | 53 + ..._generated_user_service_list_users_sync.py | 54 + ...generated_user_service_update_user_sync.py | 51 + ...ppet_metadata_google.ads.admanager.v1.json | 11633 ++++++---- .../test_ad_review_center_ad_service.py | 925 +- .../test_audience_segment_service.py | 1503 +- .../admanager_v1/test_creative_service.py | 3101 +++ .../admanager_v1/test_forecast_service.py | 3081 +++ ..._line_item_creative_association_service.py | 2461 +++ .../admanager_v1/test_line_item_service.py | 6286 +++++- .../test_line_item_template_service.py | 2343 ++ .../admanager_v1/test_native_style_service.py | 1840 +- .../gapic/admanager_v1/test_report_service.py | 2 + .../gapic/admanager_v1/test_user_service.py | 2945 ++- .../google/cloud/apphub/__init__.py | 33 +- .../google/cloud/apphub_v1/__init__.py | 36 +- .../cloud/apphub_v1/gapic_metadata.json | 60 + .../services/app_hub/async_client.py | 516 +- .../apphub_v1/services/app_hub/client.py | 541 +- .../apphub_v1/services/app_hub/pagers.py | 163 + .../services/app_hub/transports/base.py | 64 + .../services/app_hub/transports/grpc.py | 117 + .../app_hub/transports/grpc_asyncio.py | 139 + .../services/app_hub/transports/rest.py | 1138 +- .../services/app_hub/transports/rest_base.py | 79 +- .../google/cloud/apphub_v1/types/__init__.py | 32 + .../cloud/apphub_v1/types/apphub_service.py | 157 +- .../cloud/apphub_v1/types/application.py | 68 + .../google/cloud/apphub_v1/types/boundary.py | 104 + .../types/extended_metadata_schema.py | 62 + .../cloud/apphub_v1/types/properties.py | 140 + .../google/cloud/apphub_v1/types/service.py | 38 + .../google/cloud/apphub_v1/types/workload.py | 30 + ...v1_generated_app_hub_get_boundary_async.py | 53 + ..._v1_generated_app_hub_get_boundary_sync.py | 53 + ..._hub_get_extended_metadata_schema_async.py | 53 + ...p_hub_get_extended_metadata_schema_sync.py | 53 + ...ub_list_extended_metadata_schemas_async.py | 54 + ...hub_list_extended_metadata_schemas_sync.py | 54 + ...generated_app_hub_update_boundary_async.py | 60 + ..._generated_app_hub_update_boundary_sync.py | 60 + ...ippet_metadata_google.cloud.apphub.v1.json | 780 +- .../unit/gapic/apphub_v1/test_app_hub.py | 9241 +++++--- .../google/cloud/apptopology/__init__.py | 4 + .../google/cloud/apptopology_v1/__init__.py | 4 + .../cloud/apptopology_v1/gapic_metadata.json | 15 + .../services/app_topology/async_client.py | 127 + .../services/app_topology/client.py | 124 + .../services/app_topology/pagers.py | 156 + .../services/app_topology/transports/base.py | 23 + .../services/app_topology/transports/grpc.py | 27 + .../app_topology/transports/grpc_asyncio.py | 43 + .../services/app_topology/transports/rest.py | 209 + .../app_topology/transports/rest_base.py | 17 + .../cloud/apptopology_v1/types/__init__.py | 4 + .../cloud/apptopology_v1/types/service.py | 119 + ...rated_app_topology_explore_schema_async.py | 54 + ...erated_app_topology_explore_schema_sync.py | 54 + ..._metadata_google.cloud.apptopology.v1.json | 161 + .../gapic/apptopology_v1/test_app_topology.py | 1026 +- .../.repo-metadata.json | 2 +- .../README.rst | 10 +- .../docs/README.rst | 10 +- .../app_connections_service/async_client.py | 41 +- .../app_connections_service/client.py | 40 +- .../transports/grpc.py | 4 + .../transports/grpc_asyncio.py | 4 + .../transports/rest.py | 23 +- .../transports/rest_base.py | 29 +- .../types/app_connections_service.py | 62 +- .../test_app_connections_service.py | 20 + .../.repo-metadata.json | 2 +- .../README.rst | 10 +- .../docs/README.rst | 10 +- .../beyondcorp_appconnectors/__init__.py | 8 + .../beyondcorp_appconnectors_v1/__init__.py | 13 +- .../gapic_metadata.json | 15 + .../app_connectors_service/async_client.py | 158 + .../services/app_connectors_service/client.py | 153 + .../app_connectors_service/transports/base.py | 17 + .../app_connectors_service/transports/grpc.py | 35 + .../transports/grpc_asyncio.py | 40 + .../app_connectors_service/transports/rest.py | 247 + .../transports/rest_base.py | 45 +- .../types/__init__.py | 8 + .../types/app_connector_instance_config.py | 10 +- .../types/app_connectors_service.py | 85 +- .../types/resource_info.py | 63 +- ...s_service_resolve_instance_config_async.py | 53 + ...rs_service_resolve_instance_config_sync.py | 53 + ...gle.cloud.beyondcorp.appconnectors.v1.json | 161 + .../test_app_connectors_service.py | 844 +- .../.repo-metadata.json | 2 +- .../README.rst | 10 +- .../docs/README.rst | 10 +- .../app_gateways_service/async_client.py | 25 + .../services/app_gateways_service/client.py | 23 + .../app_gateways_service/transports/grpc.py | 4 + .../transports/grpc_asyncio.py | 4 + .../app_gateways_service/transports/rest.py | 21 + .../transports/rest_base.py | 29 +- .../types/app_gateways_service.py | 51 +- .../test_app_gateways_service.py | 16 + .../google/cloud/bigtable_v2/__init__.py | 20 + .../cloud/bigtable_v2/types/__init__.py | 20 + .../cloud/bigtable_v2/types/peer_info.py | 2 +- .../google/cloud/bigtable_v2/types/session.py | 522 +- .../google/cloud/compute/__init__.py | 4 + .../google/cloud/compute_v1beta/__init__.py | 4 + .../cloud/compute_v1beta/gapic_metadata.json | 5 + .../services/interconnects/client.py | 298 + .../services/interconnects/transports/base.py | 14 + .../services/interconnects/transports/rest.py | 237 + .../interconnects/transports/rest_base.py | 17 + .../cloud/compute_v1beta/types/__init__.py | 4 + .../cloud/compute_v1beta/types/compute.py | 134 +- ...a_generated_interconnects_set_name_sync.py | 54 + ..._metadata_google.cloud.compute.v1beta.json | 88 + .../compute_v1beta/test_interconnects.py | 651 + .../dialogflowcx_v3/types/fulfillment.py | 9 + .../cloud/dialogflowcx_v3/types/intent.py | 2 +- .../dialogflowcx_v3beta1/types/fulfillment.py | 9 + .../dialogflowcx_v3beta1/types/intent.py | 2 +- .../unit/gapic/dialogflowcx_v3/test_flows.py | 2 + .../unit/gapic/dialogflowcx_v3/test_pages.py | 2 + .../gapic/dialogflowcx_v3/test_playbooks.py | 3 + .../gapic/dialogflowcx_v3/test_test_cases.py | 2 + .../test_transition_route_groups.py | 2 + .../gapic/dialogflowcx_v3beta1/test_flows.py | 2 + .../gapic/dialogflowcx_v3beta1/test_pages.py | 2 + .../dialogflowcx_v3beta1/test_playbooks.py | 3 + .../dialogflowcx_v3beta1/test_test_cases.py | 2 + .../test_transition_route_groups.py | 2 + .../dialogflow_v2beta1/companion_agents.rst | 10 + .../docs/dialogflow_v2beta1/services_.rst | 1 + .../google/cloud/dialogflow/__init__.py | 4 + .../google/cloud/dialogflow_v2/__init__.py | 4 + .../services/sip_trunks/async_client.py | 2 + .../services/sip_trunks/client.py | 2 + .../cloud/dialogflow_v2/types/__init__.py | 4 + .../cloud/dialogflow_v2/types/sip_trunk.py | 221 + .../cloud/dialogflow_v2beta1/__init__.py | 50 + .../dialogflow_v2beta1/gapic_metadata.json | 109 + .../services/companion_agents/__init__.py | 22 + .../services/companion_agents/async_client.py | 1216 ++ .../services/companion_agents/client.py | 1562 ++ .../services/companion_agents/pagers.py | 197 + .../companion_agents/transports/README.rst | 10 + .../companion_agents/transports/__init__.py | 36 + .../companion_agents/transports/base.py | 323 + .../companion_agents/transports/grpc.py | 570 + .../transports/grpc_asyncio.py | 641 + .../companion_agents/transports/rest.py | 1998 ++ .../companion_agents/transports/rest_base.py | 259 + .../conversation_profiles/async_client.py | 4 + .../services/conversation_profiles/client.py | 22 + .../services/conversations/async_client.py | 6 +- .../services/conversations/client.py | 24 +- .../services/participants/async_client.py | 94 + .../services/participants/client.py | 92 + .../services/participants/transports/base.py | 17 + .../services/participants/transports/grpc.py | 33 + .../participants/transports/grpc_asyncio.py | 38 + .../services/participants/transports/rest.py | 34 + .../participants/transports/rest_base.py | 4 + .../dialogflow_v2beta1/types/__init__.py | 44 + .../dialogflow_v2beta1/types/answer_record.py | 31 + .../dialogflow_v2beta1/types/audio_config.py | 124 + .../cloud/dialogflow_v2beta1/types/ces_app.py | 4 +- .../types/companion_agent.py | 443 + .../dialogflow_v2beta1/types/conversation.py | 3 +- .../types/conversation_profile.py | 17 +- .../dialogflow_v2beta1/types/generator.py | 4 +- .../dialogflow_v2beta1/types/grounding.py | 255 + .../dialogflow_v2beta1/types/participant.py | 517 +- .../cloud/dialogflow_v2beta1/types/session.py | 8 +- .../dialogflow_v2beta1/types/tool_call.py | 7 + ...rated_sip_trunks_create_sip_trunk_async.py | 1 + ...erated_sip_trunks_create_sip_trunk_sync.py | 1 + ...rated_sip_trunks_update_sip_trunk_async.py | 1 + ...erated_sip_trunks_update_sip_trunk_sync.py | 1 + ...ion_agents_create_companion_agent_async.py | 53 + ...nion_agents_create_companion_agent_sync.py | 53 + ...ion_agents_delete_companion_agent_async.py | 50 + ...nion_agents_delete_companion_agent_sync.py | 50 + ...panion_agents_get_companion_agent_async.py | 53 + ...mpanion_agents_get_companion_agent_sync.py | 53 + ...nion_agents_list_companion_agents_async.py | 54 + ...anion_agents_list_companion_agents_sync.py | 54 + ...ion_agents_update_companion_agent_async.py | 51 + ...nion_agents_update_companion_agent_sync.py | 51 + ...nversations_batch_create_messages_async.py | 1 - ...onversations_batch_create_messages_sync.py | 1 - ...ations_generate_stateless_summary_async.py | 1 - ...sations_generate_stateless_summary_sync.py | 1 - ...ng_reactive_companion_suggestions_async.py | 67 + ...ing_reactive_companion_suggestions_sync.py | 67 + ...t_metadata_google.cloud.dialogflow.v2.json | 56 +- ...adata_google.cloud.dialogflow.v2beta1.json | 1032 +- .../gapic/dialogflow_v2/test_sip_trunks.py | 87 + .../dialogflow_v2beta1/test_answer_records.py | 74 +- .../test_companion_agents.py | 7346 +++++++ .../test_conversation_profiles.py | 134 +- .../dialogflow_v2beta1/test_conversations.py | 189 +- .../test_generator_evaluations.py | 5 +- .../dialogflow_v2beta1/test_generators.py | 10 +- .../dialogflow_v2beta1/test_participants.py | 211 + .../google/cloud/netapp/__init__.py | 28 + .../google/cloud/netapp_v1/__init__.py | 28 + .../cloud/netapp_v1/gapic_metadata.json | 75 + .../services/net_app/async_client.py | 705 +- .../netapp_v1/services/net_app/client.py | 712 +- .../netapp_v1/services/net_app/pagers.py | 156 + .../services/net_app/transports/base.py | 73 + .../services/net_app/transports/grpc.py | 152 +- .../net_app/transports/grpc_asyncio.py | 181 +- .../services/net_app/transports/rest.py | 1921 +- .../services/net_app/transports/rest_base.py | 83 + .../google/cloud/netapp_v1/types/__init__.py | 28 + .../google/cloud/netapp_v1/types/backup.py | 51 +- .../cloud/netapp_v1/types/backup_vault.py | 2 +- .../google/cloud/netapp_v1/types/common.py | 21 + .../google/cloud/netapp_v1/types/kms.py | 2 +- .../google/cloud/netapp_v1/types/ontap.py | 2 +- .../cloud/netapp_v1/types/storage_pool.py | 4 +- .../google/cloud/netapp_v1/types/volume.py | 373 +- ...1_generated_net_app_create_volume_async.py | 2 +- ...v1_generated_net_app_create_volume_sync.py | 2 +- ...enerated_net_app_get_split_status_async.py | 53 + ...generated_net_app_get_split_status_sync.py | 53 + ...rated_net_app_list_backup_configs_async.py | 54 + ...erated_net_app_list_backup_configs_sync.py | 54 + ..._generated_net_app_restore_volume_async.py | 65 + ...1_generated_net_app_restore_volume_sync.py | 65 + ..._v1_generated_net_app_start_split_async.py | 57 + ...p_v1_generated_net_app_start_split_sync.py | 57 + ...ated_net_app_update_backup_config_async.py | 58 + ...rated_net_app_update_backup_config_sync.py | 58 + ...1_generated_net_app_update_volume_async.py | 2 +- ...v1_generated_net_app_update_volume_sync.py | 2 +- ...ippet_metadata_google.cloud.netapp.v1.json | 1545 +- .../unit/gapic/netapp_v1/test_net_app.py | 18052 ++++++++++------ .../cloud/redis_cluster_v1beta1/__init__.py | 30 + .../redis_cluster_v1beta1/gapic_metadata.json | 105 + .../cloud_redis_cluster/async_client.py | 1005 +- .../services/cloud_redis_cluster/client.py | 1268 +- .../services/cloud_redis_cluster/pagers.py | 316 + .../cloud_redis_cluster/transports/base.py | 107 + .../cloud_redis_cluster/transports/grpc.py | 225 + .../transports/grpc_asyncio.py | 264 + .../cloud_redis_cluster/transports/rest.py | 2042 +- .../transports/rest_base.py | 116 + .../redis_cluster_v1beta1/types/__init__.py | 30 + .../types/cloud_redis_cluster.py | 601 + ...d_redis_cluster_create_acl_policy_async.py | 59 + ...ud_redis_cluster_create_acl_policy_sync.py | 59 + ...d_redis_cluster_delete_acl_policy_async.py | 57 + ...ud_redis_cluster_delete_acl_policy_sync.py | 57 + ...loud_redis_cluster_get_acl_policy_async.py | 53 + ...s_cluster_get_acl_policy_revision_async.py | 53 + ...is_cluster_get_acl_policy_revision_sync.py | 53 + ...cloud_redis_cluster_get_acl_policy_sync.py | 53 + ...d_redis_cluster_list_acl_policies_async.py | 54 + ...ud_redis_cluster_list_acl_policies_sync.py | 54 + ...cluster_list_acl_policy_revisions_async.py | 54 + ..._cluster_list_acl_policy_revisions_sync.py | 54 + ...d_redis_cluster_update_acl_policy_async.py | 61 + ...ud_redis_cluster_update_acl_policy_sync.py | 61 + ...ta_google.cloud.redis.cluster.v1beta1.json | 1573 +- .../test_cloud_redis_cluster.py | 15470 +++++++++---- .../blue_green_deployments_service.rst | 10 + .../docs/sqladmin_v1/services_.rst | 2 + .../sql_workload_captures_service.rst | 6 + .../blue_green_deployments_service.rst | 10 + .../docs/sqladmin_v1beta4/services_.rst | 2 + .../sql_workload_captures_service.rst | 6 + .../google/cloud/sqladmin/__init__.py | 58 + .../google/cloud/sqladmin_v1/__init__.py | 58 + .../cloud/sqladmin_v1/gapic_metadata.json | 128 + .../__init__.py | 22 + .../async_client.py | 1004 + .../blue_green_deployments_service/client.py | 1304 ++ .../blue_green_deployments_service/pagers.py | 218 + .../transports/README.rst | 10 + .../transports/__init__.py | 32 + .../transports/base.py | 245 + .../transports/grpc.py | 498 + .../transports/grpc_asyncio.py | 543 + .../sql_backup_runs_service/async_client.py | 10 + .../sql_backup_runs_service/client.py | 50 + .../sql_backups_service/async_client.py | 14 + .../services/sql_backups_service/client.py | 72 + .../sql_instances_service/async_client.py | 20 +- .../services/sql_instances_service/client.py | 76 +- .../sql_workload_captures_service/__init__.py | 22 + .../async_client.py | 849 + .../sql_workload_captures_service/client.py | 1128 + .../transports/README.rst | 10 + .../transports/__init__.py | 32 + .../transports/base.py | 240 + .../transports/grpc.py | 498 + .../transports/grpc_asyncio.py | 543 + .../cloud/sqladmin_v1/types/__init__.py | 42 + .../types/cloud_sql_blue_green_deployments.py | 652 + .../sqladmin_v1/types/cloud_sql_instances.py | 173 +- .../sqladmin_v1/types/cloud_sql_resources.py | 205 +- .../sqladmin_v1/types/cloud_sql_users.py | 8 +- .../types/cloud_sql_workload_captures.py | 326 + .../google/cloud/sqladmin_v1beta4/__init__.py | 58 + .../sqladmin_v1beta4/gapic_metadata.json | 188 + .../__init__.py | 22 + .../async_client.py | 1012 + .../blue_green_deployments_service/client.py | 1301 ++ .../blue_green_deployments_service/pagers.py | 218 + .../transports/README.rst | 10 + .../transports/__init__.py | 39 + .../transports/base.py | 245 + .../transports/grpc.py | 509 + .../transports/grpc_asyncio.py | 554 + .../transports/rest.py | 1335 ++ .../transports/rest_base.py | 182 + .../sql_backup_runs_service/async_client.py | 10 + .../sql_backup_runs_service/client.py | 50 + .../sql_backups_service/async_client.py | 14 + .../services/sql_backups_service/client.py | 72 + .../sql_instances_service/async_client.py | 18 +- .../services/sql_instances_service/client.py | 74 +- .../sql_instances_service/transports/rest.py | 2 +- .../sql_workload_captures_service/__init__.py | 22 + .../async_client.py | 853 + .../sql_workload_captures_service/client.py | 1121 + .../transports/README.rst | 10 + .../transports/__init__.py | 39 + .../transports/base.py | 240 + .../transports/grpc.py | 498 + .../transports/grpc_asyncio.py | 543 + .../transports/rest.py | 1294 ++ .../transports/rest_base.py | 180 + .../cloud/sqladmin_v1beta4/types/__init__.py | 42 + .../cloud/sqladmin_v1beta4/types/cloud_sql.py | 5 +- .../types/cloud_sql_blue_green_deployments.py | 708 + .../types/cloud_sql_resources.py | 369 +- .../sqladmin_v1beta4/types/cloud_sql_users.py | 8 +- .../types/cloud_sql_workload_captures.py | 325 + .../snippet_metadata_google.cloud.sql.v1.json | 6210 ++++-- ...pet_metadata_google.cloud.sql.v1beta4.json | 6254 ++++-- ...vice_create_blue_green_deployment_async.py | 58 + ...rvice_create_blue_green_deployment_sync.py | 58 + ...vice_delete_blue_green_deployment_async.py | 53 + ...rvice_delete_blue_green_deployment_sync.py | 53 + ...service_get_blue_green_deployment_async.py | 53 + ..._service_get_blue_green_deployment_sync.py | 53 + ...rvice_list_blue_green_deployments_async.py | 54 + ...ervice_list_blue_green_deployments_sync.py | 54 + ..._switchover_blue_green_deployment_async.py | 53 + ...e_switchover_blue_green_deployment_sync.py | 53 + ...ql_workload_captures_service_list_async.py | 54 + ...sql_workload_captures_service_list_sync.py | 54 + ...l_workload_captures_service_start_async.py | 54 + ...oad_captures_service_start_replay_async.py | 59 + ...load_captures_service_start_replay_sync.py | 59 + ...ql_workload_captures_service_start_sync.py | 54 + ...ql_workload_captures_service_stop_async.py | 54 + ...load_captures_service_stop_replay_async.py | 54 + ...kload_captures_service_stop_replay_sync.py | 54 + ...sql_workload_captures_service_stop_sync.py | 54 + ...vice_create_blue_green_deployment_async.py | 58 + ...rvice_create_blue_green_deployment_sync.py | 58 + ...vice_delete_blue_green_deployment_async.py | 53 + ...rvice_delete_blue_green_deployment_sync.py | 53 + ...service_get_blue_green_deployment_async.py | 53 + ..._service_get_blue_green_deployment_sync.py | 53 + ...rvice_list_blue_green_deployments_async.py | 54 + ...ervice_list_blue_green_deployments_sync.py | 54 + ..._switchover_blue_green_deployment_async.py | 53 + ...e_switchover_blue_green_deployment_sync.py | 53 + ...ql_workload_captures_service_list_async.py | 54 + ...sql_workload_captures_service_list_sync.py | 54 + ...l_workload_captures_service_start_async.py | 54 + ...oad_captures_service_start_replay_async.py | 59 + ...load_captures_service_start_replay_sync.py | 59 + ...ql_workload_captures_service_start_sync.py | 54 + ...ql_workload_captures_service_stop_async.py | 54 + ...load_captures_service_stop_replay_async.py | 54 + ...kload_captures_service_stop_replay_sync.py | 54 + ...sql_workload_captures_service_stop_sync.py | 54 + .../test_blue_green_deployments_service.py | 4292 ++++ .../test_sql_backup_runs_service.py | 89 +- .../sqladmin_v1/test_sql_backups_service.py | 93 + .../sqladmin_v1/test_sql_instances_service.py | 101 +- .../test_sql_workload_captures_service.py | 3473 +++ .../test_blue_green_deployments_service.py | 6419 ++++++ .../test_sql_backup_runs_service.py | 91 +- .../test_sql_backups_service.py | 111 + .../test_sql_instances_service.py | 148 +- .../test_sql_workload_captures_service.py | 5021 +++++ .../developer_knowledge/transports/base.py | 11 +- .../transports/grpc_asyncio.py | 11 +- 575 files changed, 185961 insertions(+), 27510 deletions(-) create mode 100644 packages/google-ads-admanager/docs/admanager_v1/creative_service.rst create mode 100644 packages/google-ads-admanager/docs/admanager_v1/forecast_service.rst create mode 100644 packages/google-ads-admanager/docs/admanager_v1/line_item_creative_association_service.rst create mode 100644 packages/google-ads-admanager/docs/admanager_v1/line_item_template_service.rst create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/__init__.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/client.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/pagers.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/transports/README.rst create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/transports/__init__.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/transports/base.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/transports/rest.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/transports/rest_base.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/forecast_service/__init__.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/forecast_service/client.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/forecast_service/transports/README.rst create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/forecast_service/transports/__init__.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/forecast_service/transports/base.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/forecast_service/transports/rest.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/forecast_service/transports/rest_base.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_creative_association_service/__init__.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_creative_association_service/client.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_creative_association_service/pagers.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_creative_association_service/transports/README.rst create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_creative_association_service/transports/__init__.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_creative_association_service/transports/base.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_creative_association_service/transports/rest.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_creative_association_service/transports/rest_base.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_template_service/__init__.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_template_service/client.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_template_service/pagers.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_template_service/transports/README.rst create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_template_service/transports/__init__.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_template_service/transports/base.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_template_service/transports/rest.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/line_item_template_service/transports/rest_base.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/services/user_service/pagers.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/asset_enums.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/asset_messages.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/creative_asset.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/creative_enums.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/creative_service.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/creative_third_party_data_declaration_status_enum.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/custom_creative_asset.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/date_range.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/forecast_messages.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/forecast_service.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/forecasting_enums.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/line_item_creative_association_enums.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/line_item_creative_association_messages.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/line_item_creative_association_service.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/line_item_template_messages.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/line_item_template_service.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/rich_media_studio_child_asset_type_enum.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/rich_media_studio_creative_billing_attribute_enum.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/rich_media_studio_creative_format_enum.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/rich_media_studio_messages.py create mode 100644 packages/google-ads-admanager/google/ads/admanager_v1/types/vast_redirect_type_enum.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_ad_review_center_ad_service_batch_apply_ad_review_center_custom_labels_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_ad_review_center_ad_service_fetch_ad_review_center_custom_labels_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_audience_segment_service_batch_update_audience_segments_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_audience_segment_service_update_audience_segment_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_creative_service_batch_activate_creatives_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_creative_service_batch_deactivate_creatives_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_creative_service_get_creative_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_creative_service_list_creatives_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_forecast_service_run_availability_forecast_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_forecast_service_run_delivery_forecast_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_forecast_service_run_traffic_data_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_creative_association_service_get_line_item_creative_association_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_creative_association_service_list_line_item_creative_associations_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_batch_activate_line_items_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_batch_archive_line_items_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_batch_create_line_items_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_batch_delete_line_items_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_batch_pause_line_items_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_batch_release_line_items_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_batch_reserve_and_overbook_line_items_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_batch_reserve_line_items_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_batch_resume_and_overbook_line_items_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_batch_resume_line_items_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_batch_unarchive_line_items_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_batch_update_line_items_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_create_line_item_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_service_update_line_item_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_template_service_get_line_item_template_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_line_item_template_service_list_line_item_templates_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_native_style_service_create_native_style_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_native_style_service_update_native_style_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_user_service_batch_activate_users_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_user_service_batch_create_users_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_user_service_batch_deactivate_users_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_user_service_batch_update_users_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_user_service_create_user_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_user_service_list_users_sync.py create mode 100644 packages/google-ads-admanager/samples/generated_samples/admanager_v1_generated_user_service_update_user_sync.py create mode 100644 packages/google-ads-admanager/tests/unit/gapic/admanager_v1/test_creative_service.py create mode 100644 packages/google-ads-admanager/tests/unit/gapic/admanager_v1/test_forecast_service.py create mode 100644 packages/google-ads-admanager/tests/unit/gapic/admanager_v1/test_line_item_creative_association_service.py create mode 100644 packages/google-ads-admanager/tests/unit/gapic/admanager_v1/test_line_item_template_service.py create mode 100644 packages/google-cloud-apphub/google/cloud/apphub_v1/types/boundary.py create mode 100644 packages/google-cloud-apphub/google/cloud/apphub_v1/types/extended_metadata_schema.py create mode 100644 packages/google-cloud-apphub/google/cloud/apphub_v1/types/properties.py create mode 100644 packages/google-cloud-apphub/samples/generated_samples/apphub_v1_generated_app_hub_get_boundary_async.py create mode 100644 packages/google-cloud-apphub/samples/generated_samples/apphub_v1_generated_app_hub_get_boundary_sync.py create mode 100644 packages/google-cloud-apphub/samples/generated_samples/apphub_v1_generated_app_hub_get_extended_metadata_schema_async.py create mode 100644 packages/google-cloud-apphub/samples/generated_samples/apphub_v1_generated_app_hub_get_extended_metadata_schema_sync.py create mode 100644 packages/google-cloud-apphub/samples/generated_samples/apphub_v1_generated_app_hub_list_extended_metadata_schemas_async.py create mode 100644 packages/google-cloud-apphub/samples/generated_samples/apphub_v1_generated_app_hub_list_extended_metadata_schemas_sync.py create mode 100644 packages/google-cloud-apphub/samples/generated_samples/apphub_v1_generated_app_hub_update_boundary_async.py create mode 100644 packages/google-cloud-apphub/samples/generated_samples/apphub_v1_generated_app_hub_update_boundary_sync.py create mode 100644 packages/google-cloud-apptopology/samples/generated_samples/apptopology_v1_generated_app_topology_explore_schema_async.py create mode 100644 packages/google-cloud-apptopology/samples/generated_samples/apptopology_v1_generated_app_topology_explore_schema_sync.py create mode 100644 packages/google-cloud-beyondcorp-appconnectors/samples/generated_samples/beyondcorp_v1_generated_app_connectors_service_resolve_instance_config_async.py create mode 100644 packages/google-cloud-beyondcorp-appconnectors/samples/generated_samples/beyondcorp_v1_generated_app_connectors_service_resolve_instance_config_sync.py create mode 100644 packages/google-cloud-compute-v1beta/samples/generated_samples/compute_v1beta_generated_interconnects_set_name_sync.py create mode 100644 packages/google-cloud-dialogflow/docs/dialogflow_v2beta1/companion_agents.rst create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/services/companion_agents/__init__.py create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/services/companion_agents/async_client.py create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/services/companion_agents/client.py create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/services/companion_agents/pagers.py create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/services/companion_agents/transports/README.rst create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/services/companion_agents/transports/__init__.py create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/services/companion_agents/transports/base.py create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/services/companion_agents/transports/grpc.py create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/services/companion_agents/transports/grpc_asyncio.py create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/services/companion_agents/transports/rest.py create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/services/companion_agents/transports/rest_base.py create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/types/companion_agent.py create mode 100644 packages/google-cloud-dialogflow/google/cloud/dialogflow_v2beta1/types/grounding.py create mode 100644 packages/google-cloud-dialogflow/samples/generated_samples/dialogflow_v2beta1_generated_companion_agents_create_companion_agent_async.py create mode 100644 packages/google-cloud-dialogflow/samples/generated_samples/dialogflow_v2beta1_generated_companion_agents_create_companion_agent_sync.py create mode 100644 packages/google-cloud-dialogflow/samples/generated_samples/dialogflow_v2beta1_generated_companion_agents_delete_companion_agent_async.py create mode 100644 packages/google-cloud-dialogflow/samples/generated_samples/dialogflow_v2beta1_generated_companion_agents_delete_companion_agent_sync.py create mode 100644 packages/google-cloud-dialogflow/samples/generated_samples/dialogflow_v2beta1_generated_companion_agents_get_companion_agent_async.py create mode 100644 packages/google-cloud-dialogflow/samples/generated_samples/dialogflow_v2beta1_generated_companion_agents_get_companion_agent_sync.py create mode 100644 packages/google-cloud-dialogflow/samples/generated_samples/dialogflow_v2beta1_generated_companion_agents_list_companion_agents_async.py create mode 100644 packages/google-cloud-dialogflow/samples/generated_samples/dialogflow_v2beta1_generated_companion_agents_list_companion_agents_sync.py create mode 100644 packages/google-cloud-dialogflow/samples/generated_samples/dialogflow_v2beta1_generated_companion_agents_update_companion_agent_async.py create mode 100644 packages/google-cloud-dialogflow/samples/generated_samples/dialogflow_v2beta1_generated_companion_agents_update_companion_agent_sync.py create mode 100644 packages/google-cloud-dialogflow/samples/generated_samples/dialogflow_v2beta1_generated_participants_streaming_reactive_companion_suggestions_async.py create mode 100644 packages/google-cloud-dialogflow/samples/generated_samples/dialogflow_v2beta1_generated_participants_streaming_reactive_companion_suggestions_sync.py create mode 100644 packages/google-cloud-dialogflow/tests/unit/gapic/dialogflow_v2beta1/test_companion_agents.py create mode 100644 packages/google-cloud-netapp/samples/generated_samples/netapp_v1_generated_net_app_get_split_status_async.py create mode 100644 packages/google-cloud-netapp/samples/generated_samples/netapp_v1_generated_net_app_get_split_status_sync.py create mode 100644 packages/google-cloud-netapp/samples/generated_samples/netapp_v1_generated_net_app_list_backup_configs_async.py create mode 100644 packages/google-cloud-netapp/samples/generated_samples/netapp_v1_generated_net_app_list_backup_configs_sync.py create mode 100644 packages/google-cloud-netapp/samples/generated_samples/netapp_v1_generated_net_app_restore_volume_async.py create mode 100644 packages/google-cloud-netapp/samples/generated_samples/netapp_v1_generated_net_app_restore_volume_sync.py create mode 100644 packages/google-cloud-netapp/samples/generated_samples/netapp_v1_generated_net_app_start_split_async.py create mode 100644 packages/google-cloud-netapp/samples/generated_samples/netapp_v1_generated_net_app_start_split_sync.py create mode 100644 packages/google-cloud-netapp/samples/generated_samples/netapp_v1_generated_net_app_update_backup_config_async.py create mode 100644 packages/google-cloud-netapp/samples/generated_samples/netapp_v1_generated_net_app_update_backup_config_sync.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_create_acl_policy_async.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_create_acl_policy_sync.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_delete_acl_policy_async.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_delete_acl_policy_sync.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_get_acl_policy_async.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_get_acl_policy_revision_async.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_get_acl_policy_revision_sync.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_get_acl_policy_sync.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_list_acl_policies_async.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_list_acl_policies_sync.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_list_acl_policy_revisions_async.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_list_acl_policy_revisions_sync.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_update_acl_policy_async.py create mode 100644 packages/google-cloud-redis-cluster/samples/generated_samples/redis_v1beta1_generated_cloud_redis_cluster_update_acl_policy_sync.py create mode 100644 packages/google-cloud-sql/docs/sqladmin_v1/blue_green_deployments_service.rst create mode 100644 packages/google-cloud-sql/docs/sqladmin_v1/sql_workload_captures_service.rst create mode 100644 packages/google-cloud-sql/docs/sqladmin_v1beta4/blue_green_deployments_service.rst create mode 100644 packages/google-cloud-sql/docs/sqladmin_v1beta4/sql_workload_captures_service.rst create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/blue_green_deployments_service/__init__.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/blue_green_deployments_service/async_client.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/blue_green_deployments_service/client.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/blue_green_deployments_service/pagers.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/blue_green_deployments_service/transports/README.rst create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/blue_green_deployments_service/transports/__init__.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/blue_green_deployments_service/transports/base.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/blue_green_deployments_service/transports/grpc.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/blue_green_deployments_service/transports/grpc_asyncio.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/sql_workload_captures_service/__init__.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/sql_workload_captures_service/async_client.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/sql_workload_captures_service/client.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/sql_workload_captures_service/transports/README.rst create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/sql_workload_captures_service/transports/__init__.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/sql_workload_captures_service/transports/base.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/sql_workload_captures_service/transports/grpc.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/services/sql_workload_captures_service/transports/grpc_asyncio.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/types/cloud_sql_blue_green_deployments.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1/types/cloud_sql_workload_captures.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/blue_green_deployments_service/__init__.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/blue_green_deployments_service/async_client.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/blue_green_deployments_service/client.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/blue_green_deployments_service/pagers.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/blue_green_deployments_service/transports/README.rst create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/blue_green_deployments_service/transports/__init__.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/blue_green_deployments_service/transports/base.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/blue_green_deployments_service/transports/grpc.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/blue_green_deployments_service/transports/grpc_asyncio.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/blue_green_deployments_service/transports/rest.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/blue_green_deployments_service/transports/rest_base.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/sql_workload_captures_service/__init__.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/sql_workload_captures_service/async_client.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/sql_workload_captures_service/client.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/sql_workload_captures_service/transports/README.rst create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/sql_workload_captures_service/transports/__init__.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/sql_workload_captures_service/transports/base.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/sql_workload_captures_service/transports/grpc.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/sql_workload_captures_service/transports/grpc_asyncio.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/sql_workload_captures_service/transports/rest.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/services/sql_workload_captures_service/transports/rest_base.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/types/cloud_sql_blue_green_deployments.py create mode 100644 packages/google-cloud-sql/google/cloud/sqladmin_v1beta4/types/cloud_sql_workload_captures.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_blue_green_deployments_service_create_blue_green_deployment_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_blue_green_deployments_service_create_blue_green_deployment_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_blue_green_deployments_service_delete_blue_green_deployment_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_blue_green_deployments_service_delete_blue_green_deployment_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_blue_green_deployments_service_get_blue_green_deployment_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_blue_green_deployments_service_get_blue_green_deployment_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_blue_green_deployments_service_list_blue_green_deployments_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_blue_green_deployments_service_list_blue_green_deployments_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_blue_green_deployments_service_switchover_blue_green_deployment_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_blue_green_deployments_service_switchover_blue_green_deployment_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_sql_workload_captures_service_list_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_sql_workload_captures_service_list_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_sql_workload_captures_service_start_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_sql_workload_captures_service_start_replay_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_sql_workload_captures_service_start_replay_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_sql_workload_captures_service_start_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_sql_workload_captures_service_stop_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_sql_workload_captures_service_stop_replay_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_sql_workload_captures_service_stop_replay_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1_generated_sql_workload_captures_service_stop_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_blue_green_deployments_service_create_blue_green_deployment_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_blue_green_deployments_service_create_blue_green_deployment_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_blue_green_deployments_service_delete_blue_green_deployment_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_blue_green_deployments_service_delete_blue_green_deployment_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_blue_green_deployments_service_get_blue_green_deployment_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_blue_green_deployments_service_get_blue_green_deployment_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_blue_green_deployments_service_list_blue_green_deployments_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_blue_green_deployments_service_list_blue_green_deployments_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_blue_green_deployments_service_switchover_blue_green_deployment_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_blue_green_deployments_service_switchover_blue_green_deployment_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_sql_workload_captures_service_list_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_sql_workload_captures_service_list_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_sql_workload_captures_service_start_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_sql_workload_captures_service_start_replay_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_sql_workload_captures_service_start_replay_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_sql_workload_captures_service_start_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_sql_workload_captures_service_stop_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_sql_workload_captures_service_stop_replay_async.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_sql_workload_captures_service_stop_replay_sync.py create mode 100644 packages/google-cloud-sql/samples/generated_samples/sqladmin_v1beta4_generated_sql_workload_captures_service_stop_sync.py create mode 100644 packages/google-cloud-sql/tests/unit/gapic/sqladmin_v1/test_blue_green_deployments_service.py create mode 100644 packages/google-cloud-sql/tests/unit/gapic/sqladmin_v1/test_sql_workload_captures_service.py create mode 100644 packages/google-cloud-sql/tests/unit/gapic/sqladmin_v1beta4/test_blue_green_deployments_service.py create mode 100644 packages/google-cloud-sql/tests/unit/gapic/sqladmin_v1beta4/test_sql_workload_captures_service.py diff --git a/librarian.yaml b/librarian.yaml index c014d8d18d6d..437d0339e032 100644 --- a/librarian.yaml +++ b/librarian.yaml @@ -16,8 +16,8 @@ version: v0.47.0 repo: googleapis/google-cloud-python sources: googleapis: - commit: 93d6085996d0b4ff7e7e86ca9945d5524bb380d1 - sha256: 2271ff64a808d6a9089d0f410d28fc9ce4dd35ffe6e1c2cf79dd9a8578ddc871 + commit: 6a8220ce2f0b8f51e60bb2cc0d74a14d380a823b + sha256: eb737809a4b1aa6ab7be3c2078b6cbdde44337dcdf464d35813e6345896e3343 tools: pip: - name: black diff --git a/packages/google-ads-admanager/docs/admanager_v1/creative_service.rst b/packages/google-ads-admanager/docs/admanager_v1/creative_service.rst new file mode 100644 index 000000000000..2f4e457a7ab3 --- /dev/null +++ b/packages/google-ads-admanager/docs/admanager_v1/creative_service.rst @@ -0,0 +1,10 @@ +CreativeService +--------------------------------- + +.. automodule:: google.ads.admanager_v1.services.creative_service + :members: + :inherited-members: + +.. automodule:: google.ads.admanager_v1.services.creative_service.pagers + :members: + :inherited-members: diff --git a/packages/google-ads-admanager/docs/admanager_v1/forecast_service.rst b/packages/google-ads-admanager/docs/admanager_v1/forecast_service.rst new file mode 100644 index 000000000000..b831dcd06e94 --- /dev/null +++ b/packages/google-ads-admanager/docs/admanager_v1/forecast_service.rst @@ -0,0 +1,6 @@ +ForecastService +--------------------------------- + +.. automodule:: google.ads.admanager_v1.services.forecast_service + :members: + :inherited-members: diff --git a/packages/google-ads-admanager/docs/admanager_v1/line_item_creative_association_service.rst b/packages/google-ads-admanager/docs/admanager_v1/line_item_creative_association_service.rst new file mode 100644 index 000000000000..ba6382d83d89 --- /dev/null +++ b/packages/google-ads-admanager/docs/admanager_v1/line_item_creative_association_service.rst @@ -0,0 +1,10 @@ +LineItemCreativeAssociationService +---------------------------------------------------- + +.. automodule:: google.ads.admanager_v1.services.line_item_creative_association_service + :members: + :inherited-members: + +.. automodule:: google.ads.admanager_v1.services.line_item_creative_association_service.pagers + :members: + :inherited-members: diff --git a/packages/google-ads-admanager/docs/admanager_v1/line_item_template_service.rst b/packages/google-ads-admanager/docs/admanager_v1/line_item_template_service.rst new file mode 100644 index 000000000000..3d0707efbac1 --- /dev/null +++ b/packages/google-ads-admanager/docs/admanager_v1/line_item_template_service.rst @@ -0,0 +1,10 @@ +LineItemTemplateService +----------------------------------------- + +.. automodule:: google.ads.admanager_v1.services.line_item_template_service + :members: + :inherited-members: + +.. automodule:: google.ads.admanager_v1.services.line_item_template_service.pagers + :members: + :inherited-members: diff --git a/packages/google-ads-admanager/docs/admanager_v1/services_.rst b/packages/google-ads-admanager/docs/admanager_v1/services_.rst index 61c1061eec98..298f236de4ac 100644 --- a/packages/google-ads-admanager/docs/admanager_v1/services_.rst +++ b/packages/google-ads-admanager/docs/admanager_v1/services_.rst @@ -23,6 +23,7 @@ Services for Google Ads Admanager v1 API content_bundle_service content_label_service content_service + creative_service creative_set_service creative_template_service creative_wrapper_service @@ -36,9 +37,12 @@ Services for Google Ads Admanager v1 API device_category_service device_manufacturer_service entity_signals_mapping_service + forecast_service geo_target_service label_service + line_item_creative_association_service line_item_service + line_item_template_service linked_device_service live_stream_service mcm_earnings_service diff --git a/packages/google-ads-admanager/docs/admanager_v1/user_service.rst b/packages/google-ads-admanager/docs/admanager_v1/user_service.rst index c7be2db4394e..9bae86979749 100644 --- a/packages/google-ads-admanager/docs/admanager_v1/user_service.rst +++ b/packages/google-ads-admanager/docs/admanager_v1/user_service.rst @@ -4,3 +4,7 @@ UserService .. automodule:: google.ads.admanager_v1.services.user_service :members: :inherited-members: + +.. automodule:: google.ads.admanager_v1.services.user_service.pagers + :members: + :inherited-members: diff --git a/packages/google-ads-admanager/google/ads/admanager/__init__.py b/packages/google-ads-admanager/google/ads/admanager/__init__.py index f5c68a69ab9d..8b8d6c1aa48a 100644 --- a/packages/google-ads-admanager/google/ads/admanager/__init__.py +++ b/packages/google-ads-admanager/google/ads/admanager/__init__.py @@ -64,6 +64,9 @@ ContentLabelServiceClient, ) from google.ads.admanager_v1.services.content_service.client import ContentServiceClient +from google.ads.admanager_v1.services.creative_service.client import ( + CreativeServiceClient, +) from google.ads.admanager_v1.services.creative_set_service.client import ( CreativeSetServiceClient, ) @@ -103,13 +106,22 @@ from google.ads.admanager_v1.services.entity_signals_mapping_service.client import ( EntitySignalsMappingServiceClient, ) +from google.ads.admanager_v1.services.forecast_service.client import ( + ForecastServiceClient, +) from google.ads.admanager_v1.services.geo_target_service.client import ( GeoTargetServiceClient, ) from google.ads.admanager_v1.services.label_service.client import LabelServiceClient +from google.ads.admanager_v1.services.line_item_creative_association_service.client import ( + LineItemCreativeAssociationServiceClient, +) from google.ads.admanager_v1.services.line_item_service.client import ( LineItemServiceClient, ) +from google.ads.admanager_v1.services.line_item_template_service.client import ( + LineItemTemplateServiceClient, +) from google.ads.admanager_v1.services.linked_device_service.client import ( LinkedDeviceServiceClient, ) @@ -190,6 +202,7 @@ ) from google.ads.admanager_v1.types.ad_review_center_ad_enums import ( AdReviewCenterAdStatusEnum, + ArcCreativeFormatEnum, ManualAdReviewCenterAdStatusEnum, ) from google.ads.admanager_v1.types.ad_review_center_ad_messages import AdReviewCenterAd @@ -197,8 +210,12 @@ BatchAdReviewCenterAdsOperationMetadata, BatchAllowAdReviewCenterAdsRequest, BatchAllowAdReviewCenterAdsResponse, + BatchApplyAdReviewCenterCustomLabelsRequest, + BatchApplyAdReviewCenterCustomLabelsResponse, BatchBlockAdReviewCenterAdsRequest, BatchBlockAdReviewCenterAdsResponse, + FetchAdReviewCenterCustomLabelsRequest, + FetchAdReviewCenterCustomLabelsResponse, SearchAdReviewCenterAdsRequest, SearchAdReviewCenterAdsResponse, ) @@ -304,6 +321,8 @@ UpdateApplicationRequest, ) from google.ads.admanager_v1.types.applied_label import AppliedLabel +from google.ads.admanager_v1.types.asset_enums import AssetImageDensityEnum +from google.ads.admanager_v1.types.asset_messages import Asset from google.ads.admanager_v1.types.audience_segment_enums import ( AudienceSegmentApprovalStatusEnum, AudienceSegmentLicenseTypeEnum, @@ -324,10 +343,13 @@ BatchPopulateAudienceSegmentsResponse, BatchRejectAudienceSegmentsRequest, BatchRejectAudienceSegmentsResponse, + BatchUpdateAudienceSegmentsRequest, + BatchUpdateAudienceSegmentsResponse, CreateAudienceSegmentRequest, GetAudienceSegmentRequest, ListAudienceSegmentsRequest, ListAudienceSegmentsResponse, + UpdateAudienceSegmentRequest, ) from google.ads.admanager_v1.types.bandwidth_group_messages import BandwidthGroup from google.ads.admanager_v1.types.bandwidth_group_service import ( @@ -506,11 +528,54 @@ ListContentResponse, ) from google.ads.admanager_v1.types.conversion_event_enum import ConversionEventEnum -from google.ads.admanager_v1.types.creative_messages import Creative +from google.ads.admanager_v1.types.creative_asset import ClickTag, CreativeAsset +from google.ads.admanager_v1.types.creative_enums import ( + CreativeDestinationUrlTypeEnum, + CreativeLockedOrientationEnum, + CreativeSslOverrideEnum, + CreativeSslScanResultEnum, + VastAdIdTypeEnum, +) +from google.ads.admanager_v1.types.creative_messages import ( + AdExchangeCreativeDetails, + AdSenseCreativeDetails, + AspectRatioImageCreativeDetails, + AudioCreativeDetails, + AudioRedirectCreativeDetails, + BuyerPlacementConfig, + ClickTrackingCreativeDetails, + Creative, + CustomCreativeDetails, + Html5CreativeDetails, + ImageCreativeDetails, + ImageOverlayCreativeDetails, + ImageRedirectCreativeDetails, + ImageRedirectOverlayCreativeDetails, + InternalRedirectCreativeDetails, + LegacyDfpCreativeDetails, + ProgrammaticCreativeDetails, + RichMediaStudioCreativeDetails, + SetTopBoxCreativeDetails, + TemplateCreativeDetails, + ThirdPartyCreativeDetails, + VastInfo, + VastRedirectCreativeDetails, + VideoCreativeDetails, + VideoRedirectCreativeDetails, +) from google.ads.admanager_v1.types.creative_placeholder import ( CreativePlaceholder, CreativePlaceholderCompanion, ) +from google.ads.admanager_v1.types.creative_service import ( + BatchActivateCreativesRequest, + BatchActivateCreativesResponse, + BatchDeactivateCreativesRequest, + BatchDeactivateCreativesResponse, + GetCreativeRequest, + ListCreativesRequest, + ListCreativesResponse, +) from google.ads.admanager_v1.types.creative_set_messages import CreativeSet from google.ads.admanager_v1.types.creative_set_service import ( CreateCreativeSetRequest, @@ -536,6 +601,9 @@ from google.ads.admanager_v1.types.creative_template_variable_url_type_enum import ( CreativeTemplateVariableUrlTypeEnum, ) +from google.ads.admanager_v1.types.creative_third_party_data_declaration_status_enum import ( + CreativeThirdPartyDataDeclarationStatusEnum, +) from google.ads.admanager_v1.types.creative_wrapper_enums import ( CreativeWrapperOrderingEnum, CreativeWrapperStatusEnum, @@ -557,6 +625,7 @@ ListCreativeWrappersResponse, UpdateCreativeWrapperRequest, ) +from google.ads.admanager_v1.types.custom_creative_asset import CustomCreativeAsset from google.ads.admanager_v1.types.custom_field_enums import ( CustomFieldDataTypeEnum, CustomFieldEntityTypeEnum, @@ -703,6 +772,7 @@ ) from google.ads.admanager_v1.types.dai_session_messages import DaiSession from google.ads.admanager_v1.types.dai_session_service import GetDaiSessionRequest +from google.ads.admanager_v1.types.date_range import DateRange from google.ads.admanager_v1.types.deal_buyer_permission_type_enum import ( DealBuyerPermissionTypeEnum, ) @@ -757,6 +827,38 @@ ExchangeSyndicationProductEnum, ) from google.ads.admanager_v1.types.exclusion_scope_enum import ExclusionScopeEnum +from google.ads.admanager_v1.types.forecast_messages import ( + AlternativeUnitTypeForecast, + AvailabilityForecast, + AvailabilityForecastOptions, + ContendingLineItem, + DeliveryForecast, + DeliveryForecastOptions, + DeliveryForecastResult, + ExistingLineItemList, + ForecastBreakdown, + ForecastBreakdownEntry, + ForecastBreakdownOptions, + ForecastBreakdownTarget, + GrpDemographicBreakdown, + LineItemDeliveryForecast, + TargetingCriteriaBreakdown, + TimeSeries, +) +from google.ads.admanager_v1.types.forecast_service import ( + RunAvailabilityForecastRequest, + RunAvailabilityForecastResponse, + RunDeliveryForecastRequest, + RunDeliveryForecastResponse, + RunTrafficDataRequest, + RunTrafficDataResponse, +) +from google.ads.admanager_v1.types.forecasting_enums import ( + ForecastingGrpAgeEnum, + ForecastingGrpGenderEnum, + ForecastingGrpUnitEnum, + ForecastingTargetingDimensionEnum, +) from google.ads.admanager_v1.types.frequency_cap import FrequencyCap from google.ads.admanager_v1.types.geo_target_messages import GeoTarget from google.ads.admanager_v1.types.geo_target_service import ( @@ -789,6 +891,18 @@ from google.ads.admanager_v1.types.line_item_allowed_format_enum import ( LineItemAllowedFormatEnum, ) +from google.ads.admanager_v1.types.line_item_creative_association_enums import ( + LineItemCreativeAssociationStatusEnum, +) +from google.ads.admanager_v1.types.line_item_creative_association_messages import ( + LineItemCreativeAssociation, + LineItemCreativeAssociationStats, +) +from google.ads.admanager_v1.types.line_item_creative_association_service import ( + GetLineItemCreativeAssociationRequest, + ListLineItemCreativeAssociationsRequest, + ListLineItemCreativeAssociationsResponse, +) from google.ads.admanager_v1.types.line_item_deal_info import LineItemDealInfo from google.ads.admanager_v1.types.line_item_delivery_forecast_source_enum import ( LineItemDeliveryForecastSourceEnum, @@ -802,11 +916,42 @@ ) from google.ads.admanager_v1.types.line_item_messages import LineItem from google.ads.admanager_v1.types.line_item_service import ( + BatchActivateLineItemsRequest, + BatchActivateLineItemsResponse, + BatchArchiveLineItemsRequest, + BatchArchiveLineItemsResponse, + BatchCreateLineItemsRequest, + BatchCreateLineItemsResponse, + BatchDeleteLineItemsRequest, + BatchPauseLineItemsRequest, + BatchPauseLineItemsResponse, + BatchReleaseLineItemsRequest, + BatchReleaseLineItemsResponse, + BatchReserveAndOverbookLineItemsRequest, + BatchReserveAndOverbookLineItemsResponse, + BatchReserveLineItemsRequest, + BatchReserveLineItemsResponse, + BatchResumeAndOverbookLineItemsRequest, + BatchResumeAndOverbookLineItemsResponse, + BatchResumeLineItemsRequest, + BatchResumeLineItemsResponse, + BatchUnarchiveLineItemsRequest, + BatchUnarchiveLineItemsResponse, + BatchUpdateLineItemsRequest, + BatchUpdateLineItemsResponse, + CreateLineItemRequest, GetLineItemRequest, ListLineItemsRequest, ListLineItemsResponse, + UpdateLineItemRequest, ) from google.ads.admanager_v1.types.line_item_stats import LineItemStats +from google.ads.admanager_v1.types.line_item_template_messages import LineItemTemplate +from google.ads.admanager_v1.types.line_item_template_service import ( + GetLineItemTemplateRequest, + ListLineItemTemplatesRequest, + ListLineItemTemplatesResponse, +) from google.ads.admanager_v1.types.linked_device_enums import LinkedDeviceVisibilityEnum from google.ads.admanager_v1.types.linked_device_messages import LinkedDevice from google.ads.admanager_v1.types.linked_device_service import ( @@ -1077,6 +1222,18 @@ ListRichMediaAdsCompaniesRequest, ListRichMediaAdsCompaniesResponse, ) +from google.ads.admanager_v1.types.rich_media_studio_child_asset_type_enum import ( + RichMediaStudioChildAssetTypeEnum, +) +from google.ads.admanager_v1.types.rich_media_studio_creative_billing_attribute_enum import ( + RichMediaStudioCreativeBillingAttributeEnum, +) +from google.ads.admanager_v1.types.rich_media_studio_creative_format_enum import ( + RichMediaStudioCreativeFormatEnum, +) +from google.ads.admanager_v1.types.rich_media_studio_messages import ( + RichMediaStudioChildAssetProperty, +) from google.ads.admanager_v1.types.role_enums import RoleStatusEnum from google.ads.admanager_v1.types.role_messages import Role from google.ads.admanager_v1.types.role_service import ( @@ -1227,7 +1384,22 @@ ) from google.ads.admanager_v1.types.time_unit_enum import TimeUnitEnum from google.ads.admanager_v1.types.user_messages import User -from google.ads.admanager_v1.types.user_service import GetUserRequest +from google.ads.admanager_v1.types.user_service import ( + BatchActivateUsersRequest, + BatchActivateUsersResponse, + BatchCreateUsersRequest, + BatchCreateUsersResponse, + BatchDeactivateUsersRequest, + BatchDeactivateUsersResponse, + BatchUpdateUsersRequest, + BatchUpdateUsersResponse, + CreateUserRequest, + GetUserRequest, + ListUsersRequest, + ListUsersResponse, + UpdateUserRequest, +) +from google.ads.admanager_v1.types.vast_redirect_type_enum import VastRedirectTypeEnum from google.ads.admanager_v1.types.video_position_enum import VideoPositionEnum from google.ads.admanager_v1.types.video_tracking_url import VideoTrackingUrl from google.ads.admanager_v1.types.video_transcode_status_enum import ( @@ -1273,6 +1445,7 @@ "ContentBundleServiceClient", "ContentLabelServiceClient", "ContentServiceClient", + "CreativeServiceClient", "CreativeSetServiceClient", "CreativeTemplateServiceClient", "CreativeWrapperServiceClient", @@ -1286,9 +1459,12 @@ "DeviceCategoryServiceClient", "DeviceManufacturerServiceClient", "EntitySignalsMappingServiceClient", + "ForecastServiceClient", "GeoTargetServiceClient", "LabelServiceClient", + "LineItemCreativeAssociationServiceClient", "LineItemServiceClient", + "LineItemTemplateServiceClient", "LinkedDeviceServiceClient", "LiveStreamServiceClient", "McmEarningsServiceClient", @@ -1326,13 +1502,18 @@ "ListAdBreaksResponse", "UpdateAdBreakRequest", "AdReviewCenterAdStatusEnum", + "ArcCreativeFormatEnum", "ManualAdReviewCenterAdStatusEnum", "AdReviewCenterAd", "BatchAdReviewCenterAdsOperationMetadata", "BatchAllowAdReviewCenterAdsRequest", "BatchAllowAdReviewCenterAdsResponse", + "BatchApplyAdReviewCenterCustomLabelsRequest", + "BatchApplyAdReviewCenterCustomLabelsResponse", "BatchBlockAdReviewCenterAdsRequest", "BatchBlockAdReviewCenterAdsResponse", + "FetchAdReviewCenterCustomLabelsRequest", + "FetchAdReviewCenterCustomLabelsResponse", "SearchAdReviewCenterAdsRequest", "SearchAdReviewCenterAdsResponse", "AdRuleFrequencyCapBehaviorEnum", @@ -1414,6 +1595,8 @@ "ListApplicationsResponse", "UpdateApplicationRequest", "AppliedLabel", + "AssetImageDensityEnum", + "Asset", "AudienceSegmentApprovalStatusEnum", "AudienceSegmentLicenseTypeEnum", "AudienceSegmentStatusEnum", @@ -1431,10 +1614,13 @@ "BatchPopulateAudienceSegmentsResponse", "BatchRejectAudienceSegmentsRequest", "BatchRejectAudienceSegmentsResponse", + "BatchUpdateAudienceSegmentsRequest", + "BatchUpdateAudienceSegmentsResponse", "CreateAudienceSegmentRequest", "GetAudienceSegmentRequest", "ListAudienceSegmentsRequest", "ListAudienceSegmentsResponse", + "UpdateAudienceSegmentRequest", "BandwidthGroup", "GetBandwidthGroupRequest", "ListBandwidthGroupsRequest", @@ -1566,9 +1752,47 @@ "ListContentRequest", "ListContentResponse", "ConversionEventEnum", + "ClickTag", + "CreativeAsset", + "CreativeDestinationUrlTypeEnum", + "CreativeLockedOrientationEnum", + "CreativeSslOverrideEnum", + "CreativeSslScanResultEnum", + "VastAdIdTypeEnum", + "AdExchangeCreativeDetails", + "AdSenseCreativeDetails", + "AspectRatioImageCreativeDetails", + "AudioCreativeDetails", + "AudioRedirectCreativeDetails", + "BuyerPlacementConfig", + "ClickTrackingCreativeDetails", "Creative", + "CustomCreativeDetails", + "Html5CreativeDetails", + "ImageCreativeDetails", + "ImageOverlayCreativeDetails", + "ImageRedirectCreativeDetails", + "ImageRedirectOverlayCreativeDetails", + "InternalRedirectCreativeDetails", + "LegacyDfpCreativeDetails", + "ProgrammaticCreativeDetails", + "RichMediaStudioCreativeDetails", + "SetTopBoxCreativeDetails", + "TemplateCreativeDetails", + "ThirdPartyCreativeDetails", + "VastInfo", + "VastRedirectCreativeDetails", + "VideoCreativeDetails", + "VideoRedirectCreativeDetails", "CreativePlaceholder", "CreativePlaceholderCompanion", + "BatchActivateCreativesRequest", + "BatchActivateCreativesResponse", + "BatchDeactivateCreativesRequest", + "BatchDeactivateCreativesResponse", + "GetCreativeRequest", + "ListCreativesRequest", + "ListCreativesResponse", "CreativeSet", "CreateCreativeSetRequest", "GetCreativeSetRequest", @@ -1584,6 +1808,7 @@ "ListCreativeTemplatesRequest", "ListCreativeTemplatesResponse", "CreativeTemplateVariableUrlTypeEnum", + "CreativeThirdPartyDataDeclarationStatusEnum", "CreativeWrapperOrderingEnum", "CreativeWrapperStatusEnum", "CreativeWrapperTypeEnum", @@ -1601,6 +1826,7 @@ "ListCreativeWrappersRequest", "ListCreativeWrappersResponse", "UpdateCreativeWrapperRequest", + "CustomCreativeAsset", "CustomFieldDataTypeEnum", "CustomFieldEntityTypeEnum", "CustomFieldStatusEnum", @@ -1711,6 +1937,7 @@ "TrackingPingFindingTypeEnum", "DaiSession", "GetDaiSessionRequest", + "DateRange", "DealBuyerPermissionTypeEnum", "DealPriorityTierEnum", "CompanionDeliveryOptionEnum", @@ -1745,6 +1972,32 @@ "EnvironmentTypeEnum", "ExchangeSyndicationProductEnum", "ExclusionScopeEnum", + "AlternativeUnitTypeForecast", + "AvailabilityForecast", + "AvailabilityForecastOptions", + "ContendingLineItem", + "DeliveryForecast", + "DeliveryForecastOptions", + "DeliveryForecastResult", + "ExistingLineItemList", + "ForecastBreakdown", + "ForecastBreakdownEntry", + "ForecastBreakdownOptions", + "ForecastBreakdownTarget", + "GrpDemographicBreakdown", + "LineItemDeliveryForecast", + "TargetingCriteriaBreakdown", + "TimeSeries", + "RunAvailabilityForecastRequest", + "RunAvailabilityForecastResponse", + "RunDeliveryForecastRequest", + "RunDeliveryForecastResponse", + "RunTrafficDataRequest", + "RunTrafficDataResponse", + "ForecastingGrpAgeEnum", + "ForecastingGrpGenderEnum", + "ForecastingGrpUnitEnum", + "ForecastingTargetingDimensionEnum", "FrequencyCap", "GeoTarget", "GetGeoTargetRequest", @@ -1772,6 +2025,12 @@ "ListLabelsResponse", "UpdateLabelRequest", "LineItemAllowedFormatEnum", + "LineItemCreativeAssociationStatusEnum", + "LineItemCreativeAssociation", + "LineItemCreativeAssociationStats", + "GetLineItemCreativeAssociationRequest", + "ListLineItemCreativeAssociationsRequest", + "ListLineItemCreativeAssociationsResponse", "LineItemDealInfo", "LineItemDeliveryForecastSourceEnum", "LineItemDiscount", @@ -1780,10 +2039,39 @@ "LineItemReservationStatusEnum", "LineItemTypeEnum", "LineItem", + "BatchActivateLineItemsRequest", + "BatchActivateLineItemsResponse", + "BatchArchiveLineItemsRequest", + "BatchArchiveLineItemsResponse", + "BatchCreateLineItemsRequest", + "BatchCreateLineItemsResponse", + "BatchDeleteLineItemsRequest", + "BatchPauseLineItemsRequest", + "BatchPauseLineItemsResponse", + "BatchReleaseLineItemsRequest", + "BatchReleaseLineItemsResponse", + "BatchReserveAndOverbookLineItemsRequest", + "BatchReserveAndOverbookLineItemsResponse", + "BatchReserveLineItemsRequest", + "BatchReserveLineItemsResponse", + "BatchResumeAndOverbookLineItemsRequest", + "BatchResumeAndOverbookLineItemsResponse", + "BatchResumeLineItemsRequest", + "BatchResumeLineItemsResponse", + "BatchUnarchiveLineItemsRequest", + "BatchUnarchiveLineItemsResponse", + "BatchUpdateLineItemsRequest", + "BatchUpdateLineItemsResponse", + "CreateLineItemRequest", "GetLineItemRequest", "ListLineItemsRequest", "ListLineItemsResponse", + "UpdateLineItemRequest", "LineItemStats", + "LineItemTemplate", + "GetLineItemTemplateRequest", + "ListLineItemTemplatesRequest", + "ListLineItemTemplatesResponse", "LinkedDeviceVisibilityEnum", "LinkedDevice", "GetLinkedDeviceRequest", @@ -1993,6 +2281,10 @@ "GetRichMediaAdsCompanyRequest", "ListRichMediaAdsCompaniesRequest", "ListRichMediaAdsCompaniesResponse", + "RichMediaStudioChildAssetTypeEnum", + "RichMediaStudioCreativeBillingAttributeEnum", + "RichMediaStudioCreativeFormatEnum", + "RichMediaStudioChildAssetProperty", "RoleStatusEnum", "Role", "GetRoleRequest", @@ -2113,7 +2405,20 @@ "ThirdPartyMeasurementSettings", "TimeUnitEnum", "User", + "BatchActivateUsersRequest", + "BatchActivateUsersResponse", + "BatchCreateUsersRequest", + "BatchCreateUsersResponse", + "BatchDeactivateUsersRequest", + "BatchDeactivateUsersResponse", + "BatchUpdateUsersRequest", + "BatchUpdateUsersResponse", + "CreateUserRequest", "GetUserRequest", + "ListUsersRequest", + "ListUsersResponse", + "UpdateUserRequest", + "VastRedirectTypeEnum", "VideoPositionEnum", "VideoTrackingUrl", "VideoTranscodeStatusEnum", diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/__init__.py b/packages/google-ads-admanager/google/ads/admanager_v1/__init__.py index 75fcf1616db3..746b2ac0d048 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/__init__.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/__init__.py @@ -46,6 +46,7 @@ "google.ads.admanager_v1.services.content_bundle_service", "google.ads.admanager_v1.services.content_label_service", "google.ads.admanager_v1.services.content_service", + "google.ads.admanager_v1.services.creative_service", "google.ads.admanager_v1.services.creative_set_service", "google.ads.admanager_v1.services.creative_template_service", "google.ads.admanager_v1.services.creative_wrapper_service", @@ -59,9 +60,12 @@ "google.ads.admanager_v1.services.device_category_service", "google.ads.admanager_v1.services.device_manufacturer_service", "google.ads.admanager_v1.services.entity_signals_mapping_service", + "google.ads.admanager_v1.services.forecast_service", "google.ads.admanager_v1.services.geo_target_service", "google.ads.admanager_v1.services.label_service", + "google.ads.admanager_v1.services.line_item_creative_association_service", "google.ads.admanager_v1.services.line_item_service", + "google.ads.admanager_v1.services.line_item_template_service", "google.ads.admanager_v1.services.linked_device_service", "google.ads.admanager_v1.services.live_stream_service", "google.ads.admanager_v1.services.mcm_earnings_service", @@ -115,6 +119,8 @@ "google.ads.admanager_v1.types.application_messages", "google.ads.admanager_v1.types.application_service", "google.ads.admanager_v1.types.applied_label", + "google.ads.admanager_v1.types.asset_enums", + "google.ads.admanager_v1.types.asset_messages", "google.ads.admanager_v1.types.audience_segment_enums", "google.ads.admanager_v1.types.audience_segment_messages", "google.ads.admanager_v1.types.audience_segment_service", @@ -157,8 +163,11 @@ "google.ads.admanager_v1.types.content_messages", "google.ads.admanager_v1.types.content_service", "google.ads.admanager_v1.types.conversion_event_enum", + "google.ads.admanager_v1.types.creative_asset", + "google.ads.admanager_v1.types.creative_enums", "google.ads.admanager_v1.types.creative_messages", "google.ads.admanager_v1.types.creative_placeholder", + "google.ads.admanager_v1.types.creative_service", "google.ads.admanager_v1.types.creative_set_messages", "google.ads.admanager_v1.types.creative_set_service", "google.ads.admanager_v1.types.creative_targeting", @@ -166,9 +175,11 @@ "google.ads.admanager_v1.types.creative_template_messages", "google.ads.admanager_v1.types.creative_template_service", "google.ads.admanager_v1.types.creative_template_variable_url_type_enum", + "google.ads.admanager_v1.types.creative_third_party_data_declaration_status_enum", "google.ads.admanager_v1.types.creative_wrapper_enums", "google.ads.admanager_v1.types.creative_wrapper_messages", "google.ads.admanager_v1.types.creative_wrapper_service", + "google.ads.admanager_v1.types.custom_creative_asset", "google.ads.admanager_v1.types.custom_field_enums", "google.ads.admanager_v1.types.custom_field_messages", "google.ads.admanager_v1.types.custom_field_service", @@ -190,6 +201,7 @@ "google.ads.admanager_v1.types.dai_session_enums", "google.ads.admanager_v1.types.dai_session_messages", "google.ads.admanager_v1.types.dai_session_service", + "google.ads.admanager_v1.types.date_range", "google.ads.admanager_v1.types.deal_buyer_permission_type_enum", "google.ads.admanager_v1.types.deal_priority_tier_enum", "google.ads.admanager_v1.types.delivery_enums", @@ -207,6 +219,9 @@ "google.ads.admanager_v1.types.environment_type_enum", "google.ads.admanager_v1.types.exchange_syndication_product_enum", "google.ads.admanager_v1.types.exclusion_scope_enum", + "google.ads.admanager_v1.types.forecast_messages", + "google.ads.admanager_v1.types.forecast_service", + "google.ads.admanager_v1.types.forecasting_enums", "google.ads.admanager_v1.types.frequency_cap", "google.ads.admanager_v1.types.geo_target_messages", "google.ads.admanager_v1.types.geo_target_service", @@ -219,6 +234,9 @@ "google.ads.admanager_v1.types.label_messages", "google.ads.admanager_v1.types.label_service", "google.ads.admanager_v1.types.line_item_allowed_format_enum", + "google.ads.admanager_v1.types.line_item_creative_association_enums", + "google.ads.admanager_v1.types.line_item_creative_association_messages", + "google.ads.admanager_v1.types.line_item_creative_association_service", "google.ads.admanager_v1.types.line_item_deal_info", "google.ads.admanager_v1.types.line_item_delivery_forecast_source_enum", "google.ads.admanager_v1.types.line_item_discount", @@ -226,6 +244,8 @@ "google.ads.admanager_v1.types.line_item_messages", "google.ads.admanager_v1.types.line_item_service", "google.ads.admanager_v1.types.line_item_stats", + "google.ads.admanager_v1.types.line_item_template_messages", + "google.ads.admanager_v1.types.line_item_template_service", "google.ads.admanager_v1.types.linked_device_enums", "google.ads.admanager_v1.types.linked_device_messages", "google.ads.admanager_v1.types.linked_device_service", @@ -280,6 +300,10 @@ "google.ads.admanager_v1.types.rich_media_ads_company_enums", "google.ads.admanager_v1.types.rich_media_ads_company_messages", "google.ads.admanager_v1.types.rich_media_ads_company_service", + "google.ads.admanager_v1.types.rich_media_studio_child_asset_type_enum", + "google.ads.admanager_v1.types.rich_media_studio_creative_billing_attribute_enum", + "google.ads.admanager_v1.types.rich_media_studio_creative_format_enum", + "google.ads.admanager_v1.types.rich_media_studio_messages", "google.ads.admanager_v1.types.role_enums", "google.ads.admanager_v1.types.role_messages", "google.ads.admanager_v1.types.role_service", @@ -314,6 +338,7 @@ "google.ads.admanager_v1.types.time_unit_enum", "google.ads.admanager_v1.types.user_messages", "google.ads.admanager_v1.types.user_service", + "google.ads.admanager_v1.types.vast_redirect_type_enum", "google.ads.admanager_v1.types.video_position_enum", "google.ads.admanager_v1.types.video_tracking_url", "google.ads.admanager_v1.types.video_transcode_status_enum", @@ -344,6 +369,7 @@ from .services.content_bundle_service import ContentBundleServiceClient from .services.content_label_service import ContentLabelServiceClient from .services.content_service import ContentServiceClient +from .services.creative_service import CreativeServiceClient from .services.creative_set_service import CreativeSetServiceClient from .services.creative_template_service import CreativeTemplateServiceClient from .services.creative_wrapper_service import CreativeWrapperServiceClient @@ -357,9 +383,14 @@ from .services.device_category_service import DeviceCategoryServiceClient from .services.device_manufacturer_service import DeviceManufacturerServiceClient from .services.entity_signals_mapping_service import EntitySignalsMappingServiceClient +from .services.forecast_service import ForecastServiceClient from .services.geo_target_service import GeoTargetServiceClient from .services.label_service import LabelServiceClient +from .services.line_item_creative_association_service import ( + LineItemCreativeAssociationServiceClient, +) from .services.line_item_service import LineItemServiceClient +from .services.line_item_template_service import LineItemTemplateServiceClient from .services.linked_device_service import LinkedDeviceServiceClient from .services.live_stream_service import LiveStreamServiceClient from .services.mcm_earnings_service import McmEarningsServiceClient @@ -402,6 +433,7 @@ ) from .types.ad_review_center_ad_enums import ( AdReviewCenterAdStatusEnum, + ArcCreativeFormatEnum, ManualAdReviewCenterAdStatusEnum, ) from .types.ad_review_center_ad_messages import AdReviewCenterAd @@ -409,8 +441,12 @@ BatchAdReviewCenterAdsOperationMetadata, BatchAllowAdReviewCenterAdsRequest, BatchAllowAdReviewCenterAdsResponse, + BatchApplyAdReviewCenterCustomLabelsRequest, + BatchApplyAdReviewCenterCustomLabelsResponse, BatchBlockAdReviewCenterAdsRequest, BatchBlockAdReviewCenterAdsResponse, + FetchAdReviewCenterCustomLabelsRequest, + FetchAdReviewCenterCustomLabelsResponse, SearchAdReviewCenterAdsRequest, SearchAdReviewCenterAdsResponse, ) @@ -502,6 +538,8 @@ UpdateApplicationRequest, ) from .types.applied_label import AppliedLabel +from .types.asset_enums import AssetImageDensityEnum +from .types.asset_messages import Asset from .types.audience_segment_enums import ( AudienceSegmentApprovalStatusEnum, AudienceSegmentLicenseTypeEnum, @@ -522,10 +560,13 @@ BatchPopulateAudienceSegmentsResponse, BatchRejectAudienceSegmentsRequest, BatchRejectAudienceSegmentsResponse, + BatchUpdateAudienceSegmentsRequest, + BatchUpdateAudienceSegmentsResponse, CreateAudienceSegmentRequest, GetAudienceSegmentRequest, ListAudienceSegmentsRequest, ListAudienceSegmentsResponse, + UpdateAudienceSegmentRequest, ) from .types.bandwidth_group_messages import BandwidthGroup from .types.bandwidth_group_service import ( @@ -686,11 +727,54 @@ ListContentResponse, ) from .types.conversion_event_enum import ConversionEventEnum -from .types.creative_messages import Creative +from .types.creative_asset import ClickTag, CreativeAsset +from .types.creative_enums import ( + CreativeDestinationUrlTypeEnum, + CreativeLockedOrientationEnum, + CreativeSslOverrideEnum, + CreativeSslScanResultEnum, + VastAdIdTypeEnum, +) +from .types.creative_messages import ( + AdExchangeCreativeDetails, + AdSenseCreativeDetails, + AspectRatioImageCreativeDetails, + AudioCreativeDetails, + AudioRedirectCreativeDetails, + BuyerPlacementConfig, + ClickTrackingCreativeDetails, + Creative, + CustomCreativeDetails, + Html5CreativeDetails, + ImageCreativeDetails, + ImageOverlayCreativeDetails, + ImageRedirectCreativeDetails, + ImageRedirectOverlayCreativeDetails, + InternalRedirectCreativeDetails, + LegacyDfpCreativeDetails, + ProgrammaticCreativeDetails, + RichMediaStudioCreativeDetails, + SetTopBoxCreativeDetails, + TemplateCreativeDetails, + ThirdPartyCreativeDetails, + VastInfo, + VastRedirectCreativeDetails, + VideoCreativeDetails, + VideoRedirectCreativeDetails, +) from .types.creative_placeholder import ( CreativePlaceholder, CreativePlaceholderCompanion, ) +from .types.creative_service import ( + BatchActivateCreativesRequest, + BatchActivateCreativesResponse, + BatchDeactivateCreativesRequest, + BatchDeactivateCreativesResponse, + GetCreativeRequest, + ListCreativesRequest, + ListCreativesResponse, +) from .types.creative_set_messages import CreativeSet from .types.creative_set_service import ( CreateCreativeSetRequest, @@ -713,6 +797,9 @@ from .types.creative_template_variable_url_type_enum import ( CreativeTemplateVariableUrlTypeEnum, ) +from .types.creative_third_party_data_declaration_status_enum import ( + CreativeThirdPartyDataDeclarationStatusEnum, +) from .types.creative_wrapper_enums import ( CreativeWrapperOrderingEnum, CreativeWrapperStatusEnum, @@ -734,6 +821,7 @@ ListCreativeWrappersResponse, UpdateCreativeWrapperRequest, ) +from .types.custom_creative_asset import CustomCreativeAsset from .types.custom_field_enums import ( CustomFieldDataTypeEnum, CustomFieldEntityTypeEnum, @@ -866,6 +954,7 @@ ) from .types.dai_session_messages import DaiSession from .types.dai_session_service import GetDaiSessionRequest +from .types.date_range import DateRange from .types.deal_buyer_permission_type_enum import DealBuyerPermissionTypeEnum from .types.deal_priority_tier_enum import DealPriorityTierEnum from .types.delivery_enums import ( @@ -910,6 +999,38 @@ from .types.environment_type_enum import EnvironmentTypeEnum from .types.exchange_syndication_product_enum import ExchangeSyndicationProductEnum from .types.exclusion_scope_enum import ExclusionScopeEnum +from .types.forecast_messages import ( + AlternativeUnitTypeForecast, + AvailabilityForecast, + AvailabilityForecastOptions, + ContendingLineItem, + DeliveryForecast, + DeliveryForecastOptions, + DeliveryForecastResult, + ExistingLineItemList, + ForecastBreakdown, + ForecastBreakdownEntry, + ForecastBreakdownOptions, + ForecastBreakdownTarget, + GrpDemographicBreakdown, + LineItemDeliveryForecast, + TargetingCriteriaBreakdown, + TimeSeries, +) +from .types.forecast_service import ( + RunAvailabilityForecastRequest, + RunAvailabilityForecastResponse, + RunDeliveryForecastRequest, + RunDeliveryForecastResponse, + RunTrafficDataRequest, + RunTrafficDataResponse, +) +from .types.forecasting_enums import ( + ForecastingGrpAgeEnum, + ForecastingGrpGenderEnum, + ForecastingGrpUnitEnum, + ForecastingTargetingDimensionEnum, +) from .types.frequency_cap import FrequencyCap from .types.geo_target_messages import GeoTarget from .types.geo_target_service import ( @@ -940,6 +1061,18 @@ UpdateLabelRequest, ) from .types.line_item_allowed_format_enum import LineItemAllowedFormatEnum +from .types.line_item_creative_association_enums import ( + LineItemCreativeAssociationStatusEnum, +) +from .types.line_item_creative_association_messages import ( + LineItemCreativeAssociation, + LineItemCreativeAssociationStats, +) +from .types.line_item_creative_association_service import ( + GetLineItemCreativeAssociationRequest, + ListLineItemCreativeAssociationsRequest, + ListLineItemCreativeAssociationsResponse, +) from .types.line_item_deal_info import LineItemDealInfo from .types.line_item_delivery_forecast_source_enum import ( LineItemDeliveryForecastSourceEnum, @@ -953,11 +1086,42 @@ ) from .types.line_item_messages import LineItem from .types.line_item_service import ( + BatchActivateLineItemsRequest, + BatchActivateLineItemsResponse, + BatchArchiveLineItemsRequest, + BatchArchiveLineItemsResponse, + BatchCreateLineItemsRequest, + BatchCreateLineItemsResponse, + BatchDeleteLineItemsRequest, + BatchPauseLineItemsRequest, + BatchPauseLineItemsResponse, + BatchReleaseLineItemsRequest, + BatchReleaseLineItemsResponse, + BatchReserveAndOverbookLineItemsRequest, + BatchReserveAndOverbookLineItemsResponse, + BatchReserveLineItemsRequest, + BatchReserveLineItemsResponse, + BatchResumeAndOverbookLineItemsRequest, + BatchResumeAndOverbookLineItemsResponse, + BatchResumeLineItemsRequest, + BatchResumeLineItemsResponse, + BatchUnarchiveLineItemsRequest, + BatchUnarchiveLineItemsResponse, + BatchUpdateLineItemsRequest, + BatchUpdateLineItemsResponse, + CreateLineItemRequest, GetLineItemRequest, ListLineItemsRequest, ListLineItemsResponse, + UpdateLineItemRequest, ) from .types.line_item_stats import LineItemStats +from .types.line_item_template_messages import LineItemTemplate +from .types.line_item_template_service import ( + GetLineItemTemplateRequest, + ListLineItemTemplatesRequest, + ListLineItemTemplatesResponse, +) from .types.linked_device_enums import LinkedDeviceVisibilityEnum from .types.linked_device_messages import LinkedDevice from .types.linked_device_service import ( @@ -1206,6 +1370,16 @@ ListRichMediaAdsCompaniesRequest, ListRichMediaAdsCompaniesResponse, ) +from .types.rich_media_studio_child_asset_type_enum import ( + RichMediaStudioChildAssetTypeEnum, +) +from .types.rich_media_studio_creative_billing_attribute_enum import ( + RichMediaStudioCreativeBillingAttributeEnum, +) +from .types.rich_media_studio_creative_format_enum import ( + RichMediaStudioCreativeFormatEnum, +) +from .types.rich_media_studio_messages import RichMediaStudioChildAssetProperty from .types.role_enums import RoleStatusEnum from .types.role_messages import Role from .types.role_service import GetRoleRequest, ListRolesRequest, ListRolesResponse @@ -1341,7 +1515,22 @@ from .types.third_party_measurement_settings import ThirdPartyMeasurementSettings from .types.time_unit_enum import TimeUnitEnum from .types.user_messages import User -from .types.user_service import GetUserRequest +from .types.user_service import ( + BatchActivateUsersRequest, + BatchActivateUsersResponse, + BatchCreateUsersRequest, + BatchCreateUsersResponse, + BatchDeactivateUsersRequest, + BatchDeactivateUsersResponse, + BatchUpdateUsersRequest, + BatchUpdateUsersResponse, + CreateUserRequest, + GetUserRequest, + ListUsersRequest, + ListUsersResponse, + UpdateUserRequest, +) +from .types.vast_redirect_type_enum import VastRedirectTypeEnum from .types.video_position_enum import VideoPositionEnum from .types.video_tracking_url import VideoTrackingUrl from .types.video_transcode_status_enum import VideoTranscodeStatusEnum @@ -1370,6 +1559,7 @@ "AdBreakOptimizationTypeEnum", "AdBreakServiceClient", "AdBreakStateEnum", + "AdExchangeCreativeDetails", "AdManagerError", "AdMediaDeliveryConfig", "AdRequestFindingTypeEnum", @@ -1386,6 +1576,7 @@ "AdRuleSlotBumperEnum", "AdRuleSlotMidrollFrequencyTypeEnum", "AdRuleStatusEnum", + "AdSenseCreativeDetails", "AdSpot", "AdSpotFillTypeEnum", "AdSpotServiceClient", @@ -1396,13 +1587,18 @@ "AdUnitSize", "AdUnitStatusEnum", "AdUnitTargeting", + "AlternativeUnitTypeForecast", "Application", "ApplicationApprovalStatusEnum", "ApplicationPlatformEnum", "ApplicationServiceClient", "ApplicationStoreEnum", "AppliedLabel", + "ArcCreativeFormatEnum", "ArchiveDaiEncodingProfileRequest", + "AspectRatioImageCreativeDetails", + "Asset", + "AssetImageDensityEnum", "AudienceSegment", "AudienceSegmentApprovalStatusEnum", "AudienceSegmentLicenseTypeEnum", @@ -1410,8 +1606,12 @@ "AudienceSegmentStatusEnum", "AudienceSegmentTargeting", "AudienceSegmentTypeEnum", + "AudioCreativeDetails", + "AudioRedirectCreativeDetails", "AudioSettings", "AuxiliaryAdSettings", + "AvailabilityForecast", + "AvailabilityForecastOptions", "BandwidthGroup", "BandwidthGroupServiceClient", "BandwidthTargeting", @@ -1431,6 +1631,8 @@ "BatchActivateContentBundlesResponse", "BatchActivateCreativeWrappersRequest", "BatchActivateCreativeWrappersResponse", + "BatchActivateCreativesRequest", + "BatchActivateCreativesResponse", "BatchActivateCustomFieldsRequest", "BatchActivateCustomFieldsResponse", "BatchActivateCustomTargetingKeysRequest", @@ -1443,6 +1645,8 @@ "BatchActivateDaiEncodingProfilesResponse", "BatchActivateLabelsRequest", "BatchActivateLabelsResponse", + "BatchActivateLineItemsRequest", + "BatchActivateLineItemsResponse", "BatchActivateLiveStreamsRequest", "BatchActivateLiveStreamsResponse", "BatchActivateNativeStylesRequest", @@ -1451,9 +1655,13 @@ "BatchActivatePlacementsResponse", "BatchActivateTeamsRequest", "BatchActivateTeamsResponse", + "BatchActivateUsersRequest", + "BatchActivateUsersResponse", "BatchAdReviewCenterAdsOperationMetadata", "BatchAllowAdReviewCenterAdsRequest", "BatchAllowAdReviewCenterAdsResponse", + "BatchApplyAdReviewCenterCustomLabelsRequest", + "BatchApplyAdReviewCenterCustomLabelsResponse", "BatchApproveAndOverbookOrdersRequest", "BatchApproveAndOverbookOrdersResponse", "BatchApproveAudienceSegmentsRequest", @@ -1472,6 +1680,8 @@ "BatchArchiveCdnConfigsResponse", "BatchArchiveDaiEncodingProfilesRequest", "BatchArchiveDaiEncodingProfilesResponse", + "BatchArchiveLineItemsRequest", + "BatchArchiveLineItemsResponse", "BatchArchiveLiveStreamsRequest", "BatchArchiveLiveStreamsResponse", "BatchArchiveNativeStylesRequest", @@ -1520,6 +1730,8 @@ "BatchCreateEntitySignalsMappingsResponse", "BatchCreateLabelsRequest", "BatchCreateLabelsResponse", + "BatchCreateLineItemsRequest", + "BatchCreateLineItemsResponse", "BatchCreateLiveStreamsRequest", "BatchCreateLiveStreamsResponse", "BatchCreateNativeStylesRequest", @@ -1536,6 +1748,8 @@ "BatchCreateTargetingPresetsResponse", "BatchCreateTeamsRequest", "BatchCreateTeamsResponse", + "BatchCreateUsersRequest", + "BatchCreateUsersResponse", "BatchCreateViewabilityProvidersRequest", "BatchCreateViewabilityProvidersResponse", "BatchDeactivateAdRulesRequest", @@ -1552,6 +1766,8 @@ "BatchDeactivateContentBundlesResponse", "BatchDeactivateCreativeWrappersRequest", "BatchDeactivateCreativeWrappersResponse", + "BatchDeactivateCreativesRequest", + "BatchDeactivateCreativesResponse", "BatchDeactivateCustomFieldsRequest", "BatchDeactivateCustomFieldsResponse", "BatchDeactivateCustomTargetingKeysRequest", @@ -1572,7 +1788,10 @@ "BatchDeactivateTargetingPresetsResponse", "BatchDeactivateTeamsRequest", "BatchDeactivateTeamsResponse", + "BatchDeactivateUsersRequest", + "BatchDeactivateUsersResponse", "BatchDeleteAdRulesRequest", + "BatchDeleteLineItemsRequest", "BatchDeleteOrdersRequest", "BatchDeleteOrdersResponse", "BatchDisapproveOrdersRequest", @@ -1581,6 +1800,8 @@ "BatchDisapproveOrdersWithoutReservationChangesResponse", "BatchPauseAdsLiveStreamsRequest", "BatchPauseAdsLiveStreamsResponse", + "BatchPauseLineItemsRequest", + "BatchPauseLineItemsResponse", "BatchPauseLiveStreamsRequest", "BatchPauseLiveStreamsResponse", "BatchPauseOrdersRequest", @@ -1593,12 +1814,22 @@ "BatchRejectAudienceSegmentsResponse", "BatchRejectChildPublishersRequest", "BatchRejectChildPublishersResponse", + "BatchReleaseLineItemsRequest", + "BatchReleaseLineItemsResponse", "BatchRenegotiateChildPublisherAgreementsRequest", "BatchRenegotiateChildPublisherAgreementsResponse", "BatchResendChildPublisherInvitationEmailsRequest", "BatchResendChildPublisherInvitationEmailsResponse", + "BatchReserveAndOverbookLineItemsRequest", + "BatchReserveAndOverbookLineItemsResponse", + "BatchReserveLineItemsRequest", + "BatchReserveLineItemsResponse", + "BatchResumeAndOverbookLineItemsRequest", + "BatchResumeAndOverbookLineItemsResponse", "BatchResumeAndOverbookOrdersRequest", "BatchResumeAndOverbookOrdersResponse", + "BatchResumeLineItemsRequest", + "BatchResumeLineItemsResponse", "BatchResumeOrdersRequest", "BatchResumeOrdersResponse", "BatchRetractOrdersRequest", @@ -1615,6 +1846,8 @@ "BatchSubmitSitesForApprovalResponse", "BatchUnarchiveApplicationsRequest", "BatchUnarchiveApplicationsResponse", + "BatchUnarchiveLineItemsRequest", + "BatchUnarchiveLineItemsResponse", "BatchUnarchiveOrdersRequest", "BatchUnarchiveOrdersResponse", "BatchUnarchiveSlatesRequest", @@ -1627,6 +1860,8 @@ "BatchUpdateAdUnitsResponse", "BatchUpdateApplicationsRequest", "BatchUpdateApplicationsResponse", + "BatchUpdateAudienceSegmentsRequest", + "BatchUpdateAudienceSegmentsResponse", "BatchUpdateBreakTemplatesRequest", "BatchUpdateBreakTemplatesResponse", "BatchUpdateCdnConfigsRequest", @@ -1653,6 +1888,8 @@ "BatchUpdateEntitySignalsMappingsResponse", "BatchUpdateLabelsRequest", "BatchUpdateLabelsResponse", + "BatchUpdateLineItemsRequest", + "BatchUpdateLineItemsResponse", "BatchUpdateLiveStreamsRequest", "BatchUpdateLiveStreamsResponse", "BatchUpdateNativeStylesRequest", @@ -1671,6 +1908,8 @@ "BatchUpdateTargetingPresetsResponse", "BatchUpdateTeamsRequest", "BatchUpdateTeamsResponse", + "BatchUpdateUsersRequest", + "BatchUpdateUsersResponse", "BatchUpdateViewabilityProvidersRequest", "BatchUpdateViewabilityProvidersResponse", "BatchWithdrawChildPublishersRequest", @@ -1686,6 +1925,7 @@ "BrowserLanguageTargeting", "BrowserServiceClient", "BrowserTargeting", + "BuyerPlacementConfig", "CdnConfig", "CdnConfigServiceClient", "CdnConfigStatusEnum", @@ -1699,6 +1939,8 @@ "ChildPublisherIdentityVerificationStatusEnum", "ChildPublisherOnboardingTaskEnum", "ChildPublisherServiceClient", + "ClickTag", + "ClickTrackingCreativeDetails", "CmsContent", "CmsMetadataKey", "CmsMetadataKeyServiceClient", @@ -1716,6 +1958,7 @@ "ContactServiceClient", "ContactStatusEnum", "ContainerTypeEnum", + "ContendingLineItem", "Content", "ContentBundle", "ContentBundleServiceClient", @@ -1747,6 +1990,7 @@ "CreateDaiEncodingProfileRequest", "CreateEntitySignalsMappingRequest", "CreateLabelRequest", + "CreateLineItemRequest", "CreateLiveStreamRequest", "CreateNativeStyleRequest", "CreateOrderRequest", @@ -1758,15 +2002,22 @@ "CreateSlateRequest", "CreateTargetingPresetRequest", "CreateTeamRequest", + "CreateUserRequest", "CreateViewabilityProviderRequest", "Creative", + "CreativeAsset", + "CreativeDestinationUrlTypeEnum", "CreativeFindingTypeEnum", "CreativeIdTypeEnum", + "CreativeLockedOrientationEnum", "CreativePlaceholder", "CreativePlaceholderCompanion", "CreativeRotationTypeEnum", + "CreativeServiceClient", "CreativeSet", "CreativeSetServiceClient", + "CreativeSslOverrideEnum", + "CreativeSslScanResultEnum", "CreativeTargeting", "CreativeTemplate", "CreativeTemplateServiceClient", @@ -1774,11 +2025,14 @@ "CreativeTemplateTypeEnum", "CreativeTemplateVariable", "CreativeTemplateVariableUrlTypeEnum", + "CreativeThirdPartyDataDeclarationStatusEnum", "CreativeWrapper", "CreativeWrapperOrderingEnum", "CreativeWrapperServiceClient", "CreativeWrapperStatusEnum", "CreativeWrapperTypeEnum", + "CustomCreativeAsset", + "CustomCreativeDetails", "CustomField", "CustomFieldDataTypeEnum", "CustomFieldEntityTypeEnum", @@ -1817,6 +2071,7 @@ "DaiSessionServiceClient", "DashBridge", "DataSegmentTargeting", + "DateRange", "DeactivateCustomTargetingValueRequest", "DeactivateTargetingPresetRequest", "DealBuyerPermissionTypeEnum", @@ -1827,6 +2082,9 @@ "DelegationInvitationStatusEnum", "DelegationTypeEnum", "DeleteAdBreakRequest", + "DeliveryForecast", + "DeliveryForecastOptions", + "DeliveryForecastResult", "DeliveryIndicator", "DeviceCapability", "DeviceCapabilityServiceClient", @@ -1846,11 +2104,23 @@ "EnvironmentTypeEnum", "ExchangeSyndicationProductEnum", "ExclusionScopeEnum", + "ExistingLineItemList", + "FetchAdReviewCenterCustomLabelsRequest", + "FetchAdReviewCenterCustomLabelsResponse", "FetchMcmEarningsRequest", "FetchMcmEarningsResponse", "FetchReportResultRowsRequest", "FetchReportResultRowsResponse", "FirstPartyMobileApplicationTargeting", + "ForecastBreakdown", + "ForecastBreakdownEntry", + "ForecastBreakdownOptions", + "ForecastBreakdownTarget", + "ForecastServiceClient", + "ForecastingGrpAgeEnum", + "ForecastingGrpGenderEnum", + "ForecastingGrpUnitEnum", + "ForecastingTargetingDimensionEnum", "FrequencyCap", "GeoTarget", "GeoTargetServiceClient", @@ -1874,6 +2144,7 @@ "GetContentBundleRequest", "GetContentLabelRequest", "GetContentRequest", + "GetCreativeRequest", "GetCreativeSetRequest", "GetCreativeTemplateRequest", "GetCreativeWrapperRequest", @@ -1890,7 +2161,9 @@ "GetEntitySignalsMappingRequest", "GetGeoTargetRequest", "GetLabelRequest", + "GetLineItemCreativeAssociationRequest", "GetLineItemRequest", + "GetLineItemTemplateRequest", "GetLinkedDeviceRequest", "GetLiveStreamRequest", "GetMobileCarrierRequest", @@ -1920,28 +2193,43 @@ "GetViewabilityProviderRequest", "Goal", "GoalTypeEnum", + "GrpDemographicBreakdown", "GrpProviderEnum", "GrpSettings", "GrpTargetGenderEnum", "HlsMasterPlaylistRefreshTypeEnum", "HlsSettings", "HlsSettingsPlaylistTypeEnum", + "Html5CreativeDetails", + "ImageCreativeDetails", + "ImageOverlayCreativeDetails", + "ImageRedirectCreativeDetails", + "ImageRedirectOverlayCreativeDetails", + "InternalRedirectCreativeDetails", "InventoryTargeting", "Label", "LabelFrequencyCap", "LabelServiceClient", "LabelTypeEnum", + "LegacyDfpCreativeDetails", "LineItem", "LineItemAllowedFormatEnum", "LineItemComputedStatusEnum", "LineItemCostTypeEnum", + "LineItemCreativeAssociation", + "LineItemCreativeAssociationServiceClient", + "LineItemCreativeAssociationStats", + "LineItemCreativeAssociationStatusEnum", "LineItemDealInfo", + "LineItemDeliveryForecast", "LineItemDeliveryForecastSourceEnum", "LineItemDeliveryRateTypeEnum", "LineItemDiscount", "LineItemReservationStatusEnum", "LineItemServiceClient", "LineItemStats", + "LineItemTemplate", + "LineItemTemplateServiceClient", "LineItemTypeEnum", "LinkedDevice", "LinkedDeviceServiceClient", @@ -1992,6 +2280,8 @@ "ListCreativeTemplatesResponse", "ListCreativeWrappersRequest", "ListCreativeWrappersResponse", + "ListCreativesRequest", + "ListCreativesResponse", "ListCustomFieldsRequest", "ListCustomFieldsResponse", "ListCustomTargetingKeysRequest", @@ -2014,6 +2304,10 @@ "ListGeoTargetsResponse", "ListLabelsRequest", "ListLabelsResponse", + "ListLineItemCreativeAssociationsRequest", + "ListLineItemCreativeAssociationsResponse", + "ListLineItemTemplatesRequest", + "ListLineItemTemplatesResponse", "ListLineItemsRequest", "ListLineItemsResponse", "ListLinkedDevicesRequest", @@ -2066,6 +2360,8 @@ "ListTeamsResponse", "ListThirdPartyCompaniesRequest", "ListThirdPartyCompaniesResponse", + "ListUsersRequest", + "ListUsersResponse", "ListViewabilityProvidersRequest", "ListViewabilityProvidersResponse", "LiveStream", @@ -2121,6 +2417,7 @@ "PrivateMarketplaceDealStatusEnum", "ProgrammaticBuyer", "ProgrammaticBuyerServiceClient", + "ProgrammaticCreativeDetails", "ProvisionTestNetworkRequest", "ReachPartnerEnum", "RefreshRateTypeEnum", @@ -2138,17 +2435,29 @@ "RichMediaAdsCompany", "RichMediaAdsCompanyGdprStatusEnum", "RichMediaAdsCompanyServiceClient", + "RichMediaStudioChildAssetProperty", + "RichMediaStudioChildAssetTypeEnum", + "RichMediaStudioCreativeBillingAttributeEnum", + "RichMediaStudioCreativeDetails", + "RichMediaStudioCreativeFormatEnum", "RoadblockingTypeEnum", "Role", "RoleServiceClient", "RoleStatusEnum", + "RunAvailabilityForecastRequest", + "RunAvailabilityForecastResponse", + "RunDeliveryForecastRequest", + "RunDeliveryForecastResponse", "RunReportMetadata", "RunReportRequest", "RunReportResponse", + "RunTrafficDataRequest", + "RunTrafficDataResponse", "ScheduleOptions", "SearchAdReviewCenterAdsRequest", "SearchAdReviewCenterAdsResponse", "SessionFindingSeverityEnum", + "SetTopBoxCreativeDetails", "Site", "SiteApprovalStatusEnum", "SiteDisapprovalReasonEnum", @@ -2169,6 +2478,7 @@ "TargetWindowEnum", "TargetedVideoBumperTypeEnum", "Targeting", + "TargetingCriteriaBreakdown", "TargetingPreset", "TargetingPresetServiceClient", "TargetingPresetStatusEnum", @@ -2180,13 +2490,16 @@ "TeamServiceClient", "TeamStatusEnum", "TechnologyTargeting", + "TemplateCreativeDetails", "ThirdPartyCompany", "ThirdPartyCompanyServiceClient", "ThirdPartyCompanyStatusEnum", "ThirdPartyCompanyTypeEnum", + "ThirdPartyCreativeDetails", "ThirdPartyDataDeclaration", "ThirdPartyDataDeclarationTypeEnum", "ThirdPartyMeasurementSettings", + "TimeSeries", "TimeUnitEnum", "TrackingPingFindingTypeEnum", "UnitTypeEnum", @@ -2195,6 +2508,7 @@ "UpdateAdSpotRequest", "UpdateAdUnitRequest", "UpdateApplicationRequest", + "UpdateAudienceSegmentRequest", "UpdateBreakTemplateRequest", "UpdateCdnConfigRequest", "UpdateChildPublisherRequest", @@ -2209,6 +2523,7 @@ "UpdateDaiEncodingProfileRequest", "UpdateEntitySignalsMappingRequest", "UpdateLabelRequest", + "UpdateLineItemRequest", "UpdateLiveStreamRequest", "UpdateNativeStyleRequest", "UpdateNetworkRequest", @@ -2222,13 +2537,20 @@ "UpdateSlateRequest", "UpdateTargetingPresetRequest", "UpdateTeamRequest", + "UpdateUserRequest", "UpdateViewabilityProviderRequest", "User", "UserDomainTargeting", "UserServiceClient", + "VastAdIdTypeEnum", + "VastInfo", + "VastRedirectCreativeDetails", + "VastRedirectTypeEnum", + "VideoCreativeDetails", "VideoPosition", "VideoPositionEnum", "VideoPositionTargeting", + "VideoRedirectCreativeDetails", "VideoSettings", "VideoTrackingUrl", "VideoTranscodeStatusEnum", diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/gapic_metadata.json b/packages/google-ads-admanager/google/ads/admanager_v1/gapic_metadata.json index efda67c99567..2bb6de9ec1a8 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/gapic_metadata.json +++ b/packages/google-ads-admanager/google/ads/admanager_v1/gapic_metadata.json @@ -49,11 +49,21 @@ "batch_allow_ad_review_center_ads" ] }, + "BatchApplyAdReviewCenterCustomLabels": { + "methods": [ + "batch_apply_ad_review_center_custom_labels" + ] + }, "BatchBlockAdReviewCenterAds": { "methods": [ "batch_block_ad_review_center_ads" ] }, + "FetchAdReviewCenterCustomLabels": { + "methods": [ + "fetch_ad_review_center_custom_labels" + ] + }, "SearchAdReviewCenterAds": { "methods": [ "search_ad_review_center_ads" @@ -299,6 +309,11 @@ "batch_reject_audience_segments" ] }, + "BatchUpdateAudienceSegments": { + "methods": [ + "batch_update_audience_segments" + ] + }, "CreateAudienceSegment": { "methods": [ "create_audience_segment" @@ -313,6 +328,11 @@ "methods": [ "list_audience_segments" ] + }, + "UpdateAudienceSegment": { + "methods": [ + "update_audience_segment" + ] } } } @@ -725,6 +745,35 @@ } } }, + "CreativeService": { + "clients": { + "rest": { + "libraryClient": "CreativeServiceClient", + "rpcs": { + "BatchActivateCreatives": { + "methods": [ + "batch_activate_creatives" + ] + }, + "BatchDeactivateCreatives": { + "methods": [ + "batch_deactivate_creatives" + ] + }, + "GetCreative": { + "methods": [ + "get_creative" + ] + }, + "ListCreatives": { + "methods": [ + "list_creatives" + ] + } + } + } + } + }, "CreativeSetService": { "clients": { "rest": { @@ -1177,6 +1226,30 @@ } } }, + "ForecastService": { + "clients": { + "rest": { + "libraryClient": "ForecastServiceClient", + "rpcs": { + "RunAvailabilityForecast": { + "methods": [ + "run_availability_forecast" + ] + }, + "RunDeliveryForecast": { + "methods": [ + "run_delivery_forecast" + ] + }, + "RunTrafficData": { + "methods": [ + "run_traffic_data" + ] + } + } + } + } + }, "GeoTargetService": { "clients": { "rest": { @@ -1245,11 +1318,95 @@ } } }, + "LineItemCreativeAssociationService": { + "clients": { + "rest": { + "libraryClient": "LineItemCreativeAssociationServiceClient", + "rpcs": { + "GetLineItemCreativeAssociation": { + "methods": [ + "get_line_item_creative_association" + ] + }, + "ListLineItemCreativeAssociations": { + "methods": [ + "list_line_item_creative_associations" + ] + } + } + } + } + }, "LineItemService": { "clients": { "rest": { "libraryClient": "LineItemServiceClient", "rpcs": { + "BatchActivateLineItems": { + "methods": [ + "batch_activate_line_items" + ] + }, + "BatchArchiveLineItems": { + "methods": [ + "batch_archive_line_items" + ] + }, + "BatchCreateLineItems": { + "methods": [ + "batch_create_line_items" + ] + }, + "BatchDeleteLineItems": { + "methods": [ + "batch_delete_line_items" + ] + }, + "BatchPauseLineItems": { + "methods": [ + "batch_pause_line_items" + ] + }, + "BatchReleaseLineItems": { + "methods": [ + "batch_release_line_items" + ] + }, + "BatchReserveAndOverbookLineItems": { + "methods": [ + "batch_reserve_and_overbook_line_items" + ] + }, + "BatchReserveLineItems": { + "methods": [ + "batch_reserve_line_items" + ] + }, + "BatchResumeAndOverbookLineItems": { + "methods": [ + "batch_resume_and_overbook_line_items" + ] + }, + "BatchResumeLineItems": { + "methods": [ + "batch_resume_line_items" + ] + }, + "BatchUnarchiveLineItems": { + "methods": [ + "batch_unarchive_line_items" + ] + }, + "BatchUpdateLineItems": { + "methods": [ + "batch_update_line_items" + ] + }, + "CreateLineItem": { + "methods": [ + "create_line_item" + ] + }, "GetLineItem": { "methods": [ "get_line_item" @@ -1259,6 +1416,30 @@ "methods": [ "list_line_items" ] + }, + "UpdateLineItem": { + "methods": [ + "update_line_item" + ] + } + } + } + } + }, + "LineItemTemplateService": { + "clients": { + "rest": { + "libraryClient": "LineItemTemplateServiceClient", + "rpcs": { + "GetLineItemTemplate": { + "methods": [ + "get_line_item_template" + ] + }, + "ListLineItemTemplates": { + "methods": [ + "list_line_item_templates" + ] } } } @@ -1448,6 +1629,11 @@ "batch_update_native_styles" ] }, + "CreateNativeStyle": { + "methods": [ + "create_native_style" + ] + }, "GetNativeStyle": { "methods": [ "get_native_style" @@ -1457,6 +1643,11 @@ "methods": [ "list_native_styles" ] + }, + "UpdateNativeStyle": { + "methods": [ + "update_native_style" + ] } } } @@ -2138,10 +2329,45 @@ "rest": { "libraryClient": "UserServiceClient", "rpcs": { + "BatchActivateUsers": { + "methods": [ + "batch_activate_users" + ] + }, + "BatchCreateUsers": { + "methods": [ + "batch_create_users" + ] + }, + "BatchDeactivateUsers": { + "methods": [ + "batch_deactivate_users" + ] + }, + "BatchUpdateUsers": { + "methods": [ + "batch_update_users" + ] + }, + "CreateUser": { + "methods": [ + "create_user" + ] + }, "GetUser": { "methods": [ "get_user" ] + }, + "ListUsers": { + "methods": [ + "list_users" + ] + }, + "UpdateUser": { + "methods": [ + "update_user" + ] } } } diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/client.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/client.py index 0b3186efae09..7579f3cbbdc5 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/client.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/client.py @@ -116,7 +116,7 @@ def get_transport_class( class AdReviewCenterAdServiceClient(metaclass=AdReviewCenterAdServiceClientMeta): - """Provides methods for handling AdReviewCenterAd objects.""" + """Provides methods for handling ``AdReviewCenterAd`` objects.""" # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. DEFAULT_ENDPOINT = "admanager.googleapis.com" @@ -1064,6 +1064,283 @@ def sample_batch_block_ad_review_center_ads(): # Done; return the response. return response + def fetch_ad_review_center_custom_labels( + self, + request: Optional[ + Union[ + ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsRequest, dict + ] + ] = None, + *, + parent: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), + ) -> ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse: + r"""Fetches all custom labels for a publisher. Custom + labels can help you filter and find creatives with the + associated label. For more information, see + https://support.google.com/admanager/answer/13812863. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.ads import admanager_v1 + + def sample_fetch_ad_review_center_custom_labels(): + # Create a client + client = admanager_v1.AdReviewCenterAdServiceClient() + + # Initialize request argument(s) + request = admanager_v1.FetchAdReviewCenterCustomLabelsRequest( + parent="parent_value", + ) + + # Make the request + response = client.fetch_ad_review_center_custom_labels(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.ads.admanager_v1.types.FetchAdReviewCenterCustomLabelsRequest, dict]): + The request object. Request object for ``FetchAdReviewCenterCustomLabels`` + method. + parent (str): + Required. The parent, which owns this collection of + AdReviewCenterAds custom labels. Format: + networks/{network_code}/webProperties/{web_property_code} + + Since a network can only have a single web property of + each ``ExchangeSyndicationProduct``, you can use the + ``ExchangeSyndicationProduct`` as an alias for the web + property code: + + ``networks/{network_code}/webProperties/display`` + + ``networks/{network_code}/webProperties/videoAndAudio`` + + ``networks/{network_code}/webProperties/mobileApp`` + + ``networks/{network_code}/webProperties/games`` + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be + sent along with the request as metadata. Normally, each value must be of type `str`, + but for metadata keys ending with the suffix `-bin`, the corresponding values must + be of type `bytes`. + + Returns: + google.ads.admanager_v1.types.FetchAdReviewCenterCustomLabelsResponse: + Response object for FetchAdReviewCenterCustomLabels + method. + + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + flattened_params = [parent] + has_flattened_params = ( + len([param for param in flattened_params if param is not None]) > 0 + ) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance( + request, ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsRequest + ): + request = ( + ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsRequest( + request + ) + ) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[ + self._transport.fetch_ad_review_center_custom_labels + ] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def batch_apply_ad_review_center_custom_labels( + self, + request: Optional[ + Union[ + ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsRequest, + dict, + ] + ] = None, + *, + parent: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), + ) -> ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse: + r"""Performs batch apply on custom labels associated with + Ad review center ads. Custom labels can help you filter + and find creatives with the associated label. For more + information, see + https://support.google.com/admanager/answer/13812863. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.ads import admanager_v1 + + def sample_batch_apply_ad_review_center_custom_labels(): + # Create a client + client = admanager_v1.AdReviewCenterAdServiceClient() + + # Initialize request argument(s) + request = admanager_v1.BatchApplyAdReviewCenterCustomLabelsRequest( + parent="parent_value", + ) + + # Make the request + response = client.batch_apply_ad_review_center_custom_labels(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.ads.admanager_v1.types.BatchApplyAdReviewCenterCustomLabelsRequest, dict]): + The request object. Request object for + ``BatchApplyAdReviewCenterCustomLabels`` method. + parent (str): + Required. The parent, which owns this collection of + AdReviewCenterAds. Format: + networks/{network_code}/webProperties/{web_property_code} + + Since a network can only have a single web property of + each ``ExchangeSyndicationProduct``, you can use the + ``ExchangeSyndicationProduct`` as an alias for the web + property code: + + ``networks/{network_code}/webProperties/display`` + + ``networks/{network_code}/webProperties/videoAndAudio`` + + ``networks/{network_code}/webProperties/mobileApp`` + + ``networks/{network_code}/webProperties/games`` + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be + sent along with the request as metadata. Normally, each value must be of type `str`, + but for metadata keys ending with the suffix `-bin`, the corresponding values must + be of type `bytes`. + + Returns: + google.ads.admanager_v1.types.BatchApplyAdReviewCenterCustomLabelsResponse: + Response object for BatchApplyAdReviewCenterCustomLabels + method. + + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + flattened_params = [parent] + has_flattened_params = ( + len([param for param in flattened_params if param is not None]) > 0 + ) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance( + request, + ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsRequest, + ): + request = ( + ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsRequest( + request + ) + ) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[ + self._transport.batch_apply_ad_review_center_custom_labels + ] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + def __enter__(self) -> "AdReviewCenterAdServiceClient": return self diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/transports/base.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/transports/base.py index dc3ec3bcd877..316998b5ac0b 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/transports/base.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/transports/base.py @@ -159,6 +159,16 @@ def _prep_wrapped_messages(self, client_info): default_timeout=None, client_info=client_info, ), + self.fetch_ad_review_center_custom_labels: gapic_v1.method.wrap_method( + self.fetch_ad_review_center_custom_labels, + default_timeout=None, + client_info=client_info, + ), + self.batch_apply_ad_review_center_custom_labels: gapic_v1.method.wrap_method( + self.batch_apply_ad_review_center_custom_labels, + default_timeout=None, + client_info=client_info, + ), self.cancel_operation: gapic_v1.method.wrap_method( self.cancel_operation, default_timeout=None, @@ -215,6 +225,34 @@ def batch_block_ad_review_center_ads( ]: raise NotImplementedError() + @property + def fetch_ad_review_center_custom_labels( + self, + ) -> Callable[ + [ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsRequest], + Union[ + ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse, + Awaitable[ + ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse + ], + ], + ]: + raise NotImplementedError() + + @property + def batch_apply_ad_review_center_custom_labels( + self, + ) -> Callable[ + [ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsRequest], + Union[ + ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse, + Awaitable[ + ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse + ], + ], + ]: + raise NotImplementedError() + @property def get_operation( self, diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/transports/rest.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/transports/rest.py index ea24bb1150d1..5807875fb105 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/transports/rest.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/transports/rest.py @@ -81,6 +81,14 @@ def post_batch_allow_ad_review_center_ads(self, response): logging.log(f"Received response: {response}") return response + def pre_batch_apply_ad_review_center_custom_labels(self, request, metadata): + logging.log(f"Received request: {request}") + return request, metadata + + def post_batch_apply_ad_review_center_custom_labels(self, response): + logging.log(f"Received response: {response}") + return response + def pre_batch_block_ad_review_center_ads(self, request, metadata): logging.log(f"Received request: {request}") return request, metadata @@ -89,6 +97,14 @@ def post_batch_block_ad_review_center_ads(self, response): logging.log(f"Received response: {response}") return response + def pre_fetch_ad_review_center_custom_labels(self, request, metadata): + logging.log(f"Received request: {request}") + return request, metadata + + def post_fetch_ad_review_center_custom_labels(self, response): + logging.log(f"Received response: {response}") + return response + def pre_search_ad_review_center_ads(self, request, metadata): logging.log(f"Received request: {request}") return request, metadata @@ -152,6 +168,59 @@ def post_batch_allow_ad_review_center_ads_with_metadata( """ return response, metadata + def pre_batch_apply_ad_review_center_custom_labels( + self, + request: ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsRequest, + metadata: Sequence[Tuple[str, Union[str, bytes]]], + ) -> Tuple[ + ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsRequest, + Sequence[Tuple[str, Union[str, bytes]]], + ]: + """Pre-rpc interceptor for batch_apply_ad_review_center_custom_labels + + Override in a subclass to manipulate the request or metadata + before they are sent to the AdReviewCenterAdService server. + """ + return request, metadata + + def post_batch_apply_ad_review_center_custom_labels( + self, + response: ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse, + ) -> ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse: + """Post-rpc interceptor for batch_apply_ad_review_center_custom_labels + + DEPRECATED. Please use the `post_batch_apply_ad_review_center_custom_labels_with_metadata` + interceptor instead. + + Override in a subclass to read or manipulate the response + after it is returned by the AdReviewCenterAdService server but before + it is returned to user code. This `post_batch_apply_ad_review_center_custom_labels` interceptor runs + before the `post_batch_apply_ad_review_center_custom_labels_with_metadata` interceptor. + """ + return response + + def post_batch_apply_ad_review_center_custom_labels_with_metadata( + self, + response: ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse, + metadata: Sequence[Tuple[str, Union[str, bytes]]], + ) -> Tuple[ + ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse, + Sequence[Tuple[str, Union[str, bytes]]], + ]: + """Post-rpc interceptor for batch_apply_ad_review_center_custom_labels + + Override in a subclass to read or manipulate the response or metadata after it + is returned by the AdReviewCenterAdService server but before it is returned to user code. + + We recommend only using this `post_batch_apply_ad_review_center_custom_labels_with_metadata` + interceptor in new development instead of the `post_batch_apply_ad_review_center_custom_labels` interceptor. + When both interceptors are used, this `post_batch_apply_ad_review_center_custom_labels_with_metadata` interceptor runs after the + `post_batch_apply_ad_review_center_custom_labels` interceptor. The (possibly modified) response returned by + `post_batch_apply_ad_review_center_custom_labels` will be passed to + `post_batch_apply_ad_review_center_custom_labels_with_metadata`. + """ + return response, metadata + def pre_batch_block_ad_review_center_ads( self, request: ad_review_center_ad_service.BatchBlockAdReviewCenterAdsRequest, @@ -201,6 +270,59 @@ def post_batch_block_ad_review_center_ads_with_metadata( """ return response, metadata + def pre_fetch_ad_review_center_custom_labels( + self, + request: ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsRequest, + metadata: Sequence[Tuple[str, Union[str, bytes]]], + ) -> Tuple[ + ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsRequest, + Sequence[Tuple[str, Union[str, bytes]]], + ]: + """Pre-rpc interceptor for fetch_ad_review_center_custom_labels + + Override in a subclass to manipulate the request or metadata + before they are sent to the AdReviewCenterAdService server. + """ + return request, metadata + + def post_fetch_ad_review_center_custom_labels( + self, + response: ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse, + ) -> ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse: + """Post-rpc interceptor for fetch_ad_review_center_custom_labels + + DEPRECATED. Please use the `post_fetch_ad_review_center_custom_labels_with_metadata` + interceptor instead. + + Override in a subclass to read or manipulate the response + after it is returned by the AdReviewCenterAdService server but before + it is returned to user code. This `post_fetch_ad_review_center_custom_labels` interceptor runs + before the `post_fetch_ad_review_center_custom_labels_with_metadata` interceptor. + """ + return response + + def post_fetch_ad_review_center_custom_labels_with_metadata( + self, + response: ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse, + metadata: Sequence[Tuple[str, Union[str, bytes]]], + ) -> Tuple[ + ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse, + Sequence[Tuple[str, Union[str, bytes]]], + ]: + """Post-rpc interceptor for fetch_ad_review_center_custom_labels + + Override in a subclass to read or manipulate the response or metadata after it + is returned by the AdReviewCenterAdService server but before it is returned to user code. + + We recommend only using this `post_fetch_ad_review_center_custom_labels_with_metadata` + interceptor in new development instead of the `post_fetch_ad_review_center_custom_labels` interceptor. + When both interceptors are used, this `post_fetch_ad_review_center_custom_labels_with_metadata` interceptor runs after the + `post_fetch_ad_review_center_custom_labels` interceptor. The (possibly modified) response returned by + `post_fetch_ad_review_center_custom_labels` will be passed to + `post_fetch_ad_review_center_custom_labels_with_metadata`. + """ + return response, metadata + def pre_search_ad_review_center_ads( self, request: ad_review_center_ad_service.SearchAdReviewCenterAdsRequest, @@ -312,7 +434,7 @@ class AdReviewCenterAdServiceRestStub: class AdReviewCenterAdServiceRestTransport(_BaseAdReviewCenterAdServiceRestTransport): """REST backend synchronous transport for AdReviewCenterAdService. - Provides methods for handling AdReviewCenterAd objects. + Provides methods for handling ``AdReviewCenterAd`` objects. This class defines the same methods as the primary client, so the primary client can load the underlying transport implementation @@ -591,6 +713,170 @@ def __call__( ) return resp + class _BatchApplyAdReviewCenterCustomLabels( + _BaseAdReviewCenterAdServiceRestTransport._BaseBatchApplyAdReviewCenterCustomLabels, + AdReviewCenterAdServiceRestStub, + ): + def __hash__(self): + return hash( + "AdReviewCenterAdServiceRestTransport.BatchApplyAdReviewCenterCustomLabels" + ) + + @staticmethod + def _get_response( + host, + metadata, + query_params, + session, + timeout, + transcoded_request, + body=None, + ): + uri = transcoded_request["uri"] + method = transcoded_request["method"] + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(session, method)( + "{host}{uri}".format(host=host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params, strict=True), + data=body, + ) + return response + + def __call__( + self, + request: ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), + ) -> ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse: + r"""Call the batch apply ad review + center custom labels method over HTTP. + + Args: + request (~.ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsRequest): + The request object. Request object for + ``BatchApplyAdReviewCenterCustomLabels`` method. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be + sent along with the request as metadata. Normally, each value must be of type `str`, + but for metadata keys ending with the suffix `-bin`, the corresponding values must + be of type `bytes`. + + Returns: + ~.ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse: + Response object for + ``BatchApplyAdReviewCenterCustomLabels`` method. + + """ + + http_options = _BaseAdReviewCenterAdServiceRestTransport._BaseBatchApplyAdReviewCenterCustomLabels._get_http_options() + request, metadata = ( + self._interceptor.pre_batch_apply_ad_review_center_custom_labels( + request, metadata + ) + ) + transcoded_request, body, query_params = transcode_request( + http_options, + request, + required_fields_default_values=getattr( + _BaseAdReviewCenterAdServiceRestTransport._BaseBatchApplyAdReviewCenterCustomLabels, + "_BaseBatchApplyAdReviewCenterCustomLabels__REQUIRED_FIELDS_DEFAULT_VALUES", + None, + ), + rest_numeric_enums=True, + ) + + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + request_url = "{host}{uri}".format( + host=self._host, uri=transcoded_request["uri"] + ) + method = transcoded_request["method"] + try: + request_payload = type(request).to_json(request) + except: + request_payload = None + http_request = { + "payload": request_payload, + "requestMethod": method, + "requestUrl": request_url, + "headers": dict(metadata), + } + _LOGGER.debug( + f"Sending request for google.ads.admanager_v1.AdReviewCenterAdServiceClient.BatchApplyAdReviewCenterCustomLabels", + extra={ + "serviceName": "google.ads.admanager.v1.AdReviewCenterAdService", + "rpcName": "BatchApplyAdReviewCenterCustomLabels", + "httpRequest": http_request, + "metadata": http_request["headers"], + }, + ) + + # Send the request + response = AdReviewCenterAdServiceRestTransport._BatchApplyAdReviewCenterCustomLabels._get_response( + self._host, + metadata, + query_params, + self._session, + timeout, + transcoded_request, + body, + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + # Return the response + resp = ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse() + pb_resp = ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse.pb( + resp + ) + + json_format.Parse(response.content, pb_resp, ignore_unknown_fields=True) + + resp = self._interceptor.post_batch_apply_ad_review_center_custom_labels( + resp + ) + response_metadata = [(k, str(v)) for k, v in response.headers.items()] + resp, _ = ( + self._interceptor.post_batch_apply_ad_review_center_custom_labels_with_metadata( + resp, response_metadata + ) + ) + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + try: + response_payload = ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse.to_json( + response + ) + except: + response_payload = None + http_response = { + "payload": response_payload, + "headers": dict(response.headers), + "status": response.status_code, + } + _LOGGER.debug( + "Received response for google.ads.admanager_v1.AdReviewCenterAdServiceClient.batch_apply_ad_review_center_custom_labels", + extra={ + "serviceName": "google.ads.admanager.v1.AdReviewCenterAdService", + "rpcName": "BatchApplyAdReviewCenterCustomLabels", + "metadata": http_response["headers"], + "httpResponse": http_response, + }, + ) + return resp + class _BatchBlockAdReviewCenterAds( _BaseAdReviewCenterAdServiceRestTransport._BaseBatchBlockAdReviewCenterAds, AdReviewCenterAdServiceRestStub, @@ -746,6 +1032,168 @@ def __call__( ) return resp + class _FetchAdReviewCenterCustomLabels( + _BaseAdReviewCenterAdServiceRestTransport._BaseFetchAdReviewCenterCustomLabels, + AdReviewCenterAdServiceRestStub, + ): + def __hash__(self): + return hash( + "AdReviewCenterAdServiceRestTransport.FetchAdReviewCenterCustomLabels" + ) + + @staticmethod + def _get_response( + host, + metadata, + query_params, + session, + timeout, + transcoded_request, + body=None, + ): + uri = transcoded_request["uri"] + method = transcoded_request["method"] + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(session, method)( + "{host}{uri}".format(host=host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params, strict=True), + ) + return response + + def __call__( + self, + request: ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), + ) -> ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse: + r"""Call the fetch ad review center + custom labels method over HTTP. + + Args: + request (~.ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsRequest): + The request object. Request object for ``FetchAdReviewCenterCustomLabels`` + method. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be + sent along with the request as metadata. Normally, each value must be of type `str`, + but for metadata keys ending with the suffix `-bin`, the corresponding values must + be of type `bytes`. + + Returns: + ~.ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse: + Response object for ``FetchAdReviewCenterCustomLabels`` + method. + + """ + + http_options = _BaseAdReviewCenterAdServiceRestTransport._BaseFetchAdReviewCenterCustomLabels._get_http_options() + request, metadata = ( + self._interceptor.pre_fetch_ad_review_center_custom_labels( + request, metadata + ) + ) + transcoded_request, body, query_params = transcode_request( + http_options, + request, + required_fields_default_values=getattr( + _BaseAdReviewCenterAdServiceRestTransport._BaseFetchAdReviewCenterCustomLabels, + "_BaseFetchAdReviewCenterCustomLabels__REQUIRED_FIELDS_DEFAULT_VALUES", + None, + ), + rest_numeric_enums=True, + ) + + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + request_url = "{host}{uri}".format( + host=self._host, uri=transcoded_request["uri"] + ) + method = transcoded_request["method"] + try: + request_payload = type(request).to_json(request) + except: + request_payload = None + http_request = { + "payload": request_payload, + "requestMethod": method, + "requestUrl": request_url, + "headers": dict(metadata), + } + _LOGGER.debug( + f"Sending request for google.ads.admanager_v1.AdReviewCenterAdServiceClient.FetchAdReviewCenterCustomLabels", + extra={ + "serviceName": "google.ads.admanager.v1.AdReviewCenterAdService", + "rpcName": "FetchAdReviewCenterCustomLabels", + "httpRequest": http_request, + "metadata": http_request["headers"], + }, + ) + + # Send the request + response = AdReviewCenterAdServiceRestTransport._FetchAdReviewCenterCustomLabels._get_response( + self._host, + metadata, + query_params, + self._session, + timeout, + transcoded_request, + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + # Return the response + resp = ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse() + pb_resp = ( + ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse.pb( + resp + ) + ) + + json_format.Parse(response.content, pb_resp, ignore_unknown_fields=True) + + resp = self._interceptor.post_fetch_ad_review_center_custom_labels(resp) + response_metadata = [(k, str(v)) for k, v in response.headers.items()] + resp, _ = ( + self._interceptor.post_fetch_ad_review_center_custom_labels_with_metadata( + resp, response_metadata + ) + ) + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + try: + response_payload = ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse.to_json( + response + ) + except: + response_payload = None + http_response = { + "payload": response_payload, + "headers": dict(response.headers), + "status": response.status_code, + } + _LOGGER.debug( + "Received response for google.ads.admanager_v1.AdReviewCenterAdServiceClient.fetch_ad_review_center_custom_labels", + extra={ + "serviceName": "google.ads.admanager.v1.AdReviewCenterAdService", + "rpcName": "FetchAdReviewCenterCustomLabels", + "metadata": http_response["headers"], + "httpResponse": http_response, + }, + ) + return resp + class _SearchAdReviewCenterAds( _BaseAdReviewCenterAdServiceRestTransport._BaseSearchAdReviewCenterAds, AdReviewCenterAdServiceRestStub, @@ -910,6 +1358,19 @@ def batch_allow_ad_review_center_ads( self._session, self._host, self._interceptor ) # type: ignore + @property + def batch_apply_ad_review_center_custom_labels( + self, + ) -> Callable[ + [ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsRequest], + ad_review_center_ad_service.BatchApplyAdReviewCenterCustomLabelsResponse, + ]: + # The return type is fine, but mypy isn't sophisticated enough to determine what's going on here. + # In C++ this would require a dynamic_cast + return self._BatchApplyAdReviewCenterCustomLabels( + self._session, self._host, self._interceptor + ) # type: ignore + @property def batch_block_ad_review_center_ads( self, @@ -923,6 +1384,19 @@ def batch_block_ad_review_center_ads( self._session, self._host, self._interceptor ) # type: ignore + @property + def fetch_ad_review_center_custom_labels( + self, + ) -> Callable[ + [ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsRequest], + ad_review_center_ad_service.FetchAdReviewCenterCustomLabelsResponse, + ]: + # The return type is fine, but mypy isn't sophisticated enough to determine what's going on here. + # In C++ this would require a dynamic_cast + return self._FetchAdReviewCenterCustomLabels( + self._session, self._host, self._interceptor + ) # type: ignore + @property def search_ad_review_center_ads( self, diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/transports/rest_base.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/transports/rest_base.py index e219ba67f130..e626016f97c0 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/transports/rest_base.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/ad_review_center_ad_service/transports/rest_base.py @@ -105,6 +105,23 @@ def _get_http_options(): ] return http_options + class _BaseBatchApplyAdReviewCenterCustomLabels: + def __hash__(self): # pragma: NO COVER + return NotImplementedError("__hash__ must be implemented.") + + __REQUIRED_FIELDS_DEFAULT_VALUES: Dict[str, Any] = {} + + @staticmethod + def _get_http_options(): + http_options: List[Dict[str, str]] = [ + { + "method": "post", + "uri": "/v1/{parent=networks/*/webProperties/*}/adReviewCenterAds:batchApplyCustomLabels", + "body": "*", + }, + ] + return http_options + class _BaseBatchBlockAdReviewCenterAds: def __hash__(self): # pragma: NO COVER return NotImplementedError("__hash__ must be implemented.") @@ -122,6 +139,22 @@ def _get_http_options(): ] return http_options + class _BaseFetchAdReviewCenterCustomLabels: + def __hash__(self): # pragma: NO COVER + return NotImplementedError("__hash__ must be implemented.") + + __REQUIRED_FIELDS_DEFAULT_VALUES: Dict[str, Any] = {} + + @staticmethod + def _get_http_options(): + http_options: List[Dict[str, str]] = [ + { + "method": "get", + "uri": "/v1/{parent=networks/*/webProperties/*}/adReviewCenterAds:fetchCustomLabels", + }, + ] + return http_options + class _BaseSearchAdReviewCenterAds: def __hash__(self): # pragma: NO COVER return NotImplementedError("__hash__ must be implemented.") diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/client.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/client.py index 0096bec0372f..8c44213a97c1 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/client.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/client.py @@ -68,6 +68,7 @@ _LOGGER = std_logging.getLogger(__name__) +import google.protobuf.field_mask_pb2 as field_mask_pb2 # type: ignore from google.longrunning import operations_pb2 # type: ignore from google.ads.admanager_v1.services.audience_segment_service import pagers @@ -1195,6 +1196,253 @@ def sample_batch_create_audience_segments(): # Done; return the response. return response + def update_audience_segment( + self, + request: Optional[ + Union[audience_segment_service.UpdateAudienceSegmentRequest, dict] + ] = None, + *, + audience_segment: Optional[audience_segment_messages.AudienceSegment] = None, + update_mask: Optional[field_mask_pb2.FieldMask] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), + ) -> audience_segment_messages.AudienceSegment: + r"""Updates an ``AudienceSegment`` object. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.ads import admanager_v1 + + def sample_update_audience_segment(): + # Create a client + client = admanager_v1.AudienceSegmentServiceClient() + + # Initialize request argument(s) + request = admanager_v1.UpdateAudienceSegmentRequest( + ) + + # Make the request + response = client.update_audience_segment(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.ads.admanager_v1.types.UpdateAudienceSegmentRequest, dict]): + The request object. Request object for ``UpdateAudienceSegment`` method. + audience_segment (google.ads.admanager_v1.types.AudienceSegment): + Required. The ``AudienceSegment`` to update. + This corresponds to the ``audience_segment`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + update_mask (google.protobuf.field_mask_pb2.FieldMask): + Optional. The list of fields to + update. + + This corresponds to the ``update_mask`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be + sent along with the request as metadata. Normally, each value must be of type `str`, + but for metadata keys ending with the suffix `-bin`, the corresponding values must + be of type `bytes`. + + Returns: + google.ads.admanager_v1.types.AudienceSegment: + The AudienceSegment resource. + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + flattened_params = [audience_segment, update_mask] + has_flattened_params = ( + len([param for param in flattened_params if param is not None]) > 0 + ) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance( + request, audience_segment_service.UpdateAudienceSegmentRequest + ): + request = audience_segment_service.UpdateAudienceSegmentRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if audience_segment is not None: + request.audience_segment = audience_segment + if update_mask is not None: + request.update_mask = update_mask + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.update_audience_segment] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata( + (("audience_segment.name", request.audience_segment.name),) + ), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def batch_update_audience_segments( + self, + request: Optional[ + Union[audience_segment_service.BatchUpdateAudienceSegmentsRequest, dict] + ] = None, + *, + parent: Optional[str] = None, + requests: Optional[ + MutableSequence[audience_segment_service.UpdateAudienceSegmentRequest] + ] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), + ) -> audience_segment_service.BatchUpdateAudienceSegmentsResponse: + r"""Batch updates ``AudienceSegment`` objects. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.ads import admanager_v1 + + def sample_batch_update_audience_segments(): + # Create a client + client = admanager_v1.AudienceSegmentServiceClient() + + # Initialize request argument(s) + request = admanager_v1.BatchUpdateAudienceSegmentsRequest( + parent="parent_value", + ) + + # Make the request + response = client.batch_update_audience_segments(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.ads.admanager_v1.types.BatchUpdateAudienceSegmentsRequest, dict]): + The request object. Request object for ``BatchUpdateAudienceSegments`` + method. + parent (str): + Required. The parent resource where ``AudienceSegments`` + will be updated. Format: ``networks/{network_code}`` The + parent field in the + UpdateAudienceSegmentRequest.audienceSegment must match + this field. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + requests (MutableSequence[google.ads.admanager_v1.types.UpdateAudienceSegmentRequest]): + Required. The ``AudienceSegment`` objects to update. A + maximum of 100 objects can be updated in a batch. + + This corresponds to the ``requests`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be + sent along with the request as metadata. Normally, each value must be of type `str`, + but for metadata keys ending with the suffix `-bin`, the corresponding values must + be of type `bytes`. + + Returns: + google.ads.admanager_v1.types.BatchUpdateAudienceSegmentsResponse: + Response object for BatchUpdateAudienceSegments method. + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + flattened_params = [parent, requests] + has_flattened_params = ( + len([param for param in flattened_params if param is not None]) > 0 + ) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance( + request, audience_segment_service.BatchUpdateAudienceSegmentsRequest + ): + request = audience_segment_service.BatchUpdateAudienceSegmentsRequest( + request + ) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if requests is not None: + request.requests = requests + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[ + self._transport.batch_update_audience_segments + ] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + def batch_activate_audience_segments( self, request: Optional[ diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/transports/base.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/transports/base.py index fd045a8723c1..66dae11d414d 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/transports/base.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/transports/base.py @@ -167,6 +167,16 @@ def _prep_wrapped_messages(self, client_info): default_timeout=None, client_info=client_info, ), + self.update_audience_segment: gapic_v1.method.wrap_method( + self.update_audience_segment, + default_timeout=None, + client_info=client_info, + ), + self.batch_update_audience_segments: gapic_v1.method.wrap_method( + self.batch_update_audience_segments, + default_timeout=None, + client_info=client_info, + ), self.batch_activate_audience_segments: gapic_v1.method.wrap_method( self.batch_activate_audience_segments, default_timeout=None, @@ -261,6 +271,30 @@ def batch_create_audience_segments( ]: raise NotImplementedError() + @property + def update_audience_segment( + self, + ) -> Callable[ + [audience_segment_service.UpdateAudienceSegmentRequest], + Union[ + audience_segment_messages.AudienceSegment, + Awaitable[audience_segment_messages.AudienceSegment], + ], + ]: + raise NotImplementedError() + + @property + def batch_update_audience_segments( + self, + ) -> Callable[ + [audience_segment_service.BatchUpdateAudienceSegmentsRequest], + Union[ + audience_segment_service.BatchUpdateAudienceSegmentsResponse, + Awaitable[audience_segment_service.BatchUpdateAudienceSegmentsResponse], + ], + ]: + raise NotImplementedError() + @property def batch_activate_audience_segments( self, diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/transports/rest.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/transports/rest.py index dfa1715339d2..e84794c99825 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/transports/rest.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/transports/rest.py @@ -124,6 +124,14 @@ def post_batch_reject_audience_segments(self, response): logging.log(f"Received response: {response}") return response + def pre_batch_update_audience_segments(self, request, metadata): + logging.log(f"Received request: {request}") + return request, metadata + + def post_batch_update_audience_segments(self, response): + logging.log(f"Received response: {response}") + return response + def pre_create_audience_segment(self, request, metadata): logging.log(f"Received request: {request}") return request, metadata @@ -148,6 +156,14 @@ def post_list_audience_segments(self, response): logging.log(f"Received response: {response}") return response + def pre_update_audience_segment(self, request, metadata): + logging.log(f"Received request: {request}") + return request, metadata + + def post_update_audience_segment(self, response): + logging.log(f"Received response: {response}") + return response + transport = AudienceSegmentServiceRestTransport(interceptor=MyCustomAudienceSegmentServiceInterceptor()) client = AudienceSegmentServiceClient(transport=transport) @@ -466,6 +482,58 @@ def post_batch_reject_audience_segments_with_metadata( """ return response, metadata + def pre_batch_update_audience_segments( + self, + request: audience_segment_service.BatchUpdateAudienceSegmentsRequest, + metadata: Sequence[Tuple[str, Union[str, bytes]]], + ) -> Tuple[ + audience_segment_service.BatchUpdateAudienceSegmentsRequest, + Sequence[Tuple[str, Union[str, bytes]]], + ]: + """Pre-rpc interceptor for batch_update_audience_segments + + Override in a subclass to manipulate the request or metadata + before they are sent to the AudienceSegmentService server. + """ + return request, metadata + + def post_batch_update_audience_segments( + self, response: audience_segment_service.BatchUpdateAudienceSegmentsResponse + ) -> audience_segment_service.BatchUpdateAudienceSegmentsResponse: + """Post-rpc interceptor for batch_update_audience_segments + + DEPRECATED. Please use the `post_batch_update_audience_segments_with_metadata` + interceptor instead. + + Override in a subclass to read or manipulate the response + after it is returned by the AudienceSegmentService server but before + it is returned to user code. This `post_batch_update_audience_segments` interceptor runs + before the `post_batch_update_audience_segments_with_metadata` interceptor. + """ + return response + + def post_batch_update_audience_segments_with_metadata( + self, + response: audience_segment_service.BatchUpdateAudienceSegmentsResponse, + metadata: Sequence[Tuple[str, Union[str, bytes]]], + ) -> Tuple[ + audience_segment_service.BatchUpdateAudienceSegmentsResponse, + Sequence[Tuple[str, Union[str, bytes]]], + ]: + """Post-rpc interceptor for batch_update_audience_segments + + Override in a subclass to read or manipulate the response or metadata after it + is returned by the AudienceSegmentService server but before it is returned to user code. + + We recommend only using this `post_batch_update_audience_segments_with_metadata` + interceptor in new development instead of the `post_batch_update_audience_segments` interceptor. + When both interceptors are used, this `post_batch_update_audience_segments_with_metadata` interceptor runs after the + `post_batch_update_audience_segments` interceptor. The (possibly modified) response returned by + `post_batch_update_audience_segments` will be passed to + `post_batch_update_audience_segments_with_metadata`. + """ + return response, metadata + def pre_create_audience_segment( self, request: audience_segment_service.CreateAudienceSegmentRequest, @@ -622,6 +690,58 @@ def post_list_audience_segments_with_metadata( """ return response, metadata + def pre_update_audience_segment( + self, + request: audience_segment_service.UpdateAudienceSegmentRequest, + metadata: Sequence[Tuple[str, Union[str, bytes]]], + ) -> Tuple[ + audience_segment_service.UpdateAudienceSegmentRequest, + Sequence[Tuple[str, Union[str, bytes]]], + ]: + """Pre-rpc interceptor for update_audience_segment + + Override in a subclass to manipulate the request or metadata + before they are sent to the AudienceSegmentService server. + """ + return request, metadata + + def post_update_audience_segment( + self, response: audience_segment_messages.AudienceSegment + ) -> audience_segment_messages.AudienceSegment: + """Post-rpc interceptor for update_audience_segment + + DEPRECATED. Please use the `post_update_audience_segment_with_metadata` + interceptor instead. + + Override in a subclass to read or manipulate the response + after it is returned by the AudienceSegmentService server but before + it is returned to user code. This `post_update_audience_segment` interceptor runs + before the `post_update_audience_segment_with_metadata` interceptor. + """ + return response + + def post_update_audience_segment_with_metadata( + self, + response: audience_segment_messages.AudienceSegment, + metadata: Sequence[Tuple[str, Union[str, bytes]]], + ) -> Tuple[ + audience_segment_messages.AudienceSegment, + Sequence[Tuple[str, Union[str, bytes]]], + ]: + """Post-rpc interceptor for update_audience_segment + + Override in a subclass to read or manipulate the response or metadata after it + is returned by the AudienceSegmentService server but before it is returned to user code. + + We recommend only using this `post_update_audience_segment_with_metadata` + interceptor in new development instead of the `post_update_audience_segment` interceptor. + When both interceptors are used, this `post_update_audience_segment_with_metadata` interceptor runs after the + `post_update_audience_segment` interceptor. The (possibly modified) response returned by + `post_update_audience_segment` will be passed to + `post_update_audience_segment_with_metadata`. + """ + return response, metadata + def pre_cancel_operation( self, request: operations_pb2.CancelOperationRequest, @@ -1728,6 +1848,166 @@ def __call__( ) return resp + class _BatchUpdateAudienceSegments( + _BaseAudienceSegmentServiceRestTransport._BaseBatchUpdateAudienceSegments, + AudienceSegmentServiceRestStub, + ): + def __hash__(self): + return hash( + "AudienceSegmentServiceRestTransport.BatchUpdateAudienceSegments" + ) + + @staticmethod + def _get_response( + host, + metadata, + query_params, + session, + timeout, + transcoded_request, + body=None, + ): + uri = transcoded_request["uri"] + method = transcoded_request["method"] + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(session, method)( + "{host}{uri}".format(host=host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params, strict=True), + data=body, + ) + return response + + def __call__( + self, + request: audience_segment_service.BatchUpdateAudienceSegmentsRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), + ) -> audience_segment_service.BatchUpdateAudienceSegmentsResponse: + r"""Call the batch update audience + segments method over HTTP. + + Args: + request (~.audience_segment_service.BatchUpdateAudienceSegmentsRequest): + The request object. Request object for ``BatchUpdateAudienceSegments`` + method. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be + sent along with the request as metadata. Normally, each value must be of type `str`, + but for metadata keys ending with the suffix `-bin`, the corresponding values must + be of type `bytes`. + + Returns: + ~.audience_segment_service.BatchUpdateAudienceSegmentsResponse: + Response object for ``BatchUpdateAudienceSegments`` + method. + + """ + + http_options = _BaseAudienceSegmentServiceRestTransport._BaseBatchUpdateAudienceSegments._get_http_options() + request, metadata = self._interceptor.pre_batch_update_audience_segments( + request, metadata + ) + transcoded_request, body, query_params = transcode_request( + http_options, + request, + required_fields_default_values=getattr( + _BaseAudienceSegmentServiceRestTransport._BaseBatchUpdateAudienceSegments, + "_BaseBatchUpdateAudienceSegments__REQUIRED_FIELDS_DEFAULT_VALUES", + None, + ), + rest_numeric_enums=True, + ) + + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + request_url = "{host}{uri}".format( + host=self._host, uri=transcoded_request["uri"] + ) + method = transcoded_request["method"] + try: + request_payload = type(request).to_json(request) + except: + request_payload = None + http_request = { + "payload": request_payload, + "requestMethod": method, + "requestUrl": request_url, + "headers": dict(metadata), + } + _LOGGER.debug( + f"Sending request for google.ads.admanager_v1.AudienceSegmentServiceClient.BatchUpdateAudienceSegments", + extra={ + "serviceName": "google.ads.admanager.v1.AudienceSegmentService", + "rpcName": "BatchUpdateAudienceSegments", + "httpRequest": http_request, + "metadata": http_request["headers"], + }, + ) + + # Send the request + response = AudienceSegmentServiceRestTransport._BatchUpdateAudienceSegments._get_response( + self._host, + metadata, + query_params, + self._session, + timeout, + transcoded_request, + body, + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + # Return the response + resp = audience_segment_service.BatchUpdateAudienceSegmentsResponse() + pb_resp = audience_segment_service.BatchUpdateAudienceSegmentsResponse.pb( + resp + ) + + json_format.Parse(response.content, pb_resp, ignore_unknown_fields=True) + + resp = self._interceptor.post_batch_update_audience_segments(resp) + response_metadata = [(k, str(v)) for k, v in response.headers.items()] + resp, _ = ( + self._interceptor.post_batch_update_audience_segments_with_metadata( + resp, response_metadata + ) + ) + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + try: + response_payload = audience_segment_service.BatchUpdateAudienceSegmentsResponse.to_json( + response + ) + except: + response_payload = None + http_response = { + "payload": response_payload, + "headers": dict(response.headers), + "status": response.status_code, + } + _LOGGER.debug( + "Received response for google.ads.admanager_v1.AudienceSegmentServiceClient.batch_update_audience_segments", + extra={ + "serviceName": "google.ads.admanager.v1.AudienceSegmentService", + "rpcName": "BatchUpdateAudienceSegments", + "metadata": http_response["headers"], + "httpResponse": http_response, + }, + ) + return resp + class _CreateAudienceSegment( _BaseAudienceSegmentServiceRestTransport._BaseCreateAudienceSegment, AudienceSegmentServiceRestStub, @@ -2182,6 +2462,156 @@ def __call__( ) return resp + class _UpdateAudienceSegment( + _BaseAudienceSegmentServiceRestTransport._BaseUpdateAudienceSegment, + AudienceSegmentServiceRestStub, + ): + def __hash__(self): + return hash("AudienceSegmentServiceRestTransport.UpdateAudienceSegment") + + @staticmethod + def _get_response( + host, + metadata, + query_params, + session, + timeout, + transcoded_request, + body=None, + ): + uri = transcoded_request["uri"] + method = transcoded_request["method"] + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(session, method)( + "{host}{uri}".format(host=host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params, strict=True), + data=body, + ) + return response + + def __call__( + self, + request: audience_segment_service.UpdateAudienceSegmentRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), + ) -> audience_segment_messages.AudienceSegment: + r"""Call the update audience segment method over HTTP. + + Args: + request (~.audience_segment_service.UpdateAudienceSegmentRequest): + The request object. Request object for ``UpdateAudienceSegment`` method. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be + sent along with the request as metadata. Normally, each value must be of type `str`, + but for metadata keys ending with the suffix `-bin`, the corresponding values must + be of type `bytes`. + + Returns: + ~.audience_segment_messages.AudienceSegment: + The ``AudienceSegment`` resource. + """ + + http_options = _BaseAudienceSegmentServiceRestTransport._BaseUpdateAudienceSegment._get_http_options() + request, metadata = self._interceptor.pre_update_audience_segment( + request, metadata + ) + transcoded_request, body, query_params = transcode_request( + http_options, + request, + required_fields_default_values=getattr( + _BaseAudienceSegmentServiceRestTransport._BaseUpdateAudienceSegment, + "_BaseUpdateAudienceSegment__REQUIRED_FIELDS_DEFAULT_VALUES", + None, + ), + rest_numeric_enums=True, + ) + + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + request_url = "{host}{uri}".format( + host=self._host, uri=transcoded_request["uri"] + ) + method = transcoded_request["method"] + try: + request_payload = type(request).to_json(request) + except: + request_payload = None + http_request = { + "payload": request_payload, + "requestMethod": method, + "requestUrl": request_url, + "headers": dict(metadata), + } + _LOGGER.debug( + f"Sending request for google.ads.admanager_v1.AudienceSegmentServiceClient.UpdateAudienceSegment", + extra={ + "serviceName": "google.ads.admanager.v1.AudienceSegmentService", + "rpcName": "UpdateAudienceSegment", + "httpRequest": http_request, + "metadata": http_request["headers"], + }, + ) + + # Send the request + response = AudienceSegmentServiceRestTransport._UpdateAudienceSegment._get_response( + self._host, + metadata, + query_params, + self._session, + timeout, + transcoded_request, + body, + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + # Return the response + resp = audience_segment_messages.AudienceSegment() + pb_resp = audience_segment_messages.AudienceSegment.pb(resp) + + json_format.Parse(response.content, pb_resp, ignore_unknown_fields=True) + + resp = self._interceptor.post_update_audience_segment(resp) + response_metadata = [(k, str(v)) for k, v in response.headers.items()] + resp, _ = self._interceptor.post_update_audience_segment_with_metadata( + resp, response_metadata + ) + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + try: + response_payload = ( + audience_segment_messages.AudienceSegment.to_json(response) + ) + except: + response_payload = None + http_response = { + "payload": response_payload, + "headers": dict(response.headers), + "status": response.status_code, + } + _LOGGER.debug( + "Received response for google.ads.admanager_v1.AudienceSegmentServiceClient.update_audience_segment", + extra={ + "serviceName": "google.ads.admanager.v1.AudienceSegmentService", + "rpcName": "UpdateAudienceSegment", + "metadata": http_response["headers"], + "httpResponse": http_response, + }, + ) + return resp + @property def batch_activate_audience_segments( self, @@ -2260,6 +2690,19 @@ def batch_reject_audience_segments( self._session, self._host, self._interceptor ) # type: ignore + @property + def batch_update_audience_segments( + self, + ) -> Callable[ + [audience_segment_service.BatchUpdateAudienceSegmentsRequest], + audience_segment_service.BatchUpdateAudienceSegmentsResponse, + ]: + # The return type is fine, but mypy isn't sophisticated enough to determine what's going on here. + # In C++ this would require a dynamic_cast + return self._BatchUpdateAudienceSegments( + self._session, self._host, self._interceptor + ) # type: ignore + @property def create_audience_segment( self, @@ -2293,6 +2736,17 @@ def list_audience_segments( # In C++ this would require a dynamic_cast return self._ListAudienceSegments(self._session, self._host, self._interceptor) # type: ignore + @property + def update_audience_segment( + self, + ) -> Callable[ + [audience_segment_service.UpdateAudienceSegmentRequest], + audience_segment_messages.AudienceSegment, + ]: + # The return type is fine, but mypy isn't sophisticated enough to determine what's going on here. + # In C++ this would require a dynamic_cast + return self._UpdateAudienceSegment(self._session, self._host, self._interceptor) # type: ignore + @property def cancel_operation(self): return self._CancelOperation(self._session, self._host, self._interceptor) # type: ignore diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/transports/rest_base.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/transports/rest_base.py index b6402e92b413..e235e9fb99ff 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/transports/rest_base.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/audience_segment_service/transports/rest_base.py @@ -193,6 +193,23 @@ def _get_http_options(): ] return http_options + class _BaseBatchUpdateAudienceSegments: + def __hash__(self): # pragma: NO COVER + return NotImplementedError("__hash__ must be implemented.") + + __REQUIRED_FIELDS_DEFAULT_VALUES: Dict[str, Any] = {} + + @staticmethod + def _get_http_options(): + http_options: List[Dict[str, str]] = [ + { + "method": "post", + "uri": "/v1/{parent=networks/*}/audienceSegments:batchUpdate", + "body": "*", + }, + ] + return http_options + class _BaseCreateAudienceSegment: def __hash__(self): # pragma: NO COVER return NotImplementedError("__hash__ must be implemented.") @@ -242,6 +259,23 @@ def _get_http_options(): ] return http_options + class _BaseUpdateAudienceSegment: + def __hash__(self): # pragma: NO COVER + return NotImplementedError("__hash__ must be implemented.") + + __REQUIRED_FIELDS_DEFAULT_VALUES: Dict[str, Any] = {} + + @staticmethod + def _get_http_options(): + http_options: List[Dict[str, str]] = [ + { + "method": "patch", + "uri": "/v1/{audience_segment.name=networks/*/audienceSegments/*}", + "body": "audience_segment", + }, + ] + return http_options + class _BaseCancelOperation: def __hash__(self): # pragma: NO COVER return NotImplementedError("__hash__ must be implemented.") diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/child_publisher_service/client.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/child_publisher_service/client.py index 9ade2c02a330..aa6cd251c345 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/services/child_publisher_service/client.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/child_publisher_service/client.py @@ -655,7 +655,7 @@ def sample_get_child_publisher(): Args: request (Union[google.ads.admanager_v1.types.GetChildPublisherRequest, dict]): - The request object. Request object for [GetChildPublisher][] method. + The request object. Request object for ``GetChildPublisher`` method. name (str): Required. The resource name of the [ChildPublisher][google.ads.admanager.v1.ChildPublisher]. @@ -769,10 +769,10 @@ def sample_list_child_publishers(): Args: request (Union[google.ads.admanager_v1.types.ListChildPublishersRequest, dict]): - The request object. Request object for [ListChildPublishers][] method. + The request object. Request object for ``ListChildPublishers`` method. parent (str): Required. The parent, which owns this collection of - [ChildPublisher][google.ads.admanager.v1.ChildPublisher]s. + [ChildPublishers][google.ads.admanager.v1.ChildPublisher]. Format: ``networks/{network_code}`` This corresponds to the ``parent`` field @@ -788,7 +788,7 @@ def sample_list_child_publishers(): Returns: google.ads.admanager_v1.services.child_publisher_service.pagers.ListChildPublishersPager: - Response object for [ListChildPublishers][] containing matching + Response object for ListChildPublishers containing matching [ChildPublisher][google.ads.admanager.v1.ChildPublisher] objects. @@ -896,7 +896,7 @@ def sample_create_child_publisher(): Args: request (Union[google.ads.admanager_v1.types.CreateChildPublisherRequest, dict]): - The request object. Request object for [CreateChildPublisher][] method. + The request object. Request object for ``CreateChildPublisher`` method. parent (str): Required. The parent resource where this [ChildPublisher][google.ads.admanager.v1.ChildPublisher] @@ -1025,11 +1025,11 @@ def sample_batch_create_child_publishers(): Args: request (Union[google.ads.admanager_v1.types.BatchCreateChildPublishersRequest, dict]): - The request object. Request object for [BatchCreateChildPublishers][] + The request object. Request object for ``BatchCreateChildPublishers`` method. parent (str): Required. The parent resource where - [ChildPublisher][google.ads.admanager.v1.ChildPublisher]s + [ChildPublishers][google.ads.admanager.v1.ChildPublisher] will be created. Format: ``networks/{network_code}`` The parent field in the CreateChildPublisherRequest must match this field. @@ -1056,9 +1056,7 @@ def sample_batch_create_child_publishers(): Returns: google.ads.admanager_v1.types.BatchCreateChildPublishersResponse: - Response object for [BatchCreateChildPublishers][] - method. - + Response object for BatchCreateChildPublishers method. """ # Create or coerce a protobuf request object. # - Quick check: If we got a request object, we should *not* have @@ -1154,7 +1152,7 @@ def sample_update_child_publisher(): Args: request (Union[google.ads.admanager_v1.types.UpdateChildPublisherRequest, dict]): - The request object. Request object for [UpdateChildPublisher][] method. + The request object. Request object for ``UpdateChildPublisher`` method. child_publisher (google.ads.admanager_v1.types.ChildPublisher): Required. The [ChildPublisher][google.ads.admanager.v1.ChildPublisher] @@ -1287,11 +1285,11 @@ def sample_batch_update_child_publishers(): Args: request (Union[google.ads.admanager_v1.types.BatchUpdateChildPublishersRequest, dict]): - The request object. Request object for [BatchUpdateChildPublishers][] + The request object. Request object for ``BatchUpdateChildPublishers`` method. parent (str): Required. The parent resource where - [ChildPublisher][google.ads.admanager.v1.ChildPublisher]s + [ChildPublishers][google.ads.admanager.v1.ChildPublisher] will be updated. Format: ``networks/{network_code}`` The parent field in the UpdateChildPublisherRequest must match this field. @@ -1318,9 +1316,7 @@ def sample_batch_update_child_publishers(): Returns: google.ads.admanager_v1.types.BatchUpdateChildPublishersResponse: - Response object for [BatchUpdateChildPublishers][] - method. - + Response object for BatchUpdateChildPublishers method. """ # Create or coerce a protobuf request object. # - Quick check: If we got a request object, we should *not* have @@ -1428,7 +1424,7 @@ def sample_batch_resend_child_publisher_invitation_emails(): Args: request (Union[google.ads.admanager_v1.types.BatchResendChildPublisherInvitationEmailsRequest, dict]): The request object. Request message for - [BatchResendChildPublisherInvitationEmails][] method. + ``BatchResendChildPublisherInvitationEmails`` method. parent (str): Required. Format: ``networks/{network_code}`` This corresponds to the ``parent`` field @@ -1436,7 +1432,7 @@ def sample_batch_resend_child_publisher_invitation_emails(): should not be set. names (MutableSequence[str]): Required. Resource names of the - [ChildPublisher][google.ads.admanager.v1.ChildPublisher]s + [ChildPublishers][google.ads.admanager.v1.ChildPublisher] that should be resent invitation emails. Format: ``networks/{network_code}/childPublisher/{child_publisher_id}`` @@ -1454,7 +1450,7 @@ def sample_batch_resend_child_publisher_invitation_emails(): Returns: google.ads.admanager_v1.types.BatchResendChildPublisherInvitationEmailsResponse: Response message for - [BatchResendChildPublisherInvitationEmails][] method. + BatchResendChildPublisherInvitationEmails method. """ # Create or coerce a protobuf request object. @@ -1574,7 +1570,7 @@ def sample_batch_renegotiate_child_publisher_agreements(): Args: request (Union[google.ads.admanager_v1.types.BatchRenegotiateChildPublisherAgreementsRequest, dict]): The request object. Request message for - [BatchRenegotiateChildPublisherAgreements][] method. + ``BatchRenegotiateChildPublisherAgreements`` method. parent (str): Required. Format: ``networks/{network_code}`` This corresponds to the ``parent`` field @@ -1599,7 +1595,7 @@ def sample_batch_renegotiate_child_publisher_agreements(): Returns: google.ads.admanager_v1.types.BatchRenegotiateChildPublisherAgreementsResponse: Response message for - [BatchRenegotiateChildPublisherAgreements][] method. + BatchRenegotiateChildPublisherAgreements method. """ # Create or coerce a protobuf request object. @@ -1686,7 +1682,7 @@ def batch_reject_child_publishers( that are not yet associated with an Ad Manager network. To sever the relationship from the parent publisher's side, use - [BatchWithdrawChildPublisher][]. + ``BatchWithdrawChildPublishers``. .. code-block:: python @@ -1717,7 +1713,7 @@ def sample_batch_reject_child_publishers(): Args: request (Union[google.ads.admanager_v1.types.BatchRejectChildPublishersRequest, dict]): - The request object. Request message for [BatchRejectChildPublishers][] + The request object. Request message for ``BatchRejectChildPublishers`` method. parent (str): Required. Format: ``networks/{network_code}`` @@ -1726,7 +1722,7 @@ def sample_batch_reject_child_publishers(): should not be set. names (MutableSequence[str]): Required. Resource names of the - [ChildPublisher][google.ads.admanager.v1.ChildPublisher]s + [ChildPublishers][google.ads.admanager.v1.ChildPublisher] to reject. Format: ``networks/{network_code}/childPublisher/{child_publisher_id}`` @@ -1743,9 +1739,7 @@ def sample_batch_reject_child_publishers(): Returns: google.ads.admanager_v1.types.BatchRejectChildPublishersResponse: - Response message for [BatchRejectChildPublishers][] - method. - + Response message for BatchRejectChildPublishers method. """ # Create or coerce a protobuf request object. # - Quick check: If we got a request object, we should *not* have @@ -1812,12 +1806,12 @@ def batch_withdraw_child_publishers( metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), ) -> child_publisher_service.BatchWithdrawChildPublishersResponse: r"""Batch withdraws - [ChildPublisher][google.ads.admanager.v1.ChildPublisher]s. + [ChildPublishers][google.ads.admanager.v1.ChildPublisher]. Only expired, pending, and accepted - [ChildPublisher][google.ads.admanager.v1.ChildPublisher]s can be + [ChildPublishers][google.ads.admanager.v1.ChildPublisher] can be withdrawn. Rejected or withdrawn - [ChildPublisher][google.ads.admanager.v1.ChildPublisher]s will + [ChildPublishers][google.ads.admanager.v1.ChildPublisher] will be ignored. .. code-block:: python @@ -1849,7 +1843,7 @@ def sample_batch_withdraw_child_publishers(): Args: request (Union[google.ads.admanager_v1.types.BatchWithdrawChildPublishersRequest, dict]): - The request object. Request message for [BatchWithdrawChildPublishers][] + The request object. Request message for ``BatchWithdrawChildPublishers`` method. parent (str): Required. Format: ``networks/{network_code}`` @@ -1858,7 +1852,7 @@ def sample_batch_withdraw_child_publishers(): should not be set. names (MutableSequence[str]): Required. Resource names of the - [ChildPublisher][google.ads.admanager.v1.ChildPublisher]s + [ChildPublishers][google.ads.admanager.v1.ChildPublisher] to withdraw. Format: ``networks/{network_code}/childPublisher/{child_publisher_id}`` @@ -1875,7 +1869,7 @@ def sample_batch_withdraw_child_publishers(): Returns: google.ads.admanager_v1.types.BatchWithdrawChildPublishersResponse: - Response message for [BatchWithdrawChildPublishers][] + Response message for BatchWithdrawChildPublishers method. """ diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/child_publisher_service/transports/rest.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/child_publisher_service/transports/rest.py index 0a308d9eed3f..8b159a438d61 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/services/child_publisher_service/transports/rest.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/child_publisher_service/transports/rest.py @@ -867,7 +867,7 @@ def __call__( Args: request (~.child_publisher_service.BatchCreateChildPublishersRequest): - The request object. Request object for [BatchCreateChildPublishers][] + The request object. Request object for ``BatchCreateChildPublishers`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. @@ -879,7 +879,7 @@ def __call__( Returns: ~.child_publisher_service.BatchCreateChildPublishersResponse: - Response object for [BatchCreateChildPublishers][] + Response object for ``BatchCreateChildPublishers`` method. """ @@ -1025,7 +1025,7 @@ def __call__( Args: request (~.child_publisher_service.BatchRejectChildPublishersRequest): - The request object. Request message for [BatchRejectChildPublishers][] + The request object. Request message for ``BatchRejectChildPublishers`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. @@ -1037,7 +1037,7 @@ def __call__( Returns: ~.child_publisher_service.BatchRejectChildPublishersResponse: - Response message for [BatchRejectChildPublishers][] + Response message for ``BatchRejectChildPublishers`` method. """ @@ -1186,7 +1186,7 @@ def __call__( Args: request (~.child_publisher_service.BatchRenegotiateChildPublisherAgreementsRequest): The request object. Request message for - [BatchRenegotiateChildPublisherAgreements][] method. + ``BatchRenegotiateChildPublisherAgreements`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -1198,7 +1198,7 @@ def __call__( Returns: ~.child_publisher_service.BatchRenegotiateChildPublisherAgreementsResponse: Response message for - [BatchRenegotiateChildPublisherAgreements][] method. + ``BatchRenegotiateChildPublisherAgreements`` method. """ @@ -1350,7 +1350,7 @@ def __call__( Args: request (~.child_publisher_service.BatchResendChildPublisherInvitationEmailsRequest): The request object. Request message for - [BatchResendChildPublisherInvitationEmails][] method. + ``BatchResendChildPublisherInvitationEmails`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -1362,7 +1362,7 @@ def __call__( Returns: ~.child_publisher_service.BatchResendChildPublisherInvitationEmailsResponse: Response message for - [BatchResendChildPublisherInvitationEmails][] method. + ``BatchResendChildPublisherInvitationEmails`` method. """ @@ -1513,7 +1513,7 @@ def __call__( Args: request (~.child_publisher_service.BatchUpdateChildPublishersRequest): - The request object. Request object for [BatchUpdateChildPublishers][] + The request object. Request object for ``BatchUpdateChildPublishers`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. @@ -1525,7 +1525,7 @@ def __call__( Returns: ~.child_publisher_service.BatchUpdateChildPublishersResponse: - Response object for [BatchUpdateChildPublishers][] + Response object for ``BatchUpdateChildPublishers`` method. """ @@ -1673,7 +1673,7 @@ def __call__( Args: request (~.child_publisher_service.BatchWithdrawChildPublishersRequest): - The request object. Request message for [BatchWithdrawChildPublishers][] + The request object. Request message for ``BatchWithdrawChildPublishers`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. @@ -1685,7 +1685,7 @@ def __call__( Returns: ~.child_publisher_service.BatchWithdrawChildPublishersResponse: - Response message for [BatchWithdrawChildPublishers][] + Response message for ``BatchWithdrawChildPublishers`` method. """ @@ -1830,7 +1830,7 @@ def __call__( Args: request (~.child_publisher_service.CreateChildPublisherRequest): - The request object. Request object for [CreateChildPublisher][] method. + The request object. Request object for ``CreateChildPublisher`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -1984,7 +1984,7 @@ def __call__( Args: request (~.child_publisher_service.GetChildPublisherRequest): - The request object. Request object for [GetChildPublisher][] method. + The request object. Request object for ``GetChildPublisher`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -2137,7 +2137,7 @@ def __call__( Args: request (~.child_publisher_service.ListChildPublishersRequest): - The request object. Request object for [ListChildPublishers][] method. + The request object. Request object for ``ListChildPublishers`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -2148,7 +2148,7 @@ def __call__( Returns: ~.child_publisher_service.ListChildPublishersResponse: - Response object for [ListChildPublishers][] containing + Response object for ``ListChildPublishers`` containing matching [ChildPublisher][google.ads.admanager.v1.ChildPublisher] objects. @@ -2294,7 +2294,7 @@ def __call__( Args: request (~.child_publisher_service.UpdateChildPublisherRequest): - The request object. Request object for [UpdateChildPublisher][] method. + The request object. Request object for ``UpdateChildPublisher`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/company_service/client.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/company_service/client.py index 15355e3dcc1a..a372d43f7820 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/services/company_service/client.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/company_service/client.py @@ -701,7 +701,7 @@ def sample_get_company(): Args: request (Union[google.ads.admanager_v1.types.GetCompanyRequest, dict]): - The request object. Request object for [GetCompany][] method. + The request object. Request object for ``GetCompany`` method. name (str): Required. The resource name of the Company. Format: ``networks/{network_code}/companies/{company_id}`` @@ -807,10 +807,11 @@ def sample_list_companies(): Args: request (Union[google.ads.admanager_v1.types.ListCompaniesRequest, dict]): - The request object. Request object for [ListCompanies][] method. + The request object. Request object for ``ListCompanies`` method. parent (str): Required. The parent, which owns this collection of - [Companies][]. Format: ``networks/{network_code}`` + [Companies][google.ads.admanager.v1.Company]. Format: + ``networks/{network_code}`` This corresponds to the ``parent`` field on the ``request`` instance; if ``request`` is provided, this @@ -825,9 +826,7 @@ def sample_list_companies(): Returns: google.ads.admanager_v1.services.company_service.pagers.ListCompaniesPager: - Response object for - [ListCompaniesRequest][google.ads.admanager.v1.ListCompaniesRequest] - containing matching + Response object for ListCompaniesRequest containing matching [Company][google.ads.admanager.v1.Company] objects. Iterating over this object will yield results and @@ -931,7 +930,7 @@ def sample_create_company(): Args: request (Union[google.ads.admanager_v1.types.CreateCompanyRequest, dict]): - The request object. Request object for [CreateCompany][] method. + The request object. Request object for ``CreateCompany`` method. parent (str): Required. The parent resource where this [Company][google.ads.admanager.v1.Company] will be @@ -1055,12 +1054,12 @@ def sample_batch_create_companies(): Args: request (Union[google.ads.admanager_v1.types.BatchCreateCompaniesRequest, dict]): - The request object. Request object for [BatchCreateCompanies][] method. + The request object. Request object for ``BatchCreateCompanies`` method. parent (str): - Required. The parent resource where [Companies][] will - be created. Format: ``networks/{network_code}`` The - parent field in the CreateCompanyRequest must match this - field. + Required. The parent resource where + [Companies][google.ads.admanager.v1.Company] will be + created. Format: ``networks/{network_code}`` The parent + field in the CreateCompanyRequest must match this field. This corresponds to the ``parent`` field on the ``request`` instance; if ``request`` is provided, this @@ -1083,7 +1082,7 @@ def sample_batch_create_companies(): Returns: google.ads.admanager_v1.types.BatchCreateCompaniesResponse: - Response object for [BatchCreateCompanies][] method. + Response object for BatchCreateCompanies method. """ # Create or coerce a protobuf request object. # - Quick check: If we got a request object, we should *not* have @@ -1172,7 +1171,7 @@ def sample_update_company(): Args: request (Union[google.ads.admanager_v1.types.UpdateCompanyRequest, dict]): - The request object. Request object for [UpdateCompany][] method. + The request object. Request object for ``UpdateCompany`` method. company (google.ads.admanager_v1.types.Company): Required. The [Company][google.ads.admanager.v1.Company] to update. @@ -1298,12 +1297,12 @@ def sample_batch_update_companies(): Args: request (Union[google.ads.admanager_v1.types.BatchUpdateCompaniesRequest, dict]): - The request object. Request object for [BatchUpdateCompanies][] method. + The request object. Request object for ``BatchUpdateCompanies`` method. parent (str): - Required. The parent resource where [Companies][] will - be updated. Format: ``networks/{network_code}`` The - parent field in the UpdateCompanyRequest must match this - field. + Required. The parent resource where + [Companies][google.ads.admanager.v1.Company] will be + updated. Format: ``networks/{network_code}`` The parent + field in the UpdateCompanyRequest must match this field. This corresponds to the ``parent`` field on the ``request`` instance; if ``request`` is provided, this @@ -1326,7 +1325,7 @@ def sample_batch_update_companies(): Returns: google.ads.admanager_v1.types.BatchUpdateCompaniesResponse: - Response object for [BatchUpdateCompanies][] method. + Response object for BatchUpdateCompanies method. """ # Create or coerce a protobuf request object. # - Quick check: If we got a request object, we should *not* have diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/company_service/transports/rest.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/company_service/transports/rest.py index 76eb62f544ac..2e097da25723 100644 --- a/packages/google-ads-admanager/google/ads/admanager_v1/services/company_service/transports/rest.py +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/company_service/transports/rest.py @@ -610,7 +610,7 @@ def __call__( Args: request (~.company_service.BatchCreateCompaniesRequest): - The request object. Request object for [BatchCreateCompanies][] method. + The request object. Request object for ``BatchCreateCompanies`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -621,7 +621,7 @@ def __call__( Returns: ~.company_service.BatchCreateCompaniesResponse: - Response object for [BatchCreateCompanies][] method. + Response object for ``BatchCreateCompanies`` method. """ http_options = _BaseCompanyServiceRestTransport._BaseBatchCreateCompanies._get_http_options() @@ -760,7 +760,7 @@ def __call__( Args: request (~.company_service.BatchUpdateCompaniesRequest): - The request object. Request object for [BatchUpdateCompanies][] method. + The request object. Request object for ``BatchUpdateCompanies`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -771,7 +771,7 @@ def __call__( Returns: ~.company_service.BatchUpdateCompaniesResponse: - Response object for [BatchUpdateCompanies][] method. + Response object for ``BatchUpdateCompanies`` method. """ http_options = _BaseCompanyServiceRestTransport._BaseBatchUpdateCompanies._get_http_options() @@ -909,7 +909,7 @@ def __call__( Args: request (~.company_service.CreateCompanyRequest): - The request object. Request object for [CreateCompany][] method. + The request object. Request object for ``CreateCompany`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -1055,7 +1055,7 @@ def __call__( Args: request (~.company_service.GetCompanyRequest): - The request object. Request object for [GetCompany][] method. + The request object. Request object for ``GetCompany`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -1200,7 +1200,7 @@ def __call__( Args: request (~.company_service.ListCompaniesRequest): - The request object. Request object for [ListCompanies][] method. + The request object. Request object for ``ListCompanies`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -1211,10 +1211,9 @@ def __call__( Returns: ~.company_service.ListCompaniesResponse: - Response object for - [ListCompaniesRequest][google.ads.admanager.v1.ListCompaniesRequest] - containing matching - [Company][google.ads.admanager.v1.Company] objects. + Response object for ``ListCompaniesRequest`` containing + matching [Company][google.ads.admanager.v1.Company] + objects. """ @@ -1352,7 +1351,7 @@ def __call__( Args: request (~.company_service.UpdateCompanyRequest): - The request object. Request object for [UpdateCompany][] method. + The request object. Request object for ``UpdateCompany`` method. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/__init__.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/__init__.py new file mode 100644 index 000000000000..c44896618548 --- /dev/null +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .client import CreativeServiceClient + +__all__ = ("CreativeServiceClient",) diff --git a/packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/client.py b/packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/client.py new file mode 100644 index 000000000000..7152973f8cf1 --- /dev/null +++ b/packages/google-ads-admanager/google/ads/admanager_v1/services/creative_service/client.py @@ -0,0 +1,1331 @@ +# -*- coding: utf-8 -*- +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import logging as std_logging +import os +import re +import warnings +from collections import OrderedDict +from http import HTTPStatus +from typing import ( + Callable, + Dict, + Mapping, + MutableMapping, + MutableSequence, + Optional, + Sequence, + Tuple, + Type, + Union, + cast, +) + +import google.protobuf +from google.api_core import client_options as client_options_lib +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.exceptions import MutualTLSChannelError # type: ignore +from google.auth.transport import mtls # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.ads.admanager_v1 import gapic_version as package_version +from google.ads.admanager_v1._compat import ( + get_api_endpoint, + get_default_mtls_endpoint, + get_universe_domain, + read_environment_variables, + should_use_client_cert, +) + +try: + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.Retry, object, None] # type: ignore + +try: + from google.api_core import client_logging # type: ignore + + CLIENT_LOGGING_SUPPORTED = True # pragma: NO COVER +except ImportError: # pragma: NO COVER + CLIENT_LOGGING_SUPPORTED = False + +_LOGGER = std_logging.getLogger(__name__) + +import google.protobuf.timestamp_pb2 as timestamp_pb2 # type: ignore +from google.longrunning import operations_pb2 # type: ignore + +from google.ads.admanager_v1.services.creative_service import pagers +from google.ads.admanager_v1.types import ( + applied_label, + creative_messages, + creative_service, + creative_third_party_data_declaration_status_enum, + custom_field_value, + size, + third_party_data_declaration, +) + +from .transports.base import DEFAULT_CLIENT_INFO, CreativeServiceTransport +from .transports.rest import CreativeServiceRestTransport + + +class CreativeServiceClientMeta(type): + """Metaclass for the CreativeService client. + + This provides class-level methods for building and retrieving + support objects (e.g. transport) without polluting the client instance + objects. + """ + + _transport_registry = OrderedDict() # type: Dict[str, Type[CreativeServiceTransport]] + _transport_registry["rest"] = CreativeServiceRestTransport + + def get_transport_class( + cls, + label: Optional[str] = None, + ) -> Type[CreativeServiceTransport]: + """Returns an appropriate transport class. + + Args: + label: The name of the desired transport. If none is + provided, then the first transport in the registry is used. + + Returns: + The transport class to use. + """ + # If a specific transport is requested, return that one. + if label: + return cls._transport_registry[label] + + # No transport is requested; return the default (that is, the first one + # in the dictionary). + return next(iter(cls._transport_registry.values())) + + +class CreativeServiceClient(metaclass=CreativeServiceClientMeta): + """Provides methods for handling ``Creative`` objects.""" + + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + DEFAULT_ENDPOINT = "admanager.googleapis.com" + DEFAULT_MTLS_ENDPOINT = get_default_mtls_endpoint(DEFAULT_ENDPOINT) + + _DEFAULT_ENDPOINT_TEMPLATE = "admanager.{UNIVERSE_DOMAIN}" + _DEFAULT_UNIVERSE = "googleapis.com" + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + CreativeServiceClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + CreativeServiceClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file(filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> CreativeServiceTransport: + """Returns the transport used by the client instance. + + Returns: + CreativeServiceTransport: The transport used by the client + instance. + """ + return self._transport + + @staticmethod + def asset_path( + network_code: str, + asset: str, + ) -> str: + """Returns a fully-qualified asset string.""" + return "networks/{network_code}/assets/{asset}".format( + network_code=network_code, + asset=asset, + ) + + @staticmethod + def parse_asset_path(path: str) -> Dict[str, str]: + """Parses a asset path into its component segments.""" + m = re.match(r"^networks/(?P.+?)/assets/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def company_path( + network_code: str, + company: str, + ) -> str: + """Returns a fully-qualified company string.""" + return "networks/{network_code}/companies/{company}".format( + network_code=network_code, + company=company, + ) + + @staticmethod + def parse_company_path(path: str) -> Dict[str, str]: + """Parses a company path into its component segments.""" + m = re.match( + r"^networks/(?P.+?)/companies/(?P.+?)$", path + ) + return m.groupdict() if m else {} + + @staticmethod + def creative_path( + network_code: str, + creative: str, + ) -> str: + """Returns a fully-qualified creative string.""" + return "networks/{network_code}/creatives/{creative}".format( + network_code=network_code, + creative=creative, + ) + + @staticmethod + def parse_creative_path(path: str) -> Dict[str, str]: + """Parses a creative path into its component segments.""" + m = re.match( + r"^networks/(?P.+?)/creatives/(?P.+?)$", path + ) + return m.groupdict() if m else {} + + @staticmethod + def creative_set_path( + network_code: str, + creative_set: str, + ) -> str: + """Returns a fully-qualified creative_set string.""" + return "networks/{network_code}/creativeSets/{creative_set}".format( + network_code=network_code, + creative_set=creative_set, + ) + + @staticmethod + def parse_creative_set_path(path: str) -> Dict[str, str]: + """Parses a creative_set path into its component segments.""" + m = re.match( + r"^networks/(?P.+?)/creativeSets/(?P.+?)$", path + ) + return m.groupdict() if m else {} + + @staticmethod + def custom_field_path( + network_code: str, + custom_field: str, + ) -> str: + """Returns a fully-qualified custom_field string.""" + return "networks/{network_code}/customFields/{custom_field}".format( + network_code=network_code, + custom_field=custom_field, + ) + + @staticmethod + def parse_custom_field_path(path: str) -> Dict[str, str]: + """Parses a custom_field path into its component segments.""" + m = re.match( + r"^networks/(?P.+?)/customFields/(?P.+?)$", path + ) + return m.groupdict() if m else {} + + @staticmethod + def label_path( + network_code: str, + label: str, + ) -> str: + """Returns a fully-qualified label string.""" + return "networks/{network_code}/labels/{label}".format( + network_code=network_code, + label=label, + ) + + @staticmethod + def parse_label_path(path: str) -> Dict[str, str]: + """Parses a label path into its component segments.""" + m = re.match(r"^networks/(?P.+?)/labels/(?P