diff --git a/.librarian/generator-input/.repo-metadata.json b/.librarian/generator-input/.repo-metadata.json new file mode 100644 index 000000000..f644429bc --- /dev/null +++ b/.librarian/generator-input/.repo-metadata.json @@ -0,0 +1,18 @@ +{ + "name": "storage", + "name_pretty": "Google Cloud Storage", + "product_documentation": "https://cloud.google.com/storage", + "client_documentation": "https://cloud.google.com/python/docs/reference/storage/latest", + "issue_tracker": "https://issuetracker.google.com/savedsearches/559782", + "release_level": "stable", + "language": "python", + "library_type": "GAPIC_MANUAL", + "repo": "googleapis/python-storage", + "distribution_name": "google-cloud-storage", + "api_id": "storage.googleapis.com", + "requires_billing": true, + "default_version": "v2", + "codeowner_team": "@googleapis/gcs-sdk-team", + "api_shortname": "storage", + "api_description": "is a durable and highly available object storage service. Google Cloud Storage is almost infinitely scalable and guarantees consistency: when a write succeeds, the latest copy of the object will be returned to any GET, globally." +} diff --git a/owlbot.py b/.librarian/generator-input/librarian.py similarity index 65% rename from owlbot.py rename to .librarian/generator-input/librarian.py index 67b2369ce..ad0baa2a3 100644 --- a/owlbot.py +++ b/.librarian/generator-input/librarian.py @@ -98,41 +98,11 @@ "noxfile.py", "CONTRIBUTING.rst", "README.rst", - ".kokoro/continuous/continuous.cfg", - ".kokoro/presubmit/system-3.8.cfg", - ".kokoro/presubmit/prerelease-deps.cfg", - ".kokoro/continuous/prerelease-deps.cfg", - ".github/blunderbuss.yml", # blunderbuss assignment to python squad - ".github/workflows", # exclude gh actions as credentials are needed for tests - ".github/release-please.yml", # special support for a python2 branch in this repo + ".kokoro/**", + ".github/**", ], ) -s.replace( - ".kokoro/build.sh", - "export PYTHONUNBUFFERED=1", - """export PYTHONUNBUFFERED=1 - -# Export variable to override api endpoint -export API_ENDPOINT_OVERRIDE - -# Export variable to override api endpoint version -export API_VERSION_OVERRIDE - -# Export dual region locations -export DUAL_REGION_LOC_1 -export DUAL_REGION_LOC_2 - -# Setup universe domain testing needed environment variables. -export TEST_UNIVERSE_DOMAIN_CREDENTIAL=$(realpath ${KOKORO_GFILE_DIR}/secret_manager/client-library-test-universe-domain-credential) -export TEST_UNIVERSE_DOMAIN=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-domain) -export TEST_UNIVERSE_PROJECT_ID=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-project-id) -export TEST_UNIVERSE_LOCATION=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-storage-location) - -""") - python.py_samples(skip_readmes=True) -# Use a python runtime which is available in the owlbot post processor here -# https://github.com/googleapis/synthtool/blob/master/docker/owlbot/python/Dockerfile -s.shell.run(["nox", "-s", "blacken-3.10"], hide_output=False) +s.shell.run(["nox", "-s", "blacken"], hide_output=False) diff --git a/.librarian/generator-input/noxfile.py b/.librarian/generator-input/noxfile.py new file mode 100644 index 000000000..16cf97b01 --- /dev/null +++ b/.librarian/generator-input/noxfile.py @@ -0,0 +1,408 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +import os +import pathlib +import re +import shutil + +import nox + + +BLACK_VERSION = "black==23.7.0" +BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] + +DEFAULT_PYTHON_VERSION = "3.12" +SYSTEM_TEST_PYTHON_VERSIONS = ["3.12"] +UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] +CONFORMANCE_TEST_PYTHON_VERSIONS = ["3.12"] + +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + +nox.options.sessions = [ + "blacken", + "conftest_retry", + "docfx", + "docs", + "lint", + "lint_setup_py", + "system", + # TODO(https://github.com/googleapis/python-storage/issues/1499): + # Remove or restore testing for Python 3.7/3.8 + "unit-3.9", + "unit-3.10", + "unit-3.11", + "unit-3.12", + "unit-3.13", + # cover must be last to avoid error `No data to report` + "cover", +] + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint(session): + """Run linters. + + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + # Pin flake8 to 6.0.0 + # See https://github.com/googleapis/python-storage/issues/1102 + session.install("flake8==6.0.0", BLACK_VERSION) + session.run( + "black", + "--check", + *BLACK_PATHS, + ) + session.run("flake8", "google", "tests") + + +@nox.session(python="3.14") +def blacken(session): + """Run black. + + Format code to uniform standard. + """ + session.install(BLACK_VERSION) + session.run( + "black", + *BLACK_PATHS, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint_setup_py(session): + """Verify that setup.py is valid (including RST check).""" + session.install("docutils", "pygments", "setuptools>=79.0.1") + session.run("python", "setup.py", "check", "--restructuredtext", "--strict") + + +def default(session, install_extras=True): + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + # Install all test dependencies, then install this package in-place. + session.install( + "mock", + "pytest", + "pytest-cov", + "pytest-asyncio", + "brotli", + "grpcio", + "grpcio-status", + "proto-plus", + "grpc-google-iam-v1", + "-c", + constraints_path, + ) + + if install_extras: + session.install("opentelemetry-api", "opentelemetry-sdk") + + session.install("-e", ".", "-c", constraints_path) + + # This dependency is included in setup.py for backwards compatibility only + # and the client library is expected to pass all tests without it. See + # setup.py and README for details. + session.run("pip", "uninstall", "-y", "google-resumable-media") + + # Run py.test against the unit tests. + session.run( + "py.test", + "--quiet", + f"--junitxml=unit_{session.python}_sponge_log.xml", + "--cov=google.cloud.storage", + "--cov=google.cloud", + "--cov=tests.unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + os.path.join("tests", "unit"), + os.path.join("tests", "resumable_media", "unit"), + *session.posargs, + ) + + +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +def unit(session): + """Run the unit test suite.""" + default(session) + + +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def system(session): + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + """Run the system test suite.""" + rerun_count = 0 + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") + # Environment check: Only run tests if the environment variable is set. + if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): + session.skip( + "Credentials must be set via environment variable GOOGLE_APPLICATION_CREDENTIALS" + ) + # mTLS tests requires pyopenssl. + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") == "true": + session.install("pyopenssl") + # Check if endpoint is being overriden for rerun_count + if ( + os.getenv("API_ENDPOINT_OVERRIDE", "https://storage.googleapis.com") + != "https://storage.googleapis.com" + ): + rerun_count = 3 + + # Use pre-release gRPC for system tests. + # TODO: Remove ban of 1.52.0rc1 once grpc/grpc#31885 is resolved. + session.install("--pre", "grpcio!=1.52.0rc1") + + # Install all test dependencies, then install this package into the + # virtualenv's dist-packages. + # 2021-05-06: defer installing 'google-cloud-*' to after this package, + # in order to work around Python 2.7 googolapis-common-protos + # issue. + session.install("mock", "pytest", "pytest-rerunfailures", "-c", constraints_path) + session.install("-e", ".", "-c", constraints_path) + session.install( + "google-cloud-testutils", + "google-cloud-iam", + "google-cloud-pubsub", + "google-cloud-kms", + "brotli", + "-c", + constraints_path, + ) + + # Run py.test against the system tests. + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + "--reruns={}".format(rerun_count), + os.path.join("tests", "system"), + os.path.join("tests", "resumable_media", "system"), + *session.posargs, + ) + + +@nox.session(python=CONFORMANCE_TEST_PYTHON_VERSIONS) +def conftest_retry(session): + """Run the retry conformance test suite.""" + conformance_test_folder_path = os.path.join("tests", "conformance") + conformance_test_folder_exists = os.path.exists(conformance_test_folder_path) + # Environment check: only run tests if found. + if not conformance_test_folder_exists: + session.skip("Conformance tests were not found") + + # Install all test dependencies and pytest plugin to run tests in parallel. + # Then install this package in-place. + session.install("pytest", "pytest-xdist") + session.install("-e", ".") + + # Run #CPU processes in parallel if no test session arguments are passed in. + if session.posargs: + test_cmd = [ + "py.test", + "--quiet", + conformance_test_folder_path, + *session.posargs, + ] + else: + test_cmd = ["py.test", "-n", "auto", "--quiet", conformance_test_folder_path] + + # Run py.test against the conformance tests. + session.run(*test_cmd) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def cover(session): + """Run the final coverage report. + + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=99") + + session.run("coverage", "erase") + + +@nox.session(python="3.10") +def docs(session): + """Build the docs for this library.""" + + session.install("-e", ".") + session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", + "sphinx==4.5.0", + "alabaster", + "recommonmark", + ) + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python="3.10") +def docfx(session): + """Build the docfx yaml files for this library.""" + + session.install("-e", ".") + session.install("grpcio") + session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", + "gcp-sphinx-docfx-yaml", + "alabaster", + "recommonmark", + ) + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-T", # show full traceback on exception + "-N", # no colors + "-D", + ( + "extensions=sphinx.ext.autodoc," + "sphinx.ext.autosummary," + "docfx_yaml.extension," + "sphinx.ext.intersphinx," + "sphinx.ext.coverage," + "sphinx.ext.napoleon," + "sphinx.ext.todo," + "sphinx.ext.viewcode," + "recommonmark" + ), + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1]) +@nox.parametrize( + "protobuf_implementation", + ["python", "upb"], +) +def prerelease_deps(session, protobuf_implementation): + """Run all tests with prerelease versions of dependencies installed.""" + + # Install all test dependencies + session.install("mock", "pytest", "pytest-cov", "brotli") + + # Install dependencies needed for system tests + session.install( + "google-cloud-pubsub", + "google-cloud-kms", + "google-cloud-testutils", + "google-cloud-iam", + ) + + # Install all dependencies + session.install("-e", ".[protobuf, tracing]") + + prerel_deps = [ + "google-api-core", + "google-auth", + "google-cloud-core", + "google-crc32c", + "google-resumable-media", + "opentelemetry-api", + "protobuf", + ] + + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "google-cloud-core": "google.cloud.version", + "opentelemetry-api": "opentelemetry.version", + "protobuf": "google.protobuf", + } + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + print(f"Installed {dep}") + + version_namespace = package_namespaces.get(dep) + + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) + # Remaining dependencies + other_deps = [ + "requests", + ] + session.install(*other_deps) + + session.run( + "py.test", + "tests/unit", + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + os.path.join("tests", "system"), + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) diff --git a/.librarian/generator-input/setup.py b/.librarian/generator-input/setup.py new file mode 100644 index 000000000..2c4504749 --- /dev/null +++ b/.librarian/generator-input/setup.py @@ -0,0 +1,107 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import os + +import setuptools + + +# Package metadata. + +name = "google-cloud-storage" +description = "Google Cloud Storage API client library" +# Should be one of: +# 'Development Status :: 3 - Alpha' +# 'Development Status :: 4 - Beta' +# 'Development Status :: 5 - Production/Stable' +release_status = "Development Status :: 5 - Production/Stable" +dependencies = [ + "google-auth >= 2.26.1, < 3.0.0", + "google-api-core >= 2.27.0, < 3.0.0", + "google-cloud-core >= 2.4.2, < 3.0.0", + # The dependency "google-resumable-media" is no longer used. However, the + # dependency is still included here to accommodate users who may be + # importing exception classes from the google-resumable-media without + # installing it explicitly. See the python-storage README for details on + # exceptions and importing. Users who are not importing + # google-resumable-media classes in their application can safely disregard + # this dependency. + "google-resumable-media >= 2.7.2, < 3.0.0", + "requests >= 2.22.0, < 3.0.0", + "google-crc32c >= 1.1.3, < 2.0.0", +] +extras = { + "protobuf": ["protobuf >= 3.20.2, < 7.0.0"], + "tracing": [ + "opentelemetry-api >= 1.1.0, < 2.0.0", + ], +} + + +# Setup boilerplate below this line. + +package_root = os.path.abspath(os.path.dirname(__file__)) + +version = {} +with open(os.path.join(package_root, "google/cloud/storage/version.py")) as fp: + exec(fp.read(), version) +version = version["__version__"] + +readme_filename = os.path.join(package_root, "README.rst") +with io.open(readme_filename, encoding="utf-8") as readme_file: + readme = readme_file.read() + +# Only include packages under the 'google' namespace. Do not include tests, +# benchmarks, etc. +packages = [ + package + for package in setuptools.find_namespace_packages() + if package.startswith("google") +] + + +setuptools.setup( + name=name, + version=version, + description=description, + long_description=readme, + author="Google LLC", + author_email="googleapis-packages@google.com", + license="Apache 2.0", + url="https://github.com/googleapis/python-storage", + classifiers=[ + release_status, + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Operating System :: OS Independent", + "Topic :: Internet", + ], + platforms="Posix; MacOS X; Windows", + packages=packages, + install_requires=dependencies, + extras_require=extras, + python_requires=">=3.7", + include_package_data=True, + zip_safe=False, +) diff --git a/.librarian/state.yaml b/.librarian/state.yaml new file mode 100644 index 000000000..1502e804d --- /dev/null +++ b/.librarian/state.yaml @@ -0,0 +1,31 @@ +image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:8e2c32496077054105bd06c54a59d6a6694287bc053588e24debe6da6920ad91 +libraries: + - id: google-cloud-storage + version: 3.6.0 + last_generated_commit: 5400ccce473c439885bd6bf2924fd242271bfcab + apis: + - path: google/storage/v2 + service_config: storage_v2.yaml + source_roots: + - . + preserve_regex: [] + remove_regex: + - ^.flake8 + - ^.pre-commit-config.yaml + - ^.trampolinerc + - ^.repo-metadata.json + - ^LICENSE + - ^MANIFEST.in + - ^SECURITY.md + - ^mypy.ini + - ^noxfile.py + - ^renovate.json + - ^setup.py + - ^docs/summary_overview.md + - ^google/cloud/_storage_v2 + - ^samples/generated_samples + - ^testing/constraints-3.8.txt + - ^testing/constraints-3.1.* + - ^tests/__init__.py + - ^tests/unit/gapic + tag_format: v{version} diff --git a/google/cloud/_storage_v2/gapic_version.py b/google/cloud/_storage_v2/gapic_version.py index 20a9cd975..d69b0530e 100644 --- a/google/cloud/_storage_v2/gapic_version.py +++ b/google/cloud/_storage_v2/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "0.0.0" # {x-release-please-version} +__version__ = "3.6.0" # {x-release-please-version} diff --git a/noxfile.py b/noxfile.py index 451fced3e..16cf97b01 100644 --- a/noxfile.py +++ b/noxfile.py @@ -74,9 +74,7 @@ def lint(session): session.run("flake8", "google", "tests") -# Use a python runtime which is available in the owlbot post processor here -# https://github.com/googleapis/synthtool/blob/master/docker/owlbot/python/Dockerfile -@nox.session(python=["3.10", DEFAULT_PYTHON_VERSION]) +@nox.session(python="3.14") def blacken(session): """Run black. diff --git a/samples/generated_samples/snippet_metadata_google.storage.v2.json b/samples/generated_samples/snippet_metadata_google.storage.v2.json index b2448bff2..4af7ef641 100644 --- a/samples/generated_samples/snippet_metadata_google.storage.v2.json +++ b/samples/generated_samples/snippet_metadata_google.storage.v2.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-storage", - "version": "0.0.0" + "version": "3.6.0" }, "snippets": [ { diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 9c17b387b..151762409 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -5,7 +5,7 @@ # e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0", # Then this file should have google-cloud-foo==1.14.0 google-auth==2.26.1 -google-api-core==2.15.0 +google-api-core==2.27.0 google-cloud-core==2.4.2 google-resumable-media==2.7.2 requests==2.22.0 diff --git a/testing/constraints-3.9.txt b/testing/constraints-3.9.txt index ccf6c1493..f022e9e1c 100644 --- a/testing/constraints-3.9.txt +++ b/testing/constraints-3.9.txt @@ -5,7 +5,7 @@ # e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0", # Then this file should have google-cloud-foo==1.14.0 google-auth==2.26.1 -google-api-core>=2.15.0 +google-api-core==2.27.0 google-cloud-core==2.4.2 google-resumable-media==2.7.2 requests==2.22.0