diff --git a/.librarian/generator-input/.repo-metadata.json b/.librarian/generator-input/.repo-metadata.json deleted file mode 100644 index f644429bc..000000000 --- a/.librarian/generator-input/.repo-metadata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "storage", - "name_pretty": "Google Cloud Storage", - "product_documentation": "https://cloud.google.com/storage", - "client_documentation": "https://cloud.google.com/python/docs/reference/storage/latest", - "issue_tracker": "https://issuetracker.google.com/savedsearches/559782", - "release_level": "stable", - "language": "python", - "library_type": "GAPIC_MANUAL", - "repo": "googleapis/python-storage", - "distribution_name": "google-cloud-storage", - "api_id": "storage.googleapis.com", - "requires_billing": true, - "default_version": "v2", - "codeowner_team": "@googleapis/gcs-sdk-team", - "api_shortname": "storage", - "api_description": "is a durable and highly available object storage service. Google Cloud Storage is almost infinitely scalable and guarantees consistency: when a write succeeds, the latest copy of the object will be returned to any GET, globally." -} diff --git a/.librarian/generator-input/noxfile.py b/.librarian/generator-input/noxfile.py deleted file mode 100644 index 16cf97b01..000000000 --- a/.librarian/generator-input/noxfile.py +++ /dev/null @@ -1,408 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -import os -import pathlib -import re -import shutil - -import nox - - -BLACK_VERSION = "black==23.7.0" -BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] - -DEFAULT_PYTHON_VERSION = "3.12" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.12"] -UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] -CONFORMANCE_TEST_PYTHON_VERSIONS = ["3.12"] - -CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() - -# Error if a python version is missing -nox.options.error_on_missing_interpreters = True - -nox.options.sessions = [ - "blacken", - "conftest_retry", - "docfx", - "docs", - "lint", - "lint_setup_py", - "system", - # TODO(https://github.com/googleapis/python-storage/issues/1499): - # Remove or restore testing for Python 3.7/3.8 - "unit-3.9", - "unit-3.10", - "unit-3.11", - "unit-3.12", - "unit-3.13", - # cover must be last to avoid error `No data to report` - "cover", -] - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def lint(session): - """Run linters. - - Returns a failure if the linters find linting errors or sufficiently - serious code quality issues. - """ - # Pin flake8 to 6.0.0 - # See https://github.com/googleapis/python-storage/issues/1102 - session.install("flake8==6.0.0", BLACK_VERSION) - session.run( - "black", - "--check", - *BLACK_PATHS, - ) - session.run("flake8", "google", "tests") - - -@nox.session(python="3.14") -def blacken(session): - """Run black. - - Format code to uniform standard. - """ - session.install(BLACK_VERSION) - session.run( - "black", - *BLACK_PATHS, - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def lint_setup_py(session): - """Verify that setup.py is valid (including RST check).""" - session.install("docutils", "pygments", "setuptools>=79.0.1") - session.run("python", "setup.py", "check", "--restructuredtext", "--strict") - - -def default(session, install_extras=True): - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" - ) - # Install all test dependencies, then install this package in-place. - session.install( - "mock", - "pytest", - "pytest-cov", - "pytest-asyncio", - "brotli", - "grpcio", - "grpcio-status", - "proto-plus", - "grpc-google-iam-v1", - "-c", - constraints_path, - ) - - if install_extras: - session.install("opentelemetry-api", "opentelemetry-sdk") - - session.install("-e", ".", "-c", constraints_path) - - # This dependency is included in setup.py for backwards compatibility only - # and the client library is expected to pass all tests without it. See - # setup.py and README for details. - session.run("pip", "uninstall", "-y", "google-resumable-media") - - # Run py.test against the unit tests. - session.run( - "py.test", - "--quiet", - f"--junitxml=unit_{session.python}_sponge_log.xml", - "--cov=google.cloud.storage", - "--cov=google.cloud", - "--cov=tests.unit", - "--cov-append", - "--cov-config=.coveragerc", - "--cov-report=", - "--cov-fail-under=0", - os.path.join("tests", "unit"), - os.path.join("tests", "resumable_media", "unit"), - *session.posargs, - ) - - -@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) -def unit(session): - """Run the unit test suite.""" - default(session) - - -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) -def system(session): - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" - ) - """Run the system test suite.""" - rerun_count = 0 - - # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. - if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": - session.skip("RUN_SYSTEM_TESTS is set to false, skipping") - # Environment check: Only run tests if the environment variable is set. - if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): - session.skip( - "Credentials must be set via environment variable GOOGLE_APPLICATION_CREDENTIALS" - ) - # mTLS tests requires pyopenssl. - if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") == "true": - session.install("pyopenssl") - # Check if endpoint is being overriden for rerun_count - if ( - os.getenv("API_ENDPOINT_OVERRIDE", "https://storage.googleapis.com") - != "https://storage.googleapis.com" - ): - rerun_count = 3 - - # Use pre-release gRPC for system tests. - # TODO: Remove ban of 1.52.0rc1 once grpc/grpc#31885 is resolved. - session.install("--pre", "grpcio!=1.52.0rc1") - - # Install all test dependencies, then install this package into the - # virtualenv's dist-packages. - # 2021-05-06: defer installing 'google-cloud-*' to after this package, - # in order to work around Python 2.7 googolapis-common-protos - # issue. - session.install("mock", "pytest", "pytest-rerunfailures", "-c", constraints_path) - session.install("-e", ".", "-c", constraints_path) - session.install( - "google-cloud-testutils", - "google-cloud-iam", - "google-cloud-pubsub", - "google-cloud-kms", - "brotli", - "-c", - constraints_path, - ) - - # Run py.test against the system tests. - session.run( - "py.test", - "--quiet", - f"--junitxml=system_{session.python}_sponge_log.xml", - "--reruns={}".format(rerun_count), - os.path.join("tests", "system"), - os.path.join("tests", "resumable_media", "system"), - *session.posargs, - ) - - -@nox.session(python=CONFORMANCE_TEST_PYTHON_VERSIONS) -def conftest_retry(session): - """Run the retry conformance test suite.""" - conformance_test_folder_path = os.path.join("tests", "conformance") - conformance_test_folder_exists = os.path.exists(conformance_test_folder_path) - # Environment check: only run tests if found. - if not conformance_test_folder_exists: - session.skip("Conformance tests were not found") - - # Install all test dependencies and pytest plugin to run tests in parallel. - # Then install this package in-place. - session.install("pytest", "pytest-xdist") - session.install("-e", ".") - - # Run #CPU processes in parallel if no test session arguments are passed in. - if session.posargs: - test_cmd = [ - "py.test", - "--quiet", - conformance_test_folder_path, - *session.posargs, - ] - else: - test_cmd = ["py.test", "-n", "auto", "--quiet", conformance_test_folder_path] - - # Run py.test against the conformance tests. - session.run(*test_cmd) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def cover(session): - """Run the final coverage report. - - This outputs the coverage report aggregating coverage from the unit - test runs (not system test runs), and then erases coverage data. - """ - session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=99") - - session.run("coverage", "erase") - - -@nox.session(python="3.10") -def docs(session): - """Build the docs for this library.""" - - session.install("-e", ".") - session.install( - # We need to pin to specific versions of the `sphinxcontrib-*` packages - # which still support sphinx 4.x. - # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 - # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. - "sphinxcontrib-applehelp==1.0.4", - "sphinxcontrib-devhelp==1.0.2", - "sphinxcontrib-htmlhelp==2.0.1", - "sphinxcontrib-qthelp==1.0.3", - "sphinxcontrib-serializinghtml==1.1.5", - "sphinx==4.5.0", - "alabaster", - "recommonmark", - ) - - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) - session.run( - "sphinx-build", - "-W", # warnings as errors - "-T", # show full traceback on exception - "-N", # no colors - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), - ) - - -@nox.session(python="3.10") -def docfx(session): - """Build the docfx yaml files for this library.""" - - session.install("-e", ".") - session.install("grpcio") - session.install( - # We need to pin to specific versions of the `sphinxcontrib-*` packages - # which still support sphinx 4.x. - # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 - # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. - "sphinxcontrib-applehelp==1.0.4", - "sphinxcontrib-devhelp==1.0.2", - "sphinxcontrib-htmlhelp==2.0.1", - "sphinxcontrib-qthelp==1.0.3", - "sphinxcontrib-serializinghtml==1.1.5", - "gcp-sphinx-docfx-yaml", - "alabaster", - "recommonmark", - ) - - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) - session.run( - "sphinx-build", - "-T", # show full traceback on exception - "-N", # no colors - "-D", - ( - "extensions=sphinx.ext.autodoc," - "sphinx.ext.autosummary," - "docfx_yaml.extension," - "sphinx.ext.intersphinx," - "sphinx.ext.coverage," - "sphinx.ext.napoleon," - "sphinx.ext.todo," - "sphinx.ext.viewcode," - "recommonmark" - ), - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), - ) - - -@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1]) -@nox.parametrize( - "protobuf_implementation", - ["python", "upb"], -) -def prerelease_deps(session, protobuf_implementation): - """Run all tests with prerelease versions of dependencies installed.""" - - # Install all test dependencies - session.install("mock", "pytest", "pytest-cov", "brotli") - - # Install dependencies needed for system tests - session.install( - "google-cloud-pubsub", - "google-cloud-kms", - "google-cloud-testutils", - "google-cloud-iam", - ) - - # Install all dependencies - session.install("-e", ".[protobuf, tracing]") - - prerel_deps = [ - "google-api-core", - "google-auth", - "google-cloud-core", - "google-crc32c", - "google-resumable-media", - "opentelemetry-api", - "protobuf", - ] - - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "google-cloud-core": "google.cloud.version", - "opentelemetry-api": "opentelemetry.version", - "protobuf": "google.protobuf", - } - - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - print(f"Installed {dep}") - - version_namespace = package_namespaces.get(dep) - - if version_namespace: - session.run( - "python", - "-c", - f"import {version_namespace}; print({version_namespace}.__version__)", - ) - # Remaining dependencies - other_deps = [ - "requests", - ] - session.install(*other_deps) - - session.run( - "py.test", - "tests/unit", - env={ - "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, - }, - ) - - session.run( - "py.test", - "--verbose", - f"--junitxml=system_{session.python}_sponge_log.xml", - os.path.join("tests", "system"), - *session.posargs, - env={ - "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, - }, - ) diff --git a/.librarian/generator-input/setup.py b/.librarian/generator-input/setup.py deleted file mode 100644 index 2c4504749..000000000 --- a/.librarian/generator-input/setup.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -import os - -import setuptools - - -# Package metadata. - -name = "google-cloud-storage" -description = "Google Cloud Storage API client library" -# Should be one of: -# 'Development Status :: 3 - Alpha' -# 'Development Status :: 4 - Beta' -# 'Development Status :: 5 - Production/Stable' -release_status = "Development Status :: 5 - Production/Stable" -dependencies = [ - "google-auth >= 2.26.1, < 3.0.0", - "google-api-core >= 2.27.0, < 3.0.0", - "google-cloud-core >= 2.4.2, < 3.0.0", - # The dependency "google-resumable-media" is no longer used. However, the - # dependency is still included here to accommodate users who may be - # importing exception classes from the google-resumable-media without - # installing it explicitly. See the python-storage README for details on - # exceptions and importing. Users who are not importing - # google-resumable-media classes in their application can safely disregard - # this dependency. - "google-resumable-media >= 2.7.2, < 3.0.0", - "requests >= 2.22.0, < 3.0.0", - "google-crc32c >= 1.1.3, < 2.0.0", -] -extras = { - "protobuf": ["protobuf >= 3.20.2, < 7.0.0"], - "tracing": [ - "opentelemetry-api >= 1.1.0, < 2.0.0", - ], -} - - -# Setup boilerplate below this line. - -package_root = os.path.abspath(os.path.dirname(__file__)) - -version = {} -with open(os.path.join(package_root, "google/cloud/storage/version.py")) as fp: - exec(fp.read(), version) -version = version["__version__"] - -readme_filename = os.path.join(package_root, "README.rst") -with io.open(readme_filename, encoding="utf-8") as readme_file: - readme = readme_file.read() - -# Only include packages under the 'google' namespace. Do not include tests, -# benchmarks, etc. -packages = [ - package - for package in setuptools.find_namespace_packages() - if package.startswith("google") -] - - -setuptools.setup( - name=name, - version=version, - description=description, - long_description=readme, - author="Google LLC", - author_email="googleapis-packages@google.com", - license="Apache 2.0", - url="https://github.com/googleapis/python-storage", - classifiers=[ - release_status, - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Operating System :: OS Independent", - "Topic :: Internet", - ], - platforms="Posix; MacOS X; Windows", - packages=packages, - install_requires=dependencies, - extras_require=extras, - python_requires=">=3.7", - include_package_data=True, - zip_safe=False, -) diff --git a/.librarian/state.yaml b/.librarian/state.yaml deleted file mode 100644 index 1502e804d..000000000 --- a/.librarian/state.yaml +++ /dev/null @@ -1,31 +0,0 @@ -image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:8e2c32496077054105bd06c54a59d6a6694287bc053588e24debe6da6920ad91 -libraries: - - id: google-cloud-storage - version: 3.6.0 - last_generated_commit: 5400ccce473c439885bd6bf2924fd242271bfcab - apis: - - path: google/storage/v2 - service_config: storage_v2.yaml - source_roots: - - . - preserve_regex: [] - remove_regex: - - ^.flake8 - - ^.pre-commit-config.yaml - - ^.trampolinerc - - ^.repo-metadata.json - - ^LICENSE - - ^MANIFEST.in - - ^SECURITY.md - - ^mypy.ini - - ^noxfile.py - - ^renovate.json - - ^setup.py - - ^docs/summary_overview.md - - ^google/cloud/_storage_v2 - - ^samples/generated_samples - - ^testing/constraints-3.8.txt - - ^testing/constraints-3.1.* - - ^tests/__init__.py - - ^tests/unit/gapic - tag_format: v{version} diff --git a/google/cloud/_storage_v2/gapic_version.py b/google/cloud/_storage_v2/gapic_version.py index d69b0530e..20a9cd975 100644 --- a/google/cloud/_storage_v2/gapic_version.py +++ b/google/cloud/_storage_v2/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "3.6.0" # {x-release-please-version} +__version__ = "0.0.0" # {x-release-please-version} diff --git a/noxfile.py b/noxfile.py index 16cf97b01..451fced3e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -74,7 +74,9 @@ def lint(session): session.run("flake8", "google", "tests") -@nox.session(python="3.14") +# Use a python runtime which is available in the owlbot post processor here +# https://github.com/googleapis/synthtool/blob/master/docker/owlbot/python/Dockerfile +@nox.session(python=["3.10", DEFAULT_PYTHON_VERSION]) def blacken(session): """Run black. diff --git a/.librarian/generator-input/librarian.py b/owlbot.py similarity index 65% rename from .librarian/generator-input/librarian.py rename to owlbot.py index ad0baa2a3..67b2369ce 100644 --- a/.librarian/generator-input/librarian.py +++ b/owlbot.py @@ -98,11 +98,41 @@ "noxfile.py", "CONTRIBUTING.rst", "README.rst", - ".kokoro/**", - ".github/**", + ".kokoro/continuous/continuous.cfg", + ".kokoro/presubmit/system-3.8.cfg", + ".kokoro/presubmit/prerelease-deps.cfg", + ".kokoro/continuous/prerelease-deps.cfg", + ".github/blunderbuss.yml", # blunderbuss assignment to python squad + ".github/workflows", # exclude gh actions as credentials are needed for tests + ".github/release-please.yml", # special support for a python2 branch in this repo ], ) +s.replace( + ".kokoro/build.sh", + "export PYTHONUNBUFFERED=1", + """export PYTHONUNBUFFERED=1 + +# Export variable to override api endpoint +export API_ENDPOINT_OVERRIDE + +# Export variable to override api endpoint version +export API_VERSION_OVERRIDE + +# Export dual region locations +export DUAL_REGION_LOC_1 +export DUAL_REGION_LOC_2 + +# Setup universe domain testing needed environment variables. +export TEST_UNIVERSE_DOMAIN_CREDENTIAL=$(realpath ${KOKORO_GFILE_DIR}/secret_manager/client-library-test-universe-domain-credential) +export TEST_UNIVERSE_DOMAIN=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-domain) +export TEST_UNIVERSE_PROJECT_ID=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-project-id) +export TEST_UNIVERSE_LOCATION=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-storage-location) + +""") + python.py_samples(skip_readmes=True) -s.shell.run(["nox", "-s", "blacken"], hide_output=False) +# Use a python runtime which is available in the owlbot post processor here +# https://github.com/googleapis/synthtool/blob/master/docker/owlbot/python/Dockerfile +s.shell.run(["nox", "-s", "blacken-3.10"], hide_output=False) diff --git a/samples/generated_samples/snippet_metadata_google.storage.v2.json b/samples/generated_samples/snippet_metadata_google.storage.v2.json index 4af7ef641..b2448bff2 100644 --- a/samples/generated_samples/snippet_metadata_google.storage.v2.json +++ b/samples/generated_samples/snippet_metadata_google.storage.v2.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-storage", - "version": "3.6.0" + "version": "0.0.0" }, "snippets": [ { diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 151762409..9c17b387b 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -5,7 +5,7 @@ # e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0", # Then this file should have google-cloud-foo==1.14.0 google-auth==2.26.1 -google-api-core==2.27.0 +google-api-core==2.15.0 google-cloud-core==2.4.2 google-resumable-media==2.7.2 requests==2.22.0 diff --git a/testing/constraints-3.9.txt b/testing/constraints-3.9.txt index f022e9e1c..ccf6c1493 100644 --- a/testing/constraints-3.9.txt +++ b/testing/constraints-3.9.txt @@ -5,7 +5,7 @@ # e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0", # Then this file should have google-cloud-foo==1.14.0 google-auth==2.26.1 -google-api-core==2.27.0 +google-api-core>=2.15.0 google-cloud-core==2.4.2 google-resumable-media==2.7.2 requests==2.22.0