diff --git a/scripts/split_repo_migration/README.md b/scripts/split_repo_migration/README.md deleted file mode 100644 index 2ccca1d40917..000000000000 --- a/scripts/split_repo_migration/README.md +++ /dev/null @@ -1,36 +0,0 @@ -The directory contains scripts which can be used to migrate client library code -from split repositories located at http://github.com/googleapis/python-* into -this mono repository. Each such split repo becomes a "package" (under -`packages/`) in this repo. - -These two scripts should be called one after the other for each split repo being -migrated, but this process can be done for multiple repos at once. In other -words, the order in which the PRs are created or merged does not matter: - - [single-library.git-migrate-history.sh](single-library.git-migrate-history.sh) - imports the files in the split repo into a package in this repo. In doing - so, it also imports the associated git history. This script opens a PR, which - must be merge-committed once approved, in order to preserve that history. - - [single-library.post-process.api-files.sh](single-library.post-process.api-files.sh) - applies needed package-specific migration changes that only touch files - specific to the repo being migrated. It locally commits the changes to this - repo but does not open a new PR. Typically this commit is included in the PR - created by the previous script. - -The output of this script should be merged before this script is called again, -as it modifies shared files: - - [multiple-library.post-process.sh](multiple-library.post-process.sh), for a - given list of migrated packages, applies package-specific migration changes - that touch files shared among all packages in this repo. It does this by - first calling the script below for each specified package, and then invoking - OwlBot locally to post-process all the specified packages at once. It - locally commits the changes to this repo but does not open a PR. - - [single-library.post-process.common-files.sh](single-library.post-process.common-files.sh) - applies package-specific migration changes that touch files shared among - multiple packages. It locally commits the changes to this repo but does - not open a PR. - -The following script cleans up the now-migrated split repo: -- [delete-everything-split-repo.sh](delete-everything-split-repo.sh) will remove - almost all files from a split repository and update the README.rst file to - indicate that the repository is archived. - diff --git a/scripts/split_repo_migration/delete-everything-split-repo.sh b/scripts/split_repo_migration/delete-everything-split-repo.sh deleted file mode 100755 index bac2a672080d..000000000000 --- a/scripts/split_repo_migration/delete-everything-split-repo.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script was written as part of a project to migrate -# split repositories to a mono repository google-cloud-python. -# This script will delete all files in a split repository and -# update the README to indicate that the repository is archived. - -# sourced vs execution detection obtained from https://stackoverflow.com/a/28776166 -SOURCED=0 -if [ -n "$ZSH_VERSION" ]; then - case $ZSH_EVAL_CONTEXT in *:file) SOURCED=1;; esac -elif [ -n "$KSH_VERSION" ]; then - [ "$(cd -- "$(dirname -- "$0")" && pwd -P)/$(basename -- "$0")" != "$(cd -- "$(dirname -- "${.sh.file}")" && pwd -P)/$(basename -- "${.sh.file}")" ] && SOURCED=1 -elif [ -n "$BASH_VERSION" ]; then - (return 0 2>/dev/null) && SOURCED=1 -else # All other shells: examine $0 for known shell binary filenames. - # Detects `sh` and `dash`; add additional shell filenames as needed. - case ${0##*/} in sh|-sh|dash|-dash) SOURCED=1;; esac -fi - -(( ${SOURCED} != 1 )) || { \ - echo "Please do not source this script, but execute it directly." - return -10 -} - -# We require executing the script so that an early exit (explicitly or via -e) -# does not kill the user's shell. - -# `-e` enables the script to automatically fail when a command fails -set -e - -# Ensure that both arguments are provided -if [ $# -lt 2 ] -then - echo "Usage: $0 - -Args: - split-repo-name: The name of the split repository. For example, \`python-access-approval\`. - artifact-name: The name of the artifact on PyPI. For example, \`google-cloud-access-approval\`. -" - exit 1 -fi - -# The name of the split repository. (e.g. python-access-approval) -SPLIT_REPO=$1 -# The name of the artifact on PyPI. (e.g. google-cloud-access-approval) -ARTIFACT_NAME=$2 - -SPLIT_REPO_DIR="${SPLIT_REPO_DIR:-/tmp/delete-library.${SPLIT_REPO}.$(date +"%Y-%m-%d.%H:%M:%S")}" - -git clone "git@github.com:googleapis/${SPLIT_REPO}.git" $SPLIT_REPO_DIR - -cd $SPLIT_REPO_DIR - -# Create a git branch -BRANCH_NAME="migrate-library.$(date +'%Y-%m-%d.%H-%M-%S')" -git checkout -b ${BRANCH_NAME} - -# Quick check to make sure there are no handwritten samples. -# Handwritten samples should be migrated to python-docs-samples before repository code is deleted. -# Exclude the samples/generated_samples/* directory which contains autogenerated samples. -NUM_PY_SAMPLES_FILES=$(find samples -type f -name "*.py" ! -path "samples/generated_samples/*" | grep -v "nox.*\.py" | grep -v "__init__\.py" | wc -l) - -# Fail if there are handwritten samples. -if [[ $NUM_PY_SAMPLES_FILES -ne "0" ]]; then - echo "Please migrate handwritten samples to python-docs-samples before deleting the source." - exit 1 -fi - -# Fail if this repository has a customized version of the autogenerated code. -# This specific check is to ensure that autogenerated files are not excluded. -STANDARD_EXCLUDES=$(grep owlbot.py -e 'excludes=\["\*\*/gapic_version.py"]' | wc -l) - -if [[ $STANDARD_EXCLUDES -ne "1" ]]; then - echo "A non-standard value for \`excludes\` is not supported in the repository" - exit 1 -fi - -# Fail if this repository has a customized version of the autogenerated code. -# This specific check is to make sure that the owlbot.py file does not contain `s.replace` or `sed` -# which has historically been used to modify the generated code. -NUM_STRING_REPLACEMENTS=$(grep owlbot.py -e 's.replace' | wc -l) -if [[ $NUM_STRING_REPLACEMENTS -ne "0" ]]; then - echo "s.replace() is not supported in the python mono repository" - exit 1 -fi - -NUM_STRING_REPLACEMENTS=$(grep owlbot.py -we 'sed' | wc -l) -if [[ $NUM_STRING_REPLACEMENTS -ne "0" ]]; then - echo "s.replace() is not supported in the python mono repository" - exit 1 -fi - -# Delete everything except LICENSE, SECURITY.md, README.rst, .git and .repo-metadata.json -ls -A | grep -v 'LICENSE\|SECURITY.md\|README.rst\|.git\|.repo-metadata.json' | xargs rm -rf - -# Additional cleanup for files/folders with .git prefix -rm -rf .github -rm -rf .gitignore - -# Append a prefix to the README to state that this repository has been archived and the source has moved. -README_RST="${SPLIT_REPO_DIR}/README.rst" - -if [[ ! -f $README_RST ]]; then - echo "README.rst file not found" - exit 1 -fi - -{ cat - $README_RST | sponge $README_RST ; } << EOF -:**NOTE**: **This github repository is archived. The repository contents and history have moved to** \`google-cloud-python\`_. - -.. _google-cloud-python: https://github.com/googleapis/google-cloud-python/tree/main/packages/${ARTIFACT_NAME} - - -EOF - -# Stage all files -git add . - -# Commit all files -git commit -m 'build: update README to indicate that source has moved and delete all files' - -# Push the branch. Note force push option is used to allow this script to be called multiple times. -git push origin -f ${BRANCH_NAME} diff --git a/scripts/split_repo_migration/multiple-library.post-process.sh b/scripts/split_repo_migration/multiple-library.post-process.sh deleted file mode 100755 index bac01bb0d535..000000000000 --- a/scripts/split_repo_migration/multiple-library.post-process.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script was written as part of a project to migrate -# split repositories to the mono-repository google-cloud-python. -# -# This script takes a list of packages. It will first call -# single-library.post-process.common-files.sh on each package to update the -# various metadata files migrated from the split repo. It will then cause the -# OwlBot post-processor to run once (via Docker) for all the repos together to -# update the appropriate files. -# -# This script deals with common (shared) files, and thus CANNOT be called in -# parallel for individual APIs, since each run touches shared files and would -# thus lead to merge conflicts. -# -# Pre-condition: the split repo has been copied to the path indicated when -# calling this function. -# -# INVOCATION -# multiple-library.post-process.common-files.sh DIR PACKAGE [PACKAGE...] -# where dir is the path to the monorepo (relative dirs, like "." are fine) and -# PACKAGE is the name of a (newly migrated) package in the monorepo. -# EXAMPLE from the root directory of the monorepo: -# ./scripts/split_repo_migration/multiple-library.post-process.common-files.sh . google-cloud-speech -# - -# sourced vs execution detection obtained from https://stackoverflow.com/a/28776166 -SOURCED=0 -if [ -n "$ZSH_VERSION" ]; then - case $ZSH_EVAL_CONTEXT in *:file) SOURCED=1;; esac -elif [ -n "$KSH_VERSION" ]; then - [ "$(cd -- "$(dirname -- "$0")" && pwd -P)/$(basename -- "$0")" != "$(cd -- "$(dirname -- "${.sh.file}")" && pwd -P)/$(basename -- "${.sh.file}")" ] && SOURCED=1 -elif [ -n "$BASH_VERSION" ]; then - (return 0 2>/dev/null) && SOURCED=1 -else # All other shells: examine $0 for known shell binary filenames. - # Detects `sh` and `dash`; add additional shell filenames as needed. - case ${0##*/} in sh|-sh|dash|-dash) SOURCED=1;; esac -fi - -(( ${SOURCED} != 1 )) || { \ - echo "Please do not source this script, but execute it directly." - return -10 -} - -# to allow echoing commands for debugging, precede the following with something -# like "echo" -GIT="git" - -# We require executing the script so that an early exit (explicitly or via -e) -# does not kill the user's shell. - -# `-e` enables the script to automatically fail when a command fails -set -e - -MESSAGE="" - -if [ $# -lt 2 ] -then - echo "Usage: $0 REPO_ROOT PACKAGE...." - exit 1 -fi - -PATH_MONOREPO="$(realpath "$1")" -cat <& /dev/null - -BRANCH="migration-batch-${USER}-$$" -${GIT} checkout -b "${BRANCH}" - -# variable naming convention -# PATH_* are absolute system paths -# MONOREPO_PATH_* are paths relative to the root of the monorepo -# MONOREPO_* are values used in the monorepo - -for MONOREPO_PACKAGE_NAME in "$@" -do - PATH_PACKAGE="$(realpath "packages/${MONOREPO_PACKAGE_NAME}")" - MONOREPO_PATH_PACKAGE="packages/${MONOREPO_PACKAGE_NAME}" - cat <& /dev/null # "${PATH_MONOREPO}" - -[[ -z ${MESSAGE} ]] && echo "Done." || echo -e "Done, with a message:\n\n${MESSAGE}\n" diff --git a/scripts/split_repo_migration/single-library.git-migrate-history.sh b/scripts/split_repo_migration/single-library.git-migrate-history.sh deleted file mode 100755 index ba6c012070df..000000000000 --- a/scripts/split_repo_migration/single-library.git-migrate-history.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Sample usage: -# -# git-migrate-history.sh \ -# "googleapis/python-speech" \ -# "~/temp/google-cloud-python" \ -# "" \ -# "" \ -# ".github/.OwlBot.yaml" -# -# This invocation runs migration script on python-speech, where the source path is root directory, -# nothing is ignored, and .OwlBot.yaml is being kept. - -set -ex - -[[ "${BASH_SOURCE[0]}" != "${0}" ]] && EXIT=return || EXIT=exit - -if [ $# -lt 3 ] -then - echo "Usage: $0 [folders,to,skip] [files,to,keep] [branch-name] [issue-number]" - $EXIT 1 -fi - -# source GitHub repository. format: / -SOURCE_REPO=$1 - -# destination GitHub repository. format: / -TARGET_REPO=$2 # Path to google-cloud-python - -# path in the source repo to copy code from. Defaults to the root directory -SOURCE_PATH=$3 - -# comma-separated list of files/folders to skip -IGNORE_FOLDERS=$4 - -# keep these specific files that would otherwise be deleted by IGNORE_FOLDERS -KEEP_FILES=$5 - -# override the HEAD branch name for the migration PR -BRANCH=$6 - -# GitHub issue number -ISSUE_NUMBER=$7 - -if [[ -z "${BRANCH}" ]] -then - # default the branch name to be generated from the source repo name - BRANCH=$(basename ${SOURCE_REPO})-migration -fi -export FILTER_BRANCH_SQUELCH_WARNING=1 - -# create a working directory -WORKDIR="$(mktemp -d -t code-migration-XXXXXXXXXX)" -echo "Created working directory: ${WORKDIR}" - -pushd "${WORKDIR}" # cd into workdir - -echo "Cloning source repository: ${SOURCE_REPO}" -git clone --recurse-submodules "git@github.com:${SOURCE_REPO}.git" source-repo - -pushd source-repo - -DISTRIBUTION_NAME=$(jq -r '.distribution_name' .repo-metadata.json) # -r removes quotes around the name. -TARGET_PATH="packages/${DISTRIBUTION_NAME}" - -git remote remove origin - -# prune only files within the specified directory -if [[ ! -z "${SOURCE_PATH}" ]] -then - echo "Pruning commits only including path: ${SOURCE_PATH}" - git filter-branch \ # rewrites history... unsafe - --prune-empty \ - --subdirectory-filter "${SOURCE_PATH}" -fi - -if [[ ! -z "${IGNORE_FOLDERS}" ]] -then - echo "Ignoring folder: ${IGNORE_FOLDERS}" - mkdir -p "${WORKDIR}/filtered-source" - FOLDERS=$(echo ${IGNORE_FOLDERS} | tr "," " ") - # remove files/folders we don't want - FILTER="(rm -rf ${FOLDERS} || true)" - if [[ ! -z "${KEEP_FILES}" ]] - then - KEEP_FILES_SPACES=($(echo ${KEEP_FILES} | tr "," " ")) - LAST_ELEMENT=$(( ${#KEEP_FILES_SPACES[@]} - 1 )) - KEEP_FILE_COMMANDS="" - for file in "${KEEP_FILES_SPACES[@]}" - do - if [[ $file == "${KEEP_FILES_SPACES[$LAST_ELEMENT]}" ]] - then - KEEP_FILE_COMMANDS+="git checkout -- ${file} 2>/dev/null || true" - else - KEEP_FILE_COMMANDS+="git checkout -- ${file} 2>/dev/null || true; " - fi - done - # restore files to keep, silence errors if the file doesn't exist - FILTER="${FILTER}; ${KEEP_FILE_COMMANDS}" - fi - git filter-branch \ - --force \ - --prune-empty \ - --tree-filter "${FILTER}" -fi - -# reorganize the filtered code into the desired target locations -echo "Moving files to destination path: ${TARGET_PATH}" -git filter-branch \ - --force \ - --prune-empty \ - --tree-filter \ - "git submodule update --init --recursive; find . -mindepth 2 -name .git -exec rm -rf {} +; git submodule update --init --recursive; find . -mindepth 2 -name .git -exec rm -rf {} +; shopt -s dotglob; mkdir -p ${WORKDIR}/migrated-source; mv * ${WORKDIR}/migrated-source; mkdir -p ${TARGET_PATH}; { mv ${WORKDIR}/migrated-source/* ${TARGET_PATH} || echo 'No files to move' ; }" - -# back to workdir -popd - -# merge histories -pushd $TARGET_REPO - -REMOTE="remote.${SOURCE_REPO}" -git remote add --fetch ${REMOTE} ${WORKDIR}/source-repo -git checkout -B "${BRANCH}" -git merge --allow-unrelated-histories ${REMOTE}/main --no-edit - -echo "Success" - -popd # back to workdir - -# Do a diff between source code split repo and migrated code. -git clone --recurse-submodules "git@github.com:${SOURCE_REPO}.git" source-repo-validation # Not ideal to clone again. -find source-repo-validation -name .git -exec rm -rf {} + # That folder is not needed for validation. - -DIFF_FILE="${WORKDIR}/diff.txt" -if diff -r "${TARGET_REPO}/${TARGET_PATH}" source-repo-validation > "${DIFF_FILE}" ; then - echo "No diff" -else - echo "Diff non-empty. See ${DIFF_FILE}" - $EXIT 1 -fi - -pushd "${TARGET_REPO}" >& /dev/null # To target repo - -# For postprocessing of the batch migration script. -mkdir -p owl-bot-staging/${DISTRIBUTION_NAME}/${DISTRIBUTION_NAME} -touch owl-bot-staging/${DISTRIBUTION_NAME}/${DISTRIBUTION_NAME}/${DISTRIBUTION_NAME}.txt -git add owl-bot-staging -git commit -m "Trigger owlbot post-processor" - -git push -u origin "${BRANCH}" --force - -# create pull request -if which gh > /dev/null -then - while ! gh pr create --draft --title "chore(migration): Migrate code from ${SOURCE_REPO} into ${TARGET_PATH}" --body "$(echo -e "See #${ISSUE_NUMBER}. \n\nThis PR should be merged with a merge-commit, not a squash-commit, in order to preserve the git history.")" ; do - echo "** PR creation command FAILED (${SOURCE_REPO} --> ${TARGET_PATH}) : sleeping & retrying" - sleep 30s - done -else - hub pull-request --draft -m "migrate code from ${SOURCE_REPO}" -fi - -popd >& /dev/null - -# Some of the post-processing scripts require the ${DISTRIBUTION_NAME}. We -# output it here so they can use the same value we derived, rather than risking -# diverging computations of the value. -echo "${DISTRIBUTION_NAME}" diff --git a/scripts/split_repo_migration/single-library.post-process.api-files.sh b/scripts/split_repo_migration/single-library.post-process.api-files.sh deleted file mode 100755 index a4335ddd3021..000000000000 --- a/scripts/split_repo_migration/single-library.post-process.api-files.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script was written as part of a project to migrate -# split repositories to the mono-repository google-cloud-python. -# -# This script will update the various metadata files migrated from the split -# repo into formats and locations suitable for the mono-repo. This script deals -# with API-specific file, and can be invoked on many just-migrated APIs in any -# order. There is a companion scripts. *.common-files.sh, which must be invoked -# for all APIs to finish the migration to common, but which cannot be -# parallelized, as it touches shared files. -# -# Pre-condition: the split repo has been copied to the path indicated when -# calling this function. -# -# INVOCATION from the mono-repo root directory: -# single-library.post-process.api-files.sh PACKAGE_PATH -# where PACKAGE_PATH is the path (absolute or relative) from pwd to the -# directory in google-cloud-python holding the copied split repo. Typically, if -# running from the top level of google-cloud-python, this is something like -# "packages/google-cloud-APINAME" -# EXAMPLE from the root directory of the monorepo: -# ./scripts/split_repo_migration/single-library.post-process.api-files.sh packages/google-cloud-speech -# -# For debugging/developing this script, you can have additional parameters: -# ./single-library.post-process.api-files.sh SPLIT_REPO_DIR MONOREPO_DIR MONOREPO_PACKAGE_NAME -# Example from this script's directory: -# ./single-library.post-process.api-files.sh ../../../python-speech ../../ google-cloud-speech - -# sourced vs execution detection obtained from https://stackoverflow.com/a/28776166 -SOURCED=0 -if [ -n "$ZSH_VERSION" ]; then - case $ZSH_EVAL_CONTEXT in *:file) SOURCED=1;; esac -elif [ -n "$KSH_VERSION" ]; then - [ "$(cd -- "$(dirname -- "$0")" && pwd -P)/$(basename -- "$0")" != "$(cd -- "$(dirname -- "${.sh.file}")" && pwd -P)/$(basename -- "${.sh.file}")" ] && SOURCED=1 -elif [ -n "$BASH_VERSION" ]; then - (return 0 2>/dev/null) && SOURCED=1 -else # All other shells: examine $0 for known shell binary filenames. - # Detects `sh` and `dash`; add additional shell filenames as needed. - case ${0##*/} in sh|-sh|dash|-dash) SOURCED=1;; esac -fi - -(( ${SOURCED} != 1 )) || { \ - echo "Please do not source this script, but execute it directly." - return -10 -} - -# We require executing the script so that an early exit (explicitly or via -e) -# does not kill the user's shell. - -# `-e` enables the script to automatically fail when a command fails -set -e - - -NOP="echo -n" -DEBUG="" # "yes" # set to blank for a real run, or non-blank to prevent modifying the split-repo -MESSAGE="" - -[[ -n ${DEBUG} ]] && { - # Comment out some commands for development/debugging - RM='echo "*** would run:" rm' - GIT='echo "*** would run:" git' - - # It's easier to debug if keys don't get reordered - SORT_JSON_KEYS="" -} || { - # Real commands for actual runs - RM='rm' - GIT='git' - - # Sort keys for easier look-up. - SORT_JSON_KEYS="--sort-keys" -} - -if [ $# -lt 1 ] -then - echo "Usage: $0 PATH/TO/PACKAGE_NAME" - exit 1 -fi - -# variable naming convention -# PATH_* are absolute system paths -# MONOREPO_PATH_* are paths relative to the root of the monorepo -# MONOREPO_* are values used in the monorepo - -# In a real run for a split repo already copied to the mono-repo, $1 would be -# specified as `packages/PACKAGE_NAME`. For developing this script, it's useful -# to specify $1 as the name of a completely separate split repo, in which case -# specifying $2 and $3 is also useful (see below). -PATH_PACKAGE="$(realpath "$1")" - -# The optional second argument is useful for development: it specifies a different -# directory for the monorepo (rather than deriving it from $1). -PATH_MONOREPO="${2:-$(dirname $(dirname $PATH_PACKAGE))}" -PATH_MONOREPO="$(realpath "${PATH_MONOREPO}")" - -# The optional third argument is useful for development: it specifies what the the -# package name should be for this API once migrated to the monorepo. This can be -# inferred if we've already migrated the split repo to the correct location, but -# otherwise needs to be specified, since it differs from the split repo name -# (for example, the repo "python-speech" defines a package -# "google-cloud-speech"). -MONOREPO_PACKAGE_NAME="${3:-$(basename ${PATH_PACKAGE})}" - -MONOREPO_PATH_PACKAGE="packages/${MONOREPO_PACKAGE_NAME}" - -cat <& /dev/null - -## START owlbot.yaml migration ######################################## -# variable prefix: OWY_* -# FIXME: KEEP? -OWY_MONO_PATH="${MONOREPO_PATH_PACKAGE}/.OwlBot.yaml" -echo "Migrating: ${OWY_MONO_PATH}" - -OWY_SPLIT_PATH="${PATH_PACKAGE}/.github/.OwlBot.yaml" -[[ ! -f ${OWY_SPLIT_PATH} ]] || { - mkdir -p $(dirname ${OWY_MONO_PATH}) - cp ${OWY_SPLIT_PATH} ${OWY_MONO_PATH} - - # remove `docker:` line - sed -i "/docker:/d" "${OWY_MONO_PATH}" - # remove `image:` line - sed -i "/image:/d" "${OWY_MONO_PATH}" - - # The order of lines A and B below matters, since we DO want lines affected by - # A to be further affected by B. - sed -i 's|\.\*-py/(.*)|.*-py|' "${OWY_MONO_PATH}" - sed -i "s|dest: /owl-bot-staging/\$1/\$2|dest: /owl-bot-staging/\$1|" "${OWY_MONO_PATH}" # A - sed -i "s| /owl-bot-staging/| /owl-bot-staging/${MONOREPO_PACKAGE_NAME}/|g" "${OWY_MONO_PATH}" # B - - # TODO: Review the following: For consistency with NodeJS migration script: - # - we are not removing `begin-after-commit-hash` - # - we are not removing `deep-remove-regex`, even though it refers to a non-API-specific directory. - - $RM ${OWY_SPLIT_PATH} -} -## END owlbot.yaml migration - - -## START owlbot.py deletion ######################################## -# variable prefix: OWP_* -OWP_MONO_PATH="${MONOREPO_PATH_PACKAGE}/owlbot.py" -echo "Migrating: ${OWP_MONO_PATH}" - -[[ -z ${DEBUG} ]] || { \ - OWP_SPLIT_PATH="${PATH_PACKAGE}/owlbot.py" - [[ ! -f "${OWP_SPLIT_PATH}" ]] || cp -u ${OWP_SPLIT_PATH} ${OWP_MONO_PATH} -} - -[[ ! -f "${OWP_MONO_PATH}" ]] || { - MESSAGE="${MESSAGE}\n\nWARNING: Deleted ${OWP_MONO_PATH}" - rm -rf "${OWP_MONO_PATH}" -} -## END owlbot.py deletion - -## START delete doc/changelog.md .github/ .kokoro/#################### -${RM} -f "${PATH_PACKAGE}/docs/changelog.md" -${RM} -rf "${PATH_PACKAGE}/.github" -${RM} -rf "${PATH_PACKAGE}/.kokoro" -${RM} -f "${PATH_PACKAGE}/.trampolinerc" -${RM} -f "${PATH_PACKAGE}/${MONOREPO_PACKAGE_NAME}.txt" - -## END delete doc/changelog.md .github/ .kokoro/ - -## START commit changes ############################################# -echo "Committing changes locally" -${GIT} add . -${GIT} commit -am "build: ${MONOREPO_PACKAGE_NAME} migration: adjust owlbot-related files" || \ - echo "No changes to commit" -## END commit changes - -popd >& /dev/null # "${PATH_MONOREPO}" - -[[ -z ${MESSAGE} ]] && echo "Done." || echo -e "Done, with a message:\n\n${MESSAGE}\n" - - - diff --git a/scripts/split_repo_migration/single-library.post-process.common-files.sh b/scripts/split_repo_migration/single-library.post-process.common-files.sh deleted file mode 100755 index d79546bd234e..000000000000 --- a/scripts/split_repo_migration/single-library.post-process.common-files.sh +++ /dev/null @@ -1,232 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script was written as part of a project to migrate -# split repositories to the mono-repository google-cloud-python. -# -# This script will update the various metadata files migrated from the split -# repo into formats and locations suitable for the mono-repo. This script deals -# with common (shared) files, and thus CANNOT be called in parallel for -# individual APIs, since each run touches shared files and would thus lead to -# merge conflicts. -# -# Pre-condition: the split repo has been copied to the path indicated when -# calling this function. -# -# **** NOTE: You would typically not call this directly unless debugging. Use -# multiple-library.post-process.common-files.sh instead. **** -# -# INVOCATION from the mono-repo root directory: -# single-library.post-process.common-files.sh PACKAGE_PATH -# where PACKAGE_PATH is the path (absolute or relative) from pwd to the -# directory in google-cloud-python holding the copied split repo. Typically, if -# running from the top level of google-cloud-python, this is something like -# "packages/google-cloud-APINAME" -# EXAMPLE from the root directory of the monorepo: -# ./scripts/split_repo_migration/single-library.post-process.common-files.sh packages/google-cloud-speech -# -# For debugging/developing this script, you can have additional parameters: -# ./single-library.post-process.common-files.sh SPLIT_REPO_DIR MONOREPO_DIR MONOREPO_PACKAGE_NAME -# Example from this script's directory: -# ./single-library.post-process.common-files.sh ../../../python-speech ../../ google-cloud-speech - -# sourced vs execution detection obtained from https://stackoverflow.com/a/28776166 -SOURCED=0 -if [ -n "$ZSH_VERSION" ]; then - case $ZSH_EVAL_CONTEXT in *:file) SOURCED=1;; esac -elif [ -n "$KSH_VERSION" ]; then - [ "$(cd -- "$(dirname -- "$0")" && pwd -P)/$(basename -- "$0")" != "$(cd -- "$(dirname -- "${.sh.file}")" && pwd -P)/$(basename -- "${.sh.file}")" ] && SOURCED=1 -elif [ -n "$BASH_VERSION" ]; then - (return 0 2>/dev/null) && SOURCED=1 -else # All other shells: examine $0 for known shell binary filenames. - # Detects `sh` and `dash`; add additional shell filenames as needed. - case ${0##*/} in sh|-sh|dash|-dash) SOURCED=1;; esac -fi - -(( ${SOURCED} != 1 )) || { \ - echo "Please do not source this script, but execute it directly." - return -10 -} - -# We require executing the script so that an early exit (explicitly or via -e) -# does not kill the user's shell. - -# `-e` enables the script to automatically fail when a command fails -set -e - - -NOP="echo -n" -DEBUG="" # "yes" # set to blank for a real run, or non-blank to prevent modifying the split-repo -MESSAGE="" - -# Use indent of 4 when formatting json for readability -INDENT_JSON="--indent 4" - -[[ -n ${DEBUG} ]] && { - # Comment out some commands for development/debugging - RM='echo "*** would run:" rm' - GIT='echo "*** would run:" git' - - # It's easier to debug if keys don't get reordered - SORT_JSON_KEYS="" -} || { - # Real commands for actual runs - RM='rm' - GIT='git' - - # Sort keys for easier look-up. - SORT_JSON_KEYS="--sort-keys" -} - -if [ $# -lt 1 ] -then - echo "Usage: $0 PATH/TO/PACKAGE_NAME" - exit 1 -fi - -# variable naming convention -# PATH_* are absolute system paths -# MONOREPO_PATH_* are paths relative to the root of the monorepo -# MONOREPO_* are values used in the monorepo - -# In a real run for a split repo already copied to the mono-repo, $1 would be -# specified as `packages/PACKAGE_NAME`. For developing this script, it's useful -# to specify $1 as the name of a completely separate split repo, in which case -# specifying $2 and $3 is also useful (see below). -PATH_PACKAGE="$(realpath "$1")" - -# The optional second argument is useful for development: it specifies a different -# directory for the monorepo (rather than deriving it from $1). -PATH_MONOREPO="${2:-$(dirname $(dirname $PATH_PACKAGE))}" -PATH_MONOREPO="$(realpath "${PATH_MONOREPO}")" - -# The optional third argument is useful for development: it specifies what the the -# package name should be for this API once migrated to the monorepo. This can be -# inferred if we've already migrated the split repo to the correct location, but -# otherwise needs to be specified, since it differs from the split repo name -# (for example, the repo "python-speech" defines a package -# "google-cloud-speech"). -MONOREPO_PACKAGE_NAME="${3:-$(basename ${PATH_PACKAGE})}" - -MONOREPO_PATH_PACKAGE="packages/${MONOREPO_PACKAGE_NAME}" - -cat <& /dev/null - -## START release-please config migration ######################################## -# variable prefix: RPC_* - -RPC_MONO_PATH="release-please-config.json" -echo "Migrating: ${RPC_MONO_PATH}" - -# enable this if we want sorted keys. Keep it disabled to append new entries at -# the end (useful for debugging): -RPC_SORT_KEYS="${SORT_JSON_KEYS}" - -RPC_SPLIT_PATH="${PATH_PACKAGE}/release-please-config.json" -jq ${INDENT_JSON} ".packages.\".\" += {component: \"${MONOREPO_PACKAGE_NAME}\"}" "${RPC_SPLIT_PATH}" | sponge "${RPC_SPLIT_PATH}" -RPC_NEW_OBJECT="$(jq '.packages."."' "${RPC_SPLIT_PATH}")" - -jq ${INDENT_JSON} ${RPC_SORT_KEYS} --argjson newObject "${RPC_NEW_OBJECT}" ". * {\"packages\": {\"${MONOREPO_PATH_PACKAGE}\": \$newObject}}" ${RPC_MONO_PATH} | sponge ${RPC_MONO_PATH} -$RM ${RPC_SPLIT_PATH} -## END release-please config migration - -## START release-please manifest migration ######################################## -# variable prefix: RPM_* -RPM_MONO_PATH=".release-please-manifest.json" -echo "Migrating: ${RPM_MONO_PATH}" - -# enable this if we want sorted keys. Keep it disabled to append new entries at -# the end (useful for debugging): -RPM_SORT_KEYS="${SORT_JSON_KEYS}" - -RPM_SPLIT_PATH="${PATH_PACKAGE}/.release-please-manifest.json" -RPM_VERSION="$(jq '."."' "${RPM_SPLIT_PATH}")" -jq ${INDENT_JSON} ${RPM_SORT_KEYS} ". * {\"${MONOREPO_PATH_PACKAGE}\": ${RPM_VERSION}}" ${RPM_MONO_PATH} | sponge ${RPM_MONO_PATH} -$RM ${RPM_SPLIT_PATH} -## END release-please manifest migration - -## START migrate release tags ######################################## -# We need to migrate the release tags since that is what release-please uses to -# get the next release number.We use ${RPM_VERSION} from the previous section -# variable prefix: LRT_* -echo "Replicating latest release tag" - -LRT_VERSION="${RPM_VERSION//\"/}" - -# any of the gapic_version.py files will do: they all match -LRT_VERSION_FILE="$(find ${MONOREPO_PATH_PACKAGE} -name "gapic_version.py" | head -n 1)" -LRT_SHA=$(git log --format=oneline ${LRT_VERSION_FILE} | grep release | head -n 1 | awk '{ print $1 }') -$GIT tag -f ${MONOREPO_PACKAGE_NAME}-v${LRT_VERSION} ${LRT_SHA} -$GIT push --tags -## END migrate release tags - - - - -## START .repo-metadata.json migration ######################################## -# variable prefix: RMJ_* -RMJ_MONO_PATH="${MONOREPO_PATH_PACKAGE}/.repo-metadata.json" -echo "Migrating: ${RMJ_MONO_PATH}" - -[[ -z ${DEBUG} ]] || { \ -RMJ_SPLIT_PATH="${PATH_PACKAGE}/.repo-metadata.json" -cp ${RMJ_SPLIT_PATH} ${RMJ_MONO_PATH} -} - -jq ${INDENT_JSON} '.repo = "googleapis/google-cloud-python"' ${RMJ_MONO_PATH} | sponge ${RMJ_MONO_PATH} -jq -r ".issue_tracker" "${RMJ_MONO_PATH}" | grep -q "github.com" && { - jq ${INDENT_JSON} '.issue_tracker = "https://github.com/googleapis/google-cloud-python/issues"' ${RMJ_MONO_PATH} | sponge ${RMJ_MONO_PATH} -} || { $NOP ; } -## END .repo-metadata.json migration - - -## START .pre-commit-config migration ######################################## -# variable prefix: PCC_* -PCC_MONO_PATH=".pre-commit-config.yaml" -echo "Migrating: ${PCC_MONO_PATH}" - -PCC_SPLIT_PATH="${PATH_PACKAGE}/.pre-commit-config.yaml" - -# We only copy this if it doesn't already exist in the mono-repo. -[[ ! -f "${PCC_MONO_PATH}" ]] && [[ -f "${PCC_SPLIT_PATH}" ]] && { - cp ${PCC_SPLIT_PATH} ${PCC_MONO_PATH} -} || { $NOP ; } -$RM -f ${PCC_SPLIT_PATH} -## END .pre-commit-config migration - -### Delete file that was copied over from owlbot-staging/ -${RM} -f "${PATH_PACKAGE}/${MONOREPO_PACKAGE_NAME}.txt" - - -## START commit changes ############################################# -echo "Committing changes locally" -${GIT} add . -${GIT} commit -am "migration(${MONOREPO_PACKAGE_NAME}): adjust metadata and automation configs" -## END commit changes - -popd >& /dev/null # "${PATH_MONOREPO}" - -[[ -z ${MESSAGE} ]] && echo "Done." || echo -e "Done, with a message:\n\n${MESSAGE}\n" - - -