From cbad49a8685ac0cbffa3e7daba8f6ce4f9dd35a2 Mon Sep 17 00:00:00 2001 From: Alec Dorrington Date: Sun, 27 Sep 2026 18:56:32 +0200 Subject: [PATCH] Synced deletions, but only of files which came from the source. Deletions were never propagated, as the sync workflow checked out the source with a depth of 1, so `git log --diff-filter=D` could never see a deletion. Had it done so, it would also have deleted any file in the target that merely shared a name with one deleted from the source. Both repositories are now checked out with full history, and a file is only deleted from the target if: - It originated from the source: when it was last added to the target, it matched a version which the source once had. - It hasn't been re-added since: it was last added to the target before it was deleted from the source. - It hasn't been modified downstream: it currently matches a version which the source once had. Timestamps come from first-parent history, so they reflect when a change landed on each branch rather than when it was first committed. Renames are treated as a deletion plus an addition. This is enabled by default, and can be disabled with `syncDeletions`. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/sync.yml | 3 + README.md | 24 +++++++ scripts/load-config.sh | 1 + scripts/merge-files.sh | 111 ++++++++++++++++++++++++++------- scripts/parse-config.sh | 4 ++ templates/pull-request-body.md | 3 +- 6 files changed, 124 insertions(+), 22 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index dde5daf..20a5bfb 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -54,11 +54,13 @@ jobs: repository: SgtSwagrid/github-graph path: .github-graph + # Full history of both repositories is needed to sync deletions. - name: Checkout source uses: actions/checkout@v4 with: ref: ${{ matrix.child.source.branch }} path: source + fetch-depth: 0 - name: Checkout target uses: actions/checkout@v4 @@ -67,6 +69,7 @@ jobs: ref: ${{ matrix.child.target.branch }} token: ${{ secrets[matrix.child.token] }} path: target + fetch-depth: 0 - name: Load config run: bash .github-graph/scripts/load-config.sh '${{ toJson(matrix.child) }}' diff --git a/README.md b/README.md index bb60025..74511ce 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,29 @@ i.e. `.github/workflows/sync.yml` and `.github/graph.json`, as these aren't excluded automatically. It is not necessary to ignore files which lie outside of `source.root`. +### `syncDeletions` + +Whether files which are deleted from the source should also be deleted from the target. +Can be defined for a child, or globally at the top-level. +Defaults to `true`. + +```json +{ + "syncDeletions": false +} +``` + +As the target may have files of its own which happen to share a name with one deleted from the source, +a file is only deleted if the history of both repositories shows that it's a copy of the one which was deleted. +In particular, a file is kept if: +- **It didn't originate from the source.** For example, the target had its own version of the file, which was later overwritten by the source. +- **It was re-added after the deletion.** For example, the deletion was synced, but the target later restored the file or created a new one with the same name. +- **It was modified in the target.** That is, it doesn't match any version which the source ever had. + +Any file which is kept for one of these reasons is reported in the workflow logs. +A renamed file is treated as a deletion of the old name, plus an addition of the new one. +Files in the `ignore` list are never deleted. + ### `token` The name of the GitHub Actions secret containing the access token for the target repository. @@ -246,6 +269,7 @@ That being said, if you wanted to tackle these yourself, I'd be a very grateful ### Merge semantics Updated files are never "merged", but simply overwrite whatever exists downstream. +Likewise, deleted files are deleted downstream, unless they've since been modified or re-added there (see [`syncDeletions`](#syncdeletions)). _GitHub Graph_ is only intended for use when the responsibility for each file can be unambiguously associated with a single source repository, with the understanding that copies shouldn't be modified. diff --git a/scripts/load-config.sh b/scripts/load-config.sh index 37dc45a..49730f0 100644 --- a/scripts/load-config.sh +++ b/scripts/load-config.sh @@ -30,6 +30,7 @@ jq -r ' "TARGET_ROOT=\(.target.root)", "TARGET_URL=\(.target.url)", "IGNORE=\(.ignore | tojson)", + "SYNC_DELETIONS=\(.syncDeletions)", "PR_TITLE=\(.pullRequest.title)" ' <<< "$CHILD" >> "$GITHUB_ENV" diff --git a/scripts/merge-files.sh b/scripts/merge-files.sh index 9569a56..b4f4da7 100644 --- a/scripts/merge-files.sh +++ b/scripts/merge-files.sh @@ -8,10 +8,11 @@ set -euo pipefail # - IGNORE: A JSON-formatted list of file paths to exclude from syncing. # - SOURCE_ROOT: The root directory within the source repository to copy files from. # - TARGET_ROOT: The root directory within the target repository to copy files into. +# - SYNC_DELETIONS: Whether to delete files from the target which were deleted from the source. # # Requirements: -# - source/ should already contain the source repository at the correct branch. -# - target/ should already contain the target repository at the correct branch. +# - source/ should already contain the source repository at the correct branch, with full history. +# - target/ should already contain the target repository at the correct branch, with full history. # ================================================================================================= @@ -43,6 +44,33 @@ target_path() { echo "target/$TARGET_ROOT/${1#"$prefix"}" } +# Convert a file name that is relative to SOURCE_ROOT to a normalised path relative to the target repository. +target_relative_path() { + realpath -ms --relative-to=target "$(target_path "$1")" +} + +# Output the first-parent history of a branch, as NUL-separated records of the form: +# +# There is one record for each file changed by each commit, with renames treated as a deletion plus an addition. +# Following only the first parent means that each change is dated by when it actually landed on the branch. +# - $1: The repository directory. +# - $@: Any further arguments are passed to `git log`. +file_history() { + local repo="$1" token timestamp="" blob status file + shift + while IFS= read -r -d '' token; do + if [[ "$token" =~ ^[[:space:]]*:(.*)$ ]]; then + read -r _ _ _ blob status <<< "${BASH_REMATCH[1]}" + IFS= read -r -d '' file + printf '%s\0' "$timestamp" "$blob" "$status" "$file" + elif [[ "$token" =~ ^[[:space:]]*([0-9]+)[[:space:]]*$ ]]; then + timestamp="${BASH_REMATCH[1]}" + fi + done < <( + git -C "$repo" log --first-parent --diff-merges=first-parent --no-renames --raw --no-abbrev -z --format='%ct' "$@" + ) +} + # ================================================================================================= # 1. Copy all files in the SOURCE_ROOT directory of the source repository # to the TARGET_ROOT directory of the target repository. @@ -70,31 +98,72 @@ for file in "${!SOURCE_FILES[@]}"; do done # ================================================================================================= -# 2. Delete from the target repository all files which once existed in the source repository, +# 2. Delete from the target repository all files which were synced from the source repository, # but which have since been deleted from the source repository. # Files in the ignore list or outside of SOURCE_ROOT are skipped. +# As the target may have files of its own with the same names, a file is only deleted if: +# a) It originated from the source. +# i.e. When it was last added to the target, it matched a version which the source once had. +# b) It hasn't been re-added to the target since being deleted from the source. +# i.e. It was last added to the target before it was deleted from the source. +# c) It hasn't been modified in the target. +# i.e. It currently matches a version which the source once had. # ================================================================================================= -# A list of names of all files which have been deleted from the source repository. -mapfile -t SOURCE_DELETED < <( - git -C source log --diff-filter=D --name-only --pretty=format: | sort -u -) +if [[ "$SYNC_DELETIONS" != "true" ]]; then + echo "Skipped deletions, as syncDeletions is disabled." + exit 0 +fi -for file in "${SOURCE_DELETED[@]}"; do - # Ignore because file name is empty. - if [[ -z "$file" ]]; then : - # Ignore because file is outside SOURCE_ROOT. - elif [[ "$SOURCE_ROOT" != "." && "$file" != "$SOURCE_ROOT"/* ]]; then : - # Ignore because file is in ignore list. - elif is_ignored "${file#"$SOURCE_ROOT/"}"; then : - # Ignore because file was re-added. - elif [[ -n "${SOURCE_FILES[$file]+_}" ]]; then : - # Delete the file if none of the exclusions apply. +# Provenance can't be determined without the full history of both repositories. +if [[ "$(git -C source rev-parse --is-shallow-repository)" == "true" || + "$(git -C target rev-parse --is-shallow-repository)" == "true" ]]; then + echo "::warning::Skipped deletions, as they require the full history of both repositories." + exit 0 +fi + +# For every file in SOURCE_ROOT which was ever in the source repository: +# - DELETED_AT: The time at which it was deleted from the source, or empty if it still exists. +# - SOURCE_VERSIONS: The set of versions it has had in the source, keyed by ":". +declare -A DELETED_AT SOURCE_VERSIONS +while IFS= read -r -d '' timestamp && IFS= read -r -d '' blob && IFS= read -r -d '' status && IFS= read -r -d '' file; do + if [[ "$status" == "D" ]]; then + DELETED_AT["$file"]="$timestamp" else - dest=$(target_path "$file") - if [[ -e "$dest" ]]; then - rm "$dest" - echo "Deleted: $file" + DELETED_AT["$file"]="" + SOURCE_VERSIONS["$blob:$file"]=1 + fi +done < <(file_history source --reverse -- "$SOURCE_ROOT") + +for file in "${!DELETED_AT[@]}"; do + deleted_at="${DELETED_AT[$file]}" + + # Ignore because file still exists in the source. + if [[ -z "$deleted_at" ]]; then continue; fi + # Ignore because file is in ignore list. + if is_ignored "${file#"$SOURCE_ROOT/"}"; then continue; fi + + # Ignore because file doesn't exist in the target. + dest=$(target_relative_path "$file") + current=$(git -C target rev-parse -q --verify "HEAD:$dest") || continue + + # Find when the file was last added to the target, and with which version. + added_at="" added_blob="" + while IFS= read -r -d '' timestamp && IFS= read -r -d '' blob && IFS= read -r -d '' status && IFS= read -r -d '' path; do + if [[ "$path" == "$dest" ]]; then + added_at="$timestamp" added_blob="$blob" + break fi + done < <(file_history target --diff-filter=A -- ":(literal)$dest") + + if [[ -z "$added_at" || -z "${SOURCE_VERSIONS["$added_blob:$file"]+_}" ]]; then + echo "Kept: $file (didn't originate from the source)" + elif (( added_at >= deleted_at )); then + echo "Kept: $file (re-added after being deleted from the source)" + elif [[ -z "${SOURCE_VERSIONS["$current:$file"]+_}" ]]; then + echo "Kept: $file (modified after being synced from the source)" + else + rm "target/$dest" + echo "Deleted: $file" fi done diff --git a/scripts/parse-config.sh b/scripts/parse-config.sh index 33ab80d..88a774e 100644 --- a/scripts/parse-config.sh +++ b/scripts/parse-config.sh @@ -68,6 +68,10 @@ CHILDREN=$(jq \ map( .token //= "GH_TOKEN" ) | + # Note that `//=` would also replace an explicit `false`. + map( + .syncDeletions |= (if . == null then true else . end) + ) | # Filter to only children that are triggered by the current branch. map(select(.source.branch == $ENV.GITHUB_REF_NAME)) diff --git a/templates/pull-request-body.md b/templates/pull-request-body.md index b4799b4..fa585ac 100644 --- a/templates/pull-request-body.md +++ b/templates/pull-request-body.md @@ -9,7 +9,8 @@ Note however that subsequent commits to [$SOURCE_BRANCH]($SOURCE_BRANCH_URL) may ### Warning -This change is potentially destructive, as files with matching names will be overwritten. +This change is potentially destructive, as files with matching names will be overwritten, +and files which were deleted upstream will also be deleted here. Please review carefully. ### Explanation