-
Notifications
You must be signed in to change notification settings - Fork 26
feat: add check replica command to identify broken snapshot chains #561
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
Hassanzadeh-sd
wants to merge
2
commits into
longhorn:master
Choose a base branch
from
Hassanzadeh-sd:feat-check-replica-snapshot-chain
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,6 +9,7 @@ import ( | |
|
|
||
| "github.com/longhorn/cli/pkg/consts" | ||
| local "github.com/longhorn/cli/pkg/local/preflight" | ||
| localreplica "github.com/longhorn/cli/pkg/local/replica" | ||
| "github.com/longhorn/cli/pkg/types" | ||
| "github.com/longhorn/cli/pkg/utils" | ||
| ) | ||
|
|
@@ -22,6 +23,61 @@ func NewCmdCheck(globalOpts *types.GlobalCmdOptions) *cobra.Command { | |
| utils.SetGlobalOptionsLocal(cmd, globalOpts) | ||
|
|
||
| cmd.AddCommand(newCmdCheckPreflight(globalOpts)) | ||
| cmd.AddCommand(newCmdCheckReplica(globalOpts)) | ||
|
|
||
| return cmd | ||
| } | ||
|
|
||
| func newCmdCheckReplica(globalOpts *types.GlobalCmdOptions) *cobra.Command { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This seems to only work with v1. We should specify the engine explicitly. |
||
| var localChecker = localreplica.Checker{} | ||
|
|
||
| cmd := &cobra.Command{ | ||
| Use: consts.SubCmdReplica, | ||
| Short: "Check Longhorn replica integrity", | ||
| Long: `This command checks the integrity of the snapshot chains in the Longhorn replica data directories. | ||
| It identifies broken snapshot chains, for example snapshots referencing a missing parent, disk files without metadata files, and metadata files without disk files. | ||
| The results are presented by the replica data directory names, not the actual Custom Resource (CR) names. | ||
|
|
||
| By default, this command checks all Longhorn replicas in the data directory. | ||
| You can narrow down the results by using the following options: | ||
| - --name: Specify the Longhorn replica data directory name to check a specific replica. | ||
| - --volume-name: Filter replicas by the volume they belong to.`, | ||
|
|
||
| PreRun: func(cmd *cobra.Command, args []string) { | ||
| localChecker.LogLevel = globalOpts.LogLevel | ||
|
|
||
| err := localChecker.Init() | ||
| if err != nil { | ||
| utils.CheckErr(errors.Wrap(err, "Failed to initialize replica checker")) | ||
| } | ||
| }, | ||
|
|
||
| Run: func(cmd *cobra.Command, args []string) { | ||
| err := localChecker.Run() | ||
| if err != nil { | ||
| utils.CheckErr(errors.Wrap(err, "Failed to run replica checker")) | ||
| } | ||
|
|
||
| logrus.Info("Successfully checked replica integrity") | ||
| }, | ||
|
|
||
| PostRun: func(cmd *cobra.Command, args []string) { | ||
| err := localChecker.Output() | ||
| if err != nil { | ||
| utils.CheckErr(errors.Wrap(err, "Failed to output replica checker collection")) | ||
| } | ||
|
|
||
| logrus.Info("Successfully output replica checker collection") | ||
| }, | ||
| } | ||
|
|
||
| utils.SetGlobalOptionsLocal(cmd, globalOpts) | ||
|
|
||
| cmd.Flags().StringVar(&localChecker.CurrentNodeID, consts.CmdOptNodeId, os.Getenv(consts.EnvCurrentNodeID), "Current node ID.") | ||
| cmd.Flags().StringVarP(&localChecker.OutputFilePath, consts.CmdOptOutputFile, "o", os.Getenv(consts.EnvOutputFilePath), "Output the result to a file, default to stdout.") | ||
| cmd.Flags().StringVar(&localChecker.ReplicaName, consts.CmdOptName, os.Getenv(consts.EnvLonghornReplicaName), "Specify the name of the replica to check.") | ||
| cmd.Flags().StringVar(&localChecker.VolumeName, consts.CmdOptLonghornVolumeName, os.Getenv(consts.EnvLonghornVolumeName), "Specify the name of the volume to check its replicas.") | ||
| cmd.Flags().StringVar(&localChecker.LonghornDataDirectory, consts.CmdOptLonghornDataDirectory, os.Getenv(consts.EnvLonghornDataDirectory), "Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg.") | ||
|
|
||
| return cmd | ||
| } | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| ## longhornctl check replica | ||
|
|
||
| Check Longhorn replica integrity | ||
|
|
||
| ### Synopsis | ||
|
|
||
| This command checks the integrity of the snapshot chains in the Longhorn replica data directories on each node. | ||
| It identifies broken snapshot chains, for example snapshots referencing a missing parent, disk files without metadata files, and metadata files without disk files. | ||
| The results are presented by the replica data directory names, not the actual Custom Resource (CR) names. | ||
|
|
||
| By default, this command checks all Longhorn replicas in the data directory. | ||
| You can narrow down the results by using the following options: | ||
| - --name: Specify the Longhorn replica data directory name to check a specific replica. | ||
| - --volume-name: Filter replicas by the volume they belong to. | ||
|
|
||
| ``` | ||
| longhornctl check replica [flags] | ||
| ``` | ||
|
|
||
| ### Examples | ||
|
|
||
| ``` | ||
| $ longhornctl check replica | ||
| INFO[2024-07-16T17:23:47+08:00] Initializing replica checker | ||
| INFO[2024-07-16T17:23:47+08:00] Cleaning up replica checker | ||
| INFO[2024-07-16T17:23:47+08:00] Running replica checker | ||
| INFO[2024-07-16T17:23:51+08:00] Retrieved replica check results: | ||
| replicas: | ||
| pvc-48a6457d-585e-423b-b530-bbc68a5f948a-0e2603a7: | ||
| - node: ip-10-0-2-123 | ||
| directory: /var/lib/longhorn/replicas/pvc-48a6457d-585e-423b-b530-bbc68a5f948a-0e2603a7 | ||
| volumeName: pvc-48a6457d-585e-423b-b530-bbc68a5f948a | ||
| snapshotChain: | ||
| - volume-head-001.img | ||
| - volume-snap-40b3b028-b3b3-4a35-a806-8bea77f27c00.img | ||
| errors: | ||
| - 'broken snapshot chain: disk volume-snap-40b3b028-b3b3-4a35-a806-8bea77f27c00.img references parent volume-snap-6f244bbe-2857-46e4-92e2-eb1e16a63ba1.img, but the parent metadata file is missing' | ||
| INFO[2024-07-16T17:23:51+08:00] Cleaning up replica checker | ||
| INFO[2024-07-16T17:23:51+08:00] Completed replica checker | ||
| ``` | ||
|
|
||
| ### Options | ||
|
|
||
| ``` | ||
| --data-dir string Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg. (default "/var/lib/longhorn") | ||
| -h, --help help for replica | ||
| --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") | ||
| --image-pull-secret string Secret with registry credentials for pulling images | ||
| --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. | ||
| --kubeconfig string Kubernetes config (kubeconfig) path | ||
| -l, --log-level string Log level (default "info") | ||
| --name string Specify the name of the replica to check. | ||
| --namespace string The namespace to run DaemonSet pods. (default "longhorn-system") | ||
| --node-selector string Comma-separated list of key=value pairs to match against node labels, selecting the nodes the DaemonSet will run on (e.g. env=prod,zone=us-west). | ||
| --volume-name string Specify the name of the volume to check its replicas. | ||
| ``` | ||
|
|
||
| ### Options inherited from parent commands | ||
|
|
||
| ``` | ||
| --tolerations string Semicolon-separated list of tolerations for DaemonSet pods (e.g. key=value:NoSchedule;:NoExecute). | ||
| ``` | ||
|
|
||
| ### SEE ALSO | ||
|
|
||
| * [longhornctl check](longhornctl_check.md) - Longhorn checking operations | ||
|
|
||
| ###### Auto generated by spf13/cobra on 20-Jul-2026 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,7 @@ | ||
| package consts | ||
|
|
||
| const ( | ||
| AppNameReplicaChecker = "longhorn-replica-checker" | ||
| AppNameReplicaExporter = "longhorn-replica-exporter" | ||
| AppNameReplicaGetter = "longhorn-replica-getter" | ||
| ) |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
NIT: Re-organize the import package