From 874b6090cf922ca27a2d6847a9153e40a4ec27ab Mon Sep 17 00:00:00 2001 From: mmsqe Date: Fri, 28 Aug 2026 10:06:34 +0800 Subject: [PATCH] feat: move pebble to v2 v2 fails a compaction on a block it cannot read instead of passing over it, and routes every detection through EventListener.DataCorruption. A node that arrives here without having been through the ratchet release is refused, and told to go back and take it -- the one move it cannot make is forward, since v2 rejects the format before it would migrate it. These stores open first, so this is where such a node stops. --- db.go | 2 +- go.mod | 20 +++++++++------ go.sum | 34 +++++++++++++++++-------- pebble.go | 41 +++++++++++++++++++++++++++--- pebble_test.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 142 insertions(+), 22 deletions(-) diff --git a/db.go b/db.go index 688e626..8e015de 100644 --- a/db.go +++ b/db.go @@ -38,7 +38,7 @@ const ( // - pure go // - use badgerdb build tag (go build -tags badgerdb) BadgerDBBackend BackendType = "badgerdb" - // PebbleDBDBBackend represents pebble (uses github.com/cockroachdb/pebble) + // PebbleDBDBBackend represents pebble (uses github.com/cockroachdb/pebble/v2) // - pure go PebbleDBBackend BackendType = "pebbledb" ) diff --git a/go.mod b/go.mod index 652786f..9942e44 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cometbft/cometbft-db go 1.23 require ( - github.com/cockroachdb/pebble v1.1.4 + github.com/cockroachdb/pebble/v2 v2.1.7 github.com/dgraph-io/badger/v4 v4.5.1 github.com/google/btree v1.1.3 github.com/jmhodges/levigo v1.0.0 @@ -14,11 +14,18 @@ require ( ) require ( - github.com/DataDog/zstd v1.5.6 // indirect + github.com/RaduBerinde/axisds v0.1.0 // indirect + github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect + github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect + github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258 // indirect + github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect +) + +require ( + github.com/DataDog/zstd v1.5.7 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/errors v1.11.3 // indirect - github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0 // indirect github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect @@ -28,7 +35,7 @@ require ( github.com/getsentry/sentry-go v0.31.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect github.com/google/flatbuffers v25.1.24+incompatible // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/kr/pretty v0.3.1 // indirect @@ -50,8 +57,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) - retract ( - v0.6.5 // Breaking changes were released with the wrong tag (use v0.6.6 or later). - [v1.0.2, v1.0.3] // These do not have Docker images due to the broken pipeline. + [v1.0.2, v1.0.3] // These do not have Docker images due to the broken pipeline. + v0.6.5 // Breaking changes were released with the wrong tag (use v0.6.6 or later). ) diff --git a/go.sum b/go.sum index 088686c..426e275 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,13 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= -github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= +github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/RaduBerinde/axisds v0.1.0 h1:YItk/RmU5nvlsv/awo2Fjx97Mfpt4JfgtEVAGPrLdz8= +github.com/RaduBerinde/axisds v0.1.0/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= +github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= +github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -9,18 +15,22 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= -github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0 h1:pU88SPhIFid6/k0egdR5V6eALQYq2qbSmukrkgIh/0A= -github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILMqgNeV5jiqR4j+sTuvQNHdf2chuKj1M5k= github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo= -github.com/cockroachdb/pebble v1.1.4 h1:5II1uEP4MyHLDnsrbv/EZ36arcb9Mxg3n+owhZ3GrG8= -github.com/cockroachdb/pebble v1.1.4/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= +github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= +github.com/cockroachdb/pebble/v2 v2.1.7 h1:hFQnbsniSWg9BVcNKMuaUufYPiVXY6uJvaY9grbQ9+U= +github.com/cockroachdb/pebble/v2 v2.1.7/go.mod h1:JhU5cqqYkr2BdsBHbZhRZOryAtfhcV3eNI/oBcbrxWc= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258 h1:IJ+uNItEm0qx9FE2AgIc1PMsCUtk8nbSIzhQE1t5GWw= +github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -44,6 +54,8 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/getsentry/sentry-go v0.31.1 h1:ELVc0h7gwyhnXHDouXkhqTFSO5oslsRDk0++eyE0KJ4= github.com/getsentry/sentry-go v0.31.1/go.mod h1:CYNcMMz73YigoHljQRG+qPF+eMq8gG72XcGN/p71BAY= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -64,8 +76,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e h1:4bw4WeyTYPp0smaXiJZCNnLrvVBqirQVreixayXezGc= +github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o= @@ -92,6 +104,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/linxGnu/grocksdb v1.9.8 h1:vOIKv9/+HKiqJAElJIEYv3ZLcihRxyP7Suu/Mu8Dxjs= github.com/linxGnu/grocksdb v1.9.8/go.mod h1:C3CNe9UYc9hlEM2pC82AqiGS3LRW537u9LFV4wIZuHk= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= diff --git a/pebble.go b/pebble.go index 18c4cec..f8bda31 100644 --- a/pebble.go +++ b/pebble.go @@ -2,10 +2,12 @@ package db import ( "bytes" + "context" "fmt" "path/filepath" + "strings" - "github.com/cockroachdb/pebble" + "github.com/cockroachdb/pebble/v2" ) func init() { @@ -28,12 +30,43 @@ func NewPebbleDB(name string, dir string) (*PebbleDB, error) { return NewPebbleDBWithOpts(name, dir, opts) } +// withUpgradeHint says what to do about a database this build will not open. +// +// pebble's own report is accurate and useless in the same breath: it names the +// format and says it is no longer supported, and stops there. The way out is +// not something an operator can work out from that, because the obvious move +// -- run the new build and let it migrate -- is the one thing that cannot +// work. The version is rejected before any migration would run, so nothing on +// this side of the upgrade can repair it. Only the release before this one +// can, and it does so just by being started. +// +// These stores are opened before the application's, so this is the message a +// node that upgraded out of order stops on, whatever the app database says. +// +// Matched on the text because pebble raises it with errors.Newf, with no +// sentinel to compare against. Both wordings it uses carry these two phrases; +// missing the match costs the hint, not the error. +func withUpgradeHint(err error) error { + msg := err.Error() + if !strings.Contains(msg, "format major version") || + !strings.Contains(msg, "no longer supported") { + return err + } + return fmt.Errorf("%w"+ + "\n\nthis database predates the pebble format this build requires, and"+ + "\nthis build cannot upgrade it: the version is refused before any"+ + "\nmigration would run. Start the node once on the previous release,"+ + "\nwhich moves the format forward as it opens, then return to this one."+ + "\nNothing has been changed on disk -- pebble stops before it writes.", + err) +} + func NewPebbleDBWithOpts(name string, dir string, opts *pebble.Options) (*PebbleDB, error) { dbPath := filepath.Join(dir, name+".db") opts.EnsureDefaults() p, err := pebble.Open(dbPath, opts) if err != nil { - return nil, err + return nil, withUpgradeHint(err) } return &PebbleDB{ db: p, @@ -131,7 +164,7 @@ func (db *PebbleDB) Compact(start, end []byte) (err error) { // In case the start and end keys are the same // pebbleDB will throw an error that it cannot compact. if start != nil && end != nil { - return db.db.Compact(start, end, true) + return db.db.Compact(context.Background(), start, end, true) } iter, err := db.db.NewIter(nil) if err != nil { @@ -149,7 +182,7 @@ func (db *PebbleDB) Compact(start, end []byte) (err error) { if end == nil && iter.Last() { end = append(end, iter.Key()...) } - return db.db.Compact(start, end, true) + return db.db.Compact(context.Background(), start, end, true) } // Close implements DB. diff --git a/pebble_test.go b/pebble_test.go index 0d06be1..3ac5fb4 100644 --- a/pebble_test.go +++ b/pebble_test.go @@ -1,14 +1,81 @@ package db import ( + "errors" "fmt" "os" + "path/filepath" "testing" + "github.com/cockroachdb/pebble/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// TestPebbleLegacyFormatSaysWhatToDo covers the one failure an operator meets +// by upgrading in the wrong order. These stores open before the application's, +// so this is where a node that skipped the ratchet release stops -- and the +// move that would normally follow, letting the new build migrate it, is +// exactly the one that cannot work. +// +// A legacy store is one carrying a CURRENT file, which is what pebble looks +// for; writing one is enough to reach the check without a v1 build to make a +// real one. +func TestPebbleLegacyFormatSaysWhatToDo(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "blockstore.db") + require.NoError(t, os.MkdirAll(dbPath, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dbPath, "CURRENT"), []byte("MANIFEST-000001\n"), 0o644)) + + _, err := NewPebbleDB("blockstore", dir) + require.Error(t, err) + t.Logf("operator sees:\n%v", err) + + require.ErrorContains(t, err, "format major version") + require.ErrorContains(t, err, "previous release") + require.ErrorContains(t, err, "cannot upgrade it") + require.ErrorContains(t, err, "Nothing has been changed on disk") +} + +// TestPebbleOtherOpenErrorsAreUntouched: the hint is for one failure, and must +// not be pinned to every other way an open can go wrong. +func TestPebbleOtherOpenErrorsAreUntouched(t *testing.T) { + orig := errors.New("pebble: some other trouble entirely") + require.Equal(t, orig, withUpgradeHint(orig)) +} + +// v1FormatNewest is the highest format major version pebble v1 can open. It is +// the ceiling on going back: a database carried above it can be read by this +// build and by nothing that came before it. +const v1FormatNewest = 16 + +// TestPebbleStaysAtTheVersionV1CanRead covers the FormatMajorVersion that +// NewPebbleDB deliberately does not set. Naming FormatNewest there is a +// one-line change that reads as an improvement, and it would take every +// database this build opens to a version the release before it cannot, with +// no error and no way back. Nothing else here would notice. +func TestPebbleStaysAtTheVersionV1CanRead(t *testing.T) { + dir := t.TempDir() + + db, err := NewPebbleDB("blockstore", dir) + require.NoError(t, err) + vers := db.DB().FormatMajorVersion() + require.NoError(t, db.Set([]byte("key"), []byte("value"))) + require.NoError(t, db.Close()) + + require.Equal(t, pebble.FormatMinSupported, vers, + "a new database should sit at the lowest version this build supports") + require.LessOrEqual(t, uint64(vers), uint64(v1FormatNewest), + "this is above what pebble v1 opens, so a rollback would be refused") + + again, err := NewPebbleDB("blockstore", dir) + require.NoError(t, err) + defer again.Close() + require.Equal(t, vers, again.DB().FormatMajorVersion(), + "reopening moved the format, so the way back closes on its own") +} + func TestPebbleDBBackend(t *testing.T) { name := fmt.Sprintf("test_%x", randStr(12)) dir := os.TempDir()