diff --git a/dependencies.sh b/dependencies.sh index 80f1c3ba69..ebcafe4221 100755 --- a/dependencies.sh +++ b/dependencies.sh @@ -121,6 +121,7 @@ SYSTEM_PACKAGES="build-essential \ libcurl4-openssl-dev \ libhiredis-dev \ libjemalloc-dev \ + libxxhash-dev \ pkg-config \ patchelf" diff --git a/mooncake-common/etcd/etcd_wrapper.go b/mooncake-common/etcd/etcd_wrapper.go index e57ae53628..3c8b8113ac 100644 --- a/mooncake-common/etcd/etcd_wrapper.go +++ b/mooncake-common/etcd/etcd_wrapper.go @@ -4,34 +4,76 @@ package main #include #include #include + +// Trampolines to invoke C/C++ function pointers safely from Go via cgo. +// NOTE: Calling a C function pointer by converting it to a Go func is undefined +// and can crash. Always go through a C helper like these. + +#ifndef MOONCAKE_ETCD_CALLBACK_TRAMPOLINES +#define MOONCAKE_ETCD_CALLBACK_TRAMPOLINES + +typedef void (*watch_cb_t)(void* ctx, + const char* key, size_t keySize, + const char* value, size_t valueSize, + int eventType, + long long modRev); + +static inline void call_watch_cb(void* func, + void* ctx, + const char* key, size_t keySize, + const char* value, size_t valueSize, + int eventType, + long long modRev) { + ((watch_cb_t)func)(ctx, key, keySize, value, valueSize, eventType, modRev); +} + +#endif // MOONCAKE_ETCD_CALLBACK_TRAMPOLINES */ import "C" import ( "context" + "encoding/json" "strings" "sync" "time" + "unsafe" clientv3 "go.etcd.io/etcd/client/v3" ) +// prefixWatchInfo stores cancel function and callback context for a prefix watch +type prefixWatchInfo struct { + cancel context.CancelFunc + callbackContext unsafe.Pointer + // done is closed when the watch goroutine fully exits (no more callbacks). + done chan struct{} +} + // Use different etcd client so they are not affected by each other, // and can be configured separately. var ( // etcd client for transform engine - globalClient *clientv3.Client - globalMutex sync.Mutex - globalRefCount int + globalClient *clientv3.Client + globalMutex sync.Mutex + globalRefCount int // etcd client for store - storeClient *clientv3.Client - storeMutex sync.Mutex + storeClient *clientv3.Client + storeMutex sync.Mutex // keep alive contexts for store - storeKeepAliveCtx = make(map[int64]context.CancelFunc) - storeKeepAliveMutex sync.Mutex + storeKeepAliveCtx = make(map[int64]context.CancelFunc) + storeKeepAliveMutex sync.Mutex // watch contexts for store - storeWatchCtx = make(map[string]context.CancelFunc) - storeWatchMutex sync.Mutex + storeWatchCtx = make(map[string]context.CancelFunc) + storeWatchMutex sync.Mutex + // watch contexts for prefix watch + storePrefixWatchCtx = make(map[string]prefixWatchInfo) + storePrefixWatchMutex sync.Mutex + // Valid callback contexts - track which C++ objects are still alive + // When a watch is cancelled, we remove the context from this map + // Before calling callback, we check if context is still valid + validCallbackContexts = make(map[unsafe.Pointer]bool) + validCallbackContextMutex sync.RWMutex ) //export NewEtcdClient @@ -43,30 +85,30 @@ func NewEtcdClient(endpoints *C.char, errMsg **C.char) int { return 0 } - MaxMsgSize := 32*1024*1024 - endpointStr := C.GoString(endpoints) - // Support multiple endpoints separated by comma or semicolon - // Normalize separators to semicolon first, then split - endpointStr = strings.ReplaceAll(endpointStr, ",", ";") - parts := strings.Split(endpointStr, ";") - var validEndpoints []string - for _, ep := range parts { - ep = strings.TrimSpace(ep) - if ep != "" { - validEndpoints = append(validEndpoints, ep) - } - } - if len(validEndpoints) == 0 { - *errMsg = C.CString("no valid endpoints provided") - return -1 - } - - cli, err := clientv3.New(clientv3.Config{ - Endpoints: validEndpoints, - DialTimeout: 5 * time.Second, - MaxCallSendMsgSize: MaxMsgSize, - MaxCallRecvMsgSize: MaxMsgSize, - }) + MaxMsgSize := 32 * 1024 * 1024 + endpointStr := C.GoString(endpoints) + // Support multiple endpoints separated by comma or semicolon + // Normalize separators to semicolon first, then split + endpointStr = strings.ReplaceAll(endpointStr, ",", ";") + parts := strings.Split(endpointStr, ";") + var validEndpoints []string + for _, ep := range parts { + ep = strings.TrimSpace(ep) + if ep != "" { + validEndpoints = append(validEndpoints, ep) + } + } + if len(validEndpoints) == 0 { + *errMsg = C.CString("no valid endpoints provided") + return -1 + } + + cli, err := clientv3.New(clientv3.Config{ + Endpoints: validEndpoints, + DialTimeout: 5 * time.Second, + MaxCallSendMsgSize: MaxMsgSize, + MaxCallRecvMsgSize: MaxMsgSize, + }) if err != nil { *errMsg = C.CString(err.Error()) @@ -159,11 +201,14 @@ func NewStoreEtcdClient(endpoints *C.char, errMsg **C.char) int { } endpointStr := C.GoString(endpoints) + // Support multiple endpoints separated by comma or semicolon. + endpointStr = strings.ReplaceAll(endpointStr, ",", ";") endpointList := strings.Split(endpointStr, ";") - + // Filter out any empty strings that might result from splitting var validEndpoints []string for _, ep := range endpointList { + ep = strings.TrimSpace(ep) if ep != "" { validEndpoints = append(validEndpoints, ep) } @@ -235,37 +280,37 @@ func EtcdStoreGrantLeaseWrapper(ttl int64, leaseId *int64, errMsg **C.char) int //export EtcdStoreCreateWithLeaseWrapper func EtcdStoreCreateWithLeaseWrapper(key *C.char, keySize C.int, value *C.char, valueSize C.int, leaseId int64, revisionId *int64, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - k := C.GoStringN(key, keySize) - v := C.GoStringN(value, valueSize) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - // Create a transaction - txn := storeClient.Txn(ctx) - - // Only put the key if it does not exist - resp, err := txn.If(clientv3.Compare(clientv3.CreateRevision(k), "=", 0)). - Then(clientv3.OpPut(k, v, clientv3.WithLease(clientv3.LeaseID(leaseId)))). - Commit() - - if err != nil { - *errMsg = C.CString(err.Error()) - return -1 - } - - // If the key already existed, resp.Succeeded will be false - // If we created the key, resp.Succeeded will be true - if resp.Succeeded { - *revisionId = resp.Header.Revision - return 0; - } else { - *errMsg = C.CString("etcd transaction failed") - return -2 - } + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + v := C.GoStringN(value, valueSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Create a transaction + txn := storeClient.Txn(ctx) + + // Only put the key if it does not exist + resp, err := txn.If(clientv3.Compare(clientv3.CreateRevision(k), "=", 0)). + Then(clientv3.OpPut(k, v, clientv3.WithLease(clientv3.LeaseID(leaseId)))). + Commit() + + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + // If the key already existed, resp.Succeeded will be false + // If we created the key, resp.Succeeded will be true + if resp.Succeeded { + *revisionId = resp.Header.Revision + return 0 + } else { + *errMsg = C.CString("etcd transaction failed") + return -2 + } } /* @@ -274,77 +319,77 @@ func EtcdStoreCreateWithLeaseWrapper(key *C.char, keySize C.int, value *C.char, * other than the one we want to delete. In that case, that context will * be deleted before being cancelled and will not be able to be cancelled * anymore. -*/ + */ func cancelAndDeleteWatch(k string) int { - storeWatchMutex.Lock() - defer storeWatchMutex.Unlock() - - if cancel, exists := storeWatchCtx[k]; exists { - cancel() - delete(storeWatchCtx, k) - return 0 - } + storeWatchMutex.Lock() + defer storeWatchMutex.Unlock() + + if cancel, exists := storeWatchCtx[k]; exists { + cancel() + delete(storeWatchCtx, k) + return 0 + } return -1 } //export EtcdStoreWatchUntilDeletedWrapper func EtcdStoreWatchUntilDeletedWrapper(key *C.char, keySize C.int, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - k := C.GoStringN(key, keySize) - - // Create a context with cancel function - ctx, cancel := context.WithCancel(context.Background()) - - // Store the cancel function - storeWatchMutex.Lock() - if _, exists := storeWatchCtx[k]; exists { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storeWatchMutex.Lock() + if _, exists := storeWatchCtx[k]; exists { storeWatchMutex.Unlock() - *errMsg = C.CString("This key is already being watched") - return -1 - } - storeWatchCtx[k] = cancel - storeWatchMutex.Unlock() + *errMsg = C.CString("This key is already being watched") + return -1 + } + storeWatchCtx[k] = cancel + storeWatchMutex.Unlock() // Make sure to delete from the map before returning defer cancelAndDeleteWatch(k) - // Start watching the key - watchChan := storeClient.Watch(ctx, k) - - // Wait for the key to be deleted - for { - select { - case watchResp, ok := <-watchChan: - if !ok { - // Channel closed unexpectedly - *errMsg = C.CString("watch channel closed unexpectedly") - return -1 - } - for _, event := range watchResp.Events { - if event.Type == clientv3.EventTypeDelete { - // Clean up the context when done - return 0 - } - } - case <-ctx.Done(): - // Context was cancelled + // Start watching the key + watchChan := storeClient.Watch(ctx, k) + + // Wait for the key to be deleted + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + // Channel closed unexpectedly + *errMsg = C.CString("watch channel closed unexpectedly") + return -1 + } + for _, event := range watchResp.Events { + if event.Type == clientv3.EventTypeDelete { + // Clean up the context when done + return 0 + } + } + case <-ctx.Done(): + // Context was cancelled *errMsg = C.CString("watch context cancelled") - return -2 - } - } + return -2 + } + } } //export EtcdStoreCancelWatchWrapper func EtcdStoreCancelWatchWrapper(key *C.char, keySize C.int, errMsg **C.char) int { - k := C.GoStringN(key, keySize) - if cancelAndDeleteWatch(k) == -1 { - *errMsg = C.CString("no watch context found for the given key") - return -1 - } - return 0 + k := C.GoStringN(key, keySize) + if cancelAndDeleteWatch(k) == -1 { + *errMsg = C.CString("no watch context found for the given key") + return -1 + } + return 0 } /* @@ -353,76 +398,746 @@ func EtcdStoreCancelWatchWrapper(key *C.char, keySize C.int, errMsg **C.char) in * other than the one we want to delete. In that case, that context will * be deleted before being cancelled and will not be able to be cancelled * anymore. -*/ + */ func cancelAndDeleteKeepAlive(leaseId int64) int { - storeKeepAliveMutex.Lock() - defer storeKeepAliveMutex.Unlock() - - if cancel, exists := storeKeepAliveCtx[leaseId]; exists { - cancel() - delete(storeKeepAliveCtx, leaseId) - return 0 - } + storeKeepAliveMutex.Lock() + defer storeKeepAliveMutex.Unlock() + + if cancel, exists := storeKeepAliveCtx[leaseId]; exists { + cancel() + delete(storeKeepAliveCtx, leaseId) + return 0 + } return -1 } //export EtcdStoreKeepAliveWrapper func EtcdStoreKeepAliveWrapper(leaseId int64, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - - // Create a context with cancel function - ctx, cancel := context.WithCancel(context.Background()) - - // Store the cancel function - storeKeepAliveMutex.Lock() + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storeKeepAliveMutex.Lock() if _, exists := storeKeepAliveCtx[leaseId]; exists { storeKeepAliveMutex.Unlock() - *errMsg = C.CString("This lease id is already being kept alive") - return -1 - } - storeKeepAliveCtx[leaseId] = cancel - storeKeepAliveMutex.Unlock() + *errMsg = C.CString("This lease id is already being kept alive") + return -1 + } + storeKeepAliveCtx[leaseId] = cancel + storeKeepAliveMutex.Unlock() // Make sure to delete from the map before returning - defer cancelAndDeleteKeepAlive(leaseId) - - // Start keep alive - keepAliveChan, err := storeClient.KeepAlive(ctx, clientv3.LeaseID(leaseId)) - if err != nil { - *errMsg = C.CString(err.Error()) - return -1 - } - - // Wait for keep alive responses - for { - select { - case resp, ok := <-keepAliveChan: - if !ok { - *errMsg = C.CString("keep alive channel closed") - return -1 - } - if resp == nil { - *errMsg = C.CString("keep alive response is nil") - return -1 - } - // Keep alive successful, continue - case <-ctx.Done(): + defer cancelAndDeleteKeepAlive(leaseId) + + // Start keep alive + keepAliveChan, err := storeClient.KeepAlive(ctx, clientv3.LeaseID(leaseId)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + // Wait for keep alive responses + for { + select { + case resp, ok := <-keepAliveChan: + if !ok { + *errMsg = C.CString("keep alive channel closed") + return -1 + } + if resp == nil { + *errMsg = C.CString("keep alive response is nil") + return -1 + } + // Keep alive successful, continue + case <-ctx.Done(): // Context cancelled *errMsg = C.CString("keep alive context cancelled") - return -2 - } - } + return -2 + } + } } //export EtcdStoreCancelKeepAliveWrapper func EtcdStoreCancelKeepAliveWrapper(leaseId int64, errMsg **C.char) int { - if cancelAndDeleteKeepAlive(leaseId) == -1 { - *errMsg = C.CString("no keep alive context found for the given lease ID") - return -1 - } - return 0 + if cancelAndDeleteKeepAlive(leaseId) == -1 { + *errMsg = C.CString("no keep alive context found for the given lease ID") + return -1 + } + return 0 +} + +//export EtcdStorePutWrapper +func EtcdStorePutWrapper(key *C.char, keySize C.int, value *C.char, valueSize C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + v := C.GoStringN(value, valueSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := storeClient.Put(ctx, k, v) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + return 0 +} + +// Create key if absent (CAS on CreateRevision==0). +// Return: +// - 0 on success +// - -2 if key already exists +// - -1 on error +// +//export EtcdStoreCreateWrapper +func EtcdStoreCreateWrapper(key *C.char, keySize C.int, value *C.char, valueSize C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + v := C.GoStringN(value, valueSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + txn := storeClient.Txn(ctx) + resp, err := txn.If(clientv3.Compare(clientv3.CreateRevision(k), "=", 0)). + Then(clientv3.OpPut(k, v)). + Commit() + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + if resp.Succeeded { + return 0 + } + *errMsg = C.CString("key already exists") + return -2 +} + +//export EtcdStoreBatchCreateWrapper +func EtcdStoreBatchCreateWrapper(keys **C.char, values **C.char, count C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + + n := int(count) + if n == 0 { + return 0 + } + + // Unsafe casting to access C arrays as Go slices + keyPtrs := (*[1 << 28]*C.char)(unsafe.Pointer(keys))[:n:n] + valPtrs := (*[1 << 28]*C.char)(unsafe.Pointer(values))[:n:n] + + ops := make([]clientv3.Op, 0, n) + cmps := make([]clientv3.Cmp, 0, n) + for i := 0; i < n; i++ { + k := C.GoString(keyPtrs[i]) + v := C.GoString(valPtrs[i]) + ops = append(ops, clientv3.OpPut(k, v)) + // Ensure none of the keys exist + cmps = append(cmps, clientv3.Compare(clientv3.CreateRevision(k), "=", 0)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Use Txn to ensure atomicity of the batch + resp, err := storeClient.Txn(ctx).If(cmps...).Then(ops...).Commit() + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + if !resp.Succeeded { + *errMsg = C.CString("transaction failed: one or more keys already exist") + return -2 + } + return 0 +} + +//export EtcdStoreGetWithPrefixWrapper +func EtcdStoreGetWithPrefixWrapper(prefix *C.char, prefixSize C.int, keys **C.char, keySizes **C.int, values **C.char, valueSizes **C.int, count *C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := storeClient.Get(ctx, p, clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + if len(resp.Kvs) == 0 { + *count = 0 + return 0 + } + + // Allocate arrays for keys and values + keyCount := len(resp.Kvs) + *count = C.int(keyCount) + + // Allocate memory for arrays + keysArray := (*[1 << 30]*C.char)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof((*C.char)(nil))))) + keySizesArray := (*[1 << 30]C.int)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof(C.int(0))))) + valuesArray := (*[1 << 30]*C.char)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof((*C.char)(nil))))) + valueSizesArray := (*[1 << 30]C.int)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof(C.int(0))))) + + for i, kv := range resp.Kvs { + keysArray[i] = C.CString(string(kv.Key)) + keySizesArray[i] = C.int(len(kv.Key)) + valuesArray[i] = C.CString(string(kv.Value)) + valueSizesArray[i] = C.int(len(kv.Value)) + } + + *keys = (*C.char)(unsafe.Pointer(keysArray)) + *keySizes = (*C.int)(unsafe.Pointer(keySizesArray)) + *values = (*C.char)(unsafe.Pointer(valuesArray)) + *valueSizes = (*C.int)(unsafe.Pointer(valueSizesArray)) + + return 0 +} + +//export EtcdStoreGetRangeAsJsonWrapper +func EtcdStoreGetRangeAsJsonWrapper(startKey *C.char, startKeySize C.int, endKey *C.char, endKeySize C.int, limit C.int, outJson **C.char, outJsonSize *C.int, revisionId *C.longlong, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + start := C.GoStringN(startKey, startKeySize) + end := C.GoStringN(endKey, endKeySize) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + opts := []clientv3.OpOption{ + clientv3.WithRange(end), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), + } + if limit > 0 { + opts = append(opts, clientv3.WithLimit(int64(limit))) + } + resp, err := storeClient.Get(ctx, start, opts...) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + if resp != nil && resp.Header != nil { + *revisionId = C.longlong(resp.Header.Revision) + } else { + *revisionId = 0 + } + + type kvPair struct { + Key string `json:"key"` + Value string `json:"value"` + } + kvs := make([]kvPair, 0, len(resp.Kvs)) + for _, kv := range resp.Kvs { + kvs = append(kvs, kvPair{Key: string(kv.Key), Value: string(kv.Value)}) + } + b, jerr := json.Marshal(kvs) + if jerr != nil { + *errMsg = C.CString(jerr.Error()) + return -1 + } + + *outJson = C.CString(string(b)) + *outJsonSize = C.int(len(b)) + return 0 +} + +//export EtcdStoreGetFirstKeyWithPrefixWrapper +func EtcdStoreGetFirstKeyWithPrefixWrapper(prefix *C.char, prefixSize C.int, firstKey **C.char, firstKeySize *C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + resp, err := storeClient.Get(ctx, p, clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), clientv3.WithLimit(1)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + if len(resp.Kvs) == 0 { + *errMsg = C.CString("no key found with prefix") + return -2 + } + kv := resp.Kvs[0] + *firstKey = C.CString(string(kv.Key)) + *firstKeySize = C.int(len(kv.Key)) + return 0 +} + +//export EtcdStoreGetLastKeyWithPrefixWrapper +func EtcdStoreGetLastKeyWithPrefixWrapper(prefix *C.char, prefixSize C.int, lastKey **C.char, lastKeySize *C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + resp, err := storeClient.Get( + ctx, p, + clientv3.WithPrefix(), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortDescend), + clientv3.WithLimit(1), + ) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + if len(resp.Kvs) == 0 { + *errMsg = C.CString("no key found with prefix") + return -2 + } + kv := resp.Kvs[0] + *lastKey = C.CString(string(kv.Key)) + *lastKeySize = C.int(len(kv.Key)) + return 0 +} + +//export EtcdStoreDeleteRangeWrapper +func EtcdStoreDeleteRangeWrapper(startKey *C.char, startKeySize C.int, endKey *C.char, endKeySize C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + start := C.GoStringN(startKey, startKeySize) + end := C.GoStringN(endKey, endKeySize) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := storeClient.Delete(ctx, start, clientv3.WithRange(end)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + return 0 +} + +//export EtcdStoreWatchWithPrefixFromRevisionWrapper +func EtcdStoreWatchWithPrefixFromRevisionWrapper(prefix *C.char, prefixSize C.int, startRevision C.longlong, callbackContext unsafe.Pointer, callbackFunc unsafe.Pointer, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + if callbackFunc == nil { + *errMsg = C.CString("callback function is nil") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + + ctx, cancel := context.WithCancel(context.Background()) + + storePrefixWatchMutex.Lock() + if _, exists := storePrefixWatchCtx[p]; exists { + storePrefixWatchMutex.Unlock() + *errMsg = C.CString("This prefix is already being watched") + return -1 + } + doneCh := make(chan struct{}) + storePrefixWatchCtx[p] = prefixWatchInfo{ + cancel: cancel, + callbackContext: callbackContext, + done: doneCh, + } + storePrefixWatchMutex.Unlock() + + // Register callback context as valid + validCallbackContextMutex.Lock() + validCallbackContexts[callbackContext] = true + validCallbackContextMutex.Unlock() + + go func(doneCh chan struct{}) { + defer func() { + // Unregister callback context when goroutine exits + validCallbackContextMutex.Lock() + delete(validCallbackContexts, callbackContext) + validCallbackContextMutex.Unlock() + + // Remove watch entry and signal completion + storePrefixWatchMutex.Lock() + delete(storePrefixWatchCtx, p) + storePrefixWatchMutex.Unlock() + close(doneCh) + }() + + opts := []clientv3.OpOption{clientv3.WithPrefix()} + if startRevision > 0 { + opts = append(opts, clientv3.WithRev(int64(startRevision))) + } + watchChan := storeClient.Watch(ctx, p, opts...) + + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + // Channel closed. Check if context was cancelled. + // If cancelled, don't call callback as C++ object may be destroyed. + select { + case <-ctx.Done(): + // Context was cancelled, just return without calling callback + return + default: + // Check if callback context is still valid + validCallbackContextMutex.RLock() + _, stillValid := validCallbackContexts[callbackContext] + validCallbackContextMutex.RUnlock() + + if !stillValid { + // Context is no longer valid, skip callback + return + } + + // Channel closed unexpectedly (not cancelled). Notify C++ watcher to reconnect. + func() { + defer func() { + if r := recover(); r != nil { + // C++ callback caused panic (likely object destroyed) + // Remove from valid contexts + validCallbackContextMutex.Lock() + delete(validCallbackContexts, callbackContext) + validCallbackContextMutex.Unlock() + } + }() + // Call the C callback function via C trampoline (safe ABI) + C.call_watch_cb(callbackFunc, callbackContext, nil, 0, nil, 0, C.int(2) /*WATCH_BROKEN*/, C.longlong(0)) + }() + return + } + } + if watchResp.Err() != nil { + // Watch error. Check if context was cancelled before calling callback. + select { + case <-ctx.Done(): + // Context was cancelled, just return without calling callback + return + default: + // Check if callback context is still valid + validCallbackContextMutex.RLock() + _, stillValid := validCallbackContexts[callbackContext] + validCallbackContextMutex.RUnlock() + + if !stillValid { + // Context is no longer valid, skip callback + return + } + + // Watch error (not cancelled). Notify C++ watcher to reconnect. + func() { + defer func() { + if r := recover(); r != nil { + // C++ callback caused panic (likely object destroyed) + // Remove from valid contexts + validCallbackContextMutex.Lock() + delete(validCallbackContexts, callbackContext) + validCallbackContextMutex.Unlock() + } + }() + // Call the C callback function via C trampoline (safe ABI) + C.call_watch_cb(callbackFunc, callbackContext, nil, 0, nil, 0, C.int(2) /*WATCH_BROKEN*/, C.longlong(0)) + }() + return + } + } + + // Use response-level revision as a more stable resume point. + // (It can be >= individual event's ModRevision.) + // Note: watchResp.Header is a value type, not a pointer, so we can directly access it. + respRev := int64(0) + if watchResp.Header.Revision > 0 { + respRev = watchResp.Header.Revision + } + + for _, event := range watchResp.Events { + // Check if context was cancelled before processing each event + select { + case <-ctx.Done(): + // Context was cancelled, stop processing events + return + default: + // Continue processing + } + + keyStr := string(event.Kv.Key) + keyPtr := C.CString(keyStr) + keySize := C.size_t(len(keyStr)) + + var valuePtr *C.char + var valueSize C.size_t + var eventType C.int + + if event.Type == clientv3.EventTypePut { + eventType = C.int(0) + valueStr := string(event.Kv.Value) + valuePtr = C.CString(valueStr) + valueSize = C.size_t(len(valueStr)) + } else if event.Type == clientv3.EventTypeDelete { + eventType = C.int(1) + valuePtr = nil + valueSize = 0 + } + + modRev := C.longlong(0) + if event.Kv != nil { + evRev := event.Kv.ModRevision + if respRev > evRev { + evRev = respRev + } + modRev = C.longlong(evRev) + } else if respRev > 0 { + modRev = C.longlong(respRev) + } + + // Check context again before calling callback + select { + case <-ctx.Done(): + // Context was cancelled, free allocated memory and return + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + return + default: + // Check if callback context is still valid before calling + // This prevents calling callback after C++ object is destroyed + validCallbackContextMutex.RLock() + _, stillValid := validCallbackContexts[callbackContext] + validCallbackContextMutex.RUnlock() + + if !stillValid { + // Context is no longer valid (object destroyed), skip callback + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + return + } + + // Double-check context wasn't cancelled between check and call + select { + case <-ctx.Done(): + // Context was cancelled, free memory and return + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + return + default: + // Final check: verify context is still valid immediately before calling + // This minimizes the time window between check and call + validCallbackContextMutex.RLock() + _, stillValid := validCallbackContexts[callbackContext] + validCallbackContextMutex.RUnlock() + + if !stillValid { + // Context was invalidated between previous check and now, skip callback + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + return + } + + // Call callback with panic recovery to prevent crash if C++ object is destroyed + func() { + defer func() { + if r := recover(); r != nil { + // C++ callback caused panic (likely object destroyed) + // Remove from valid contexts to prevent future callbacks + validCallbackContextMutex.Lock() + delete(validCallbackContexts, callbackContext) + validCallbackContextMutex.Unlock() + } + }() + // Callback signature: + // void cb(void* ctx, char* key, size_t keySize, char* value, size_t valueSize, int eventType, long long modRev) + // Call the C callback function via C trampoline (safe ABI) + C.call_watch_cb(callbackFunc, callbackContext, keyPtr, keySize, valuePtr, valueSize, eventType, modRev) + }() + } + } + + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + } + case <-ctx.Done(): + return + } + } + }(doneCh) + + return 0 +} + +func cancelAndDeletePrefixWatch(p string) int { + // NOTE: We intentionally do NOT delete the prefix entry here. + // The watch goroutine owns deletion + closing `done`, so callers can Wait safely. + storePrefixWatchMutex.Lock() + watchInfo, exists := storePrefixWatchCtx[p] + storePrefixWatchMutex.Unlock() + + if !exists { + return -1 + } + + // CRITICAL: Invalidate callback context first, then cancel watch. + validCallbackContextMutex.Lock() + delete(validCallbackContexts, watchInfo.callbackContext) + validCallbackContextMutex.Unlock() + + watchInfo.cancel() + return 0 +} + +//export EtcdStoreWaitWatchWithPrefixStoppedWrapper +func EtcdStoreWaitWatchWithPrefixStoppedWrapper(prefix *C.char, prefixSize C.int, timeoutMs C.int, errMsg **C.char) int { + p := C.GoStringN(prefix, prefixSize) + storePrefixWatchMutex.Lock() + watchInfo, exists := storePrefixWatchCtx[p] + storePrefixWatchMutex.Unlock() + + // If there is no watch, it's already stopped (idempotent). + if !exists { + return 0 + } + + timeout := time.Duration(timeoutMs) * time.Millisecond + if timeout <= 0 { + timeout = 5000 * time.Millisecond + } + + select { + case <-watchInfo.done: + return 0 + case <-time.After(timeout): + if errMsg != nil { + *errMsg = C.CString("timeout waiting for prefix watch to stop") + } + return -1 + } +} + +//export EtcdStoreCancelWatchWithPrefixWrapper +func EtcdStoreCancelWatchWithPrefixWrapper(prefix *C.char, prefixSize C.int, errMsg **C.char) int { + p := C.GoStringN(prefix, prefixSize) + // Idempotent cancel: callers may cancel pre-emptively before starting a watch. + // If no context exists, treat it as success. + _ = cancelAndDeletePrefixWatch(p) + // Intentionally does not wait; use EtcdStoreWaitWatchWithPrefixStoppedWrapper. + _ = errMsg + return 0 +} + +// EtcdStoreSnapshotTxnWrapper atomically: +// 1. Writes snapshot data key-value pair +// 2. Writes snapshot sequence_id key-value pair +// 3. Updates the "latest" pointer to the new snapshot +// 4. Deletes the old snapshot data if oldSnapshotId is provided +// +//export EtcdStoreSnapshotTxnWrapper +func EtcdStoreSnapshotTxnWrapper( + snapshotDataKey *C.char, snapshotDataKeySize C.int, + snapshotDataValue *C.char, snapshotDataValueSize C.int, + snapshotSeqKey *C.char, snapshotSeqKeySize C.int, + snapshotSeqValue *C.char, snapshotSeqValueSize C.int, + latestKey *C.char, latestKeySize C.int, + latestValue *C.char, latestValueSize C.int, + oldSnapshotDataKey *C.char, oldSnapshotDataKeySize C.int, + oldSnapshotSeqKey *C.char, oldSnapshotSeqKeySize C.int, + errMsg **C.char) int { + + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + + // Convert C strings to Go strings + dataKey := C.GoStringN(snapshotDataKey, snapshotDataKeySize) + dataValue := C.GoStringN(snapshotDataValue, snapshotDataValueSize) + seqKey := C.GoStringN(snapshotSeqKey, snapshotSeqKeySize) + seqValue := C.GoStringN(snapshotSeqValue, snapshotSeqValueSize) + latKey := C.GoStringN(latestKey, latestKeySize) + latValue := C.GoStringN(latestValue, latestValueSize) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Build the transaction operations + ops := []clientv3.Op{ + clientv3.OpPut(dataKey, dataValue), + clientv3.OpPut(seqKey, seqValue), + clientv3.OpPut(latKey, latValue), + } + + // Add delete operations for old snapshot if provided + if oldSnapshotDataKeySize > 0 { + oldDataKey := C.GoStringN(oldSnapshotDataKey, oldSnapshotDataKeySize) + ops = append(ops, clientv3.OpDelete(oldDataKey)) + } + if oldSnapshotSeqKeySize > 0 { + oldSeqKey := C.GoStringN(oldSnapshotSeqKey, oldSnapshotSeqKeySize) + ops = append(ops, clientv3.OpDelete(oldSeqKey)) + } + + // Execute the transaction (unconditional - no If clause) + _, err := storeClient.Txn(ctx).Then(ops...).Commit() + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + return 0 +} + +// EtcdStoreDeleteSnapshotWrapper deletes a snapshot's data and sequence_id keys. +// +//export EtcdStoreDeleteSnapshotWrapper +func EtcdStoreDeleteSnapshotWrapper( + snapshotDataKey *C.char, snapshotDataKeySize C.int, + snapshotSeqKey *C.char, snapshotSeqKeySize C.int, + errMsg **C.char) int { + + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + + dataKey := C.GoStringN(snapshotDataKey, snapshotDataKeySize) + seqKey := C.GoStringN(snapshotSeqKey, snapshotSeqKeySize) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + ops := []clientv3.Op{ + clientv3.OpDelete(dataKey), + clientv3.OpDelete(seqKey), + } + + _, err := storeClient.Txn(ctx).Then(ops...).Commit() + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + return 0 } func main() {} diff --git a/mooncake-store/conf/master.yaml b/mooncake-store/conf/master.yaml index 24e73752d3..8821177903 100644 --- a/mooncake-store/conf/master.yaml +++ b/mooncake-store/conf/master.yaml @@ -17,7 +17,7 @@ etcd_endpoints: "http://localhost:2379" root_fs_dir: "" cluster_id: "mooncake_cluster" memory_allocator: "offset" -client_live_ttl_sec: 60 +client_live_ttl_sec: 300 # Increased from 60 to 300 seconds (5 minutes) to prevent client expiration enable_http_metadata_server: false http_metadata_server_host: "0.0.0.0" diff --git a/mooncake-store/include/etcd_helper.h b/mooncake-store/include/etcd_helper.h index 1f272142ac..cdfea74c13 100644 --- a/mooncake-store/include/etcd_helper.h +++ b/mooncake-store/include/etcd_helper.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include "types.h" @@ -50,6 +52,16 @@ class EtcdHelper { EtcdLeaseId lease_id, EtcdRevisionId& revision_id); + /* + * @brief Batch create key-value pairs in a single transaction. + * Fails if any key already exists. + * @param keys: The vector of keys. + * @param values: The vector of values. + * @return: Error code. + */ + static ErrorCode BatchCreate(const std::vector& keys, + const std::vector& values); + /* * @brief Grant a lease from the etcd. * @param lease_ttl: The ttl of the lease, in seconds. @@ -90,6 +102,160 @@ class EtcdHelper { */ static ErrorCode CancelKeepAlive(EtcdLeaseId lease_id); + /* + * @brief Put a key-value pair to etcd. + * @param key: The key to put. + * @param key_size: The size of the key in bytes. + * @param value: The value to put. + * @param value_size: The size of the value in bytes. + * @return: Error code. + */ + static ErrorCode Put(const char* key, const size_t key_size, + const char* value, const size_t value_size); + + /* + * @brief Create a key-value pair in etcd if the key does not already exist. + * This is implemented via etcd transaction (CreateRevision == 0). + * @return: OK on success; ETCD_TRANSACTION_FAIL if key already exists. + */ + static ErrorCode Create(const char* key, const size_t key_size, + const char* value, const size_t value_size); + + /* + * @brief Get all key-value pairs with a given prefix. + * @param prefix: The prefix to search for. + * @param prefix_size: The size of the prefix in bytes. + * @param keys: Output param, vector of keys. + * @param values: Output param, vector of values. + * @return: Error code. + */ + static ErrorCode GetWithPrefix(const char* prefix, const size_t prefix_size, + std::vector& keys, + std::vector& values); + + /* + * @brief Range get in etcd and return result as a JSON array string. + * This avoids complex cross-language memory management for key/value arrays. + * @param start_key: Start key (inclusive). + * @param start_key_size: Size in bytes. + * @param end_key: End key (exclusive). + * @param end_key_size: Size in bytes. + * @param limit: Maximum number of kvs to return (0 means no limit). + * @param json: Output JSON string, format: [{"key":"...","value":"..."}] + * @param revision_id: Output etcd revision of this read (resp.Header.Revision). + */ + static ErrorCode GetRangeAsJson(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size, + size_t limit, + std::string& json, + EtcdRevisionId& revision_id); + + /* + * @brief Get the first key with a given prefix (sorted by key). + * @param prefix: The prefix to search for. + * @param prefix_size: The size of the prefix in bytes. + * @param first_key: Output param, the first key found. + * @return: Error code. ETCD_KEY_NOT_EXIST if no key found. + */ + static ErrorCode GetFirstKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& first_key); + + /* + * @brief Get the last key with a given prefix (sorted by key descending). + * @param prefix: The prefix to search for. + * @param prefix_size: The size of the prefix in bytes. + * @param last_key: Output param, the last key found. + * @return: Error code. ETCD_KEY_NOT_EXIST if no key found. + */ + static ErrorCode GetLastKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& last_key); + + /* + * @brief Delete a range of keys from etcd. + * @param start_key: The start key (inclusive). + * @param start_key_size: The size of the start key in bytes. + * @param end_key: The end key (exclusive). + * @param end_key_size: The size of the end key in bytes. + * @return: Error code. + */ + static ErrorCode DeleteRange(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size); + + /* + * @brief Watch all keys with a given prefix from a specific etcd revision. + * Callback includes `mod_revision` for precise resume. + * (Implementation may pass max(event.ModRevision, watchResp.Header.Revision).) + * @param callback_func: void cb(void* ctx, const char* key, size_t key_size, + * const char* value, size_t value_size, + * int event_type, int64_t mod_revision) + * event_type: 0=PUT, 1=DELETE, 2=WATCH_BROKEN (watch ended; reconnect) + */ + static ErrorCode WatchWithPrefixFromRevision( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)); + + /* + * @brief Cancel watching a prefix. + * @param prefix: The prefix to stop watching. + * @param prefix_size: The size of the prefix in bytes. + * @return: Error code. + */ + static ErrorCode CancelWatchWithPrefix(const char* prefix, + const size_t prefix_size); + + /* + * @brief Wait until a prefix watch goroutine fully exits (no more callbacks). + * This should be used after CancelWatchWithPrefix to avoid shutdown races. + * @param timeout_ms: Wait timeout in milliseconds. + */ + static ErrorCode WaitWatchWithPrefixStopped(const char* prefix, + const size_t prefix_size, + int timeout_ms); + + /* + * @brief Atomically save a snapshot to etcd and delete old snapshot. + * This is a transaction that: + * 1. Writes snapshot data (metadata JSON) + * 2. Writes snapshot sequence_id + * 3. Updates the "latest" pointer + * 4. Deletes old snapshot data and seq_id (if provided) + * @param snapshot_data_key: Key for snapshot data + * @param snapshot_data_value: JSON serialized snapshot data + * @param snapshot_seq_key: Key for snapshot sequence_id + * @param snapshot_seq_value: String representation of sequence_id + * @param latest_key: Key for "latest" pointer + * @param latest_value: New snapshot ID to set as latest + * @param old_snapshot_data_key: Key of old snapshot data to delete (empty to skip) + * @param old_snapshot_seq_key: Key of old snapshot seq to delete (empty to skip) + * @return: Error code. + */ + static ErrorCode SaveSnapshotTxn( + const std::string& snapshot_data_key, + const std::string& snapshot_data_value, + const std::string& snapshot_seq_key, + const std::string& snapshot_seq_value, + const std::string& latest_key, + const std::string& latest_value, + const std::string& old_snapshot_data_key, + const std::string& old_snapshot_seq_key); + + /* + * @brief Delete a snapshot's data and sequence_id keys. + * @param snapshot_data_key: Key for snapshot data + * @param snapshot_seq_key: Key for snapshot sequence_id + * @return: Error code. + */ + static ErrorCode DeleteSnapshot(const std::string& snapshot_data_key, + const std::string& snapshot_seq_key); + private: // Variables that are used to ensure the etcd client // is only connected once. diff --git a/mooncake-store/include/etcd_oplog_store.h b/mooncake-store/include/etcd_oplog_store.h new file mode 100644 index 0000000000..83ea2f9f15 --- /dev/null +++ b/mooncake-store/include/etcd_oplog_store.h @@ -0,0 +1,232 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "types.h" + +namespace mooncake { + +/** + * @brief Store for OpLog entries in etcd. + * + * This class is responsible for writing OpLog entries to etcd and reading them back. + * OpLog entries are stored with keys in the format: + * /oplog/{cluster_id}/{sequence_id} + * + * The latest sequence_id is also stored at: + * /oplog/{cluster_id}/latest + */ +class EtcdOpLogStore { + public: + /** + * @brief Constructor. + * @param cluster_id: The cluster ID for this OpLog store. + * @param enable_latest_seq_batch_update: Whether to start background thread + * to batch-update `/latest`. Readers (Standby) should set this to false + * to avoid unnecessary thread creation. + */ + explicit EtcdOpLogStore(const std::string& cluster_id, + bool enable_latest_seq_batch_update = false); + + /** + * @brief Write an OpLog entry to etcd. + * @param entry: The OpLog entry to write. + * @param sync: If true, wait until the entry is persisted to etcd. + * If false, buffer it and return immediately (Group Commit). + * @return: Error code. + */ + ErrorCode WriteOpLog(const OpLogEntry& entry, bool sync = true); + + /** + * @brief Read an OpLog entry from etcd by sequence_id. + * @param sequence_id: The sequence ID of the entry to read. + * @param entry: Output param, the OpLog entry. + * @return: Error code. + */ + ErrorCode ReadOpLog(uint64_t sequence_id, OpLogEntry& entry); + + /** + * @brief Read OpLog entries starting from a given sequence_id. + * @param start_sequence_id: The starting sequence ID (exclusive). + * @param limit: Maximum number of entries to read (default: 1000). + * @param entries: Output param, vector of OpLog entries. + * @return: Error code. + */ + ErrorCode ReadOpLogSince(uint64_t start_sequence_id, size_t limit, + std::vector& entries); + + // Like ReadOpLogSince, but also returns the etcd revision for consistent + // "read then watch(from revision+1)" startup. + ErrorCode ReadOpLogSinceWithRevision(uint64_t start_sequence_id, size_t limit, + std::vector& entries, + EtcdRevisionId& revision_id); + + /** + * @brief Get the latest sequence_id from etcd. + * @param sequence_id: Output param, the latest sequence_id. + * @return: Error code. ETCD_KEY_NOT_EXIST if no OpLog exists yet. + */ + ErrorCode GetLatestSequenceId(uint64_t& sequence_id); + + // Stronger (than `/latest`) best-effort query: return the maximum existing + // sequence_id by scanning etcd keys under /oplog/{cluster_id}/ with + // descending key order. + // Return ETCD_KEY_NOT_EXIST if no OpLog exists yet. + ErrorCode GetMaxSequenceId(uint64_t& sequence_id); + + /** + * @brief Update the latest sequence_id in etcd. + * @param sequence_id: The latest sequence_id to update. + * @return: Error code. + */ + ErrorCode UpdateLatestSequenceId(uint64_t sequence_id); + + /** + * @brief Record the sequence_id corresponding to a snapshot. + * @param snapshot_id: The snapshot ID. + * @param sequence_id: The sequence_id at which the snapshot was taken. + * @return: Error code. + */ + ErrorCode RecordSnapshotSequenceId(const std::string& snapshot_id, + uint64_t sequence_id); + + /** + * @brief Get the sequence_id for a given snapshot. + * @param snapshot_id: The snapshot ID. + * @param sequence_id: Output param, the sequence_id. + * @return: Error code. ETCD_KEY_NOT_EXIST if snapshot not found. + */ + ErrorCode GetSnapshotSequenceId(const std::string& snapshot_id, + uint64_t& sequence_id); + + /** + * @brief Clean up OpLog entries before a given sequence_id. + * @param before_sequence_id: All entries with sequence_id < before_sequence_id + * will be deleted. + * @return: Error code. + */ + ErrorCode CleanupOpLogBefore(uint64_t before_sequence_id); + + /** + * @brief Destructor - stops batch update thread. + */ + ~EtcdOpLogStore(); + + private: + /** + * @brief Build the etcd key for an OpLog entry. + * @param sequence_id: The sequence ID. + * @return: The etcd key. + */ + std::string BuildOpLogKey(uint64_t sequence_id) const; + + /** + * @brief Build the etcd key for the latest sequence_id. + * @return: The etcd key. + */ + std::string BuildLatestKey() const; + + /** + * @brief Build the etcd key for a snapshot sequence_id. + * @param snapshot_id: The snapshot ID. + * @return: The etcd key. + */ + std::string BuildSnapshotKey(const std::string& snapshot_id) const; + + // Best-effort: find the minimum existing OpLog sequence_id in etcd. + // Used for robust cleanup (Scheme 3) so we don't rely on a persisted + // "cleaned_upto" marker. + std::optional GetMinSequenceId() const; + + // Best-effort: find the maximum existing OpLog sequence_id in etcd. + std::optional GetMaxSequenceIdInternal() const; + + /** + * @brief Serialize an OpLogEntry to JSON string. + * @param entry: The OpLog entry to serialize. + * @return: The JSON string. + */ + std::string SerializeOpLogEntry(const OpLogEntry& entry) const; + + /** + * @brief Deserialize a JSON string to OpLogEntry. + * @param json_str: The JSON string. + * @param entry: Output param, the OpLog entry. + * @return: true if successful, false otherwise. + */ + bool DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) const; + + /** + * @brief Batch update thread function. + * Periodically updates latest_sequence_id in etcd. + */ + void BatchUpdateThread(); + + /** + * @brief Trigger immediate batch update if threshold is reached. + */ + void TriggerBatchUpdateIfNeeded(); + + /** + * @brief Perform the actual batch update to etcd. + */ + void DoBatchUpdate(); + + std::string cluster_id_; + static constexpr const char* kOpLogPrefix = "/oplog/"; + static constexpr const char* kLatestSuffix = "/latest"; + static constexpr const char* kSnapshotPrefix = "/oplog/"; + static constexpr const char* kSnapshotSuffix = "/snapshot/"; + + // Batch update mechanism for latest_sequence_id + const bool enable_latest_seq_batch_update_{false}; + std::atomic pending_latest_seq_id_{0}; + std::atomic pending_count_{0}; + std::atomic batch_update_running_{false}; + std::mutex batch_update_mutex_; + std::thread batch_update_thread_; + std::chrono::steady_clock::time_point last_update_time_; + + // Batch update configuration + static constexpr size_t kBatchSize = 100; // Update every 100 entries + static constexpr int kBatchIntervalMs = 1000; // Or every 1 second + + // Group Commit / Batch Write support + struct BatchEntry { + std::string key; + std::string value; + uint64_t sequence_id; + bool is_sync; // Track if entry requires sync + }; + + void BatchWriteThread(); + void FlushBatch(); + + mutable std::mutex batch_mutex_; + std::deque pending_batch_; + std::condition_variable cv_batch_updated_; // Notify background thread + std::condition_variable cv_sync_completed_; // Notify sync waiters + std::atomic batch_write_running_{false}; + std::thread batch_write_thread_; + std::atomic last_persisted_seq_id_{0}; + + // Configs for OpLog batching + static constexpr size_t kOpLogBatchSizeLimit = 1 * 1024 * 1024; // 1MB payload limit (soft) + static constexpr size_t kOpLogBatchCountLimit = 100; // 100 entries + static constexpr int kOpLogBatchTimeoutMs = 10; // 10ms max latency for Async + static constexpr int kSyncWaitTimeoutMs = 3000; // 3s timeout for Sync writes + static constexpr int kFlushRetryCount = 3; // Retries for failed flush + static constexpr int kFlushRetryIntervalMs = 50; // Retry interval +}; + +} // namespace mooncake diff --git a/mooncake-store/include/etcd_snapshot_provider.h b/mooncake-store/include/etcd_snapshot_provider.h new file mode 100644 index 0000000000..d620b464f0 --- /dev/null +++ b/mooncake-store/include/etcd_snapshot_provider.h @@ -0,0 +1,105 @@ +#pragma once + +#include +#include +#include +#include + +#include "snapshot_provider.h" +#include "metadata_store.h" +#include "types.h" + +namespace mooncake { + +/** + * @brief EtcdSnapshotProvider is an implementation of SnapshotProvider that + * stores and loads metadata snapshots from etcd. + * + * Key structure: + * /snapshot/{cluster_id}/latest -> snapshot_id (string) + * /snapshot/{cluster_id}/{snapshot_id}/data -> JSON serialized metadata + * /snapshot/{cluster_id}/{snapshot_id}/seq -> sequence_id (string) + * + * This implementation: + * - Uses etcd transactions to atomically save new snapshot and delete old one + * - Only keeps the latest snapshot (deletes previous snapshot when saving new one) + * - Serializes metadata to JSON format + */ +class EtcdSnapshotProvider : public SnapshotProvider { + public: + /** + * @brief Constructor. + * @param cluster_id: The cluster ID for this snapshot provider. + */ + explicit EtcdSnapshotProvider(const std::string& cluster_id); + + /** + * @brief Load the latest available snapshot for `cluster_id`. + * @param cluster_id: The cluster ID (must match the one in constructor). + * @param snapshot_id: Output param, opaque identifier of the snapshot. + * @param snapshot_sequence_id: Output param, global OpLog sequence_id at snapshot boundary. + * @param snapshot: Output param, full metadata baseline as key -> StandbyObjectMetadata. + * @return true on success, false if no snapshot available or on error. + */ + bool LoadLatestSnapshot( + const std::string& cluster_id, std::string& snapshot_id, + uint64_t& snapshot_sequence_id, + std::vector>& snapshot) override; + + /** + * @brief Save a snapshot to etcd. + * Atomically writes new snapshot data and deletes the previous snapshot. + * @param snapshot_id: Unique identifier for this snapshot (e.g., timestamp). + * @param sequence_id: The OpLog sequence_id at which this snapshot was taken. + * @param metadata: The metadata entries to save (key -> StandbyObjectMetadata). + * @return ErrorCode::OK on success, error code on failure. + */ + ErrorCode SaveSnapshot( + const std::string& snapshot_id, + uint64_t sequence_id, + const std::vector>& metadata); + + /** + * @brief Get the current latest snapshot ID without loading data. + * @param snapshot_id: Output param, the latest snapshot ID. + * @return true if a snapshot exists, false otherwise. + */ + bool GetLatestSnapshotId(std::string& snapshot_id) const; + + /** + * @brief Dump current snapshot status for debugging. + * Logs the latest snapshot ID, sequence_id, and data size. + * Does NOT load the full metadata. + */ + void DumpSnapshotStatus() const; + + /** + * @brief Get snapshot statistics without loading full data. + * @param snapshot_id: Output - latest snapshot ID (empty if none). + * @param sequence_id: Output - sequence_id of latest snapshot (0 if none). + * @param data_size_bytes: Output - size of snapshot data in bytes (0 if none). + * @return true if a snapshot exists, false otherwise. + */ + bool GetSnapshotStats(std::string& snapshot_id, + uint64_t& sequence_id, + size_t& data_size_bytes) const; + + private: + // Build etcd key paths + std::string BuildLatestKey() const; + std::string BuildDataKey(const std::string& snapshot_id) const; + std::string BuildSeqKey(const std::string& snapshot_id) const; + + // Serialize metadata to JSON + std::string SerializeMetadata( + const std::vector>& metadata) const; + + // Deserialize metadata from JSON + bool DeserializeMetadata( + const std::string& json, + std::vector>& metadata) const; + + std::string cluster_id_; +}; + +} // namespace mooncake diff --git a/mooncake-store/include/ha_helper.h b/mooncake-store/include/ha_helper.h index 897ba53a5c..a117cb29ef 100644 --- a/mooncake-store/include/ha_helper.h +++ b/mooncake-store/include/ha_helper.h @@ -3,12 +3,15 @@ #include +#include +#include #include #include #include -#include "types.h" +#include "hot_standby_service.h" #include "master_config.h" +#include "types.h" namespace mooncake { @@ -23,7 +26,18 @@ class MasterViewHelper { public: MasterViewHelper(const MasterViewHelper&) = delete; MasterViewHelper& operator=(const MasterViewHelper&) = delete; - MasterViewHelper(); + // cluster_id source of truth: + // - If provided, use it. + // - Else fall back to env MC_STORE_CLUSTER_ID. + // - Else fall back to DEFAULT_CLUSTER_ID. + explicit MasterViewHelper(const std::string& cluster_id = std::string()); + + // Update cluster_id (and derived master_view_key_) before using the helper. + // This is mainly for client-side etcd:// usage where cluster_id may be passed + // via connection string. + void SetClusterId(const std::string& cluster_id); + + const std::string& GetMasterViewKey() const { return master_view_key_; } /* * @brief Connect to the etcd cluster. This function should be called at @@ -60,6 +74,7 @@ class MasterViewHelper { ViewVersionId& version); private: + void BuildMasterViewKeyFromClusterId(const std::string& cluster_id); std::string master_view_key_; }; @@ -77,10 +92,27 @@ class MasterServiceSupervisor { ~MasterServiceSupervisor(); private: + /** + * @brief Start HotStandbyService when there is an existing leader + * @param mv_helper MasterViewHelper instance + * @param current_leader Current leader address + */ + void StartStandbyService(MasterViewHelper& mv_helper, + const std::string& current_leader); + + /** + * @brief Stop HotStandbyService + */ + void StopStandbyService(); + // coro_rpc server thread std::thread server_thread_; MasterServiceSupervisorConfig config_; + + // HotStandbyService for standby mode + std::unique_ptr standby_service_; + std::atomic standby_running_{false}; }; } // namespace mooncake diff --git a/mooncake-store/include/ha_metric_manager.h b/mooncake-store/include/ha_metric_manager.h new file mode 100644 index 0000000000..63a4f3a6af --- /dev/null +++ b/mooncake-store/include/ha_metric_manager.h @@ -0,0 +1,205 @@ +#pragma once + +#include +#include +#include +#include + +#include "ylt/metric/counter.hpp" +#include "ylt/metric/gauge.hpp" +#include "ylt/metric/histogram.hpp" + +namespace mooncake { + +/** + * @brief Singleton manager for High Availability (HA) related metrics. + * + * This class provides metrics for monitoring the health and performance + * of the OpLog replication system, including: + * - OpLog sequence tracking + * - Standby replication lag + * - Error counters (checksum failures, skipped entries) + * - Performance histograms (etcd write latency) + * - Queue sizes (pending mutations) + */ +class HAMetricManager { + public: + // --- Singleton Access --- + static HAMetricManager& instance(); + + HAMetricManager(const HAMetricManager&) = delete; + HAMetricManager& operator=(const HAMetricManager&) = delete; + HAMetricManager(HAMetricManager&&) = delete; + HAMetricManager& operator=(HAMetricManager&&) = delete; + + // ========== OpLog Sequence Metrics (Gauge) ========== + + /** + * @brief Set the latest OpLog sequence ID on Primary + */ + void set_oplog_last_sequence_id(int64_t seq_id); + int64_t get_oplog_last_sequence_id(); + + /** + * @brief Set the Standby's applied sequence ID + */ + void set_oplog_applied_sequence_id(int64_t seq_id); + int64_t get_oplog_applied_sequence_id(); + + /** + * @brief Set the replication lag (entries behind Primary) + */ + void set_oplog_standby_lag(int64_t lag); + int64_t get_oplog_standby_lag(); + + /** + * @brief Set the number of pending (out-of-order) entries in OpLogApplier + */ + void set_oplog_pending_entries(int64_t count); + int64_t get_oplog_pending_entries(); + + /** + * @brief Set the pending mutation queue size (retry queue) + */ + void set_pending_mutation_queue_size(int64_t size); + int64_t get_pending_mutation_queue_size(); + + // ========== Error Counters ========== + + /** + * @brief Increment counter for skipped OpLog entries + */ + void inc_oplog_skipped_entries(int64_t val = 1); + int64_t get_oplog_skipped_entries_total(); + + /** + * @brief Increment counter for checksum verification failures + */ + void inc_oplog_checksum_failures(int64_t val = 1); + int64_t get_oplog_checksum_failures_total(); + + /** + * @brief Increment counter for gap resolve attempts + */ + void inc_oplog_gap_resolve_attempts(int64_t val = 1); + int64_t get_oplog_gap_resolve_attempts_total(); + + /** + * @brief Increment counter for successful gap resolves + */ + void inc_oplog_gap_resolve_success(int64_t val = 1); + int64_t get_oplog_gap_resolve_success_total(); + + /** + * @brief Increment counter for etcd write failures + */ + void inc_oplog_etcd_write_failures(int64_t val = 1); + int64_t get_oplog_etcd_write_failures_total(); + + /** + * @brief Increment counter for etcd write retries + */ + void inc_oplog_etcd_write_retries(int64_t val = 1); + int64_t get_oplog_etcd_write_retries_total(); + + /** + * @brief Increment counter for watch disconnections + */ + void inc_oplog_watch_disconnections(int64_t val = 1); + int64_t get_oplog_watch_disconnections_total(); + + /** + * @brief Increment counter for successfully applied OpLog entries + */ + void inc_oplog_applied_entries(int64_t val = 1); + int64_t get_oplog_applied_entries_total(); + /** + * @brief Increase the total number of OpLog batch commits (Group Commit) + */ + void inc_oplog_batch_commits(int64_t count = 1); + int64_t get_oplog_batch_commits_total(); + + /** + * @brief Increase the number of sync batch commits (triggered by DELETE/Sync ops) + */ + void inc_oplog_sync_batch_commits(int64_t count = 1); + int64_t get_oplog_sync_batch_commits_total(); + + // ========== Latency Histograms ========== + + /** + * @brief Record etcd write latency in microseconds + */ + void observe_oplog_etcd_write_latency_us(int64_t latency_us); + + /** + * @brief Record OpLog apply latency in microseconds + */ + void observe_oplog_apply_latency_us(int64_t latency_us); + + // ========== State Machine Metrics ========== + + /** + * @brief Set the current Standby state (as integer for Prometheus) + * @param state_value Integer representation of StandbyState + */ + void set_standby_state(int64_t state_value); + int64_t get_standby_state(); + + /** + * @brief Increment state transition counter + */ + void inc_state_transitions(int64_t val = 1); + int64_t get_state_transitions_total(); + + // ========== Serialization ========== + + /** + * @brief Serializes all HA metrics into Prometheus text format. + * @return A string containing the metrics in Prometheus format. + */ + std::string serialize_metrics(); + + /** + * @brief Generates a concise, human-readable summary of HA metrics. + * @return A string containing the formatted summary. + */ + std::string get_summary_string(); + + private: + // --- Private Constructor & Destructor --- + HAMetricManager(); + ~HAMetricManager() = default; + + // --- Metric Members --- + + // OpLog Sequence Gauges + ylt::metric::gauge_t oplog_last_sequence_id_; + ylt::metric::gauge_t oplog_applied_sequence_id_; + ylt::metric::gauge_t oplog_standby_lag_; + ylt::metric::gauge_t oplog_pending_entries_; + ylt::metric::gauge_t pending_mutation_queue_size_; + + // Error Counters + ylt::metric::counter_t oplog_skipped_entries_total_; + ylt::metric::counter_t oplog_checksum_failures_total_; + ylt::metric::counter_t oplog_gap_resolve_attempts_total_; + ylt::metric::counter_t oplog_gap_resolve_success_total_; + ylt::metric::counter_t oplog_etcd_write_failures_total_; + ylt::metric::counter_t oplog_etcd_write_retries_total_; + ylt::metric::counter_t oplog_watch_disconnections_total_; + ylt::metric::counter_t oplog_applied_entries_total_; + ylt::metric::counter_t oplog_batch_commits_total_; + ylt::metric::counter_t oplog_sync_batch_commits_total_; + + // Latency Histograms (buckets in microseconds: 100us, 500us, 1ms, 5ms, 10ms, 50ms, 100ms, 500ms, 1s) + ylt::metric::histogram_t oplog_etcd_write_latency_us_; + ylt::metric::histogram_t oplog_apply_latency_us_; + + // State Machine + ylt::metric::gauge_t standby_state_; + ylt::metric::counter_t state_transitions_total_; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/hot_standby_service.h b/mooncake-store/include/hot_standby_service.h new file mode 100644 index 0000000000..29544775ed --- /dev/null +++ b/mooncake-store/include/hot_standby_service.h @@ -0,0 +1,244 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "metadata_store.h" +#include "oplog_applier.h" +#include "oplog_manager.h" +#include "oplog_watcher.h" +#include "snapshot_provider.h" +#include "standby_state_machine.h" +#include "types.h" + +namespace mooncake { + +// Forward declarations +class MasterService; +class ReplicationStream; + +/** + * @brief Configuration for HotStandbyService + */ +struct HotStandbyConfig { + std::string standby_id; + std::string primary_address; + uint32_t replication_port{0}; + uint32_t verification_interval_sec{30}; + uint32_t max_replication_lag_entries{1000}; + bool enable_verification{true}; + + // Snapshot bootstrap (optional): + // If provided, Standby will try to load a snapshot first, then replay OpLog + // from snapshot_sequence_id. + bool enable_snapshot_bootstrap{false}; +}; + +/** + * @brief Sync status information for HotStandbyService + */ +struct StandbySyncStatus { + uint64_t applied_seq_id{0}; + uint64_t primary_seq_id{0}; + uint64_t lag_entries{0}; + std::chrono::milliseconds lag_time{0}; + bool is_syncing{false}; + bool is_connected{false}; + StandbyState state{StandbyState::STOPPED}; + std::chrono::milliseconds time_in_state{0}; +}; + +/** + * @brief HotStandbyService manages standby replication and promotion + * + * This service runs on Standby Master nodes and is responsible for: + * - Connecting to Primary and receiving OpLog entries + * - Applying OpLog entries to local metadata store + * - Periodically verifying data consistency with Primary + * - Promoting to Primary when elected as new Leader + * + * For now, this is a skeleton implementation without actual network + * communication. The gRPC integration will be added later. + */ +class HotStandbyService { + public: + explicit HotStandbyService(const HotStandbyConfig& config); + ~HotStandbyService(); + + /** + * @brief Start connecting to Primary and begin replication + * @param primary_address Address of the Primary Master (not used with etcd-based sync) + * @param etcd_endpoints Comma-separated etcd endpoints + * @param cluster_id Cluster identifier for OpLog path + * @return ErrorCode::OK on success + */ + ErrorCode Start(const std::string& primary_address, + const std::string& etcd_endpoints, + const std::string& cluster_id); + + /** + * @brief Stop replication and disconnect from Primary + */ + void Stop(); + + /** + * @brief Get current synchronization status + * @return StandbySyncStatus with current sync state + */ + StandbySyncStatus GetSyncStatus() const; + + /** + * @brief Check if standby is ready for promotion + * @return true if replication lag is within threshold + */ + bool IsReadyForPromotion() const; + + /** + * @brief Promote this standby to Primary + * + * This method should be called after successful leader election. + * It returns a MasterService instance initialized with the replicated + * metadata, ready to serve as the new Primary. + * + * @return Unique pointer to MasterService, or nullptr on failure + */ + std::unique_ptr Promote(); + + /** + * @brief Get the number of metadata entries in the local store + */ + size_t GetMetadataCount() const; + + /** + * @brief Get the latest applied sequence ID after promotion + * + * This should be called after Promote() to get the sequence_id + * that the new Primary's OpLogManager should start from. + * + * @return Latest applied sequence ID, or 0 if not available + */ + uint64_t GetLatestAppliedSequenceId() const; + + // Export a point-in-time snapshot of all replicated metadata. + // This is used by MasterServiceSupervisor to initialize the new Primary + // after leader election (fast recovery). + bool ExportMetadataSnapshot( + std::vector>& out) const; + + // Inject a snapshot provider (from external snapshot implementation). + void SetSnapshotProvider(std::unique_ptr provider); + + /** + * @brief Get current state from state machine + */ + StandbyState GetState() const { return state_machine_.GetState(); } + + /** + * @brief Get state machine for monitoring/debugging + */ + const StandbyStateMachine& GetStateMachine() const { return state_machine_; } + + /** + * @brief Callback for OpLogWatcher state changes + * @param event The event to process + */ + void OnWatcherEvent(StandbyEvent event); + + private: + /** + * @brief Main replication loop (runs in background thread) + */ + void ReplicationLoop(); + + /** + * @brief Verification loop (runs in background thread) + */ + void VerificationLoop(); + + /** + * @brief Apply a single OpLog entry to local metadata store + * @param entry The OpLog entry to apply + * @deprecated Use OpLogApplier instead + */ + void ApplyOpLogEntry(const OpLogEntry& entry); + + /** + * @brief Connect to Primary and establish replication stream + * @return true on success, false on failure + */ + bool ConnectToPrimary(); + + /** + * @brief Disconnect from Primary + */ + void DisconnectFromPrimary(); + + /** + * @brief Process a batch of OpLog entries received from Primary + * @param entries Batch of OpLog entries + */ + void ProcessOpLogBatch(const std::vector& entries); + + HotStandbyConfig config_; + + // Simple in-memory metadata store implementation + class StandbyMetadataStore : public MetadataStore { + public: + bool PutMetadata(const std::string& key, + const StandbyObjectMetadata& metadata) override; + bool Put(const std::string& key, + const std::string& payload = std::string()) override; + const StandbyObjectMetadata* GetMetadata(const std::string& key) const override; + bool Remove(const std::string& key) override; + bool Exists(const std::string& key) const override; + size_t GetKeyCount() const override; + + // Snapshot for promotion/restore. + void Snapshot( + std::vector>& out) const; + + private: + mutable std::mutex mutex_; + std::unordered_map store_; + }; + std::unique_ptr metadata_store_; + std::unique_ptr snapshot_provider_{std::make_unique()}; + + // OpLog replication components + std::unique_ptr oplog_applier_; + std::unique_ptr oplog_watcher_; + + // Configuration for etcd-based OpLog sync + std::string etcd_endpoints_; + std::string cluster_id_; + + // Replication state + std::shared_ptr replication_stream_; + std::atomic applied_seq_id_{0}; + std::atomic primary_seq_id_{0}; + + // State machine for managing service lifecycle + StandbyStateMachine state_machine_; + + // Helper methods for state machine + bool IsRunning() const { return state_machine_.IsRunning(); } + bool IsConnected() const { return state_machine_.IsConnected(); } + + // Background threads + std::thread replication_thread_; + std::thread verification_thread_; + + // Synchronization + mutable std::mutex mutex_; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/master_config.h b/mooncake-store/include/master_config.h index e1e486414d..cee94f470c 100644 --- a/mooncake-store/include/master_config.h +++ b/mooncake-store/include/master_config.h @@ -45,6 +45,9 @@ struct MasterConfig { // Storage backend eviction configuration bool enable_disk_eviction; uint64_t quota_bytes; + + // Periodic snapshot configuration (HA only) + uint64_t snapshot_interval_sec; }; class MasterServiceSupervisorConfig { @@ -79,6 +82,7 @@ class MasterServiceSupervisorConfig { uint64_t put_start_release_timeout_sec = DEFAULT_PUT_START_RELEASE_TIMEOUT; bool enable_disk_eviction = true; uint64_t quota_bytes = 0; + uint64_t snapshot_interval_sec = 600; // Default: 10 minutes MasterServiceSupervisorConfig() = default; @@ -120,6 +124,7 @@ class MasterServiceSupervisorConfig { put_start_release_timeout_sec = config.put_start_release_timeout_sec; enable_disk_eviction = config.enable_disk_eviction; quota_bytes = config.quota_bytes; + snapshot_interval_sec = config.snapshot_interval_sec; validate(); } @@ -190,6 +195,7 @@ class WrappedMasterServiceConfig { uint64_t put_start_release_timeout_sec = DEFAULT_PUT_START_RELEASE_TIMEOUT; bool enable_disk_eviction = true; uint64_t quota_bytes = 0; + uint64_t snapshot_interval_sec = 600; // Default: 10 minutes WrappedMasterServiceConfig() = default; @@ -226,6 +232,7 @@ class WrappedMasterServiceConfig { put_start_discard_timeout_sec = config.put_start_discard_timeout_sec; put_start_release_timeout_sec = config.put_start_release_timeout_sec; + snapshot_interval_sec = config.snapshot_interval_sec; } // From MasterServiceSupervisorConfig, enable_ha is set to true @@ -256,6 +263,7 @@ class WrappedMasterServiceConfig { quota_bytes = config.quota_bytes; put_start_discard_timeout_sec = config.put_start_discard_timeout_sec; put_start_release_timeout_sec = config.put_start_release_timeout_sec; + snapshot_interval_sec = config.snapshot_interval_sec; } }; @@ -284,6 +292,7 @@ class MasterServiceConfigBuilder { uint64_t quota_bytes_ = 0; uint64_t put_start_discard_timeout_sec_ = DEFAULT_PUT_START_DISCARD_TIMEOUT; uint64_t put_start_release_timeout_sec_ = DEFAULT_PUT_START_RELEASE_TIMEOUT; + uint64_t snapshot_interval_sec_ = 600; // Default: 10 minutes public: MasterServiceConfigBuilder() = default; @@ -369,6 +378,12 @@ class MasterServiceConfigBuilder { return *this; } + MasterServiceConfigBuilder& set_snapshot_interval_sec( + uint64_t snapshot_interval_sec) { + snapshot_interval_sec_ = snapshot_interval_sec; + return *this; + } + MasterServiceConfig build() const; }; @@ -393,6 +408,7 @@ class MasterServiceConfig { uint64_t put_start_release_timeout_sec = DEFAULT_PUT_START_RELEASE_TIMEOUT; bool enable_disk_eviction = true; uint64_t quota_bytes = 0; + uint64_t snapshot_interval_sec = 600; // Default: 10 minutes MasterServiceConfig() = default; @@ -416,6 +432,7 @@ class MasterServiceConfig { quota_bytes = config.quota_bytes; put_start_discard_timeout_sec = config.put_start_discard_timeout_sec; put_start_release_timeout_sec = config.put_start_release_timeout_sec; + snapshot_interval_sec = config.snapshot_interval_sec; } // Static factory method to create a builder @@ -442,6 +459,7 @@ inline MasterServiceConfig MasterServiceConfigBuilder::build() const { config.put_start_release_timeout_sec = put_start_release_timeout_sec_; config.enable_disk_eviction = enable_disk_eviction_; config.quota_bytes = quota_bytes_; + config.snapshot_interval_sec = snapshot_interval_sec_; return config; } diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 35e728f0c5..847d511d99 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -4,7 +4,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -13,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -25,11 +29,17 @@ #include "master_config.h" #include "rpc_types.h" #include "replica.h" +#include "oplog_manager.h" +#include "metadata_store.h" namespace mooncake { // Forward declarations class AllocationStrategy; class EvictionStrategy; +class BufferAllocatorBase; +class EtcdSnapshotProvider; +struct StandbyObjectMetadata; +// ReplicationService forward declaration removed - using etcd-based OpLog sync instead /* * @brief MasterService is the main class for the master server. @@ -94,6 +104,13 @@ class MasterService { */ auto GetAllKeys() -> tl::expected, ErrorCode>; + // Restore metadata from a Standby snapshot (fast failover). + // NOTE: This is used only on the node that was running HotStandbyService + // right before it was promoted to leader. + void RestoreFromStandbySnapshot( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id); + /** * @brief Fetch all segments, each node has a unique real client with fixed * segment name : segment name, preferred format : {ip}:{port}, bad format : @@ -271,6 +288,14 @@ class MasterService { */ tl::expected GetStorageConfig() const; + /** + * @brief Get OpLogManager reference for external access + * @return Reference to the OpLogManager instance + */ + OpLogManager& GetOpLogManager(); + + // SetReplicationService removed - using etcd-based OpLog sync instead + /** * @brief Mounts a file storage segment into the master. * @param enable_offloading If true, enables offloading (write-to-file). @@ -301,6 +326,21 @@ class MasterService { -> tl::expected; private: + /** + * @brief Helper function to append OpLog entry + * @param type Operation type + * @param key Object key + * @param payload Optional payload data + */ + void AppendOpLogAndNotify(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Durable OpLog append: must succeed (write to etcd) for operations that may + // free/reuse memory (e.g. REMOVE). See OpLogManager::AppendAndPersist. + auto AppendOpLogAndNotifyDurable(OpType type, const std::string& key, + const std::string& payload = std::string()) + -> tl::expected; + // Resolve the key to a sanitized format for storage std::string SanitizeKey(const std::string& key) const; std::string ResolvePath(const std::string& key) const; @@ -477,6 +517,95 @@ class MasterService { } }; + /** + * @brief Serialize ObjectMetadata to JSON string for OpLog payload + * @param metadata The metadata to serialize + * @return JSON string containing the serialized metadata + */ + std::string SerializeMetadataForOpLog(const ObjectMetadata& metadata) const; + + // Serialize metadata but exclude MEMORY replicas. + // Used for eviction: when memory replicas are freed/reused, Standby must not + // keep stale memory descriptors. We persist a PUT_END containing only + // remaining (DISK/LOCAL_DISK) replicas before freeing memory. + std::string SerializeMetadataForOpLogWithoutMemReplicas( + const ObjectMetadata& metadata) const; + + // Serialize metadata from a caller-provided replica descriptor list. + // This is used when we need to persist an updated replica set *before* + // mutating local replicas (which may free/reuse memory). + std::string SerializeMetadataForOpLogFromReplicaDescriptors( + const UUID& client_id, uint64_t size, + const std::vector& replicas) const; + + // Pending durable mutations (etcd write retry queue) + // -------------------------------------------------- + // In HA mode, freeing/reusing MEMORY replicas before Standby observes the + // corresponding OpLog update can cause stale descriptors on Standby. + // If durable etcd write fails, we enqueue a pending mutation and retry + // asynchronously to avoid long-term memory retention. + enum class PendingMutationKind : uint8_t { + EVICT_MEM_REPLICAS = 1, // drop MEMORY replicas; persist PUT_END or REMOVE + CLEAR_ALL_REPLICAS = 2, // remove the whole key; persist REMOVE + CLEAR_REPLICAS_ON_SEGMENT = 3, // remove COMPLETE replicas on segment; persist PUT_END/REMOVE + }; + struct PendingMutation { + PendingMutationKind kind{PendingMutationKind::EVICT_MEM_REPLICAS}; + std::string key; + std::string segment_name; // only for CLEAR_REPLICAS_ON_SEGMENT + // OpLog entry to persist. If sequence_id==0, this is a deferred action and + // the worker will allocate a new OpLogEntry at execution time. + // If sequence_id>0, sequence_id is pre-allocated and MUST be persisted as-is + // (implements: "enqueue time seq_id fixed and smaller"). + OpLogEntry oplog_entry; + uint32_t attempt{0}; + std::chrono::steady_clock::time_point next_retry_at{}; + }; + + void EnqueuePendingMutation(PendingMutation m); + void PendingMutationWorker(); + bool ProcessPendingMutationOnce(PendingMutation& m); + + // Helper for etcd durable write (HA only): + // - Persist a pre-allocated OpLogEntry with small synchronous retries. + // - On failure, enqueue a PendingMutation (caller decides whether to proceed + // with local state changes; we do NOT block per-key). + ErrorCode PersistOpLogEntryWithSyncRetries(const OpLogEntry& entry) const; + void EnqueueRetryOnPersistFailure(const char* ctx, const OpLogEntry& entry, + ErrorCode persist_err, + PendingMutationKind kind, + const std::string& segment_name = std::string()); + + // Higher-level helper that also handles: + // - STORE_USE_ETCD compile-time switch + // - enable_ha_ runtime switch + // + // Behavior: + // - HA + STORE_USE_ETCD: AllocateEntry -> Persist (sync retries) -> enqueue on failure + // - non-HA: Append to in-memory OpLog buffer only + // - HA but STORE_USE_ETCD disabled: no-op (best-effort; see constructor warning) + void AppendOrPersistOrEnqueue(const char* ctx, OpType type, + const std::string& key, + const std::string& payload, + PendingMutationKind kind, + const std::string& segment_name = std::string()); + + // Lazy-payload variant: payload is computed only when needed. + // This is useful to avoid expensive metadata serialization when: + // - HA is enabled but STORE_USE_ETCD is disabled at compile time (no-op), or + // - the branch will not publish OpLog at all. + void AppendOrPersistOrEnqueueLazy( + const char* ctx, OpType type, const std::string& key, + const std::function& payload_factory, + PendingMutationKind kind, + const std::string& segment_name = std::string()); + + // NOTE: + // We intentionally do NOT block subsequent operations for the same key when a + // durable OpLog write fails. Failed entries are retried asynchronously with the + // original pre-allocated sequence_id, and Standby handles gaps via timeout + + // late-arrival policy (apply late REMOVE/PUT_REVOKE, discard late PUT_END). + static constexpr size_t kNumShards = 1024; // Number of metadata shards // Sharded metadata maps and their mutexes @@ -631,11 +760,41 @@ class MasterService { // Segment management SegmentManager segment_manager_; BufferAllocatorType memory_allocator_type_; + + // Keep dummy allocators alive for memory replicas restored from standby. + // AllocatedBuffer stores allocator as weak_ptr; without an owning shared_ptr, + // the allocator would expire immediately and transport_endpoint_ would be lost + // when re-serializing Replica descriptors. + std::unordered_map> + standby_allocator_keepalive_; + + // Operation log manager for hot-standby replication. It records + // state-changing operations so that a standby master can replay them. + OpLogManager oplog_manager_; + + // ReplicationService removed - using etcd-based OpLog sync instead + std::shared_ptr allocation_strategy_; + // Pending durable mutation retry queue (HA only). + std::mutex pending_mutations_mutex_; + std::condition_variable pending_mutations_cv_; + std::deque pending_mutations_; + std::atomic pending_mutations_running_{false}; + std::thread pending_mutations_thread_; + static constexpr size_t kMaxPendingMutations = 10000; // Max queue size to prevent unbounded growth + + // Periodic snapshot thread (HA only). + // This thread periodically takes a metadata snapshot and stores it in etcd. + void SnapshotThreadFunc(); + std::thread snapshot_thread_; + std::atomic snapshot_running_{false}; + std::unique_ptr snapshot_provider_; + // Discarded replicas management const std::chrono::seconds put_start_discard_timeout_sec_; const std::chrono::seconds put_start_release_timeout_sec_; + const uint64_t snapshot_interval_sec_; class DiscardedReplicas { public: DiscardedReplicas() = delete; diff --git a/mooncake-store/include/metadata_store.h b/mooncake-store/include/metadata_store.h new file mode 100644 index 0000000000..1e9066c685 --- /dev/null +++ b/mooncake-store/include/metadata_store.h @@ -0,0 +1,113 @@ +#pragma once + +#include +#include +#include +#include + +#include "replica.h" +#include "types.h" + +namespace mooncake { + +/** + * @brief Metadata structure for Standby to store and restore object information + * + * This structure contains all essential metadata information needed by Standby + * to immediately serve as Primary when promoted. + */ +struct StandbyObjectMetadata { + UUID client_id{0, 0}; + uint64_t size{0}; + std::vector replicas; + // NOTE: Lease information is NOT stored because: + // 1. Standby does not perform eviction, so lease info is not used + // 2. After promotion, new Primary should grant fresh leases, not restore old ones + uint64_t last_sequence_id{0}; // Last OpLog sequence ID that modified this key + + StandbyObjectMetadata() = default; + + // Check if this metadata has valid replicas + bool HasReplicas() const { return !replicas.empty(); } +}; + +/** + * @brief Payload structure for struct_pack serialization (msgpack binary format) + * + * Now uses UUID directly since struct_pack natively supports std::pair. + */ +struct MetadataPayload { + UUID client_id{0, 0}; + uint64_t size{0}; + std::vector replicas; + // NOTE: Lease information removed - not needed by Standby + + YLT_REFL(MetadataPayload, client_id, size, replicas); + + // Convert to StandbyObjectMetadata + StandbyObjectMetadata ToStandbyMetadata(uint64_t sequence_id) const { + StandbyObjectMetadata meta; + meta.client_id = client_id; + meta.size = size; + meta.replicas = replicas; + meta.last_sequence_id = sequence_id; + return meta; + } +}; + +/** + * @brief Abstract interface for metadata storage on Standby + * + * This interface provides basic operations for storing and managing object metadata. + * In a full implementation, this would mirror MasterService's metadata_shards_ structure. + */ +class MetadataStore { + public: + virtual ~MetadataStore() = default; + + /** + * @brief Put or update metadata for a key with structured metadata + * @param key Object key + * @param metadata Structured metadata object + * @return true on success, false on failure + */ + virtual bool PutMetadata(const std::string& key, const StandbyObjectMetadata& metadata) = 0; + + /** + * @brief Put or update metadata for a key (legacy interface for backward compatibility) + * @param key Object key + * @param payload Optional payload data (JSON serialized metadata) + * @return true on success, false on failure + */ + virtual bool Put(const std::string& key, const std::string& payload = std::string()) = 0; + + /** + * @brief Get metadata for a key + * @param key Object key + * @return Pointer to metadata if found, nullptr otherwise + */ + virtual const StandbyObjectMetadata* GetMetadata(const std::string& key) const = 0; + + /** + * @brief Remove metadata for a key + * @param key Object key + * @return true if key was found and removed, false otherwise + */ + virtual bool Remove(const std::string& key) = 0; + + /** + * @brief Check if a key exists + * @param key Object key + * @return true if key exists, false otherwise + */ + virtual bool Exists(const std::string& key) const = 0; + + /** + * @brief Get the count of keys in the store + * @return Number of keys + */ + virtual size_t GetKeyCount() const = 0; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/oplog_applier.h b/mooncake-store/include/oplog_applier.h new file mode 100644 index 0000000000..993ef81f5c --- /dev/null +++ b/mooncake-store/include/oplog_applier.h @@ -0,0 +1,170 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "metadata_store.h" + +namespace mooncake { + +// Forward declaration +class EtcdOpLogStore; + +/** + * @brief Apply OpLog entries to Standby metadata store with ordering guarantee + * + * This class applies OpLog entries to the Standby metadata store, + * ensuring both global and per-key ordering. + */ +class OpLogApplier { + public: + /** + * @brief Constructor + * @param metadata_store Metadata store to apply changes to + * @param cluster_id Cluster ID for accessing etcd OpLog (optional, for requesting missing OpLog) + */ + explicit OpLogApplier(MetadataStore* metadata_store, + const std::string& cluster_id = std::string()); + + /** + * @brief Apply a single OpLog entry (with ordering checks) + * @param entry OpLog entry to apply + * @return true on success, false on failure or ordering violation + */ + bool ApplyOpLogEntry(const OpLogEntry& entry); + + /** + * @brief Apply multiple OpLog entries + * @param entries OpLog entries to apply + * @return Number of successfully applied entries + */ + size_t ApplyOpLogEntries(const std::vector& entries); + + /** + * @brief Get the current sequence ID for a key (DEPRECATED) + * @param key Object key + * @return Always returns 0 - global sequence_id is used for ordering + * @deprecated Use global sequence_id for ordering + */ + uint64_t GetKeySequenceId(const std::string& key) const; + + /** + * @brief Get the expected global sequence ID + * @return Expected global sequence ID + */ + uint64_t GetExpectedSequenceId() const; + + /** + * @brief Recover from a given sequence ID + * @param last_applied_sequence_id Last applied sequence ID + */ + void Recover(uint64_t last_applied_sequence_id); + + /** + * @brief Process pending entries (entries with non-continuous sequence IDs) + * @return Number of entries processed + */ + size_t ProcessPendingEntries(); + + // Promotion helper: + // Try to resolve current gaps ONCE (no waiting) by fetching missing/skipped + // sequence_ids from etcd. If an entry arrives late: + // - REMOVE / PUT_REVOKE: delete the key + // - PUT_END: discard + // + // This is used during Standby promotion so we don't block promotion on gaps, + // but still best-effort clean up potentially stale metadata. + struct GapResolveResult { + size_t attempted{0}; + size_t fetched{0}; + size_t applied_deletes{0}; + }; + GapResolveResult TryResolveGapsOnceForPromotion(size_t max_ids = 1024); + + private: + /** + * @brief Check if the entry's sequence order is valid + * @param entry OpLog entry + * @return true if order is valid, false otherwise + */ + bool CheckSequenceOrder(const OpLogEntry& entry); + + /** + * @brief Apply PUT_END operation + * @param entry OpLog entry + */ + void ApplyPutEnd(const OpLogEntry& entry); + + /** + * @brief Apply PUT_REVOKE operation + * @param entry OpLog entry + */ + void ApplyPutRevoke(const OpLogEntry& entry); + + /** + * @brief Apply REMOVE operation + * @param entry OpLog entry + */ + void ApplyRemove(const OpLogEntry& entry); + + /** + * @brief Request missing OpLog entry from etcd + * @param missing_seq_id Missing sequence ID + * @return true if entry was found and applied, false otherwise + */ + bool RequestMissingOpLog(uint64_t missing_seq_id); + + /** + * @brief Schedule wait for missing entries + * @param missing_seq_id Missing sequence ID + */ + void ScheduleWaitForMissingEntries(uint64_t missing_seq_id); + + MetadataStore* metadata_store_; + + // EtcdOpLogStore for requesting missing OpLog entries (optional) + std::string cluster_id_; + mutable std::mutex etcd_oplog_store_mutex_; + mutable std::unique_ptr etcd_oplog_store_; + + /** + * @brief Get or create EtcdOpLogStore instance (lazy initialization) + * @return Pointer to EtcdOpLogStore, or nullptr if cluster_id is not set + */ + EtcdOpLogStore* GetEtcdOpLogStore() const; + + // Note: key_sequence_map_ has been removed. + // Global sequence_id is sufficient for ordering guarantee. + + // Track pending entries (entries with non-continuous sequence IDs) + mutable std::mutex pending_mutex_; + std::map pending_entries_; + + // Track missing sequence IDs that we're waiting for + std::map missing_sequence_ids_; + + // Sequence IDs we chose to skip (gap-timeout). If the late entry arrives: + // - REMOVE / PUT_REVOKE: delete the key (safe) + // - PUT_END: discard (do not resurrect potentially stale metadata) + std::map skipped_sequence_ids_; + + // Next expected global sequence_id. Read frequently from monitoring thread, + // updated by watch/apply thread. Use atomic to avoid data races. + std::atomic expected_sequence_id_{1}; + + // Constants for missing entry handling + // IMPORTANT: request must happen BEFORE skip, otherwise we will never request. + static constexpr int kMissingEntryRequestSeconds = 1; // request from etcd after 1s + static constexpr int kMissingEntrySkipSeconds = 3; // skip after 3s (avoid global stall) + static constexpr int kMaxPendingEntries = 1000; // Max pending entries before giving up +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h new file mode 100644 index 0000000000..a0ff8e719d --- /dev/null +++ b/mooncake-store/include/oplog_manager.h @@ -0,0 +1,145 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "types.h" + +namespace mooncake { + +// Forward declaration +class EtcdOpLogStore; + +// Operation types for hot-standby replication. +// This is a minimal subset that can be extended later. +enum class OpType : uint8_t { + PUT_END = 1, + PUT_REVOKE = 2, + REMOVE = 3, + // Deprecated: LEASE_RENEW is intentionally not recorded in OpLog in the + // current etcd-based hot-standby design (Standby relies on Primary DELETE operations). + LEASE_RENEW = 4, +}; + +// A single operation log entry. +// Note: Payload contains JSON serialized MetadataPayload (defined in metadata_store.h) +// for PUT_END operations, allowing Standby to restore complete metadata. +struct OpLogEntry { + uint64_t sequence_id{0}; // Monotonically increasing global sequence + uint64_t timestamp_ms{0}; // Logical timestamp in milliseconds + OpType op_type{OpType::PUT_END}; + std::string object_key; // Target object key + std::string payload; // Serialized extra data (optional) + uint32_t checksum{0}; // Checksum of payload (implementation-defined) + uint32_t prefix_hash{0}; // Hash of the entire key (for verification and optimization) +}; + +/** + * @brief In-memory operation log manager. + * + * This class is intentionally simple: it keeps a bounded deque of OpLogEntry + * and provides append / get-since primitives. It can later be extended to + * or to spill to disk if needed. In the new etcd-based design, OpLog will be written to etcd. + */ +class OpLogManager { + public: + OpLogManager(); + + // Set the EtcdOpLogStore for writing OpLog to etcd (optional). + // If not set, OpLog will only be stored in memory buffer. + void SetEtcdOpLogStore(std::shared_ptr etcd_oplog_store); + + // Append a new entry and return the assigned sequence_id. + uint64_t Append(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Allocate a new OpLogEntry with a reserved sequence_id, append it to the + // in-memory buffer, and return the full entry. + // + // IMPORTANT: This will advance last_seq_id_ even if the caller later fails + // to persist it to etcd. This supports "seq pre-allocation" semantics where + // retries use the same (smaller) sequence_id. + OpLogEntry AllocateEntry(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Persist an already-allocated entry to etcd using its sequence_id. + // Does NOT modify sequence counters. + ErrorCode PersistEntryToEtcd(const OpLogEntry& entry) const; + + // Append a new entry and durably persist it to etcd (if EtcdOpLogStore is set). + // + // This is intended for operations that may free/reuse memory (e.g. REMOVE), + // where best-effort replication is unsafe: Standby must observe the DELETE + // before promotion, otherwise it may return stale descriptors that point to + // reused memory and cause silent data corruption. + // + // Design (updated for seq pre-allocation): + // - sequence_id is allocated first and never reused. + // - If etcd write fails, caller may retry PersistEntryToEtcd with the same + // entry (sequence_id fixed and "smaller" than later entries). + tl::expected AppendAndPersist( + OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Get the latest assigned sequence id. Returns 0 if no entry exists. + uint64_t GetLastSequenceId() const; + + // Set the initial sequence ID (used when promoting Standby to Primary). + // This ensures the new Primary's OpLogManager continues from the correct sequence_id. + void SetInitialSequenceId(uint64_t sequence_id); + + // Current number of entries in the buffer. + size_t GetEntryCount() const; + + // Cleanup OpLog entries before the given sequence_id. + // This removes entries from both the in-memory buffer and etcd (if configured). + // Should be called after a snapshot is successfully saved. + // Returns the number of entries cleaned up from memory buffer. + size_t CleanupBefore(uint64_t before_sequence_id); + + // Verify checksum of an OpLogEntry payload. + // Returns true if checksum matches, false otherwise. + // This is public so OpLogWatcher and OpLogApplier can validate entries. + static bool VerifyChecksum(const OpLogEntry& entry); + + // Basic DoS protection for externally sourced OpLog entries (etcd watch / reads). + // Enforce conservative bounds on key/payload sizes before parsing/applying. + static constexpr size_t kMaxObjectKeySize = 4096; // 4 KiB + static constexpr size_t kMaxPayloadSize = 10 * 1024 * 1024; // 10 MiB + + // Validate OpLogEntry key/payload sizes. If invalid, returns false and + // optionally sets a human-readable reason. + static bool ValidateEntrySize(const OpLogEntry& entry, + std::string* reason = nullptr); + + private: + static uint64_t NowMs(); + static uint32_t ComputeChecksum(const std::string& data); + static uint32_t ComputePrefixHash(const std::string& key); + + mutable std::shared_mutex mutex_; + std::deque buffer_; + uint64_t first_seq_id_{1}; // sequence_id of buffer_.front() + uint64_t last_seq_id_{0}; // last assigned sequence_id + + // Note: We removed key_sequence_map_ and key_remove_time_map_. + // Global sequence_id is sufficient for ordering guarantee. + // All operations are applied in sequence_id order, which ensures consistency. + + // Optional etcd OpLog store for persistent storage + std::shared_ptr etcd_oplog_store_; + + // Simple bounds to avoid unbounded memory growth. + static constexpr size_t kMaxBufferEntries_ = 100000; +}; + +} // namespace mooncake + + diff --git a/mooncake-store/include/oplog_watcher.h b/mooncake-store/include/oplog_watcher.h new file mode 100644 index 0000000000..9c430e87be --- /dev/null +++ b/mooncake-store/include/oplog_watcher.h @@ -0,0 +1,158 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "standby_state_machine.h" +#include "types.h" + +namespace mooncake { + +// Forward declaration +class OpLogApplier; + +// Callback type for state events +using WatcherStateCallback = std::function; + +/** + * @brief Watch etcd for OpLog changes and apply them to Standby + * + * This class watches etcd for new OpLog entries and forwards them + * to OpLogApplier for processing. + */ +class OpLogWatcher { + public: + /** + * @brief Constructor + * @param etcd_endpoints Comma-separated etcd endpoints + * @param cluster_id Cluster identifier + * @param applier OpLog applier to process entries + */ + OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier); + + ~OpLogWatcher(); + + /** + * @brief Start watching etcd for OpLog changes + */ + void Start(); + + /** + * @brief Start from a known last-applied sequence_id. + * + * It will read historical OpLogs at a consistent etcd revision, then start + * watch from revision+1 to close the gap between "read" and "watch". + */ + bool StartFromSequenceId(uint64_t start_seq_id); + + /** + * @brief Stop watching + */ + void Stop(); + + /** + * @brief Get the last processed sequence ID + * @return Last processed sequence ID + */ + uint64_t GetLastProcessedSequenceId() const; + + /** + * @brief Set callback for state events + * @param callback Callback function to invoke on state events + */ + void SetStateCallback(WatcherStateCallback callback) { + state_callback_ = std::move(callback); + } + + /** + * @brief Check if watch is healthy + */ + bool IsWatchHealthy() const { return watch_healthy_.load(); } + + private: + /** + * @brief Notify state callback + */ + void NotifyStateEvent(StandbyEvent event) { + if (state_callback_) { + state_callback_(event); + } + } + + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries, + EtcdRevisionId& revision_id); + // Callback includes etcd KV mod_revision for precise resume. + static void WatchCallback(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, int event_type, + int64_t mod_revision); + + /** + * @brief Watch etcd OpLog changes (runs in background thread) + */ + void WatchOpLog(); + + /** + * @brief Process a Watch event + * @param key etcd key + * @param value etcd value (JSON string for PUT events, empty for DELETE events) + * @param event_type Event type (0 = PUT, 1 = DELETE) + */ + void HandleWatchEvent(const std::string& key, const std::string& value, + int event_type); + void HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision); + + /** + * @brief Deserialize OpLogEntry from JSON string + * @param json_str JSON string + * @param entry Output OpLog entry + * @return true on success, false on failure + */ + bool DeserializeOpLogEntry(const std::string& json_str, OpLogEntry& entry); + + /** + * @brief Attempt to reconnect after watch failure + */ + void TryReconnect(); + + /** + * @brief Sync missed OpLog entries after reconnection + * @return true if sync was successful + */ + bool SyncMissedEntries(); + + // Next watch revision (0 means from now). Updated by consistent reads. + std::atomic next_watch_revision_{0}; + + std::string etcd_endpoints_; + std::string cluster_id_; + OpLogApplier* applier_; + std::atomic running_{false}; + std::thread watch_thread_; + std::atomic last_processed_sequence_id_{0}; + + // Error handling and recovery + std::atomic consecutive_errors_{0}; + std::atomic reconnect_count_{0}; + std::atomic watch_healthy_{false}; + + // State callback for notifying HotStandbyService + WatcherStateCallback state_callback_; + + // Constants for error handling + static constexpr int kMaxConsecutiveErrors = 10; + static constexpr int kReconnectDelayMs = 1000; + static constexpr int kMaxReconnectDelayMs = 30000; + static constexpr int kSyncBatchSize = 1000; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/replica.h b/mooncake-store/include/replica.h index 5793e90f96..9905e6701b 100644 --- a/mooncake-store/include/replica.h +++ b/mooncake-store/include/replica.h @@ -375,10 +375,11 @@ inline Replica::Descriptor Replica::get_descriptor() const { desc.descriptor_variant = std::move(disk_desc); } else if (is_local_disk_replica()) { const auto& disk_data = std::get(data_); - LocalDiskDescriptor local_disk_desc; - local_disk_desc.client_id = disk_data.client_id; - local_disk_desc.object_size = disk_data.object_size; - local_disk_desc.transport_endpoint = disk_data.transport_endpoint; + LocalDiskDescriptor local_disk_desc{ + disk_data.client_id, + disk_data.object_size, + disk_data.transport_endpoint, + }; desc.descriptor_variant = std::move(local_disk_desc); } diff --git a/mooncake-store/include/rpc_service.h b/mooncake-store/include/rpc_service.h index 1cf5d7a24f..465a6d5661 100644 --- a/mooncake-store/include/rpc_service.h +++ b/mooncake-store/include/rpc_service.h @@ -4,15 +4,22 @@ #include #include #include +#include #include #include #include #include "master_service.h" +#include "metadata_store.h" #include "types.h" #include "rpc_types.h" #include "master_config.h" +// Forward declaration +namespace mooncake { +// ReplicationService forward declaration removed - using etcd-based OpLog sync instead +} + namespace mooncake { extern const uint64_t kMetricReportIntervalSeconds; @@ -25,6 +32,11 @@ class WrappedMasterService { void init_http_server(); + // Restore metadata and OpLog sequence from a promoted Standby (fast failover). + void RestoreFromStandby( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id); + tl::expected ExistKey(const std::string& key); tl::expected @@ -109,11 +121,15 @@ class WrappedMasterService { const UUID& client_id, const std::vector& keys, const std::vector& metadatas); + // GetReplicationService removed - using etcd-based OpLog sync instead + private: MasterService master_service_; std::thread metric_report_thread_; coro_http::coro_http_server http_server_; std::atomic metric_report_running_; + + // ReplicationService removed - using etcd-based OpLog sync instead }; void RegisterRpcService(coro_rpc::coro_rpc_server& server, diff --git a/mooncake-store/include/snapshot_provider.h b/mooncake-store/include/snapshot_provider.h new file mode 100644 index 0000000000..ae8041511c --- /dev/null +++ b/mooncake-store/include/snapshot_provider.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +#include "metadata_store.h" + +namespace mooncake { + +/** + * @brief SnapshotProvider is an abstraction for loading metadata snapshots. + * + * Assumption: snapshot functionality exists (implemented by another team), but + * may not be synced into this repo yet. We keep Mooncake-store code progressing + * by depending on this narrow interface. + * + * Snapshot semantics for hot-standby: + * - A snapshot represents a consistent metadata baseline at `snapshot_sequence_id`. + * - Standby should: load snapshot -> recover applier to snapshot_sequence_id -> + * replay OpLog entries with sequence_id > snapshot_sequence_id. + */ +class SnapshotProvider { + public: + virtual ~SnapshotProvider() = default; + + // Load the latest available snapshot for `cluster_id`. + // Returns true on success and fills: + // - snapshot_id: opaque identifier (e.g. timestamp/version) + // - snapshot_sequence_id: global OpLog sequence_id at snapshot boundary + // - snapshot: full metadata baseline as key -> StandbyObjectMetadata + virtual bool LoadLatestSnapshot( + const std::string& cluster_id, std::string& snapshot_id, + uint64_t& snapshot_sequence_id, + std::vector>& snapshot) = 0; +}; + +// Default no-op provider: behaves as if "no snapshot available". +class NoopSnapshotProvider final : public SnapshotProvider { + public: + bool LoadLatestSnapshot( + const std::string& /*cluster_id*/, std::string& snapshot_id, + uint64_t& snapshot_sequence_id, + std::vector>& snapshot) override { + snapshot_id.clear(); + snapshot_sequence_id = 0; + snapshot.clear(); + return false; + } +}; + +} // namespace mooncake + + diff --git a/mooncake-store/include/standby_state_machine.h b/mooncake-store/include/standby_state_machine.h new file mode 100644 index 0000000000..649b462566 --- /dev/null +++ b/mooncake-store/include/standby_state_machine.h @@ -0,0 +1,334 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace mooncake { + +/** + * @brief Standby service states + * + * State transition diagram: + * + * ┌─────────┐ + * │ STOPPED │◄──────────────────────────────────────┐ + * └────┬────┘ │ + * │ Start() │ Stop()/Error + * ▼ │ + * ┌─────────────┐ │ + * │ CONNECTING │◄──────────────────────┐ │ + * └──────┬──────┘ │ │ + * │ Connected │ Reconnect │ + * ▼ │ │ + * ┌─────────────┐ Error/Gap ┌────┴─────┐ │ + * │ SYNCING │─────────────────►│RECOVERING│─────┤ + * └──────┬──────┘ └──────────┘ │ + * │ Sync complete │ + * ▼ │ + * ┌─────────────┐ Watch broken ┌────────────┐ │ + * │ WATCHING │─────────────────►│RECONNECTING│───┤ + * └──────┬──────┘ └────────────┘ │ + * │ Promote() │ + * ▼ │ + * ┌─────────────┐ │ + * │ PROMOTING │───────────────────────────────────┤ + * └──────┬──────┘ │ + * │ Success │ + * ▼ │ + * ┌─────────────┐ │ + * │ PROMOTED │───────────────────────────────────┘ + * └─────────────┘ + */ +enum class StandbyState : uint8_t { + // Initial state, service not started + STOPPED = 0, + + // Connecting to etcd cluster + CONNECTING = 1, + + // Initial sync: reading historical OpLog entries + SYNCING = 2, + + // Normal operation: watching for new OpLog entries + WATCHING = 3, + + // Recovering from error: re-syncing missed entries + RECOVERING = 4, + + // Reconnecting after watch failure + RECONNECTING = 5, + + // Promotion in progress: final catch-up before becoming Primary + PROMOTING = 6, + + // Successfully promoted to Primary + PROMOTED = 7, + + // Fatal error, cannot recover + FAILED = 8, +}; + +/** + * @brief Get human-readable state name + */ +inline const char* StandbyStateToString(StandbyState state) { + switch (state) { + case StandbyState::STOPPED: + return "STOPPED"; + case StandbyState::CONNECTING: + return "CONNECTING"; + case StandbyState::SYNCING: + return "SYNCING"; + case StandbyState::WATCHING: + return "WATCHING"; + case StandbyState::RECOVERING: + return "RECOVERING"; + case StandbyState::RECONNECTING: + return "RECONNECTING"; + case StandbyState::PROMOTING: + return "PROMOTING"; + case StandbyState::PROMOTED: + return "PROMOTED"; + case StandbyState::FAILED: + return "FAILED"; + default: + return "UNKNOWN"; + } +} + +/** + * @brief Events that trigger state transitions + */ +enum class StandbyEvent : uint8_t { + // User/system actions + START, // Start() called + STOP, // Stop() called + PROMOTE, // Promote() called + + // Connection events + CONNECTED, // Successfully connected to etcd + CONNECTION_FAILED, // Failed to connect to etcd + DISCONNECTED, // Connection lost + + // Sync events + SYNC_COMPLETE, // Initial sync completed + SYNC_FAILED, // Sync failed + + // Watch events + WATCH_HEALTHY, // Watch is healthy and receiving events + WATCH_BROKEN, // Watch connection broken + + // Recovery events + RECOVERY_SUCCESS, // Successfully recovered from error + RECOVERY_FAILED, // Recovery failed + + // Promotion events + PROMOTION_SUCCESS, // Successfully promoted + PROMOTION_FAILED, // Promotion failed + + // Error events + MAX_ERRORS_REACHED, // Too many consecutive errors + FATAL_ERROR, // Unrecoverable error +}; + +inline const char* StandbyEventToString(StandbyEvent event) { + switch (event) { + case StandbyEvent::START: + return "START"; + case StandbyEvent::STOP: + return "STOP"; + case StandbyEvent::PROMOTE: + return "PROMOTE"; + case StandbyEvent::CONNECTED: + return "CONNECTED"; + case StandbyEvent::CONNECTION_FAILED: + return "CONNECTION_FAILED"; + case StandbyEvent::DISCONNECTED: + return "DISCONNECTED"; + case StandbyEvent::SYNC_COMPLETE: + return "SYNC_COMPLETE"; + case StandbyEvent::SYNC_FAILED: + return "SYNC_FAILED"; + case StandbyEvent::WATCH_HEALTHY: + return "WATCH_HEALTHY"; + case StandbyEvent::WATCH_BROKEN: + return "WATCH_BROKEN"; + case StandbyEvent::RECOVERY_SUCCESS: + return "RECOVERY_SUCCESS"; + case StandbyEvent::RECOVERY_FAILED: + return "RECOVERY_FAILED"; + case StandbyEvent::PROMOTION_SUCCESS: + return "PROMOTION_SUCCESS"; + case StandbyEvent::PROMOTION_FAILED: + return "PROMOTION_FAILED"; + case StandbyEvent::MAX_ERRORS_REACHED: + return "MAX_ERRORS_REACHED"; + case StandbyEvent::FATAL_ERROR: + return "FATAL_ERROR"; + default: + return "UNKNOWN"; + } +} + +/** + * @brief State transition result + */ +struct StateTransitionResult { + bool allowed{false}; + StandbyState old_state{StandbyState::STOPPED}; + StandbyState new_state{StandbyState::STOPPED}; + std::string reason; +}; + +/** + * @brief Callback for state transition notifications + */ +using StateChangeCallback = + std::function; + +/** + * @brief Standby State Machine + * + * Thread-safe state machine for managing Standby service lifecycle. + * All state transitions are explicit and logged. + */ +class StandbyStateMachine { + public: + StandbyStateMachine(); + + /** + * @brief Get current state (thread-safe) + */ + StandbyState GetState() const { return current_state_.load(std::memory_order_acquire); } + + /** + * @brief Check if in a specific state + */ + bool IsInState(StandbyState state) const { return GetState() == state; } + + /** + * @brief Check if service is running (SYNCING, WATCHING, RECOVERING, RECONNECTING, + * PROMOTING) + */ + bool IsRunning() const { + StandbyState s = GetState(); + return s == StandbyState::SYNCING || s == StandbyState::WATCHING || + s == StandbyState::RECOVERING || s == StandbyState::RECONNECTING || + s == StandbyState::PROMOTING; + } + + /** + * @brief Check if connected to etcd + */ + bool IsConnected() const { + StandbyState s = GetState(); + return s == StandbyState::SYNCING || s == StandbyState::WATCHING || + s == StandbyState::RECOVERING || s == StandbyState::PROMOTING; + } + + /** + * @brief Check if watch is healthy + */ + bool IsWatchHealthy() const { return GetState() == StandbyState::WATCHING; } + + /** + * @brief Check if ready for promotion + */ + bool IsReadyForPromotion() const { return GetState() == StandbyState::WATCHING; } + + /** + * @brief Process an event and perform state transition + * @param event The event to process + * @return Result indicating if transition was allowed and new state + */ + StateTransitionResult ProcessEvent(StandbyEvent event); + + /** + * @brief Register a callback for state change notifications + */ + void RegisterCallback(StateChangeCallback callback); + + /** + * @brief State transition record for debugging + */ + struct TransitionRecord { + std::chrono::steady_clock::time_point timestamp; + StandbyState from_state; + StandbyState to_state; + StandbyEvent event; + }; + + /** + * @brief Get state transition history (for debugging) + */ + std::vector GetTransitionHistory(size_t max_records = 100) const; + + /** + * @brief Get time spent in current state + */ + std::chrono::milliseconds GetTimeInCurrentState() const; + + /** + * @brief Get consecutive error count + */ + int GetConsecutiveErrors() const { return consecutive_errors_.load(); } + + /** + * @brief Increment consecutive error count + * @return New error count + */ + int IncrementErrors(); + + /** + * @brief Reset consecutive error count + */ + void ResetErrors() { consecutive_errors_.store(0); } + + /** + * @brief Get reconnect attempt count + */ + int GetReconnectCount() const { return reconnect_count_.load(); } + + /** + * @brief Increment reconnect count + */ + void IncrementReconnectCount() { reconnect_count_.fetch_add(1); } + + /** + * @brief Reset reconnect count + */ + void ResetReconnectCount() { reconnect_count_.store(0); } + + // Constants + static constexpr int kMaxConsecutiveErrors = 10; + static constexpr int kMaxReconnectAttempts = 100; + + private: + /** + * @brief Check if a transition is valid and get new state + */ + StateTransitionResult ValidateTransition(StandbyState from, StandbyEvent event) const; + + /** + * @brief Notify all registered callbacks + */ + void NotifyCallbacks(StandbyState old_state, StandbyState new_state, StandbyEvent event); + + std::atomic current_state_{StandbyState::STOPPED}; + std::atomic consecutive_errors_{0}; + std::atomic reconnect_count_{0}; + std::chrono::steady_clock::time_point state_enter_time_; + + mutable std::mutex mutex_; + std::vector callbacks_; + std::vector transition_history_; + + static constexpr size_t kMaxHistorySize = 1000; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/types.h b/mooncake-store/include/types.h index 8fbdb806bc..e5834cab2e 100644 --- a/mooncake-store/include/types.h +++ b/mooncake-store/include/types.h @@ -7,11 +7,13 @@ #include #include #include +#include #include #include "Slab.h" #include "ylt/struct_json/json_reader.h" #include "ylt/struct_json/json_writer.h" +#include "ylt/struct_pack.hpp" #ifdef STORE_USE_ETCD #include "libetcd_wrapper.h" @@ -23,6 +25,65 @@ namespace mooncake { static constexpr uint64_t WRONG_VERSION = 0; static constexpr uint64_t DEFAULT_VALUE = UINT64_MAX; static constexpr uint64_t ERRNO_BASE = DEFAULT_VALUE - 1000; + +// Sequence ID comparison utilities for wrap-around safety. +// These functions use signed difference to correctly handle uint64_t overflow +// (from UINT64_MAX wrapping to 0). Assumes sequence IDs won't differ by more +// than 2^63, which is reasonable for practical systems. +// +// Example: If sequence_id wraps from UINT64_MAX to 0, then: +// IsSequenceNewer(0, UINT64_MAX) = true (0 is newer after wrap) +// IsSequenceNewer(UINT64_MAX, 0) = false (UINT64_MAX is older before wrap) +// +// Note: We use 'static inline' here to give these small helpers internal +// linkage and avoid any potential ODR/linkage issues in large codebases. +static inline bool IsSequenceNewer(uint64_t a, uint64_t b) { + // Cast to int64_t to get signed difference, then check if positive. + // This correctly handles wrap-around: if a wrapped from UINT64_MAX to 0, + // then (int64_t)(a - b) will be positive (assuming gap < 2^63). + return static_cast(a - b) > 0; +} + +static inline bool IsSequenceOlder(uint64_t a, uint64_t b) { + return static_cast(a - b) < 0; +} + +static inline bool IsSequenceEqual(uint64_t a, uint64_t b) { + return a == b; +} + +static inline bool IsSequenceNewerOrEqual(uint64_t a, uint64_t b) { + return a == b || static_cast(a - b) > 0; +} + +static inline bool IsSequenceOlderOrEqual(uint64_t a, uint64_t b) { + return a == b || static_cast(a - b) < 0; +} + +// Cluster ID validation utilities. +// +// cluster_id is used to construct etcd key prefixes (e.g. "/oplog//..."). +// To avoid key-prefix injection / accidental cross-cluster overlap, we restrict the +// allowed characters to a conservative safe set. We validate the "component" form +// (without trailing slash). Trailing slashes should be normalized away before +// validation. +static inline bool IsValidClusterIdComponent(const std::string& cluster_id) { + if (cluster_id.empty()) { + return false; + } + if (cluster_id.size() > 128) { + return false; + } + for (unsigned char c : cluster_id) { + const bool ok = + (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || c == '_' || c == '-' || c == '.'; + if (!ok) { + return false; + } + } + return true; +} static constexpr uint64_t DEFAULT_DEFAULT_KV_LEASE_TTL = 5000; // in milliseconds static constexpr uint64_t DEFAULT_KV_SOFT_PIN_TTL_MS = diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 05968b56df..da9d7034b9 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -16,8 +16,6 @@ set(MOONCAKE_STORE_SOURCES ha_helper.cpp segment.cpp transfer_task.cpp - etcd_helper.cpp - ha_helper.cpp rpc_service.cpp offset_allocator.cpp posix_file.cpp @@ -26,10 +24,29 @@ set(MOONCAKE_STORE_SOURCES dummy_client.cpp http_metadata_server.cpp file_storage.cpp + oplog_manager.cpp + etcd_oplog_store.cpp + etcd_snapshot_provider.cpp + oplog_watcher.cpp + oplog_applier.cpp + hot_standby_service.cpp + standby_state_machine.cpp + ha_metric_manager.cpp + # replication_service.cpp removed - using etcd-based OpLog sync instead ) set(EXTRA_LIBS "") +# Find xxHash (required for ComputeChecksum) +find_path(XXHASH_INCLUDE_DIR NAMES xxhash.h PATHS /usr/include /usr/local/include) +find_library(XXHASH_LIBRARY NAMES xxhash libxxhash PATHS /usr/lib /usr/local/lib /usr/lib64) +if (XXHASH_INCLUDE_DIR AND XXHASH_LIBRARY) + message(STATUS "Found xxHash: include=${XXHASH_INCLUDE_DIR} lib=${XXHASH_LIBRARY}") + list(APPEND MASTER_EXTRA_INCS ${XXHASH_INCLUDE_DIR}) +else() + message(FATAL_ERROR "xxHash library/header not found. Please install xxhash (development headers) and try again.") +endif() + if(USE_3FS) add_subdirectory(hf3fs) list(APPEND MOONCAKE_STORE_SOURCES ${HF3FS_SOURCES}) @@ -43,6 +60,8 @@ endif() # The cache_allocator library include_directories(${Python3_INCLUDE_DIRS}) add_library(mooncake_store ${MOONCAKE_STORE_SOURCES}) +target_include_directories(mooncake_store PUBLIC ${XXHASH_INCLUDE_DIR}) +target_link_libraries(mooncake_store PUBLIC ${XXHASH_LIBRARY}) target_link_libraries(mooncake_store PUBLIC transfer_engine cachelib_memory_allocator ${ETCD_WRAPPER_LIB} glog::glog gflags::gflags ${EXTRA_LIBS} ) @@ -88,7 +107,21 @@ target_link_options(mooncake_master PRIVATE -Os -s) # Client server binary add_executable(mooncake_client real_client_main.cpp) target_link_libraries(mooncake_client PRIVATE mooncake_store) -target_compile_options(mooncake_master PRIVATE -Os) +target_compile_options(mooncake_client PRIVATE -Os) target_link_options(mooncake_client PRIVATE -Os -s) -install(TARGETS mooncake_master mooncake_client DESTINATION bin) +# Mock client simulator binary +add_executable(mock_client mock_client_main.cpp) +target_link_libraries(mock_client PRIVATE + mooncake_store + cachelib_memory_allocator + glog::glog + gflags::gflags + pthread +) +if (STORE_USE_ETCD) + add_dependencies(mock_client build_etcd_wrapper) + target_link_libraries(mock_client PRIVATE ${ETCD_WRAPPER_LIB}) +endif() + +install(TARGETS mooncake_master mooncake_client mock_client DESTINATION bin) diff --git a/mooncake-store/src/client_service.cpp b/mooncake-store/src/client_service.cpp index c62789b634..498aaca90b 100644 --- a/mooncake-store/src/client_service.cpp +++ b/mooncake-store/src/client_service.cpp @@ -175,7 +175,30 @@ tl::expected CheckRegisterMemoryParams(const void* addr, ErrorCode Client::ConnectToMaster(const std::string& master_server_entry) { if (master_server_entry.find("etcd://") == 0) { + // Support optional cluster_id in connection string: + // etcd://?cluster_id= + // If not provided, MasterViewHelper will fall back to env MC_STORE_CLUSTER_ID, + // then DEFAULT_CLUSTER_ID. std::string etcd_entry = master_server_entry.substr(strlen("etcd://")); + std::string cluster_id; + { + const size_t qpos = etcd_entry.find('?'); + if (qpos != std::string::npos) { + const std::string query = etcd_entry.substr(qpos + 1); + etcd_entry = etcd_entry.substr(0, qpos); + const std::string k = "cluster_id="; + const size_t kpos = query.find(k); + if (kpos != std::string::npos) { + const size_t vpos = kpos + k.size(); + size_t vend = query.find('&', vpos); + if (vend == std::string::npos) vend = query.size(); + cluster_id = query.substr(vpos, vend - vpos); + } + } + } + if (!cluster_id.empty()) { + master_view_helper_.SetClusterId(cluster_id); + } // Get master address from etcd auto err = master_view_helper_.ConnectToEtcd(etcd_entry); @@ -1739,6 +1762,10 @@ void Client::PingThreadMain(bool is_ha_mode, // Increment after a ping failure, reset after a ping success int ping_fail_count = 0; + // Track the last known view version to detect master changes in HA mode + ViewVersionId last_known_view_version = 0; + bool is_first_ping = true; + auto remount_segment = [this]() { // This lock must be held until the remount rpc is finished, // otherwise there will be corner cases, e.g., a segment is @@ -1774,6 +1801,55 @@ void Client::PingThreadMain(bool is_ha_mode, // Reset ping failure count ping_fail_count = 0; auto& ping_response = ping_result.value(); + + // In HA mode, actively check if view_version_id changed (master switched) + if (is_ha_mode) { + if (!is_first_ping && + ping_response.view_version_id != last_known_view_version) { + LOG(INFO) << "Detected master view version change: " + << last_known_view_version << " -> " + << ping_response.view_version_id + << ", fetching latest master view and reconnecting"; + + std::string new_master_address; + ViewVersionId new_version = 0; + auto err = master_view_helper_.GetMasterView( + new_master_address, new_version); + if (err == ErrorCode::OK) { + if (new_master_address != current_master_address) { + err = master_client_.Connect(new_master_address); + if (err == ErrorCode::OK) { + current_master_address = new_master_address; + last_known_view_version = new_version; + LOG(INFO) << "Reconnected to new master: " + << new_master_address + << ", view_version: " << new_version; + } else { + LOG(ERROR) << "Failed to connect to new master " + << new_master_address << ": " + << toString(err); + } + } else { + // Same address but version changed (e.g., master + // restarted), update version + last_known_view_version = new_version; + LOG(INFO) << "Master address unchanged but view " + "version updated: " + << new_version; + } + } else { + LOG(ERROR) << "Failed to get new master view: " + << toString(err); + } + } else { + // Update last known view version + last_known_view_version = ping_response.view_version_id; + if (is_first_ping) { + is_first_ping = false; + } + } + } + if (ping_response.client_status == ClientStatus::NEED_REMOUNT && !remount_segment_future.valid()) { // Ensure at most one remount segment thread is running @@ -1820,6 +1896,7 @@ void Client::PingThreadMain(bool is_ha_mode, } current_master_address = master_address; + last_known_view_version = next_version; // Update version on reconnect LOG(INFO) << "Reconnected to master " << master_address; ping_fail_count = 0; } else { diff --git a/mooncake-store/src/etcd_helper.cpp b/mooncake-store/src/etcd_helper.cpp index 5417de5afc..cba7b69670 100644 --- a/mooncake-store/src/etcd_helper.cpp +++ b/mooncake-store/src/etcd_helper.cpp @@ -89,6 +89,46 @@ ErrorCode EtcdHelper::CreateWithLease(const char* key, const size_t key_size, } } +ErrorCode EtcdHelper::BatchCreate(const std::vector& keys, + const std::vector& values) { + if (keys.size() != values.size()) { + return ErrorCode::INVALID_PARAMS; + } + if (keys.empty()) { + return ErrorCode::OK; + } + + std::vector c_keys; + std::vector c_values; + c_keys.reserve(keys.size()); + c_values.reserve(values.size()); + + for (const auto& key : keys) { + c_keys.push_back(const_cast(key.c_str())); + } + for (const auto& val : values) { + c_values.push_back(const_cast(val.c_str())); + } + + char* err_msg = nullptr; + int ret = EtcdStoreBatchCreateWrapper(c_keys.data(), c_values.data(), + (int)keys.size(), &err_msg); + if (ret == -2) { + if (err_msg) { + LOG(ERROR) << "BatchCreate transaction failed: " << err_msg; + free(err_msg); + } + return ErrorCode::ETCD_TRANSACTION_FAIL; + } else if (ret != 0) { + if (err_msg) { + LOG(ERROR) << "BatchCreate failed: " << err_msg; + free(err_msg); + } + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + ErrorCode EtcdHelper::GrantLease(int64_t lease_ttl, EtcdLeaseId& lease_id) { char* err_msg = nullptr; if (0 != EtcdStoreGrantLeaseWrapper(lease_ttl, &lease_id, &err_msg)) { @@ -152,6 +192,250 @@ ErrorCode EtcdHelper::CancelKeepAlive(EtcdLeaseId lease_id) { } return ErrorCode::OK; } + +ErrorCode EtcdHelper::Put(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + char* err_msg = nullptr; + int ret = EtcdStorePutWrapper((char*)key, (int)key_size, (char*)value, + (int)value_size, &err_msg); + if (ret != 0) { + LOG(ERROR) << "key=" << std::string(key, key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::Create(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + char* err_msg = nullptr; + int ret = EtcdStoreCreateWrapper((char*)key, (int)key_size, (char*)value, + (int)value_size, &err_msg); + if (ret == -2) { + free(err_msg); + return ErrorCode::ETCD_TRANSACTION_FAIL; + } + if (ret != 0) { + LOG(ERROR) << "key=" << std::string(key, key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::GetWithPrefix(const char* prefix, const size_t prefix_size, + std::vector& keys, + std::vector& values) { + // TODO: Implement GetWithPrefix - need to simplify Go wrapper interface first + // For now, return error as this requires complex memory management + LOG(ERROR) << "GetWithPrefix not yet implemented - requires Go wrapper interface simplification"; + return ErrorCode::INTERNAL_ERROR; +} + +ErrorCode EtcdHelper::GetRangeAsJson(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size, + size_t limit, + std::string& json, + EtcdRevisionId& revision_id) { + char* err_msg = nullptr; + char* json_ptr = nullptr; + int json_size = 0; + // Go wrapper takes int limit. + int ret = EtcdStoreGetRangeAsJsonWrapper((char*)start_key, (int)start_key_size, + (char*)end_key, (int)end_key_size, + (int)limit, &json_ptr, &json_size, + (GoInt64*)&revision_id, &err_msg); + if (ret != 0) { + LOG(ERROR) << "start_key=" << std::string(start_key, start_key_size) + << ", end_key=" << std::string(end_key, end_key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + json = std::string(json_ptr, json_size); + free(json_ptr); + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& first_key) { + char* err_msg = nullptr; + char* first_key_ptr = nullptr; + int first_key_size = 0; + int ret = EtcdStoreGetFirstKeyWithPrefixWrapper((char*)prefix, (int)prefix_size, + &first_key_ptr, &first_key_size, + &err_msg); + if (ret == -2) { + free(err_msg); + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + first_key = std::string(first_key_ptr, first_key_size); + free(first_key_ptr); + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::GetLastKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& last_key) { + char* err_msg = nullptr; + char* last_key_ptr = nullptr; + int last_key_size = 0; + int ret = EtcdStoreGetLastKeyWithPrefixWrapper((char*)prefix, (int)prefix_size, + &last_key_ptr, &last_key_size, + &err_msg); + if (ret == -2) { + free(err_msg); + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + last_key = std::string(last_key_ptr, last_key_size); + free(last_key_ptr); + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::DeleteRange(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size) { + char* err_msg = nullptr; + int ret = EtcdStoreDeleteRangeWrapper((char*)start_key, (int)start_key_size, + (char*)end_key, (int)end_key_size, + &err_msg); + if (ret != 0) { + LOG(ERROR) << "start_key=" << std::string(start_key, start_key_size) + << ", end_key=" << std::string(end_key, end_key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::WatchWithPrefixFromRevision( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)) { + char* err_msg = nullptr; + void* callback_func_ptr = reinterpret_cast(callback_func); + int ret = EtcdStoreWatchWithPrefixFromRevisionWrapper( + (char*)prefix, (int)prefix_size, (GoInt64)start_revision, callback_context, + callback_func_ptr, &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", start_revision=" << (int64_t)start_revision + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix, + const size_t prefix_size) { + char* err_msg = nullptr; + int ret = EtcdStoreCancelWatchWithPrefixWrapper((char*)prefix, (int)prefix_size, + &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::WaitWatchWithPrefixStopped(const char* prefix, + const size_t prefix_size, + int timeout_ms) { + char* err_msg = nullptr; + int ret = EtcdStoreWaitWatchWithPrefixStoppedWrapper((char*)prefix, + (int)prefix_size, + timeout_ms, &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", timeout_ms=" << timeout_ms + << ", error=" << (err_msg ? err_msg : "unknown"); + if (err_msg) { + free(err_msg); + } + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::SaveSnapshotTxn( + const std::string& snapshot_data_key, + const std::string& snapshot_data_value, + const std::string& snapshot_seq_key, + const std::string& snapshot_seq_value, + const std::string& latest_key, + const std::string& latest_value, + const std::string& old_snapshot_data_key, + const std::string& old_snapshot_seq_key) { + char* err_msg = nullptr; + int ret = EtcdStoreSnapshotTxnWrapper( + const_cast(snapshot_data_key.c_str()), + static_cast(snapshot_data_key.size()), + const_cast(snapshot_data_value.c_str()), + static_cast(snapshot_data_value.size()), + const_cast(snapshot_seq_key.c_str()), + static_cast(snapshot_seq_key.size()), + const_cast(snapshot_seq_value.c_str()), + static_cast(snapshot_seq_value.size()), + const_cast(latest_key.c_str()), + static_cast(latest_key.size()), + const_cast(latest_value.c_str()), + static_cast(latest_value.size()), + const_cast(old_snapshot_data_key.c_str()), + static_cast(old_snapshot_data_key.size()), + const_cast(old_snapshot_seq_key.c_str()), + static_cast(old_snapshot_seq_key.size()), + &err_msg); + if (ret != 0) { + LOG(ERROR) << "SaveSnapshotTxn failed: " << (err_msg ? err_msg : "unknown"); + if (err_msg) { + free(err_msg); + } + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::DeleteSnapshot(const std::string& snapshot_data_key, + const std::string& snapshot_seq_key) { + char* err_msg = nullptr; + int ret = EtcdStoreDeleteSnapshotWrapper( + const_cast(snapshot_data_key.c_str()), + static_cast(snapshot_data_key.size()), + const_cast(snapshot_seq_key.c_str()), + static_cast(snapshot_seq_key.size()), + &err_msg); + if (ret != 0) { + LOG(ERROR) << "DeleteSnapshot failed: " << (err_msg ? err_msg : "unknown"); + if (err_msg) { + free(err_msg); + } + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} #else ErrorCode EtcdHelper::ConnectToEtcdStoreClient( const std::string& etcd_endpoints) { @@ -174,6 +458,12 @@ ErrorCode EtcdHelper::CreateWithLease(const char* key, const size_t key_size, return ErrorCode::ETCD_OPERATION_ERROR; } +ErrorCode EtcdHelper::BatchCreate(const std::vector& keys, + const std::vector& values) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + ErrorCode EtcdHelper::GrantLease(int64_t lease_ttl, EtcdLeaseId& lease_id) { LOG(FATAL) << "Etcd is not enabled in compilation"; return ErrorCode::ETCD_OPERATION_ERROR; @@ -200,6 +490,132 @@ ErrorCode EtcdHelper::CancelKeepAlive(EtcdLeaseId lease_id) { return ErrorCode::ETCD_OPERATION_ERROR; } +ErrorCode EtcdHelper::Put(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::Create(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + (void)key; + (void)key_size; + (void)value; + (void)value_size; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::GetWithPrefix(const char* prefix, const size_t prefix_size, + std::vector& keys, + std::vector& values) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::GetRangeAsJson(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size, + size_t limit, + std::string& json, + EtcdRevisionId& revision_id) { + (void)start_key; + (void)start_key_size; + (void)end_key; + (void)end_key_size; + (void)limit; + (void)json; + (void)revision_id; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} +ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& first_key) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::GetLastKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& last_key) { + (void)prefix; + (void)prefix_size; + (void)last_key; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::DeleteRange(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::WatchWithPrefixFromRevision( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)) { + (void)prefix; + (void)prefix_size; + (void)start_revision; + (void)callback_context; + (void)callback_func; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix, + const size_t prefix_size) { + (void)prefix; + (void)prefix_size; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::WaitWatchWithPrefixStopped(const char* prefix, + const size_t prefix_size, + int timeout_ms) { + (void)prefix; + (void)prefix_size; + (void)timeout_ms; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::SaveSnapshotTxn( + const std::string& snapshot_data_key, + const std::string& snapshot_data_value, + const std::string& snapshot_seq_key, + const std::string& snapshot_seq_value, + const std::string& latest_key, + const std::string& latest_value, + const std::string& old_snapshot_data_key, + const std::string& old_snapshot_seq_key) { + (void)snapshot_data_key; + (void)snapshot_data_value; + (void)snapshot_seq_key; + (void)snapshot_seq_value; + (void)latest_key; + (void)latest_value; + (void)old_snapshot_data_key; + (void)old_snapshot_seq_key; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::DeleteSnapshot(const std::string& snapshot_data_key, + const std::string& snapshot_seq_key) { + (void)snapshot_data_key; + (void)snapshot_seq_key; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + #endif -} // namespace mooncake \ No newline at end of file +} // namespace mooncake diff --git a/mooncake-store/src/etcd_oplog_store.cpp b/mooncake-store/src/etcd_oplog_store.cpp new file mode 100644 index 0000000000..052e38811d --- /dev/null +++ b/mooncake-store/src/etcd_oplog_store.cpp @@ -0,0 +1,785 @@ +#include "etcd_oplog_store.h" + +#include +#include +#include + +#include "ha_metric_manager.h" + +#if __has_include() +#include // Ubuntu +#else +#include // CentOS +#endif + +#include "etcd_helper.h" + +namespace mooncake { + +EtcdOpLogStore::EtcdOpLogStore(const std::string& cluster_id, + bool enable_latest_seq_batch_update) + : cluster_id_(cluster_id), + enable_latest_seq_batch_update_(enable_latest_seq_batch_update), + last_update_time_(std::chrono::steady_clock::now()) { + // Normalize cluster_id to avoid accidental double slashes in etcd keys when + // caller passes a trailing '/' (master_view_key uses trailing '/', OpLog keys don't). + while (!cluster_id_.empty() && cluster_id_.back() == '/') { + cluster_id_.pop_back(); + } + + if (!cluster_id_.empty() && !IsValidClusterIdComponent(cluster_id_)) { + LOG(FATAL) << "Invalid cluster_id for EtcdOpLogStore: '" << cluster_id_ + << "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } + + // Initialize /latest key to 0 if it doesn't exist (first startup). + // This avoids "key not found" errors when querying the latest sequence ID. + // Important: Only initialize if the key doesn't exist to avoid overwriting existing data. + if (!cluster_id_.empty()) { + std::string latest_key = BuildLatestKey(); + std::string existing_value; + EtcdRevisionId revision_id; + ErrorCode get_err = EtcdHelper::Get(latest_key.c_str(), latest_key.size(), + existing_value, revision_id); + if (get_err == ErrorCode::ETCD_KEY_NOT_EXIST) { + // Key doesn't exist, safe to initialize to 0 + std::string initial_value = "0"; + ErrorCode create_err = EtcdHelper::Create(latest_key.c_str(), latest_key.size(), + initial_value.c_str(), initial_value.size()); + if (create_err == ErrorCode::OK) { + LOG(INFO) << "Initialized /latest key to 0 for cluster_id=" << cluster_id_; + } else if (create_err == ErrorCode::ETCD_TRANSACTION_FAIL) { + // Race condition: another instance created it between Get and Create + LOG(INFO) << "/latest key was created by another instance for cluster_id=" + << cluster_id_; + } else { + // Other errors (e.g., etcd not connected) are logged but don't fail construction + // The key will be created when the first OpLog entry is written + LOG(WARNING) << "Failed to initialize /latest key (error=" << create_err + << "), will be created on first OpLog write"; + } + } else if (get_err == ErrorCode::OK) { + // Key already exists, do nothing - preserve existing value + LOG(INFO) << "/latest key already exists (value=" << existing_value + << ") for cluster_id=" << cluster_id_; + } else { + // Other errors (e.g., etcd not connected) are logged but don't fail construction + LOG(WARNING) << "Failed to check /latest key existence (error=" << get_err + << "), will be created on first OpLog write"; + } + } + + // Start batch update thread only for writers. + if (enable_latest_seq_batch_update_) { + batch_update_running_.store(true); + batch_update_thread_ = + std::thread(&EtcdOpLogStore::BatchUpdateThread, this); + } + + // Start OpLog batch write thread + batch_write_running_.store(true); + batch_write_thread_ = std::thread(&EtcdOpLogStore::BatchWriteThread, this); +} + +EtcdOpLogStore::~EtcdOpLogStore() { + // Stop OpLog batch write thread + batch_write_running_.store(false); + cv_batch_updated_.notify_all(); + if (batch_write_thread_.joinable()) { + batch_write_thread_.join(); + } + + // Attempt final flush + { + std::lock_guard lock(batch_mutex_); + if (!pending_batch_.empty()) { + FlushBatch(); + } + } + + if (!enable_latest_seq_batch_update_) { + return; + } + + // Stop batch update thread + batch_update_running_.store(false); + if (batch_update_thread_.joinable()) { + batch_update_thread_.join(); + } + + // Perform final update if there are pending updates + if (pending_count_.load() > 0) { + DoBatchUpdate(); + } +} + +ErrorCode EtcdOpLogStore::WriteOpLog(const OpLogEntry& entry, bool sync) { + std::string key = BuildOpLogKey(entry.sequence_id); + std::string value = SerializeOpLogEntry(entry); + + { + std::unique_lock lock(batch_mutex_); + pending_batch_.push_back({std::move(key), std::move(value), entry.sequence_id, sync}); + + bool should_notify = false; + if (sync) { + // Strategy 2+: Sync writes (DELETE) trigger immediate flush + should_notify = true; + } else { + // Async writes (PUT_END): trigger if threshold reached + if (pending_batch_.size() >= kOpLogBatchCountLimit) { + should_notify = true; + } + } + + if (should_notify) { + cv_batch_updated_.notify_one(); + } + + if (sync) { + // Wait for persistence + uint64_t target_seq = entry.sequence_id; + bool success = cv_sync_completed_.wait_for( + lock, std::chrono::milliseconds(kSyncWaitTimeoutMs), [&] { + return last_persisted_seq_id_.load() >= target_seq; + }); + if (!success) { + LOG(ERROR) << "Timeout waiting for OpLog persistence, seq=" + << target_seq; + return ErrorCode::ETCD_OPERATION_ERROR; + } + } + } + + // Update /latest pointer logic + // We defer this to the batch flush or just queue it up here? + // Original logic: + if (!enable_latest_seq_batch_update_) { + // Direct update (may be slow, but it's what config asked for) + // Warning: This is now done AFTER op log write, which is correct order. + return UpdateLatestSequenceId(entry.sequence_id); + } + + // For batch update, we update the pending counter + pending_latest_seq_id_.store(entry.sequence_id); + size_t count = pending_count_.fetch_add(1) + 1; + if (count >= kBatchSize) { + DoBatchUpdate(); + } + + return ErrorCode::OK; +} + +void EtcdOpLogStore::BatchWriteThread() { + while (batch_write_running_.load()) { + std::unique_lock lock(batch_mutex_); + if (pending_batch_.empty()) { + // Wait for signal or timeout (Group Commit time window) + cv_batch_updated_.wait_for(lock, std::chrono::milliseconds(kOpLogBatchTimeoutMs)); + } + + if (!batch_write_running_.load() && pending_batch_.empty()) { + break; + } + + if (!pending_batch_.empty()) { + FlushBatch(); + } + } +} + +void EtcdOpLogStore::FlushBatch() { + // Note: batch_mutex_ is LOCKED when entering this function + std::deque batch_to_write; + batch_to_write.swap(pending_batch_); + + // Unlock to allow new appends while we perform IO + batch_mutex_.unlock(); + + if (batch_to_write.empty()) { + // Should not happen usually + batch_mutex_.lock(); + return; + } + + std::vector keys; + std::vector values; + keys.reserve(batch_to_write.size()); + values.reserve(batch_to_write.size()); + + uint64_t max_seq = 0; + bool has_sync_entry = false; + for (const auto& entry : batch_to_write) { + keys.push_back(entry.key); + if (entry.is_sync) { + has_sync_entry = true; + } + values.push_back(entry.value); + if (entry.sequence_id > max_seq) { + max_seq = entry.sequence_id; + } + } + + // Perform Batch IO + ErrorCode err = ErrorCode::OK; + for (int i = 0; i <= kFlushRetryCount; ++i) { + err = EtcdHelper::BatchCreate(keys, values); + if (err == ErrorCode::OK) { + break; + } + if (i < kFlushRetryCount) { + LOG(WARNING) << "Failed to flush OpLog batch (attempt " << i + 1 + << "/" << kFlushRetryCount + 1 << "), retrying..."; + std::this_thread::sleep_for(std::chrono::milliseconds(kFlushRetryIntervalMs)); + } + } + + // Re-lock to update state + batch_mutex_.lock(); + + if (err == ErrorCode::OK) { + if (max_seq > last_persisted_seq_id_.load()) { + last_persisted_seq_id_.store(max_seq); + } + + // Update HA metrics + HAMetricManager::instance().inc_oplog_batch_commits(); + if (has_sync_entry) { + HAMetricManager::instance().inc_oplog_sync_batch_commits(); + } + + if (batch_to_write.size() > 1) { + LOG(INFO) << "HA Strategy: Group Commit flush success. batch_size=" + << batch_to_write.size() << ", max_seq=" << max_seq; + } else { + VLOG(3) << "HA Strategy: Group Commit flush success. batch_size=1, max_seq=" + << max_seq; + if (!has_sync_entry) { + // Log occasionally if we are flushing single async entries (inefficiency indicator) + LOG_EVERY_N(INFO, 1000) << "Note: Frequent single-entry async flushes detected (sample)."; + } + } + } else { + LOG(ERROR) << "Failed to flush OpLog batch, count=" << batch_to_write.size(); + } + + // Wake up all waiting threads (Strategy 2+: DELETE waiters) + cv_sync_completed_.notify_all(); +} + + +ErrorCode EtcdOpLogStore::ReadOpLog(uint64_t sequence_id, + OpLogEntry& entry) { + std::string key = BuildOpLogKey(sequence_id); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry, sequence_id=" + << sequence_id; + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::ReadOpLogSince(uint64_t start_sequence_id, + size_t limit, + std::vector& entries) { + EtcdRevisionId rev = 0; + return ReadOpLogSinceWithRevision(start_sequence_id, limit, entries, rev); +} + +ErrorCode EtcdOpLogStore::ReadOpLogSinceWithRevision(uint64_t start_sequence_id, + size_t limit, + std::vector& entries, + EtcdRevisionId& revision_id) { + entries.clear(); + entries.reserve(limit); + + // Range is limited to OpLog entry keys only. + const std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/"; + std::string current_start_key = BuildOpLogKey(start_sequence_id + 1); + + // Compute prefix range end (etcd prefix end). + auto prefix_end = [](std::string p) -> std::string { + for (int i = static_cast(p.size()) - 1; i >= 0; --i) { + unsigned char c = static_cast(p[i]); + if (c < 0xFF) { + p[i] = static_cast(c + 1); + p.resize(i + 1); + return p; + } + } + return std::string(1, '\0'); + }; + const std::string end_key = prefix_end(prefix); + + // Pagination: + // - Use range-get with limit + // - Start next page from lastKey + '\0' (lexicographically just after lastKey) + // This avoids repeating the last key without adding new Go/C++ APIs. + revision_id = 0; + while (entries.size() < limit) { + const size_t page_limit = limit - entries.size(); + std::string json; + EtcdRevisionId page_rev = 0; + ErrorCode err = + EtcdHelper::GetRangeAsJson(current_start_key.c_str(), + current_start_key.size(), end_key.c_str(), + end_key.size(), page_limit, json, page_rev); + if (err != ErrorCode::OK) { + return err; + } + if (page_rev > revision_id) { + revision_id = page_rev; + } + + // Parse kv list: [{"key":"...","value":"..."}] + Json::Value root; + Json::CharReaderBuilder reader; + std::string errs; + std::istringstream s(json); + if (!Json::parseFromStream(reader, s, &root, &errs)) { + LOG(ERROR) << "Failed to parse range JSON: " << errs; + return ErrorCode::INTERNAL_ERROR; + } + if (!root.isArray()) { + return ErrorCode::INTERNAL_ERROR; + } + if (root.empty()) { + break; // no more data + } + + std::string last_key_in_page; + for (const auto& kv : root) { + const std::string key = kv.get("key", "").asString(); + last_key_in_page = key; + if (key.empty() || key.find("/latest") != std::string::npos || + key.find("/snapshot/") != std::string::npos) { + continue; + } + + // Parse seq from key suffix and filter (handles legacy keys too). + size_t pos = key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= key.size()) { + continue; + } + uint64_t seq = 0; + try { + seq = static_cast(std::stoull(key.substr(pos + 1))); + } catch (...) { + continue; + } + if (IsSequenceOlderOrEqual(seq, start_sequence_id)) { + continue; + } + + OpLogEntry entry; + const std::string value = kv.get("value", "").asString(); + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry from key=" << key; + return ErrorCode::INTERNAL_ERROR; + } + entries.push_back(std::move(entry)); + if (entries.size() >= limit) { + break; + } + } + + // Advance start key for next page. + if (last_key_in_page.empty()) { + break; + } + current_start_key = last_key_in_page; + current_start_key.push_back('\0'); + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::GetLatestSequenceId(uint64_t& sequence_id) { + std::string key = BuildLatestKey(); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + try { + sequence_id = std::stoull(value); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse latest sequence_id: " << e.what(); + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::GetMaxSequenceId(uint64_t& sequence_id) { + auto max_seq_opt = GetMaxSequenceIdInternal(); + if (!max_seq_opt.has_value()) { + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + sequence_id = max_seq_opt.value(); + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::UpdateLatestSequenceId(uint64_t sequence_id) { + std::string key = BuildLatestKey(); + std::string value = std::to_string(sequence_id); + return EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), value.size()); +} + +ErrorCode EtcdOpLogStore::RecordSnapshotSequenceId( + const std::string& snapshot_id, uint64_t sequence_id) { + std::string key = BuildSnapshotKey(snapshot_id); + std::string value = std::to_string(sequence_id); + return EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), value.size()); +} + +ErrorCode EtcdOpLogStore::GetSnapshotSequenceId( + const std::string& snapshot_id, uint64_t& sequence_id) { + std::string key = BuildSnapshotKey(snapshot_id); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + try { + sequence_id = std::stoull(value); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse snapshot sequence_id: " << e.what(); + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::CleanupOpLogBefore(uint64_t before_sequence_id) { + // Robust cleanup (Scheme 3): + // - Determine current minimum sequence_id in etcd + // - DeleteRange [min_key, before_key) + // + // IMPORTANT: This relies on lexicographical ordering of keys, so the + // sequence_id portion MUST be fixed-width (zero-padded). + auto min_seq_opt = GetMinSequenceId(); + if (!min_seq_opt.has_value()) { + return ErrorCode::OK; // nothing to cleanup + } + + uint64_t min_seq = min_seq_opt.value(); + if (before_sequence_id <= min_seq) { + return ErrorCode::OK; + } + + std::string start_key = BuildOpLogKey(min_seq); + std::string end_key = BuildOpLogKey(before_sequence_id); // delete < before_sequence_id + + return EtcdHelper::DeleteRange(start_key.c_str(), start_key.size(), + end_key.c_str(), end_key.size()); +} + +std::string EtcdOpLogStore::BuildOpLogKey(uint64_t sequence_id) const { + std::ostringstream oss; + // Fixed-width encoding for correct etcd lexicographical range operations. + // 20 digits is enough for uint64_t max (18446744073709551615). + oss << kOpLogPrefix << cluster_id_ << "/" + << std::setw(20) << std::setfill('0') << sequence_id; + return oss.str(); +} + +std::optional EtcdOpLogStore::GetMinSequenceId() const { + std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/"; + std::string first_key; + ErrorCode err = + EtcdHelper::GetFirstKeyWithPrefix(prefix.c_str(), prefix.size(), first_key); + if (err != ErrorCode::OK) { + return std::nullopt; + } + + // Skip non-entry keys if any (e.g. "/latest" or "/snapshot/..."). + // Entries are expected to be ".../<20-digit-seq>". + // If the first key isn't an entry key, fall back to nullopt (safe no-op). + if (first_key.find("/latest") != std::string::npos || + first_key.find("/snapshot/") != std::string::npos) { + return std::nullopt; + } + + size_t pos = first_key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= first_key.size()) { + return std::nullopt; + } + std::string seq_str = first_key.substr(pos + 1); + try { + return static_cast(std::stoull(seq_str)); + } catch (...) { + return std::nullopt; + } +} + +std::optional EtcdOpLogStore::GetMaxSequenceIdInternal() const { + // Entry keys are fixed-width 20-digit numbers, which (in practice) start with '0'. + // Use "/0" to avoid picking up "/latest" which is lexicographically after digits. + std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/0"; + std::string last_key; + ErrorCode err = + EtcdHelper::GetLastKeyWithPrefix(prefix.c_str(), prefix.size(), last_key); + if (err != ErrorCode::OK) { + return std::nullopt; + } + + size_t pos = last_key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= last_key.size()) { + return std::nullopt; + } + std::string seq_str = last_key.substr(pos + 1); + try { + return static_cast(std::stoull(seq_str)); + } catch (...) { + return std::nullopt; + } +} + +std::string EtcdOpLogStore::BuildLatestKey() const { + std::ostringstream oss; + oss << kOpLogPrefix << cluster_id_ << kLatestSuffix; + return oss.str(); +} + +std::string EtcdOpLogStore::BuildSnapshotKey( + const std::string& snapshot_id) const { + std::ostringstream oss; + oss << kOpLogPrefix << cluster_id_ << kSnapshotSuffix << snapshot_id + << "/sequence_id"; + return oss.str(); +} + +namespace { + +// Base64 encoding for binary payload +// JsonCpp treats strings as UTF-8, so we must encode binary data +std::string Base64Encode(const std::string& data) { + static const char base64_chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + std::string result; + result.reserve(((data.size() + 2) / 3) * 4); + + size_t i = 0; + size_t data_len = data.size(); + + // Process 3 bytes at a time + while (i + 2 < data_len) { + uint32_t octet_a = static_cast(data[i++]); + uint32_t octet_b = static_cast(data[i++]); + uint32_t octet_c = static_cast(data[i++]); + + uint32_t triple = (octet_a << 16) | (octet_b << 8) | octet_c; + + result.push_back(base64_chars[(triple >> 18) & 0x3F]); + result.push_back(base64_chars[(triple >> 12) & 0x3F]); + result.push_back(base64_chars[(triple >> 6) & 0x3F]); + result.push_back(base64_chars[triple & 0x3F]); + } + + // Handle remaining bytes + size_t remaining = data_len - i; + if (remaining > 0) { + uint32_t octet_a = static_cast(data[i++]); + uint32_t octet_b = (remaining > 1) ? static_cast(data[i++]) : 0; + uint32_t octet_c = 0; + + uint32_t triple = (octet_a << 16) | (octet_b << 8) | octet_c; + + result.push_back(base64_chars[(triple >> 18) & 0x3F]); + result.push_back(base64_chars[(triple >> 12) & 0x3F]); + result.push_back((remaining > 1) ? base64_chars[(triple >> 6) & 0x3F] : '='); + result.push_back('='); + } + + return result; +} + +std::string Base64Decode(const std::string& encoded) { + static const unsigned char decode_table[256] = { + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 64, 64, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 64, 64, 64, + 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 64, + 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 + }; + + std::string result; + result.reserve((encoded.size() * 3) / 4); + + size_t i = 0; + while (i < encoded.size()) { + // Skip whitespace and invalid chars + while (i < encoded.size() && (encoded[i] == ' ' || encoded[i] == '\n' || encoded[i] == '\r' || encoded[i] == '\t')) { + i++; + } + if (i >= encoded.size()) break; + + uint32_t sextet_a = decode_table[static_cast(encoded[i++])]; + if (i >= encoded.size() || sextet_a == 64) break; + + uint32_t sextet_b = decode_table[static_cast(encoded[i++])]; + if (sextet_b == 64) break; + + uint32_t sextet_c = (i < encoded.size()) ? decode_table[static_cast(encoded[i++])] : 64; + uint32_t sextet_d = (i < encoded.size()) ? decode_table[static_cast(encoded[i++])] : 64; + + uint32_t triple = (sextet_a << 18) | (sextet_b << 12) | + ((sextet_c != 64) ? (sextet_c << 6) : 0) | + ((sextet_d != 64) ? sextet_d : 0); + + result.push_back(static_cast((triple >> 16) & 0xFF)); + if (sextet_c != 64) { + result.push_back(static_cast((triple >> 8) & 0xFF)); + } + if (sextet_d != 64) { + result.push_back(static_cast(triple & 0xFF)); + } + } + + return result; +} + +} // namespace + +std::string EtcdOpLogStore::SerializeOpLogEntry( + const OpLogEntry& entry) const { + Json::Value root; + root["sequence_id"] = static_cast(entry.sequence_id); + root["timestamp_ms"] = static_cast(entry.timestamp_ms); + root["op_type"] = static_cast(entry.op_type); + root["object_key"] = entry.object_key; + // CRITICAL: Base64 encode binary payload to prevent UTF-8 corruption in JSON + root["payload"] = Base64Encode(entry.payload); + root["checksum"] = static_cast(entry.checksum); + root["prefix_hash"] = static_cast(entry.prefix_hash); + + Json::StreamWriterBuilder builder; + builder["indentation"] = ""; // Compact format + std::unique_ptr writer(builder.newStreamWriter()); + std::ostringstream oss; + writer->write(root, &oss); + return oss.str(); +} + +bool EtcdOpLogStore::DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) const { + Json::Value root; + Json::CharReaderBuilder builder; + std::unique_ptr reader(builder.newCharReader()); + std::string errors; + + if (!reader->parse(json_str.data(), json_str.data() + json_str.size(), + &root, &errors)) { + LOG(ERROR) << "Failed to parse JSON: " << errors; + return false; + } + + try { + entry.sequence_id = root["sequence_id"].asUInt64(); + entry.timestamp_ms = root["timestamp_ms"].asUInt64(); + entry.op_type = static_cast(root["op_type"].asInt()); + entry.object_key = root["object_key"].asString(); + // CRITICAL: Base64 decode payload to restore binary data + entry.payload = Base64Decode(root["payload"].asString()); + entry.checksum = root["checksum"].asUInt(); + entry.prefix_hash = root["prefix_hash"].asUInt(); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to deserialize OpLogEntry: " << e.what(); + return false; + } + + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "EtcdOpLogStore: entry size rejected, sequence_id=" + << entry.sequence_id << ", key=" << entry.object_key + << ", reason=" << size_reason; + return false; + } + + return true; +} + +void EtcdOpLogStore::BatchUpdateThread() { + if (!enable_latest_seq_batch_update_) { + return; + } + while (batch_update_running_.load()) { + std::this_thread::sleep_for( + std::chrono::milliseconds(kBatchIntervalMs)); + + // Check if we need to update based on time interval + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast( + now - last_update_time_).count(); + + if (pending_count_.load() > 0 && elapsed >= kBatchIntervalMs) { + DoBatchUpdate(); + } + } +} + +void EtcdOpLogStore::TriggerBatchUpdateIfNeeded() { + // This method is kept for potential future use (e.g., manual trigger) + // Currently, DoBatchUpdate() is called directly from WriteOpLog + // when batch size threshold is reached + if (pending_count_.load() >= kBatchSize) { + DoBatchUpdate(); + } +} + +void EtcdOpLogStore::DoBatchUpdate() { + if (!enable_latest_seq_batch_update_) { + return; + } + std::lock_guard lock(batch_update_mutex_); + + // Get the pending sequence_id and reset counters + uint64_t seq_id_to_update = pending_latest_seq_id_.load(); + size_t count = pending_count_.exchange(0); + + if (count == 0) { + return; // Nothing to update + } + + // Update latest_sequence_id in etcd + ErrorCode err = UpdateLatestSequenceId(seq_id_to_update); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to batch update latest_sequence_id=" + << seq_id_to_update << ", error=" << err + << ". Will retry in next batch."; + // Restore the count so it will be retried + pending_count_.fetch_add(count); + } else { + last_update_time_ = std::chrono::steady_clock::now(); + VLOG(2) << "Batch updated latest_sequence_id=" << seq_id_to_update + << " (count=" << count << " entries)"; + } +} + +} // namespace mooncake + diff --git a/mooncake-store/src/etcd_snapshot_provider.cpp b/mooncake-store/src/etcd_snapshot_provider.cpp new file mode 100644 index 0000000000..3355cf00e7 --- /dev/null +++ b/mooncake-store/src/etcd_snapshot_provider.cpp @@ -0,0 +1,450 @@ +#include "etcd_snapshot_provider.h" + +#include +#include + +#if __has_include() +#include // Ubuntu +#else +#include // CentOS +#endif + +#include "etcd_helper.h" + +namespace mooncake { + +// Key prefix for snapshots +static constexpr const char* kSnapshotPrefix = "/snapshot/"; +static constexpr const char* kLatestSuffix = "/latest"; +static constexpr const char* kDataSuffix = "/data"; +static constexpr const char* kSeqSuffix = "/seq"; + +// Helper to get current timestamp in milliseconds for timing +static inline int64_t GetCurrentTimeMs() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +EtcdSnapshotProvider::EtcdSnapshotProvider(const std::string& cluster_id) + : cluster_id_(cluster_id) { + // Normalize cluster_id to avoid double slashes + while (!cluster_id_.empty() && cluster_id_.back() == '/') { + cluster_id_.pop_back(); + } + LOG(INFO) << "[EtcdSnapshotProvider] Initialized with cluster_id=" << cluster_id_ + << ", latest_key=" << BuildLatestKey(); +} + +std::string EtcdSnapshotProvider::BuildLatestKey() const { + return std::string(kSnapshotPrefix) + cluster_id_ + kLatestSuffix; +} + +std::string EtcdSnapshotProvider::BuildDataKey(const std::string& snapshot_id) const { + return std::string(kSnapshotPrefix) + cluster_id_ + "/" + snapshot_id + kDataSuffix; +} + +std::string EtcdSnapshotProvider::BuildSeqKey(const std::string& snapshot_id) const { + return std::string(kSnapshotPrefix) + cluster_id_ + "/" + snapshot_id + kSeqSuffix; +} + +std::string EtcdSnapshotProvider::SerializeMetadata( + const std::vector>& metadata) const { + int64_t start_time = GetCurrentTimeMs(); + size_t memory_replicas = 0, disk_replicas = 0, local_disk_replicas = 0; + + Json::Value root(Json::arrayValue); + + for (const auto& [key, meta] : metadata) { + Json::Value entry; + entry["key"] = key; + entry["client_id_high"] = static_cast(meta.client_id.first); + entry["client_id_low"] = static_cast(meta.client_id.second); + entry["size"] = static_cast(meta.size); + entry["last_sequence_id"] = static_cast(meta.last_sequence_id); + + // Serialize replicas + Json::Value replicas_array(Json::arrayValue); + for (const auto& replica_desc : meta.replicas) { + Json::Value replica_json; + // Check which variant type this is + if (std::holds_alternative(replica_desc.descriptor_variant)) { + const auto& mem_desc = std::get(replica_desc.descriptor_variant); + replica_json["type"] = "memory"; + // Serialize AllocatedBuffer::Descriptor + const auto& buf_desc = mem_desc.buffer_descriptor; + replica_json["size"] = static_cast(buf_desc.size_); + replica_json["buffer_address"] = static_cast(buf_desc.buffer_address_); + replica_json["transport_endpoint"] = buf_desc.transport_endpoint_; + ++memory_replicas; + } else if (std::holds_alternative(replica_desc.descriptor_variant)) { + const auto& disk_desc = std::get(replica_desc.descriptor_variant); + replica_json["type"] = "disk"; + replica_json["file_path"] = disk_desc.file_path; + replica_json["object_size"] = static_cast(disk_desc.object_size); + ++disk_replicas; + } else if (std::holds_alternative(replica_desc.descriptor_variant)) { + const auto& local_desc = std::get(replica_desc.descriptor_variant); + replica_json["type"] = "local_disk"; + replica_json["client_id_high"] = static_cast(local_desc.client_id.first); + replica_json["client_id_low"] = static_cast(local_desc.client_id.second); + replica_json["object_size"] = static_cast(local_desc.object_size); + replica_json["transport_endpoint"] = local_desc.transport_endpoint; + ++local_disk_replicas; + } + replica_json["status"] = static_cast(replica_desc.status); + replicas_array.append(replica_json); + } + entry["replicas"] = replicas_array; + + root.append(entry); + } + + Json::StreamWriterBuilder writer; + writer["indentation"] = ""; // Compact format + std::string result = Json::writeString(writer, root); + + int64_t elapsed = GetCurrentTimeMs() - start_time; + VLOG(1) << "[EtcdSnapshotProvider] SerializeMetadata: entries=" << metadata.size() + << ", memory_replicas=" << memory_replicas + << ", disk_replicas=" << disk_replicas + << ", local_disk_replicas=" << local_disk_replicas + << ", json_size=" << result.size() << " bytes" + << ", elapsed=" << elapsed << "ms"; + + return result; +} + +bool EtcdSnapshotProvider::DeserializeMetadata( + const std::string& json, + std::vector>& metadata) const { + int64_t start_time = GetCurrentTimeMs(); + size_t memory_replicas = 0, disk_replicas = 0, local_disk_replicas = 0; + + Json::CharReaderBuilder reader; + Json::Value root; + std::string errs; + std::istringstream stream(json); + + if (!Json::parseFromStream(reader, stream, &root, &errs)) { + LOG(ERROR) << "[EtcdSnapshotProvider] Failed to parse snapshot JSON: " << errs + << ", json_size=" << json.size() + << ", first_100_chars=" << json.substr(0, std::min(json.size(), size_t(100))); + return false; + } + + if (!root.isArray()) { + LOG(ERROR) << "[EtcdSnapshotProvider] Snapshot JSON root is not an array" + << ", type=" << root.type(); + return false; + } + + VLOG(1) << "[EtcdSnapshotProvider] DeserializeMetadata: parsing " << root.size() << " entries"; + + metadata.clear(); + metadata.reserve(root.size()); + + for (const auto& entry : root) { + std::string key = entry["key"].asString(); + StandbyObjectMetadata meta; + meta.client_id.first = entry["client_id_high"].asUInt64(); + meta.client_id.second = entry["client_id_low"].asUInt64(); + meta.size = entry["size"].asUInt64(); + meta.last_sequence_id = entry["last_sequence_id"].asUInt64(); + + // Deserialize replicas + const Json::Value& replicas_array = entry["replicas"]; + for (const auto& replica_json : replicas_array) { + Replica::Descriptor replica_desc; + std::string type = replica_json["type"].asString(); + + if (type == "memory") { + MemoryDescriptor mem_desc; + mem_desc.buffer_descriptor.size_ = replica_json["size"].asUInt64(); + mem_desc.buffer_descriptor.buffer_address_ = replica_json["buffer_address"].asUInt64(); + mem_desc.buffer_descriptor.transport_endpoint_ = replica_json["transport_endpoint"].asString(); + replica_desc.descriptor_variant = mem_desc; + ++memory_replicas; + } else if (type == "disk") { + DiskDescriptor disk_desc; + disk_desc.file_path = replica_json["file_path"].asString(); + disk_desc.object_size = replica_json["object_size"].asUInt64(); + replica_desc.descriptor_variant = disk_desc; + ++disk_replicas; + } else if (type == "local_disk") { + LocalDiskDescriptor local_desc; + local_desc.client_id.first = replica_json["client_id_high"].asUInt64(); + local_desc.client_id.second = replica_json["client_id_low"].asUInt64(); + local_desc.object_size = replica_json["object_size"].asUInt64(); + local_desc.transport_endpoint = replica_json["transport_endpoint"].asString(); + replica_desc.descriptor_variant = local_desc; + ++local_disk_replicas; + } else { + LOG(WARNING) << "[EtcdSnapshotProvider] Unknown replica type: " << type + << " for key=" << key; + continue; + } + + replica_desc.status = static_cast(replica_json["status"].asInt()); + meta.replicas.push_back(replica_desc); + } + + metadata.emplace_back(key, std::move(meta)); + } + + int64_t elapsed = GetCurrentTimeMs() - start_time; + VLOG(1) << "[EtcdSnapshotProvider] DeserializeMetadata: entries=" << metadata.size() + << ", memory_replicas=" << memory_replicas + << ", disk_replicas=" << disk_replicas + << ", local_disk_replicas=" << local_disk_replicas + << ", json_size=" << json.size() << " bytes" + << ", elapsed=" << elapsed << "ms"; + + return true; +} + +bool EtcdSnapshotProvider::GetLatestSnapshotId(std::string& snapshot_id) const { + std::string latest_key = BuildLatestKey(); + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(latest_key.c_str(), latest_key.size(), + snapshot_id, revision_id); + if (err == ErrorCode::OK && !snapshot_id.empty()) { + VLOG(1) << "[EtcdSnapshotProvider] GetLatestSnapshotId: snapshot_id=" << snapshot_id + << ", etcd_revision=" << revision_id; + return true; + } + VLOG(1) << "[EtcdSnapshotProvider] GetLatestSnapshotId: no snapshot found" + << ", error=" << static_cast(err); + return false; +} + +bool EtcdSnapshotProvider::LoadLatestSnapshot( + const std::string& cluster_id, std::string& snapshot_id, + uint64_t& snapshot_sequence_id, + std::vector>& snapshot) { + + int64_t start_time = GetCurrentTimeMs(); + LOG(INFO) << "[EtcdSnapshotProvider] LoadLatestSnapshot: starting for cluster_id=" << cluster_id; + + if (cluster_id != cluster_id_) { + LOG(ERROR) << "[EtcdSnapshotProvider] Cluster ID mismatch: expected=" << cluster_id_ + << ", got=" << cluster_id; + return false; + } + + // Get the latest snapshot ID + if (!GetLatestSnapshotId(snapshot_id)) { + LOG(INFO) << "[EtcdSnapshotProvider] No snapshot available for cluster_id=" << cluster_id_; + snapshot_id.clear(); + snapshot_sequence_id = 0; + snapshot.clear(); + return false; + } + + // Get the sequence_id + std::string seq_key = BuildSeqKey(snapshot_id); + std::string seq_value; + EtcdRevisionId revision_id; + VLOG(1) << "[EtcdSnapshotProvider] LoadLatestSnapshot: fetching seq_key=" << seq_key; + ErrorCode err = EtcdHelper::Get(seq_key.c_str(), seq_key.size(), + seq_value, revision_id); + if (err != ErrorCode::OK) { + LOG(ERROR) << "[EtcdSnapshotProvider] Failed to get snapshot sequence_id" + << ", snapshot_id=" << snapshot_id + << ", seq_key=" << seq_key + << ", error=" << static_cast(err); + return false; + } + + try { + snapshot_sequence_id = std::stoull(seq_value); + } catch (const std::exception& e) { + LOG(ERROR) << "[EtcdSnapshotProvider] Failed to parse snapshot sequence_id" + << ", seq_value=" << seq_value + << ", error=" << e.what(); + return false; + } + + // Get the snapshot data + std::string data_key = BuildDataKey(snapshot_id); + std::string data_value; + VLOG(1) << "[EtcdSnapshotProvider] LoadLatestSnapshot: fetching data_key=" << data_key; + err = EtcdHelper::Get(data_key.c_str(), data_key.size(), + data_value, revision_id); + if (err != ErrorCode::OK) { + LOG(ERROR) << "[EtcdSnapshotProvider] Failed to get snapshot data" + << ", snapshot_id=" << snapshot_id + << ", data_key=" << data_key + << ", error=" << static_cast(err); + return false; + } + + VLOG(1) << "[EtcdSnapshotProvider] LoadLatestSnapshot: data_size=" << data_value.size() + << ", deserializing..."; + + // Deserialize the metadata + if (!DeserializeMetadata(data_value, snapshot)) { + LOG(ERROR) << "[EtcdSnapshotProvider] Failed to deserialize snapshot data" + << ", snapshot_id=" << snapshot_id + << ", data_size=" << data_value.size(); + return false; + } + + int64_t elapsed = GetCurrentTimeMs() - start_time; + LOG(INFO) << "[EtcdSnapshotProvider] LoadLatestSnapshot: SUCCESS" + << ", snapshot_id=" << snapshot_id + << ", sequence_id=" << snapshot_sequence_id + << ", entries=" << snapshot.size() + << ", data_size=" << data_value.size() << " bytes" + << ", elapsed=" << elapsed << "ms"; + return true; +} + +ErrorCode EtcdSnapshotProvider::SaveSnapshot( + const std::string& snapshot_id, + uint64_t sequence_id, + const std::vector>& metadata) { + + int64_t start_time = GetCurrentTimeMs(); + LOG(INFO) << "[EtcdSnapshotProvider] SaveSnapshot: starting" + << ", snapshot_id=" << snapshot_id + << ", sequence_id=" << sequence_id + << ", entries=" << metadata.size(); + + // Get the current latest snapshot ID (to delete later) + std::string old_snapshot_id; + bool has_old_snapshot = GetLatestSnapshotId(old_snapshot_id); + + // Don't delete if the old snapshot is the same as the new one + if (has_old_snapshot && old_snapshot_id == snapshot_id) { + VLOG(1) << "[EtcdSnapshotProvider] SaveSnapshot: old snapshot same as new, skipping delete"; + has_old_snapshot = false; + } + + // Build keys for new snapshot + std::string latest_key = BuildLatestKey(); + std::string data_key = BuildDataKey(snapshot_id); + std::string seq_key = BuildSeqKey(snapshot_id); + + VLOG(1) << "[EtcdSnapshotProvider] SaveSnapshot: keys" + << ", latest_key=" << latest_key + << ", data_key=" << data_key + << ", seq_key=" << seq_key; + + // Build keys for old snapshot (if exists) + std::string old_data_key; + std::string old_seq_key; + if (has_old_snapshot) { + old_data_key = BuildDataKey(old_snapshot_id); + old_seq_key = BuildSeqKey(old_snapshot_id); + VLOG(1) << "[EtcdSnapshotProvider] SaveSnapshot: will delete old snapshot" + << ", old_snapshot_id=" << old_snapshot_id + << ", old_data_key=" << old_data_key + << ", old_seq_key=" << old_seq_key; + } + + // Serialize metadata + int64_t serialize_start = GetCurrentTimeMs(); + std::string data_value = SerializeMetadata(metadata); + int64_t serialize_elapsed = GetCurrentTimeMs() - serialize_start; + std::string seq_value = std::to_string(sequence_id); + + LOG(INFO) << "[EtcdSnapshotProvider] SaveSnapshot: serialized" + << ", snapshot_id=" << snapshot_id + << ", sequence_id=" << sequence_id + << ", entries=" << metadata.size() + << ", data_size=" << data_value.size() << " bytes" + << ", serialize_time=" << serialize_elapsed << "ms" + << (has_old_snapshot ? ", deleting_old=" + old_snapshot_id : ""); + + // Execute the atomic transaction + int64_t txn_start = GetCurrentTimeMs(); + ErrorCode err = EtcdHelper::SaveSnapshotTxn( + data_key, data_value, + seq_key, seq_value, + latest_key, snapshot_id, + old_data_key, old_seq_key); + int64_t txn_elapsed = GetCurrentTimeMs() - txn_start; + + if (err != ErrorCode::OK) { + LOG(ERROR) << "[EtcdSnapshotProvider] SaveSnapshot: FAILED" + << ", snapshot_id=" << snapshot_id + << ", error=" << static_cast(err) + << ", txn_time=" << txn_elapsed << "ms"; + return err; + } + + int64_t total_elapsed = GetCurrentTimeMs() - start_time; + LOG(INFO) << "[EtcdSnapshotProvider] SaveSnapshot: SUCCESS" + << ", snapshot_id=" << snapshot_id + << ", sequence_id=" << sequence_id + << ", entries=" << metadata.size() + << ", data_size=" << data_value.size() << " bytes" + << ", serialize_time=" << serialize_elapsed << "ms" + << ", txn_time=" << txn_elapsed << "ms" + << ", total_time=" << total_elapsed << "ms"; + return ErrorCode::OK; +} + +void EtcdSnapshotProvider::DumpSnapshotStatus() const { + std::string snapshot_id; + uint64_t sequence_id = 0; + size_t data_size_bytes = 0; + + bool has_snapshot = GetSnapshotStats(snapshot_id, sequence_id, data_size_bytes); + + if (has_snapshot) { + LOG(INFO) << "[EtcdSnapshotProvider] === Snapshot Status ===" + << "\n cluster_id: " << cluster_id_ + << "\n latest_key: " << BuildLatestKey() + << "\n snapshot_id: " << snapshot_id + << "\n sequence_id: " << sequence_id + << "\n data_size: " << data_size_bytes << " bytes" + << "\n data_key: " << BuildDataKey(snapshot_id) + << "\n seq_key: " << BuildSeqKey(snapshot_id); + } else { + LOG(INFO) << "[EtcdSnapshotProvider] === Snapshot Status ===" + << "\n cluster_id: " << cluster_id_ + << "\n latest_key: " << BuildLatestKey() + << "\n status: NO SNAPSHOT AVAILABLE"; + } +} + +bool EtcdSnapshotProvider::GetSnapshotStats(std::string& snapshot_id, + uint64_t& sequence_id, + size_t& data_size_bytes) const { + snapshot_id.clear(); + sequence_id = 0; + data_size_bytes = 0; + + // Get latest snapshot ID + if (!GetLatestSnapshotId(snapshot_id)) { + return false; + } + + // Get sequence_id + std::string seq_key = BuildSeqKey(snapshot_id); + std::string seq_value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(seq_key.c_str(), seq_key.size(), + seq_value, revision_id); + if (err == ErrorCode::OK) { + try { + sequence_id = std::stoull(seq_value); + } catch (...) { + // Ignore parse errors + } + } + + // Get data size (fetch data to get size, but don't parse it) + std::string data_key = BuildDataKey(snapshot_id); + std::string data_value; + err = EtcdHelper::Get(data_key.c_str(), data_key.size(), + data_value, revision_id); + if (err == ErrorCode::OK) { + data_size_bytes = data_value.size(); + } + + return true; +} + +} // namespace mooncake diff --git a/mooncake-store/src/ha_helper.cpp b/mooncake-store/src/ha_helper.cpp index 07906772af..74bdf5e71c 100644 --- a/mooncake-store/src/ha_helper.cpp +++ b/mooncake-store/src/ha_helper.cpp @@ -1,16 +1,65 @@ #include "ha_helper.h" + +#include + +#include +#include + #include "etcd_helper.h" +#include "etcd_snapshot_provider.h" +#include "hot_standby_service.h" #include "rpc_service.h" namespace mooncake { -MasterViewHelper::MasterViewHelper() { - std::string cluster_id; +namespace { +std::string ResolveClusterIdForMasterView(const std::string& cluster_id) { + std::string resolved; + if (!cluster_id.empty()) { + resolved = cluster_id; + } else { const char* cluster_id_env = std::getenv("MC_STORE_CLUSTER_ID"); if (cluster_id_env != nullptr && strlen(cluster_id_env) > 0) { - cluster_id = cluster_id_env; + resolved = std::string(cluster_id_env); } else { - cluster_id = "mooncake"; + resolved = DEFAULT_CLUSTER_ID; + } + } + + // Validate resolved cluster_id (even if it's the default). + // Strip trailing slashes for validation. + std::string normalized = resolved; + while (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); + } + if (!normalized.empty() && !IsValidClusterIdComponent(normalized)) { + LOG(FATAL) << "Invalid cluster_id resolved for MasterViewHelper: '" << resolved + << "' (normalized: '" << normalized + << "'). Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } + + return resolved; +} +} // namespace + +MasterViewHelper::MasterViewHelper(const std::string& cluster_id) { + BuildMasterViewKeyFromClusterId(ResolveClusterIdForMasterView(cluster_id)); +} + +void MasterViewHelper::SetClusterId(const std::string& cluster_id) { + BuildMasterViewKeyFromClusterId(ResolveClusterIdForMasterView(cluster_id)); +} + +void MasterViewHelper::BuildMasterViewKeyFromClusterId( + const std::string& cluster_id_in) { + std::string cluster_id = cluster_id_in; + // Normalize cluster_id for validation: strip trailing slashes. + while (!cluster_id.empty() && cluster_id.back() == '/') { + cluster_id.pop_back(); + } + if (!IsValidClusterIdComponent(cluster_id)) { + LOG(FATAL) << "Invalid cluster_id for MasterViewHelper: '" << cluster_id + << "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; } // Ensure the cluster_id ends with '/' if not empty if (!cluster_id.empty() && cluster_id.back() != '/') { @@ -125,19 +174,77 @@ int MasterServiceSupervisor::Start() { } LOG(INFO) << "Init leader election helper..."; - MasterViewHelper mv_helper; + MasterViewHelper mv_helper(config_.cluster_id); if (mv_helper.ConnectToEtcd(config_.etcd_endpoints) != ErrorCode::OK) { LOG(ERROR) << "Failed to connect to etcd endpoints: " << config_.etcd_endpoints; return -1; } - LOG(INFO) << "Trying to elect self as leader..."; + +#ifdef STORE_USE_ETCD + // Connect to etcd for OpLog sync + if (EtcdHelper::ConnectToEtcdStoreClient(config_.etcd_endpoints.c_str()) != + ErrorCode::OK) { + LOG(ERROR) << "Failed to connect to etcd store client: " + << config_.etcd_endpoints; + return -1; + } +#endif + + LOG(INFO) << "Checking for existing leader..."; EtcdLeaseId lease_id = 0; - // view_version will be updated by ElectLeader and then used in - // WrappedMasterService ViewVersionId view_version = 0; + + // Check if there is already a leader + std::string current_leader; + ViewVersionId current_version = 0; + auto ret = mv_helper.GetMasterView(current_leader, current_version); + bool had_standby = false; + + if (ret == ErrorCode::OK) { + // There is an existing leader, start Standby service + LOG(INFO) << "Found existing leader: " << current_leader + << ", starting Standby service..."; + StartStandbyService(mv_helper, current_leader); + had_standby = true; + + // Watch until leader is deleted + LOG(INFO) << "Watching for leadership change..."; + auto watch_ret = EtcdHelper::WatchUntilDeleted( + mv_helper.GetMasterViewKey().c_str(), mv_helper.GetMasterViewKey().size()); + + if (watch_ret != ErrorCode::OK) { + LOG(ERROR) << "Error watching for leadership change: " << watch_ret; + // Stop Standby service on watch error and retry. + StopStandbyService(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + LOG(INFO) << "Leader disappeared, trying to elect self as leader..."; + } else { + LOG(INFO) << "No existing leader found, trying to elect self as leader..."; + } + + // Try to elect self as leader mv_helper.ElectLeader(config_.local_hostname, view_version, lease_id); + // If we were running as Standby, finalize catch-up and snapshot metadata now. + std::vector> standby_snapshot; + uint64_t standby_last_seq_id = 0; +#ifdef STORE_USE_ETCD + if (had_standby && standby_service_ && standby_running_.load()) { + LOG(INFO) << "Finalizing standby state for promotion..."; + standby_service_->Promote(); // does final catch-up sync + stops watcher + standby_last_seq_id = standby_service_->GetLatestAppliedSequenceId(); + standby_service_->ExportMetadataSnapshot(standby_snapshot); + // We are now leader; standby service is no longer needed. + StopStandbyService(); + LOG(INFO) << "Standby snapshot ready: keys=" << standby_snapshot.size() + << ", last_seq_id=" << standby_last_seq_id; + } +#endif + // Start a thread to keep the leader alive auto keep_leader_thread = std::thread([&server, &mv_helper, lease_id]() { @@ -154,6 +261,14 @@ int MasterServiceSupervisor::Start() { LOG(INFO) << "Starting master service..."; mooncake::WrappedMasterService wrapped_master_service( mooncake::WrappedMasterServiceConfig(config_, view_version)); + + // Restore from promoted standby snapshot if available. +#ifdef STORE_USE_ETCD + if (standby_last_seq_id > 0 || !standby_snapshot.empty()) { + wrapped_master_service.RestoreFromStandby(standby_snapshot, standby_last_seq_id); + } +#endif + mooncake::RegisterRpcService(server, wrapped_master_service); // Metric reporting is now handled by WrappedMasterService. @@ -186,7 +301,60 @@ int MasterServiceSupervisor::Start() { return 0; } +void MasterServiceSupervisor::StartStandbyService(MasterViewHelper& mv_helper, + const std::string& current_leader) { +#ifdef STORE_USE_ETCD + if (standby_running_.load()) { + LOG(WARNING) << "Standby service is already running"; + return; + } + + HotStandbyConfig standby_config; + standby_config.standby_id = config_.local_hostname; + standby_config.primary_address = current_leader; + standby_config.verification_interval_sec = 30; + standby_config.max_replication_lag_entries = 1000; + standby_config.enable_verification = false; // Disable verification for now + standby_config.enable_snapshot_bootstrap = true; + + standby_service_ = std::make_unique(standby_config); + // Inject EtcdSnapshotProvider for bootstrap + standby_service_->SetSnapshotProvider( + std::make_unique(config_.cluster_id)); + + ErrorCode err = standby_service_->Start( + current_leader, config_.etcd_endpoints, config_.cluster_id); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to start Standby service: " << err; + standby_service_.reset(); + return; + } + + standby_running_.store(true); + LOG(INFO) << "Standby service started successfully"; +#else + LOG(WARNING) << "STORE_USE_ETCD is not enabled, cannot start Standby service"; +#endif +} + +void MasterServiceSupervisor::StopStandbyService() { +#ifdef STORE_USE_ETCD + if (!standby_running_.load()) { + return; + } + + if (standby_service_) { + standby_service_->Stop(); + standby_service_.reset(); + } + + standby_running_.store(false); + LOG(INFO) << "Standby service stopped"; +#endif +} + MasterServiceSupervisor::~MasterServiceSupervisor() { + StopStandbyService(); if (server_thread_.joinable()) { server_thread_.join(); } diff --git a/mooncake-store/src/ha_metric_manager.cpp b/mooncake-store/src/ha_metric_manager.cpp new file mode 100644 index 0000000000..ddf129c607 --- /dev/null +++ b/mooncake-store/src/ha_metric_manager.cpp @@ -0,0 +1,305 @@ +#include "ha_metric_manager.h" + +#include + +#include +#include + +namespace mooncake { + +// --- Singleton Instance --- +HAMetricManager& HAMetricManager::instance() { + static HAMetricManager static_instance; + return static_instance; +} + +// --- Constructor --- +HAMetricManager::HAMetricManager() + // OpLog Sequence Gauges + : oplog_last_sequence_id_( + "ha_oplog_last_sequence_id", + "Latest OpLog sequence ID written by Primary"), + oplog_applied_sequence_id_( + "ha_oplog_applied_sequence_id", + "Latest OpLog sequence ID applied by Standby"), + oplog_standby_lag_( + "ha_oplog_standby_lag", + "Number of OpLog entries Standby is behind Primary"), + oplog_pending_entries_( + "ha_oplog_pending_entries", + "Number of out-of-order entries waiting in OpLogApplier"), + pending_mutation_queue_size_( + "ha_pending_mutation_queue_size", + "Number of mutations pending etcd write retry"), + + // Error Counters + oplog_skipped_entries_total_( + "ha_oplog_skipped_entries_total", + "Total number of OpLog entries skipped due to timeout"), + oplog_checksum_failures_total_( + "ha_oplog_checksum_failures_total", + "Total number of OpLog entries with checksum verification failures"), + oplog_gap_resolve_attempts_total_( + "ha_oplog_gap_resolve_attempts_total", + "Total number of attempts to resolve missing OpLog entries"), + oplog_gap_resolve_success_total_( + "ha_oplog_gap_resolve_success_total", + "Total number of successfully resolved missing OpLog entries"), + oplog_etcd_write_failures_total_( + "ha_oplog_etcd_write_failures_total", + "Total number of failed etcd write operations"), + oplog_etcd_write_retries_total_( + "ha_oplog_etcd_write_retries_total", + "Total number of etcd write retry attempts"), + oplog_watch_disconnections_total_( + "ha_oplog_watch_disconnections_total", + "Total number of OpLog watch disconnections"), + oplog_applied_entries_total_( + "ha_oplog_applied_entries_total", + "Total number of OpLog entries successfully applied"), + oplog_batch_commits_total_( + "ha_oplog_batch_commits_total", + "Total number of Group Commit batches flushed to etcd"), + oplog_sync_batch_commits_total_( + "ha_oplog_sync_batch_commits_total", + "Total number of sync batches (triggered by DELETE/Sync ops)"), + + // Latency Histograms (buckets in microseconds) + // 100us, 500us, 1ms, 5ms, 10ms, 50ms, 100ms, 500ms, 1s, 5s + oplog_etcd_write_latency_us_( + "ha_oplog_etcd_write_latency_us", + "Latency of etcd write operations in microseconds", + {100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000, 5000000}), + oplog_apply_latency_us_( + "ha_oplog_apply_latency_us", + "Latency of OpLog entry application in microseconds", + {10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000}), + + // State Machine + standby_state_( + "ha_standby_state", + "Current state of the Standby service (0=STOPPED, 1=CONNECTING, " + "2=SYNCING, 3=WATCHING, 4=RECOVERING, 5=RECONNECTING, " + "6=PROMOTING, 7=PROMOTED, 8=FAILED)"), + state_transitions_total_( + "ha_state_transitions_total", + "Total number of Standby state machine transitions") { + // Initialize gauges to 0 for proper Prometheus output + oplog_last_sequence_id_.update(0); + oplog_applied_sequence_id_.update(0); + oplog_standby_lag_.update(0); + oplog_pending_entries_.update(0); + pending_mutation_queue_size_.update(0); + standby_state_.update(0); +} + +// ========== OpLog Sequence Metrics (Gauge) ========== + +void HAMetricManager::set_oplog_last_sequence_id(int64_t seq_id) { + oplog_last_sequence_id_.update(seq_id); +} + +int64_t HAMetricManager::get_oplog_last_sequence_id() { + return static_cast(oplog_last_sequence_id_.value()); +} + +void HAMetricManager::set_oplog_applied_sequence_id(int64_t seq_id) { + oplog_applied_sequence_id_.update(seq_id); +} + +int64_t HAMetricManager::get_oplog_applied_sequence_id() { + return static_cast(oplog_applied_sequence_id_.value()); +} + +void HAMetricManager::set_oplog_standby_lag(int64_t lag) { + oplog_standby_lag_.update(lag); +} + +int64_t HAMetricManager::get_oplog_standby_lag() { + return static_cast(oplog_standby_lag_.value()); +} + +void HAMetricManager::set_oplog_pending_entries(int64_t count) { + oplog_pending_entries_.update(count); +} + +int64_t HAMetricManager::get_oplog_pending_entries() { + return static_cast(oplog_pending_entries_.value()); +} + +void HAMetricManager::set_pending_mutation_queue_size(int64_t size) { + pending_mutation_queue_size_.update(size); +} + +int64_t HAMetricManager::get_pending_mutation_queue_size() { + return static_cast(pending_mutation_queue_size_.value()); +} + +// ========== Error Counters ========== + +void HAMetricManager::inc_oplog_skipped_entries(int64_t val) { + oplog_skipped_entries_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_skipped_entries_total() { + return static_cast(oplog_skipped_entries_total_.value()); +} + +void HAMetricManager::inc_oplog_checksum_failures(int64_t val) { + oplog_checksum_failures_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_checksum_failures_total() { + return static_cast(oplog_checksum_failures_total_.value()); +} + +void HAMetricManager::inc_oplog_gap_resolve_attempts(int64_t val) { + oplog_gap_resolve_attempts_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_gap_resolve_attempts_total() { + return static_cast(oplog_gap_resolve_attempts_total_.value()); +} + +void HAMetricManager::inc_oplog_gap_resolve_success(int64_t val) { + oplog_gap_resolve_success_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_gap_resolve_success_total() { + return static_cast(oplog_gap_resolve_success_total_.value()); +} + +void HAMetricManager::inc_oplog_etcd_write_failures(int64_t val) { + oplog_etcd_write_failures_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_etcd_write_failures_total() { + return static_cast(oplog_etcd_write_failures_total_.value()); +} + +void HAMetricManager::inc_oplog_etcd_write_retries(int64_t val) { + oplog_etcd_write_retries_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_etcd_write_retries_total() { + return static_cast(oplog_etcd_write_retries_total_.value()); +} + +void HAMetricManager::inc_oplog_watch_disconnections(int64_t val) { + oplog_watch_disconnections_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_watch_disconnections_total() { + return static_cast(oplog_watch_disconnections_total_.value()); +} + +void HAMetricManager::inc_oplog_applied_entries(int64_t val) { + oplog_applied_entries_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_applied_entries_total() { + return static_cast(oplog_applied_entries_total_.value()); +} + +void HAMetricManager::inc_oplog_batch_commits(int64_t val) { + oplog_batch_commits_total_.inc(val); +} + +void HAMetricManager::inc_oplog_sync_batch_commits(int64_t val) { + oplog_sync_batch_commits_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_batch_commits_total() { + return static_cast(oplog_batch_commits_total_.value()); +} + +int64_t HAMetricManager::get_oplog_sync_batch_commits_total() { + return static_cast(oplog_sync_batch_commits_total_.value()); +} + +// ========== Latency Histograms ========== + +void HAMetricManager::observe_oplog_etcd_write_latency_us(int64_t latency_us) { + oplog_etcd_write_latency_us_.observe(latency_us); +} + +void HAMetricManager::observe_oplog_apply_latency_us(int64_t latency_us) { + oplog_apply_latency_us_.observe(latency_us); +} + +// ========== State Machine Metrics ========== + +void HAMetricManager::set_standby_state(int64_t state_value) { + standby_state_.update(state_value); +} + +int64_t HAMetricManager::get_standby_state() { + return static_cast(standby_state_.value()); +} + +void HAMetricManager::inc_state_transitions(int64_t val) { + state_transitions_total_.inc(val); +} + +int64_t HAMetricManager::get_state_transitions_total() { + return static_cast(state_transitions_total_.value()); +} + +// ========== Serialization ========== + +std::string HAMetricManager::serialize_metrics() { + std::stringstream ss; + + // Helper lambda to serialize a metric + auto serialize_metric = [&ss](auto& metric) { + std::string metric_str; + metric.serialize(metric_str); + ss << metric_str; + }; + + // Gauges + serialize_metric(oplog_last_sequence_id_); + serialize_metric(oplog_applied_sequence_id_); + serialize_metric(oplog_standby_lag_); + serialize_metric(oplog_pending_entries_); + serialize_metric(pending_mutation_queue_size_); + serialize_metric(standby_state_); + + // Counters + serialize_metric(oplog_skipped_entries_total_); + serialize_metric(oplog_checksum_failures_total_); + serialize_metric(oplog_gap_resolve_attempts_total_); + serialize_metric(oplog_gap_resolve_success_total_); + serialize_metric(oplog_etcd_write_failures_total_); + serialize_metric(oplog_etcd_write_retries_total_); + serialize_metric(oplog_watch_disconnections_total_); + serialize_metric(oplog_applied_entries_total_); + serialize_metric(state_transitions_total_); + + // Histograms + serialize_metric(oplog_etcd_write_latency_us_); + serialize_metric(oplog_apply_latency_us_); + + return ss.str(); +} + +std::string HAMetricManager::get_summary_string() { + std::stringstream ss; + ss << "HA Metrics Summary: "; + ss << "last_seq=" << get_oplog_last_sequence_id(); + ss << ", applied_seq=" << get_oplog_applied_sequence_id(); + ss << ", lag=" << get_oplog_standby_lag(); + ss << ", pending=" << get_oplog_pending_entries(); + ss << ", mutation_queue=" << get_pending_mutation_queue_size(); + ss << ", batch_commits=" << get_oplog_batch_commits_total(); + ss << ", sync_commits=" << get_oplog_sync_batch_commits_total(); + ss << ", skipped=" << get_oplog_skipped_entries_total(); + ss << ", checksum_fail=" << get_oplog_checksum_failures_total(); + ss << ", etcd_fail=" << get_oplog_etcd_write_failures_total(); + ss << ", watch_disconn=" << get_oplog_watch_disconnections_total(); + ss << ", state=" << get_standby_state(); + return ss.str(); +} + +} // namespace mooncake + diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp new file mode 100644 index 0000000000..3ff3940bab --- /dev/null +++ b/mooncake-store/src/hot_standby_service.cpp @@ -0,0 +1,605 @@ +#include "hot_standby_service.h" + +#include + +#include +#include + +#include "etcd_helper.h" +#include "etcd_oplog_store.h" +#include "ha_metric_manager.h" +#include "master_service.h" +#include "oplog_applier.h" +#include "oplog_manager.h" +#include "oplog_watcher.h" + +namespace mooncake { + +HotStandbyService::HotStandbyService(const HotStandbyConfig& config) + : config_(config) { + metadata_store_ = std::make_unique(); + // OpLogApplier will be re-created in Start() with the resolved cluster_id + // to enable etcd-based operations (e.g. requesting missing OpLog entries). + // Here we construct a minimal instance so that local metadata operations + // are available before etcd wiring is completed. + oplog_applier_ = std::make_unique(metadata_store_.get()); + + // Register callback for state change logging and metrics. + state_machine_.RegisterCallback([](StandbyState old_state, StandbyState new_state, StandbyEvent event) { + LOG(INFO) << "HotStandbyService state changed: " + << StandbyStateToString(old_state) << " -> " + << StandbyStateToString(new_state) + << " (event: " << StandbyEventToString(event) << ")"; + + // Update HA metrics + HAMetricManager::instance().set_standby_state(static_cast(new_state)); + HAMetricManager::instance().inc_state_transitions(); + + // Track watch disconnections + if (event == StandbyEvent::WATCH_BROKEN || event == StandbyEvent::DISCONNECTED) { + HAMetricManager::instance().inc_oplog_watch_disconnections(); + } + }); +} + +// StandbyMetadataStore implementation +bool HotStandbyService::StandbyMetadataStore::PutMetadata( + const std::string& key, const StandbyObjectMetadata& metadata) { + std::lock_guard lock(mutex_); + store_[key] = metadata; + VLOG(2) << "StandbyMetadataStore: stored metadata for key=" << key + << ", replicas=" << metadata.replicas.size() + << ", size=" << metadata.size; + return true; +} + +bool HotStandbyService::StandbyMetadataStore::Put(const std::string& key, + const std::string& payload) { + // Legacy interface - create empty metadata + StandbyObjectMetadata metadata; + std::lock_guard lock(mutex_); + store_[key] = metadata; + return true; +} + +const StandbyObjectMetadata* HotStandbyService::StandbyMetadataStore::GetMetadata( + const std::string& key) const { + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it != store_.end()) { + return &it->second; + } + return nullptr; +} + +bool HotStandbyService::StandbyMetadataStore::Remove(const std::string& key) { + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it != store_.end()) { + store_.erase(it); + return true; + } + return false; +} + +bool HotStandbyService::StandbyMetadataStore::Exists( + const std::string& key) const { + std::lock_guard lock(mutex_); + return store_.find(key) != store_.end(); +} + +size_t HotStandbyService::StandbyMetadataStore::GetKeyCount() const { + std::lock_guard lock(mutex_); + return store_.size(); +} + +void HotStandbyService::StandbyMetadataStore::Snapshot( + std::vector>& out) const { + std::lock_guard lock(mutex_); + out.clear(); + out.reserve(store_.size()); + for (const auto& kv : store_) { + out.emplace_back(kv.first, kv.second); + } +} + +HotStandbyService::~HotStandbyService() { + // Always ensure threads are joined, regardless of state + // This prevents std::terminate() if threads are still joinable + Stop(); + + // Double-check: ensure all threads are joined even if Stop() had early return + if (replication_thread_.joinable()) { + replication_thread_.join(); + } + if (verification_thread_.joinable()) { + verification_thread_.join(); + } +} + +ErrorCode HotStandbyService::Start(const std::string& primary_address, + const std::string& etcd_endpoints, + const std::string& cluster_id) { + std::lock_guard lock(mutex_); + + // Use state machine to check if already running + if (IsRunning()) { + LOG(WARNING) << "HotStandbyService is already running"; + return ErrorCode::OK; + } + + // Trigger START event + auto result = state_machine_.ProcessEvent(StandbyEvent::START); + if (!result.allowed) { + LOG(ERROR) << "Cannot start HotStandbyService: " << result.reason; + return ErrorCode::INTERNAL_ERROR; // State machine rejected START + } + + config_.primary_address = primary_address; + etcd_endpoints_ = etcd_endpoints; + cluster_id_ = cluster_id; + +#ifdef STORE_USE_ETCD + // Connect to etcd + ErrorCode err = EtcdHelper::ConnectToEtcdStoreClient(etcd_endpoints.c_str()); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to connect to etcd: " << etcd_endpoints; + state_machine_.ProcessEvent(StandbyEvent::CONNECTION_FAILED); + return err; + } + + // Transition to SYNCING state + state_machine_.ProcessEvent(StandbyEvent::CONNECTED); + + // Preserve existing local state if HotStandbyService is restarted in-process: + // - metadata_store_ may already contain real-time metadata + // - oplog_applier_ may already have expected_sequence_id_ + uint64_t local_last_seq_id = 0; + if (oplog_applier_) { + uint64_t expected = oplog_applier_->GetExpectedSequenceId(); + local_last_seq_id = expected > 0 ? expected - 1 : 0; + } + const bool has_local_metadata = + metadata_store_ && metadata_store_->GetKeyCount() > 0; + const bool has_local_state = has_local_metadata && local_last_seq_id > 0; + + // Recreate OpLogApplier with cluster_id (for requesting missing OpLog). + // If we had local state, recover to keep sequence continuity. + oplog_applier_ = std::make_unique(metadata_store_.get(), cluster_id); + if (has_local_state) { + LOG(INFO) << "Standby warm start: reuse local metadata (keys=" + << metadata_store_->GetKeyCount() + << "), recover last_seq_id=" << local_last_seq_id; + oplog_applier_->Recover(local_last_seq_id); + } + + // Create OpLogWatcher with state machine callback + oplog_watcher_ = std::make_unique( + etcd_endpoints, cluster_id, oplog_applier_.get()); + + // Register callback for watcher events + oplog_watcher_->SetStateCallback([this](StandbyEvent event) { + OnWatcherEvent(event); + }); + + // Bootstrap: + // - If we already have local state (warm start), do NOT reload snapshot. + // - Otherwise (cold start/new standby), try snapshot (if enabled) then replay OpLog. + uint64_t baseline_seq_id = has_local_state ? local_last_seq_id : 0; + if (!has_local_state && config_.enable_snapshot_bootstrap && snapshot_provider_) { + std::string snapshot_id; + uint64_t snapshot_seq_id = 0; + std::vector> snapshot; + if (snapshot_provider_->LoadLatestSnapshot(cluster_id_, snapshot_id, snapshot_seq_id, + snapshot)) { + LOG(INFO) << "Loaded snapshot: snapshot_id=" << snapshot_id + << ", snapshot_seq_id=" << snapshot_seq_id + << ", keys=" << snapshot.size(); + // Apply snapshot into local standby store. + for (const auto& kv : snapshot) { + metadata_store_->PutMetadata(kv.first, kv.second); + } + // Align applier to snapshot boundary. + oplog_applier_->Recover(snapshot_seq_id); + baseline_seq_id = snapshot_seq_id; + } else { + LOG(INFO) << "No snapshot available (or provider not ready), falling back to OpLog-only bootstrap"; + } + } + + // Read historical OpLog entries since baseline_seq_id. + uint64_t last_applied_seq_id = baseline_seq_id; + + // Start OpLogWatcher with a consistent "read then watch(from revision+1)" sequence. + if (!oplog_watcher_->StartFromSequenceId(last_applied_seq_id)) { + LOG(WARNING) << "Failed to start OpLogWatcher from sequence_id=" + << last_applied_seq_id << ", continuing anyway"; + state_machine_.ProcessEvent(StandbyEvent::SYNC_FAILED); + } else { + // Transition to WATCHING state after successful sync + state_machine_.ProcessEvent(StandbyEvent::SYNC_COMPLETE); + } + + // Start background threads + replication_thread_ = std::thread(&HotStandbyService::ReplicationLoop, this); + if (config_.enable_verification) { + verification_thread_ = + std::thread(&HotStandbyService::VerificationLoop, this); + } + + LOG(INFO) << "HotStandbyService started, watching etcd OpLog for cluster: " + << cluster_id << ", state=" << StandbyStateToString(GetState()); + return ErrorCode::OK; +#else + state_machine_.ProcessEvent(StandbyEvent::FATAL_ERROR); + LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot start HotStandbyService"; + return ErrorCode::INTERNAL_ERROR; +#endif +} + +void HotStandbyService::OnWatcherEvent(StandbyEvent event) { + state_machine_.ProcessEvent(event); +} + +void HotStandbyService::Stop() { + // Check if already stopped (to avoid duplicate processing) + bool was_running = IsRunning(); + StandbyState current_state = GetState(); + + if (!was_running && current_state != StandbyState::PROMOTING && + !replication_thread_.joinable() && !verification_thread_.joinable()) { + // Already fully stopped and threads are joined + return; + } + + // Trigger STOP event + state_machine_.ProcessEvent(StandbyEvent::STOP); + + // Stop OpLogWatcher + if (oplog_watcher_) { + oplog_watcher_->Stop(); + oplog_watcher_.reset(); + } + + // Wait for threads to finish + if (replication_thread_.joinable()) { + replication_thread_.join(); + } + if (verification_thread_.joinable()) { + verification_thread_.join(); + } + + LOG(INFO) << "HotStandbyService stopped, final_state=" << StandbyStateToString(GetState()); +} + +StandbySyncStatus HotStandbyService::GetSyncStatus() const { + StandbySyncStatus status; + + // Get applied sequence ID from OpLogApplier + if (oplog_applier_) { + status.applied_seq_id = oplog_applier_->GetExpectedSequenceId() - 1; + if (status.applied_seq_id == 0) { + status.applied_seq_id = applied_seq_id_.load(); // Fallback + } + } else { + status.applied_seq_id = applied_seq_id_.load(); + } + + // Primary sequence ID (best-effort): updated by ReplicationLoop via etcd `/latest`. + status.primary_seq_id = primary_seq_id_.load(); + + // Use state machine for connection status + status.is_connected = IsConnected(); + status.state = GetState(); + status.time_in_state = state_machine_.GetTimeInCurrentState(); + + if (status.primary_seq_id > status.applied_seq_id) { + status.lag_entries = status.primary_seq_id - status.applied_seq_id; + } else { + status.lag_entries = 0; + } + + // Lag time is currently reported as 0; if needed we can extend the + // protocol to propagate primary timestamps and compute a real value. + status.lag_time = std::chrono::milliseconds(0); + status.is_syncing = IsRunning() && IsConnected(); + + return status; +} + +bool HotStandbyService::IsReadyForPromotion() const { + // Use state machine to check if ready for promotion + if (!state_machine_.IsReadyForPromotion()) { + return false; + } + + StandbySyncStatus status = GetSyncStatus(); + + // Allow promotion even with large lag - the new Primary can continue + // syncing remaining OpLog entries from etcd after promotion. + // Log a warning if lag is large, but don't block promotion. + if (status.lag_entries > config_.max_replication_lag_entries) { + LOG(WARNING) << "Standby has large replication lag: " << status.lag_entries + << " entries (threshold: " << config_.max_replication_lag_entries + << "). Promotion will proceed, but remaining OpLog entries " + << "will be synced after promotion."; + } + + return true; +} + +std::unique_ptr HotStandbyService::Promote() { + std::lock_guard lock(mutex_); + + if (!IsReadyForPromotion()) { + LOG(ERROR) << "Standby is not ready for promotion, state=" + << StandbyStateToString(GetState()); + return nullptr; + } + + // Trigger PROMOTE event + auto result = state_machine_.ProcessEvent(StandbyEvent::PROMOTE); + if (!result.allowed) { + LOG(ERROR) << "Cannot promote: " << result.reason; + return nullptr; + } + + StandbySyncStatus status = GetSyncStatus(); + uint64_t current_applied_seq_id = status.applied_seq_id; + + LOG(INFO) << "Promoting Standby to Primary. Applied seq_id: " + << current_applied_seq_id + << ", lag: " << status.lag_entries << " entries" + << ", state: " << StandbyStateToString(GetState()); + + // Final catch-up sync before promotion. + // IMPORTANT: + // - Do NOT rely on `lag_entries` here because primary_seq_id_ is best-effort. + // - Stop OpLogWatcher first to avoid concurrent Apply from watch callbacks. + if (oplog_watcher_) { + oplog_watcher_->Stop(); + } + + // Best-effort: resolve any outstanding gaps with retry before promotion. + // Do NOT block promotion if gaps cannot be fetched after max retries. + static constexpr int kMaxGapResolveRetries = 3; + if (oplog_applier_) { + for (int retry = 0; retry < kMaxGapResolveRetries; ++retry) { + auto res = oplog_applier_->TryResolveGapsOnceForPromotion(/*max_ids=*/1024); + if (res.attempted == 0) { + // No gaps to resolve + break; + } + LOG(INFO) << "Promotion gap resolve (attempt " << (retry + 1) << "/" + << kMaxGapResolveRetries << "): attempted=" << res.attempted + << ", fetched=" << res.fetched + << ", applied_deletes=" << res.applied_deletes; + if (res.fetched == res.attempted) { + // All gaps resolved successfully + break; + } + // Some gaps failed, retry after short delay + if (retry + 1 < kMaxGapResolveRetries) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } + } + + LOG(INFO) << "Final catch-up sync from etcd before promotion..."; + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + const size_t batch_size = 1000; + + // P0 fix: Prevent underflow when current_applied_seq_id is 0 + // ReadOpLogSince reads entries with seq > given_seq, so we pass current_applied_seq_id directly + uint64_t read_from_seq = current_applied_seq_id; // Will read entries with seq > read_from_seq + + // P1 fix: Add timeout control to prevent infinite blocking + static constexpr size_t kMaxCatchUpBatches = 100; // Max 100 batches * 1000 = 100k entries + static constexpr auto kMaxCatchUpDuration = std::chrono::seconds(30); + auto catch_up_start = std::chrono::steady_clock::now(); + + size_t total_applied = 0; + size_t batch_count = 0; + + for (;;) { + // Check timeout + auto elapsed = std::chrono::steady_clock::now() - catch_up_start; + if (elapsed > kMaxCatchUpDuration) { + LOG(WARNING) << "Final catch-up: timeout after " + << std::chrono::duration_cast(elapsed).count() + << "s. Proceeding with promotion. total_applied=" << total_applied; + break; + } + + // Check batch limit + if (batch_count >= kMaxCatchUpBatches) { + LOG(WARNING) << "Final catch-up: reached max batch limit (" << kMaxCatchUpBatches + << "). Proceeding with promotion. total_applied=" << total_applied; + break; + } + + std::vector batch; + ErrorCode read_err = oplog_store.ReadOpLogSince(read_from_seq, batch_size, batch); + if (read_err != ErrorCode::OK) { + LOG(WARNING) << "Final catch-up: failed to read OpLog since seq=" + << read_from_seq << ", err=" << static_cast(read_err) + << ". Proceeding with promotion."; + break; + } + if (batch.empty()) { + break; + } + size_t applied = oplog_applier_->ApplyOpLogEntries(batch); + total_applied += applied; + read_from_seq = batch.back().sequence_id; // Next read will get entries > this seq + ++batch_count; + } + LOG(INFO) << "Final catch-up sync done. total_applied=" << total_applied + << ", batches=" << batch_count; + + // Transition to PROMOTED state + state_machine_.ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + + // Stop replication (OpLogWatcher will stop watching). + // Note: This will trigger STOP event, transitioning to STOPPED. + Stop(); + + // Design note: MasterService creation and initialization are handled by + // MasterServiceSupervisor::Start() after leader election. The + // responsibility of HotStandbyService::Promote() is limited to ensuring + // that all remaining OpLog entries are applied before the new Primary + // starts serving requests. + + LOG(INFO) << "Standby promoted to Primary successfully. " + << "All remaining OpLog entries have been synced."; + + // Return nullptr - actual MasterService creation happens externally + // The caller (MasterServiceSupervisor) will create the MasterService + // with the appropriate configuration. + return nullptr; +} + +size_t HotStandbyService::GetMetadataCount() const { + std::lock_guard lock(mutex_); + return metadata_store_ ? metadata_store_->GetKeyCount() : 0; +} + +uint64_t HotStandbyService::GetLatestAppliedSequenceId() const { + std::lock_guard lock(mutex_); + if (oplog_applier_) { + uint64_t expected_seq = oplog_applier_->GetExpectedSequenceId(); + // GetExpectedSequenceId returns the next expected sequence_id, + // so the latest applied is expected_seq - 1 + return expected_seq > 0 ? expected_seq - 1 : 0; + } + return applied_seq_id_.load(); +} + +bool HotStandbyService::ExportMetadataSnapshot( + std::vector>& out) const { + std::lock_guard lock(mutex_); + if (!metadata_store_) { + out.clear(); + return false; + } + metadata_store_->Snapshot(out); + return true; +} + +void HotStandbyService::SetSnapshotProvider(std::unique_ptr provider) { + std::lock_guard lock(mutex_); + if (provider) { + snapshot_provider_ = std::move(provider); + } else { + snapshot_provider_ = std::make_unique(); + } +} + +void HotStandbyService::ReplicationLoop() { + LOG(INFO) << "Replication loop started (etcd-based OpLog sync)"; + + // With etcd-based OpLog sync, OpLogWatcher handles the actual watching + // in its own thread. This loop now just monitors the status and updates + // metrics. + + while (IsRunning()) { + if (!IsConnected()) { + // Not connected - wait a bit before checking again + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + // Update applied_seq_id from OpLogApplier + if (oplog_applier_) { + uint64_t current_applied = oplog_applier_->GetExpectedSequenceId() - 1; + if (current_applied > 0) { + applied_seq_id_.store(current_applied); + } + } + + // Update primary_seq_id by querying etcd `/latest` (best-effort). + // Note: `/latest` is batch-updated on Primary, so this is for monitoring only. +#ifdef STORE_USE_ETCD + if (!cluster_id_.empty()) { + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + uint64_t latest_seq = 0; + ErrorCode err = oplog_store.GetLatestSequenceId(latest_seq); + if (err == ErrorCode::OK) { + primary_seq_id_.store(latest_seq); + } + } +#endif + + // Sleep and check again + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + + LOG(INFO) << "Replication loop stopped"; +} + +void HotStandbyService::VerificationLoop() { + LOG(INFO) << "Verification loop started"; + + while (IsRunning()) { + std::this_thread::sleep_for( + std::chrono::seconds(config_.verification_interval_sec)); + + if (!IsConnected()) { + continue; + } + + // Verification is not yet implemented. When enabled, this loop is + // expected to: + // 1) sample keys from the local metadata store, + // 2) calculate checksums, + // 3) send a verification request to the Primary, and + // 4) handle any mismatches that are detected. + VLOG(1) << "Verification check skipped (feature not implemented), state=" + << StandbyStateToString(GetState()); + } + + LOG(INFO) << "Verification loop stopped"; +} + +void HotStandbyService::ApplyOpLogEntry(const OpLogEntry& entry) { + // NOTE: This method is deprecated. OpLog entries are now applied via + // OpLogApplier, which is called by OpLogWatcher. This method is kept + // for backward compatibility but should not be used in the new etcd-based + // implementation. + + // Update applied_seq_id for status tracking + applied_seq_id_.store(entry.sequence_id); + + // The actual application is handled by OpLogApplier via OpLogWatcher + VLOG(2) << "ApplyOpLogEntry called (deprecated), sequence_id=" + << entry.sequence_id << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; +} + +void HotStandbyService::ProcessOpLogBatch( + const std::vector& entries) { + for (const auto& entry : entries) { + ApplyOpLogEntry(entry); + } +} + +bool HotStandbyService::ConnectToPrimary() { + // With etcd-based OpLog sync, connection is handled by OpLogWatcher + // This method is kept for compatibility but is no longer used + LOG(INFO) << "ConnectToPrimary called (no-op with etcd-based sync)"; + return true; +} + +void HotStandbyService::DisconnectFromPrimary() { + // With etcd-based OpLog sync, disconnection is handled by OpLogWatcher + // This method is kept for compatibility + if (IsConnected()) { + state_machine_.ProcessEvent(StandbyEvent::DISCONNECTED); + replication_stream_.reset(); + LOG(INFO) << "Disconnected from Primary (etcd-based sync), state=" + << StandbyStateToString(GetState()); + } +} + +} // namespace mooncake + diff --git a/mooncake-store/src/master.cpp b/mooncake-store/src/master.cpp index 73e4fc65ed..d3727abed4 100644 --- a/mooncake-store/src/master.cpp +++ b/mooncake-store/src/master.cpp @@ -100,6 +100,9 @@ DEFINE_bool(enable_disk_eviction, true, DEFINE_uint64( quota_bytes, 0, "Quota for storage backend in bytes (0 = use default 90% of capacity)"); +DEFINE_uint64( + snapshot_interval_sec, 600, + "Interval in seconds for periodic metadata snapshot (HA only, 0 = disabled)"); void InitMasterConf(const mooncake::DefaultConfig& default_config, mooncake::MasterConfig& master_config) { @@ -175,6 +178,9 @@ void InitMasterConf(const mooncake::DefaultConfig& default_config, FLAGS_enable_disk_eviction); default_config.GetUInt64("quota_bytes", &master_config.quota_bytes, FLAGS_quota_bytes); + default_config.GetUInt64("snapshot_interval_sec", + &master_config.snapshot_interval_sec, + FLAGS_snapshot_interval_sec); } void LoadConfigFromCmdline(mooncake::MasterConfig& master_config, @@ -360,6 +366,11 @@ void LoadConfigFromCmdline(mooncake::MasterConfig& master_config, !conf_set) { master_config.quota_bytes = FLAGS_quota_bytes; } + if ((google::GetCommandLineFlagInfo("snapshot_interval_sec", &info) && + !info.is_default) || + !conf_set) { + master_config.snapshot_interval_sec = FLAGS_snapshot_interval_sec; + } } // Function to start HTTP metadata server @@ -471,7 +482,8 @@ int main(int argc, char* argv[]) { << ", put_start_discard_timeout_sec=" << master_config.put_start_discard_timeout_sec << ", put_start_release_timeout_sec=" - << master_config.put_start_release_timeout_sec; + << master_config.put_start_release_timeout_sec + << ", snapshot_interval_sec=" << master_config.snapshot_interval_sec; // Start HTTP metadata server if enabled std::unique_ptr http_metadata_server; diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index e049cb10eb..10b31f4c64 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -2,19 +2,132 @@ #include #include -#include #include #include +#include #include + +#include "allocator.h" +#include "etcd_helper.h" +#include "etcd_oplog_store.h" +#include "etcd_snapshot_provider.h" +#include "ha_metric_manager.h" #include "master_metric_manager.h" +#include "metadata_store.h" // For MetadataPayload #include "segment.h" #include "types.h" +// replication_service.h removed - using etcd-based OpLog sync instead namespace mooncake { +namespace { + +// A minimal allocator implementation used only to keep AllocatedBuffer handles +// "valid" after standby promotion. It does NOT own memory. +class DummyBufferAllocator final : public BufferAllocatorBase { + public: + DummyBufferAllocator(std::string segment_name, std::string transport_endpoint) + : segment_name_(std::move(segment_name)), + transport_endpoint_(std::move(transport_endpoint)) {} + + std::unique_ptr allocate(size_t /*size*/) override { + return nullptr; + } + void deallocate(AllocatedBuffer* /*handle*/) override { + // no-op: we don't own memory + } + size_t capacity() const override { return kAllocatorUnknownFreeSpace; } + size_t size() const override { return 0; } + std::string getSegmentName() const override { return segment_name_; } + std::string getTransportEndpoint() const override { return transport_endpoint_; } + size_t getLargestFreeRegion() const override { return kAllocatorUnknownFreeSpace; } + + private: + std::string segment_name_; + std::string transport_endpoint_; +}; + +static Replica ReplicaFromDescriptor( + const Replica::Descriptor& desc, + const std::shared_ptr& allocator_keepalive) { + if (desc.is_memory_replica()) { + const auto& mem = desc.get_memory_descriptor(); + const auto& bd = mem.buffer_descriptor; + if (!allocator_keepalive) { + // This would make the buffer handle invalid immediately (allocator stored + // as weak_ptr in AllocatedBuffer). Callers restoring from standby should + // always provide a keepalive allocator. + LOG(ERROR) << "ReplicaFromDescriptor(memory) missing keepalive allocator, " + << "transport_endpoint=" << bd.transport_endpoint_; + } + + auto buf = std::make_unique( + allocator_keepalive, reinterpret_cast(bd.buffer_address_), + static_cast(bd.size_)); + return Replica(std::move(buf), desc.status); + } + if (desc.is_disk_replica()) { + const auto& disk = desc.get_disk_descriptor(); + return Replica(disk.file_path, disk.object_size, desc.status); + } + const auto& ld = desc.get_local_disk_descriptor(); + return Replica(ld.client_id, ld.object_size, ld.transport_endpoint, desc.status); +} + +} // namespace + MasterService::MasterService() : MasterService(MasterServiceConfig()) {} +std::string MasterService::SerializeMetadataForOpLog(const ObjectMetadata& metadata) const { + MetadataPayload payload; + payload.client_id = metadata.client_id; + payload.size = metadata.size; + + // Extract replica descriptors - get them all at once + payload.replicas.reserve(metadata.replicas.size()); + for (const auto& replica : metadata.replicas) { + payload.replicas.push_back(replica.get_descriptor()); + } + + // NOTE: Lease information is NOT serialized because: + // 1. Standby does not perform eviction, so lease info is not used + // 2. After promotion, new Primary should grant fresh leases, not restore old ones + + // Serialize using struct_pack (msgpack binary format) + auto result = struct_pack::serialize(payload); + return std::string(result.begin(), result.end()); +} + +std::string MasterService::SerializeMetadataForOpLogWithoutMemReplicas( + const ObjectMetadata& metadata) const { + MetadataPayload payload; + payload.client_id = metadata.client_id; + payload.size = metadata.size; + + payload.replicas.reserve(metadata.replicas.size()); + for (const auto& replica : metadata.replicas) { + if (replica.type() == ReplicaType::MEMORY) { + continue; + } + payload.replicas.push_back(replica.get_descriptor()); + } + + auto result = struct_pack::serialize(payload); + return std::string(result.begin(), result.end()); +} + +std::string MasterService::SerializeMetadataForOpLogFromReplicaDescriptors( + const UUID& client_id, uint64_t size, + const std::vector& replicas) const { + MetadataPayload payload; + payload.client_id = client_id; + payload.size = size; + payload.replicas = replicas; + auto result = struct_pack::serialize(payload); + return std::string(result.begin(), result.end()); +} + MasterService::MasterService(const MasterServiceConfig& config) : default_kv_lease_ttl_(config.default_kv_lease_ttl), default_kv_soft_pin_ttl_(config.default_kv_soft_pin_ttl), @@ -33,7 +146,8 @@ MasterService::MasterService(const MasterServiceConfig& config) memory_allocator_type_(config.memory_allocator), allocation_strategy_(std::make_shared()), put_start_discard_timeout_sec_(config.put_start_discard_timeout_sec), - put_start_release_timeout_sec_(config.put_start_release_timeout_sec) { + put_start_release_timeout_sec_(config.put_start_release_timeout_sec), + snapshot_interval_sec_(config.snapshot_interval_sec) { if (eviction_ratio_ < 0.0 || eviction_ratio_ > 1.0) { LOG(ERROR) << "Eviction ratio must be between 0.0 and 1.0, " << "current value: " << eviction_ratio_; @@ -72,6 +186,181 @@ MasterService::MasterService(const MasterServiceConfig& config) MasterMetricManager::instance().inc_total_file_capacity( global_file_segment_size_); } + + // Initialize EtcdOpLogStore if HA is enabled + // Note: This requires STORE_USE_ETCD to be enabled at compile time + // Note: etcd connection should be established before MasterService construction + // (e.g., in MasterServiceSupervisor), so we can use the existing connection +#ifdef STORE_USE_ETCD + if (enable_ha_ && !cluster_id_.empty()) { + // Try to create EtcdOpLogStore - if etcd is not connected, operations will fail + // but we can still use memory buffer as fallback + // Writer: enable batch update for `/latest` to reduce etcd write pressure. + auto etcd_oplog_store = + std::make_shared(cluster_id_, /*enable_latest_seq_batch_update=*/true); + oplog_manager_.SetEtcdOpLogStore(etcd_oplog_store); + // Fence against restart/promotion regressions: initialize OpLogManager + // to the maximum existing sequence_id in etcd so we don't collide/overwrite. + uint64_t max_seq = 0; + if (etcd_oplog_store->GetMaxSequenceId(max_seq) == ErrorCode::OK) { + oplog_manager_.SetInitialSequenceId(max_seq); + } + LOG(INFO) << "EtcdOpLogStore initialized for cluster_id=" + << cluster_id_ << " (etcd connection should be established " + << "before MasterService construction)"; + } else if (enable_ha_) { + LOG(WARNING) << "HA mode enabled but cluster_id is empty, " + "OpLog will only be stored in memory buffer"; + } +#else + if (enable_ha_) { + LOG(WARNING) << "HA mode enabled but STORE_USE_ETCD is not enabled at " + "compile time, OpLog will only be stored in memory buffer. " + "Recompile with -DSTORE_USE_ETCD=ON to enable etcd support."; + } +#endif + + // Start pending durable mutation retry thread (HA only). +#ifdef STORE_USE_ETCD + if (enable_ha_) { + pending_mutations_running_.store(true); + pending_mutations_thread_ = + std::thread(&MasterService::PendingMutationWorker, this); + } +#endif + + // Start periodic snapshot thread (HA only). +#ifdef STORE_USE_ETCD + if (enable_ha_ && !cluster_id_.empty() && snapshot_interval_sec_ > 0) { + snapshot_provider_ = std::make_unique(cluster_id_); + snapshot_running_.store(true); + snapshot_thread_ = std::thread(&MasterService::SnapshotThreadFunc, this); + LOG(INFO) << "Started periodic snapshot thread, interval_sec=" << snapshot_interval_sec_ + << ", cluster_id=" << cluster_id_; + } else if (enable_ha_ && snapshot_interval_sec_ == 0) { + LOG(INFO) << "Periodic snapshot disabled (snapshot_interval_sec=0)"; + } +#endif +} + +// Helper function to append an OpLog entry. +// In the current etcd-based design: +// - OpLogManager always appends to its in-memory buffer +// - If EtcdOpLogStore is configured (HA mode), OpLogManager also writes to etcd +// synchronously (best-effort; see OpLogManager::Append). +void MasterService::AppendOpLogAndNotify(OpType type, const std::string& key, + const std::string& payload) { + oplog_manager_.Append(type, key, payload); +} + +auto MasterService::AppendOpLogAndNotifyDurable(OpType type, const std::string& key, + const std::string& payload) + -> tl::expected { +#ifdef STORE_USE_ETCD + // In HA mode, EtcdOpLogStore should have been configured into OpLogManager. + // For safety, treat missing store as an error for durable ops. + // Best-effort synchronous retries to absorb transient etcd blips. + // + // IMPORTANT: + // sequence_id must be allocated ONCE (pre-allocation) and retried with the same + // OpLogEntry, otherwise multiple attempts would allocate multiple sequence_ids + // for a single logical operation. + const OpLogEntry entry = oplog_manager_.AllocateEntry(type, key, payload); + ErrorCode err = PersistOpLogEntryWithSyncRetries(entry); + if (err == ErrorCode::OK) { + return entry.sequence_id; + } + return tl::make_unexpected(err); +#else + (void)type; + (void)key; + (void)payload; + return tl::make_unexpected(ErrorCode::ETCD_OPERATION_ERROR); +#endif +} + +void MasterService::RestoreFromStandbySnapshot( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id) { + // 1) Ensure OpLog sequence continues without regression after failover. + // Prefer reading the true max seq from etcd (stronger than standby_last_seq), + // fall back to caller-provided initial_oplog_sequence_id. + uint64_t start_seq = initial_oplog_sequence_id; +#ifdef STORE_USE_ETCD + if (enable_ha_ && !cluster_id_.empty()) { + EtcdOpLogStore store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + uint64_t max_seq = 0; + if (store.GetMaxSequenceId(max_seq) == ErrorCode::OK) { + start_seq = std::max(start_seq, max_seq); + } + } +#endif + oplog_manager_.SetInitialSequenceId(start_seq); + + // 2) Restore metadata entries. + // Keep dummy allocators alive for restored memory replicas. AllocatedBuffer + // only holds a weak_ptr to allocator, so without this keepalive map the + // allocator would expire immediately and transport_endpoint_ would be lost. + standby_allocator_keepalive_.clear(); + auto get_keepalive_allocator = + [this](const std::string& transport_endpoint) + -> std::shared_ptr { + auto it = standby_allocator_keepalive_.find(transport_endpoint); + if (it != standby_allocator_keepalive_.end()) { + return it->second; + } + auto alloc = std::make_shared( + /*segment_name=*/std::string(), transport_endpoint); + standby_allocator_keepalive_.emplace(transport_endpoint, alloc); + return alloc; + }; + + const auto now = std::chrono::steady_clock::now(); + size_t restored = 0; + for (const auto& kv : snapshot) { + const std::string& key = kv.first; + const StandbyObjectMetadata& sm = kv.second; + + std::vector replicas; + replicas.reserve(sm.replicas.size()); + for (const auto& rd : sm.replicas) { + if (rd.is_memory_replica()) { + const auto& bd = rd.get_memory_descriptor().buffer_descriptor; + replicas.emplace_back( + ReplicaFromDescriptor(rd, get_keepalive_allocator(bd.transport_endpoint_))); + } else { + replicas.emplace_back(ReplicaFromDescriptor(rd, nullptr)); + } + } + + // NOTE: Lease information is NOT restored because: + // 1. Standby does not use lease info (no eviction) + // 2. New Primary should grant fresh leases after promotion + // 3. Restoring old lease TTLs could cause immediate eviction if they're expired + const bool enable_soft_pin = false; // Will be set by new Primary if needed + + const size_t shard_idx = getShardIndex(key); + MutexLocker lock(&metadata_shards_[shard_idx].mutex); + + // Overwrite existing key if any. + metadata_shards_[shard_idx].metadata.erase(key); + auto [it, inserted] = metadata_shards_[shard_idx].metadata.emplace( + std::piecewise_construct, std::forward_as_tuple(key), + std::forward_as_tuple(sm.client_id, now, static_cast(sm.size), + std::move(replicas), enable_soft_pin)); + (void)inserted; + + // Lease will be granted by new Primary when objects are accessed + // (via GetReplicaList, ExistKey, etc.) + + // Objects restored from PUT_END are expected to be completed. + metadata_shards_[shard_idx].processing_keys.erase(key); + + restored++; + } + + LOG(INFO) << "Restored metadata from standby snapshot: restored_keys=" + << restored << ", initial_oplog_sequence_id=" << initial_oplog_sequence_id; } MasterService::~MasterService() { @@ -84,6 +373,243 @@ MasterService::~MasterService() { if (client_monitor_thread_.joinable()) { client_monitor_thread_.join(); } + +#ifdef STORE_USE_ETCD + if (pending_mutations_running_.load()) { + pending_mutations_running_.store(false); + pending_mutations_cv_.notify_all(); + if (pending_mutations_thread_.joinable()) { + pending_mutations_thread_.join(); + } + } + if (snapshot_running_.load()) { + snapshot_running_.store(false); + if (snapshot_thread_.joinable()) { + snapshot_thread_.join(); + } + } +#endif +} + +void MasterService::EnqueuePendingMutation(PendingMutation m) { + m.attempt = 0; + m.next_retry_at = std::chrono::steady_clock::now(); + size_t queue_size = 0; + { + std::lock_guard lg(pending_mutations_mutex_); + if (pending_mutations_.size() >= kMaxPendingMutations) { + // Queue full: drop oldest mutation to prevent unbounded growth. + // Log warning for monitoring. + LOG(WARNING) << "PendingMutation queue full (size=" << pending_mutations_.size() + << "), dropping oldest mutation. key=" << pending_mutations_.front().key + << ", seq=" << pending_mutations_.front().oplog_entry.sequence_id; + pending_mutations_.pop_front(); + } + pending_mutations_.push_back(std::move(m)); + queue_size = pending_mutations_.size(); + } + HAMetricManager::instance().set_pending_mutation_queue_size(static_cast(queue_size)); + pending_mutations_cv_.notify_one(); +} + +ErrorCode MasterService::PersistOpLogEntryWithSyncRetries( + const OpLogEntry& entry) const { +#ifdef STORE_USE_ETCD + static constexpr int kSyncRetries = 3; + static constexpr int kBaseBackoffMs = 20; + ErrorCode persist_err = ErrorCode::ETCD_OPERATION_ERROR; + + auto start_time = std::chrono::steady_clock::now(); + + for (int attempt = 0; attempt < kSyncRetries; ++attempt) { + persist_err = oplog_manager_.PersistEntryToEtcd(entry); + if (persist_err == ErrorCode::OK) { + break; + } + if (attempt > 0) { + HAMetricManager::instance().inc_oplog_etcd_write_retries(); + } + std::this_thread::sleep_for( + std::chrono::milliseconds(kBaseBackoffMs * (1 << attempt))); + } + + auto end_time = std::chrono::steady_clock::now(); + auto latency_us = std::chrono::duration_cast( + end_time - start_time).count(); + HAMetricManager::instance().observe_oplog_etcd_write_latency_us(latency_us); + + if (persist_err == ErrorCode::OK) { + HAMetricManager::instance().set_oplog_last_sequence_id( + static_cast(entry.sequence_id)); + } else { + HAMetricManager::instance().inc_oplog_etcd_write_failures(); + } + + return persist_err; +#else + (void)entry; + return ErrorCode::ETCD_OPERATION_ERROR; +#endif +} + +void MasterService::EnqueueRetryOnPersistFailure( + const char* ctx, const OpLogEntry& entry, ErrorCode persist_err, + PendingMutationKind kind, const std::string& segment_name) { +#ifdef STORE_USE_ETCD + LOG(ERROR) << ctx << ": failed to persist OpLog to etcd, key=" + << entry.object_key << ", seq=" << entry.sequence_id + << ", err=" << persist_err << ". Enqueue retry."; + EnqueuePendingMutation(PendingMutation{ + kind, + entry.object_key, + segment_name, + /*oplog_entry=*/entry}); +#else + (void)ctx; + (void)entry; + (void)persist_err; + (void)kind; + (void)segment_name; +#endif +} + +void MasterService::AppendOrPersistOrEnqueue( + const char* ctx, OpType type, const std::string& key, + const std::string& payload, PendingMutationKind kind, + const std::string& segment_name) { +#ifdef STORE_USE_ETCD + if (enable_ha_) { + const OpLogEntry entry = oplog_manager_.AllocateEntry(type, key, payload); + ErrorCode persist_err = PersistOpLogEntryWithSyncRetries(entry); + if (persist_err != ErrorCode::OK) { + EnqueueRetryOnPersistFailure(ctx, entry, persist_err, kind, segment_name); + } + } else { + AppendOpLogAndNotify(type, key, payload); + } +#else + // No etcd support at compile time: + // - non-HA: keep best-effort in-memory OpLog for debugging/consistency + // - HA: no-op (constructor already warns) + if (!enable_ha_) { + AppendOpLogAndNotify(type, key, payload); + } + (void)ctx; + (void)kind; + (void)segment_name; +#endif +} + +void MasterService::AppendOrPersistOrEnqueueLazy( + const char* ctx, OpType type, const std::string& key, + const std::function& payload_factory, + PendingMutationKind kind, const std::string& segment_name) { + std::string payload; + bool payload_ready = false; + auto get_payload = [&]() -> const std::string& { + if (!payload_ready) { + payload = payload_factory ? payload_factory() : std::string(); + payload_ready = true; + } + return payload; + }; + +#ifdef STORE_USE_ETCD + if (enable_ha_) { + const OpLogEntry entry = oplog_manager_.AllocateEntry(type, key, get_payload()); + ErrorCode persist_err = PersistOpLogEntryWithSyncRetries(entry); + if (persist_err != ErrorCode::OK) { + EnqueueRetryOnPersistFailure(ctx, entry, persist_err, kind, segment_name); + } + } else { + AppendOpLogAndNotify(type, key, get_payload()); + } +#else + // No etcd support at compile time: + // - non-HA: keep best-effort in-memory OpLog for debugging/consistency + // - HA: no-op (constructor already warns) + if (!enable_ha_) { + AppendOpLogAndNotify(type, key, get_payload()); + } + (void)ctx; + (void)kind; + (void)segment_name; +#endif +} + +// Return true if processed successfully (done), false if should retry later. +bool MasterService::ProcessPendingMutationOnce(PendingMutation& m) { +#ifndef STORE_USE_ETCD + (void)m; + return true; +#else + const auto now = std::chrono::steady_clock::now(); + if (m.next_retry_at > now) { + return false; + } + + // Retrier responsibility: + // only persist the original pre-allocated OpLogEntry (fixed sequence_id) to etcd. + // Do NOT mutate local metadata here because the caller may have already moved on. + if (m.oplog_entry.sequence_id == 0) { + LOG(WARNING) << "PendingMutation has no pre-allocated OpLogEntry, drop. key=" + << m.key << ", kind=" << static_cast(m.kind); + return true; + } + + ErrorCode err = oplog_manager_.PersistEntryToEtcd(m.oplog_entry); + if (err != ErrorCode::OK) { + return false; + } + return true; +#endif +} + +void MasterService::PendingMutationWorker() { +#ifndef STORE_USE_ETCD + return; +#else + while (pending_mutations_running_.load()) { + PendingMutation m; + bool has_item = false; + { + std::unique_lock lk(pending_mutations_mutex_); + pending_mutations_cv_.wait_for(lk, std::chrono::milliseconds(200), [&] { + return !pending_mutations_running_.load() || !pending_mutations_.empty(); + }); + if (!pending_mutations_running_.load()) { + break; + } + if (pending_mutations_.empty()) { + continue; + } + m = std::move(pending_mutations_.front()); + pending_mutations_.pop_front(); + has_item = true; + } + if (!has_item) { + continue; + } + + const bool done = ProcessPendingMutationOnce(m); + if (done) { + continue; + } + + // Retry with exponential backoff (cap at 30s). + m.attempt++; + const uint32_t exp = std::min(m.attempt, 8); + const auto delay = std::chrono::milliseconds(200u * (1u << exp)); + const auto capped = std::min(delay, std::chrono::milliseconds(30000)); + m.next_retry_at = std::chrono::steady_clock::now() + capped; + + { + std::lock_guard lg(pending_mutations_mutex_); + pending_mutations_.push_back(std::move(m)); + } + pending_mutations_cv_.notify_one(); + } +#endif } auto MasterService::MountSegment(const Segment& segment, const UUID& client_id) @@ -172,10 +698,40 @@ void MasterService::ClearInvalidHandles() { MutexLocker lock(&shard.mutex); auto it = shard.metadata.begin(); while (it != shard.metadata.end()) { + // CleanupStaleHandles may remove MEMORY replicas whose allocator has + // become invalid (segment unmounted). If key remains valid (has disk + // replicas), Standby must receive an updated metadata payload that + // excludes those MEMORY replicas (Scheme A). if (CleanupStaleHandles(it->second)) { - // If the object is empty, we need to erase the iterator + // No replicas remain after cleanup -> key should be deleted. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("ClearInvalidHandles(REMOVE)", + OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::REMOVE, it->first); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, it->first); + } +#endif it = shard.metadata.erase(it); } else { + // Still has some replicas. If HA is enabled, publish updated + // metadata WITHOUT MEMORY replicas (safe superset update). +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueueLazy( + "ClearInvalidHandles(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } +#endif ++it; } } @@ -231,6 +787,9 @@ auto MasterService::ExistKey(const std::string& key) // client. metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); + // Note: LEASE_RENEW is not recorded in OpLog since Standby does not + // perform eviction. Standby will receive DELETE events from Primary + // when objects are evicted. return true; } } @@ -387,6 +946,23 @@ auto MasterService::BatchReplicaClear( continue; } + // HA safety (Scheme A): + // This operation may free/reuse MEMORY replicas. Persist REMOVE to etcd + // BEFORE actually erasing local metadata. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("BatchReplicaClear(all)", OpType::REMOVE, + key, std::string(), + PendingMutationKind::CLEAR_ALL_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#endif + // Before erasing, decrement cache metrics for each COMPLETE replica for (const auto& replica : metadata.replicas) { if (replica.status() == ReplicaStatus::COMPLETE) { @@ -433,6 +1009,45 @@ auto MasterService::BatchReplicaClear( continue; } + // HA safety (Scheme A): + // Removing replicas may free/reuse MEMORY replicas. Persist updated metadata + // BEFORE mutating metadata.replicas (which may free memory). +#ifdef STORE_USE_ETCD + if (enable_ha_) { + // Build the remaining replica descriptor list after removal. + std::vector remove_mask(metadata.replicas.size(), false); + for (size_t idx : replicas_to_remove) { + if (idx < remove_mask.size()) { + remove_mask[idx] = true; + } + } + std::vector remaining; + remaining.reserve(metadata.replicas.size()); + for (size_t i = 0; i < metadata.replicas.size(); ++i) { + if (remove_mask[i]) { + continue; + } + remaining.emplace_back(metadata.replicas[i].get_descriptor()); + } + + if (remaining.empty()) { + AppendOrPersistOrEnqueue("BatchReplicaClear(partial REMOVE)", + OpType::REMOVE, key, std::string(), + PendingMutationKind::CLEAR_REPLICAS_ON_SEGMENT, + segment_name); + } else { + const std::string payload = + SerializeMetadataForOpLogFromReplicaDescriptors( + metadata.client_id, static_cast(metadata.size), + remaining); + AppendOrPersistOrEnqueue("BatchReplicaClear(partial PUT_END)", + OpType::PUT_END, key, payload, + PendingMutationKind::CLEAR_REPLICAS_ON_SEGMENT, + segment_name); + } + } +#endif + // Remove replicas on the specified segment (in reverse order to // maintain indices) for (auto it = replicas_to_remove.rbegin(); @@ -450,7 +1065,21 @@ auto MasterService::BatchReplicaClear( // If no valid replicas remain, erase the entire metadata if (metadata.replicas.empty() || !metadata.IsValid()) { +#ifndef STORE_USE_ETCD + // Non-HA: keep old behavior; HA already persisted REMOVE above. + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#endif accessor.Erase(); + } else { +#ifndef STORE_USE_ETCD + // Non-HA: best-effort update to keep future behavior consistent. + if (!enable_ha_) { + const std::string payload = SerializeMetadataForOpLog(metadata); + AppendOpLogAndNotify(OpType::PUT_END, key, payload); + } +#endif } cleared_keys.emplace_back(key); @@ -501,6 +1130,9 @@ auto MasterService::GetReplicaListByRegex(const std::string& regex_pattern) results.emplace(key, std::move(replica_list)); metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); + // Note: LEASE_RENEW is not recorded in OpLog since Standby does not + // perform eviction. Standby will receive DELETE events from Primary + // when objects are evicted. } } } @@ -542,6 +1174,9 @@ auto MasterService::GetReplicaList(std::string_view key) // Grant a lease to the object so it will not be removed // when the client is reading it. metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); + // Note: LEASE_RENEW is not recorded in OpLog since Standby does not + // perform eviction. Standby will receive DELETE events from Primary + // when objects are evicted. return GetReplicaListResponse(std::move(replica_list), default_kv_lease_ttl_); @@ -596,8 +1231,39 @@ auto MasterService::PutStart(const UUID& client_id, const std::string& key, } metadata_shards_[shard_idx].processing_keys.erase(key); metadata_shards_[shard_idx].metadata.erase(it); + } else if (metadata.HasCompletedReplicas()) { + // Allow overwriting existing key even if lease is still active. + // This enables update operations. Note that overwriting an object + // with an active lease may cause clients reading it to see stale + // data, but this is acceptable for update operations. + if (metadata.lease_timeout >= now) { + // Lease is still active, log a warning but allow overwrite + auto remaining_lease_sec = std::chrono::duration_cast( + metadata.lease_timeout - now).count(); + LOG(WARNING) << "key=" << key + << ", info=overwriting_object_with_active_lease" + << ", remaining_lease_sec=" << remaining_lease_sec + << " (clients reading this key may see stale data)"; + } else { + // Lease has expired, this is normal + LOG(INFO) << "key=" << key + << ", info=overwriting_expired_object" + << ", lease_expired_at=" + << std::chrono::duration_cast( + metadata.lease_timeout.time_since_epoch()) + .count(); + } + // Remove the old object to allow new PutStart + metadata_shards_[shard_idx].processing_keys.erase(key); + metadata_shards_[shard_idx].metadata.erase(it); } else { - LOG(INFO) << "key=" << key << ", info=object_already_exists"; + // Object exists but has not completed replicas and PutStart has not expired + LOG(INFO) << "key=" << key << ", info=object_already_exists" + << ", has_completed_replicas=false" + << ", put_start_time=" + << std::chrono::duration_cast( + metadata.put_start_time.time_since_epoch()) + .count(); return tl::make_unexpected(ErrorCode::OBJECT_ALREADY_EXISTS); } } @@ -696,6 +1362,13 @@ auto MasterService::PutEnd(const UUID& client_id, const std::string& key, // at beginning. 2. If this object has soft pin enabled, set it to be soft // pinned. metadata.GrantLease(0, default_kv_soft_pin_ttl_); + + // Record OpLog entry for PUT_END so that standbys can replay this change. + // Serialize metadata (replicas, size, lease) to payload so Standby can restore + // complete metadata when promoted to Primary. + std::string metadata_payload = SerializeMetadataForOpLog(metadata); + AppendOpLogAndNotify(OpType::PUT_END, key, metadata_payload); + return {}; } @@ -760,6 +1433,23 @@ auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, return tl::make_unexpected(ErrorCode::INVALID_WRITE); } + // HA behavior: + // Do NOT block subsequent ops for the same key if etcd write fails. + // We allocate sequence_id once and retry persisting this OpLogEntry + // asynchronously if needed. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("PutRevoke", OpType::PUT_REVOKE, key, std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::PUT_REVOKE, key); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::PUT_REVOKE, key); + } +#endif + if (replica_type == ReplicaType::MEMORY) { MasterMetricManager::instance().dec_mem_cache_nums(); } else if (replica_type == ReplicaType::DISK) { @@ -776,6 +1466,7 @@ auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, if (metadata.IsValid() == false) { accessor.Erase(); } + return {}; } @@ -819,8 +1510,25 @@ auto MasterService::Remove(const std::string& key) return tl::make_unexpected(ErrorCode::REPLICA_IS_NOT_READY); } - // Remove object metadata + // HA behavior: + // If etcd write fails, enqueue retry but still proceed with local remove. + // Standby will handle gaps via timeout + late-arrival policy. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("Remove", OpType::REMOVE, key, std::string(), + PendingMutationKind::CLEAR_ALL_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#endif + + // Remove object metadata (may deallocate memory replicas) accessor.Erase(); + return {}; } @@ -1268,9 +1976,39 @@ void MasterService::BatchEvict(double evict_ratio_target, continue; } if (it->second.lease_timeout <= target_timeout) { - // Evict this object + // Evict this object (MEMORY replicas only). + // + // Scheme A: + // - If key remains valid after removing MEMORY replicas, + // durably persist a PUT_END carrying the updated metadata + // (without MEMORY replicas) before freeing memory. + // - If key becomes invalid (only had MEMORY replicas), + // durably persist REMOVE before freeing memory. total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + + if (enable_ha_) { + const bool has_non_mem_replica = + std::any_of(it->second.replicas.begin(), + it->second.replicas.end(), + [](const Replica& r) { + return r.type() != ReplicaType::MEMORY; + }); + if (has_non_mem_replica) { + AppendOrPersistOrEnqueueLazy( + "BatchEvict(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOrPersistOrEnqueue( + "BatchEvict(REMOVE)", OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } + } + it->second.EraseReplica( ReplicaType::MEMORY); // Erase memory replicas if (it->second.IsValid() == false) { @@ -1332,9 +2070,32 @@ void MasterService::BatchEvict(double evict_ratio_target, !it->second.HasDiffRepStatus(ReplicaStatus::COMPLETE, ReplicaType::MEMORY) && it->second.HasMemReplica()) { - // Evict this object + // Evict this object (MEMORY replicas only). See Scheme A above. total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + + if (enable_ha_) { + const bool has_non_mem_replica = + std::any_of(it->second.replicas.begin(), + it->second.replicas.end(), + [](const Replica& r) { + return r.type() != ReplicaType::MEMORY; + }); + if (has_non_mem_replica) { + AppendOrPersistOrEnqueueLazy( + "BatchEvict(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOrPersistOrEnqueue( + "BatchEvict(REMOVE)", OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } + } + it->second.EraseReplica( ReplicaType::MEMORY); // Erase memory replicas if (it->second.IsValid() == false) { @@ -1388,6 +2149,29 @@ void MasterService::BatchEvict(double evict_ratio_target, it->second.lease_timeout <= soft_target_timeout) { total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + + if (enable_ha_) { + const bool has_non_mem_replica = + std::any_of(it->second.replicas.begin(), + it->second.replicas.end(), + [](const Replica& r) { + return r.type() != ReplicaType::MEMORY; + }); + if (has_non_mem_replica) { + AppendOrPersistOrEnqueueLazy( + "BatchEvict(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOrPersistOrEnqueue( + "BatchEvict(REMOVE)", OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } + } + it->second.EraseReplica( ReplicaType::MEMORY); // Erase memory replicas if (it->second.IsValid() == false) { @@ -1449,12 +2233,24 @@ void MasterService::ClientMonitorFunc() { // Find out expired clients std::vector expired_clients; for (auto it = client_ttl.begin(); it != client_ttl.end();) { + auto remaining_sec = std::chrono::duration_cast( + it->second - now).count(); if (it->second < now) { LOG(INFO) << "client_id=" << it->first - << ", action=client_expired"; + << ", action=client_expired" + << ", ttl_sec=" << client_live_ttl_sec_ + << ", last_ping_was_sec_ago=" << -remaining_sec; expired_clients.push_back(it->first); it = client_ttl.erase(it); } else { + // Log warning if TTL is getting close to expiration (within 10 seconds) + if (remaining_sec < 10 && remaining_sec > 0) { + LOG(WARNING) << "client_id=" << it->first + << ", action=ttl_low" + << ", remaining_sec=" << remaining_sec + << ", ttl_sec=" << client_live_ttl_sec_ + << " (client may expire soon if no ping received)"; + } ++it; } } @@ -1559,4 +2355,104 @@ std::string MasterService::ResolvePath(const std::string& key) const { return full_path.lexically_normal().string(); } +OpLogManager& MasterService::GetOpLogManager() { + return oplog_manager_; +} + +// SetReplicationService removed - using etcd-based OpLog sync instead + +#ifdef STORE_USE_ETCD +void MasterService::SnapshotThreadFunc() { + LOG(INFO) << "SnapshotThreadFunc started, interval_sec=" << snapshot_interval_sec_ + << ". First snapshot will run after " << snapshot_interval_sec_ << " seconds."; + + while (snapshot_running_.load()) { + // Sleep for the configured interval (in 1-second increments for responsiveness) + for (uint64_t i = 0; i < snapshot_interval_sec_ && snapshot_running_.load(); ++i) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + + if (!snapshot_running_.load()) { + break; + } + + LOG(INFO) << "Starting periodic snapshot capture..."; + + auto start_time = std::chrono::steady_clock::now(); + + // Collect all metadata from shards + std::vector> snapshot_data; + size_t total_keys = 0; + + for (size_t i = 0; i < kNumShards; i++) { + MutexLocker lock(&metadata_shards_[i].mutex); + for (const auto& [key, metadata] : metadata_shards_[i].metadata) { + StandbyObjectMetadata standby_meta; + standby_meta.client_id = metadata.client_id; + standby_meta.size = metadata.size; + // Convert replicas to descriptors + standby_meta.replicas.reserve(metadata.replicas.size()); + for (const auto& replica : metadata.replicas) { + standby_meta.replicas.push_back(replica.get_descriptor()); + } + // Use current max sequence_id from OpLogManager as last_sequence_id + standby_meta.last_sequence_id = oplog_manager_.GetLastSequenceId(); + + snapshot_data.emplace_back(key, std::move(standby_meta)); + total_keys++; + } + } + + // Get current oplog sequence_id as snapshot boundary + uint64_t snapshot_seq_id = oplog_manager_.GetLastSequenceId(); + + // Generate snapshot_id based on timestamp + auto now = std::chrono::system_clock::now(); + auto timestamp = std::chrono::duration_cast( + now.time_since_epoch()).count(); + std::string snapshot_id = std::to_string(timestamp); + + // Save snapshot + auto save_start = std::chrono::steady_clock::now(); + ErrorCode err = snapshot_provider_->SaveSnapshot(snapshot_id, snapshot_seq_id, snapshot_data); + auto save_duration = std::chrono::duration_cast( + std::chrono::steady_clock::now() - save_start).count(); + + auto total_duration = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_time).count(); + + if (err == ErrorCode::OK) { + LOG(INFO) << "Periodic snapshot saved successfully" + << ", snapshot_id=" << snapshot_id + << ", sequence_id=" << snapshot_seq_id + << ", total_keys=" << total_keys + << ", collect_time_ms=" << (total_duration - save_duration) + << ", save_time_ms=" << save_duration + << ", total_time_ms=" << total_duration; + + // Cleanup OpLog entries before the snapshot sequence_id. + // This is safe because the snapshot contains all metadata up to this point, + // and Standby can use snapshot + OpLog entries >= snapshot_seq_id to recover. + auto cleanup_start = std::chrono::steady_clock::now(); + size_t cleaned_count = oplog_manager_.CleanupBefore(snapshot_seq_id); + auto cleanup_duration = std::chrono::duration_cast( + std::chrono::steady_clock::now() - cleanup_start).count(); + + LOG(INFO) << "OpLog cleanup after snapshot" + << ", cleaned_entries=" << cleaned_count + << ", before_sequence_id=" << snapshot_seq_id + << ", cleanup_time_ms=" << cleanup_duration; + } else { + LOG(ERROR) << "Failed to save periodic snapshot" + << ", snapshot_id=" << snapshot_id + << ", sequence_id=" << snapshot_seq_id + << ", total_keys=" << total_keys + << ", error=" << static_cast(err); + } + } + + LOG(INFO) << "SnapshotThreadFunc stopped"; +} +#endif + } // namespace mooncake \ No newline at end of file diff --git a/mooncake-store/src/mock_client_main.cpp b/mooncake-store/src/mock_client_main.cpp new file mode 100644 index 0000000000..3106b484ec --- /dev/null +++ b/mooncake-store/src/mock_client_main.cpp @@ -0,0 +1,494 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "master_client.h" +#include "replica.h" +#include "segment.h" +#include "types.h" + +DEFINE_string(master_address, "127.0.0.1:50051", + "Master service address (IP:Port)"); +DEFINE_int32(write_interval_ms, 1000, "Interval between writes in milliseconds"); +DEFINE_int32(max_keys, 100, "Maximum number of different keys to use"); +DEFINE_int32(delete_interval, 10, + "Delete one key every N writes (0 to disable deletion)"); +DEFINE_bool(enable_new_keys, true, + "Allow new keys every 100 writes (beyond the max_keys limit)"); +DEFINE_int32(value_size, 1024, "Size of value (kvcache) in bytes"); +DEFINE_bool(mount_segment, true, + "Mount a segment to master_service before starting operations"); +DEFINE_string(segment_name, "mock_client_segment", + "Name of the segment to mount"); +DEFINE_int64(segment_size, 1024 * 1024 * 64, + "Size of the segment to mount in bytes (default: 64MB)"); +DEFINE_int64(segment_base, 0x300000000, + "Base address of the segment (virtual address, default: 0x300000000)"); +DEFINE_int32(ping_interval_sec, 20, + "Interval between ping operations in seconds (should be less than " + "master_service client_live_ttl_sec / 2, default: 20)"); + +namespace mooncake { + +// Generate a key name using timestamp +std::string GenerateKey(uint64_t timestamp) { + return "mock_key_" + std::to_string(timestamp); +} + +// Mock client simulator +class MockClientSimulator { + public: + MockClientSimulator(const std::string& master_address) + : master_address_(master_address), + client_id_(generate_uuid()), + client_(client_id_), + write_count_(0), + key_index_(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()), + running_(true) { + // Connect to master service + ErrorCode err = client_.Connect(master_address); + if (err != ErrorCode::OK) { + LOG(FATAL) << "Failed to connect to master service: " + << master_address << ", error=" << static_cast(err); + } + + // Default replication config + config_.replica_num = 1; + config_.with_soft_pin = false; + + LOG(INFO) << "Mock client simulator initialized"; + LOG(INFO) << " master_address: " << master_address; + LOG(INFO) << " client_id: " << client_id_; + LOG(INFO) << " max_keys: " << FLAGS_max_keys << " (ignored, keys will grow unlimited)"; + LOG(INFO) << " write_interval_ms: " << FLAGS_write_interval_ms; + LOG(INFO) << " delete_interval: " << FLAGS_delete_interval; + LOG(INFO) << " value_size: " << FLAGS_value_size; + LOG(INFO) << " mount_segment: " << FLAGS_mount_segment; + LOG(INFO) << " ping_interval_sec: " << FLAGS_ping_interval_sec; + + // Mount segment if enabled + if (FLAGS_mount_segment) { + Segment segment; + segment.id = generate_uuid(); + segment.name = FLAGS_segment_name; + segment.base = static_cast(FLAGS_segment_base); + segment.size = static_cast(FLAGS_segment_size); + segment.te_endpoint = FLAGS_segment_name; + + LOG(INFO) << "[Mount] Attempting to mount segment:"; + LOG(INFO) << " name: " << segment.name; + LOG(INFO) << " id: " << segment.id; + LOG(INFO) << " base: 0x" << std::hex << segment.base << std::dec; + LOG(INFO) << " size: " << segment.size << " bytes (" + << (segment.size / (1024 * 1024)) << " MB)"; + LOG(INFO) << " te_endpoint: " << segment.te_endpoint; + + auto mount_result = client_.MountSegment(segment); + if (mount_result.has_value()) { + LOG(INFO) << "[Mount] Segment mounted successfully"; + mounted_segment_ = segment; + + // Immediately send a ping after mounting to register the client + LOG(INFO) << "[Mount] Sending initial ping to register client..."; + auto ping_result = client_.Ping(); + if (ping_result.has_value()) { + LOG(INFO) << "[Mount] Initial ping SUCCESS - client registered"; + } else { + LOG(WARNING) << "[Mount] Initial ping FAILED: error=" + << static_cast(ping_result.error()) + << " (this may cause segment to be unmounted)"; + } + } else { + LOG(WARNING) << "[Mount] Failed to mount segment: error=" + << static_cast(mount_result.error()); + LOG(WARNING) << "[Mount] PutStart operations may fail with " + "NO_AVAILABLE_HANDLE error"; + LOG(WARNING) << "[Mount] You may need to start mooncake_client " + "to provide real storage"; + } + } else { + LOG(INFO) << "[Mount] Segment mounting disabled (--nomount_segment)"; + LOG(INFO) << "[Mount] PutStart operations may fail if no segment is " + "available"; + } + } + + void Run() { + LOG(INFO) << "=== Starting mock client simulation ==="; + LOG(INFO) << "Press Ctrl+C to stop"; + + // Start ping thread IMMEDIATELY to keep connection alive + // This is critical: if Run() is called with delay after constructor, + // the client may expire before ping thread starts + std::thread ping_thread([this]() { + LOG(INFO) << "[Ping] Starting ping thread (interval=" + << FLAGS_ping_interval_sec << "s)"; + + // Send first ping IMMEDIATELY (no sleep) to ensure client is registered + // This compensates for any delay between constructor and Run() call + LOG(INFO) << "[Ping] Sending immediate ping to register client..."; + auto first_ping_result = client_.Ping(); + if (first_ping_result.has_value()) { + LOG(INFO) << "[Ping] Immediate ping SUCCESS - client registered"; + } else { + LOG(WARNING) << "[Ping] Immediate ping FAILED: error=" + << static_cast(first_ping_result.error()) + << " (client may expire if this persists)"; + } + + // Now start periodic pings + while (running_.load()) { + std::this_thread::sleep_for( + std::chrono::seconds(FLAGS_ping_interval_sec)); + if (!running_.load()) { + break; + } + LOG(INFO) << "[Ping] Sending periodic ping to master_service..."; + auto ping_result = client_.Ping(); + if (ping_result.has_value()) { + LOG(INFO) << "[Ping] Ping SUCCESS - connection kept alive"; + } else { + LOG(WARNING) << "[Ping] Ping FAILED: error=" + << static_cast(ping_result.error()) + << " (segment may be unmounted if this persists)"; + } + } + LOG(INFO) << "[Ping] Ping thread stopped"; + }); + + int success_count = 0; + int failure_count = 0; + int new_key_count = 0; // Count of successfully created new keys + int update_count = 0; // Count of successfully updated existing keys + int delete_count = 0; + int delete_success_count = 0; + int delete_failure_count = 0; + auto start_time = std::chrono::steady_clock::now(); + const int stats_interval = 50; // Print stats every N operations + + // Track unique keys that have been successfully written + std::unordered_set written_keys; + + while (running_.load()) { + int current_write_count = write_count_.load(); + LOG(INFO) << "--- Operation #" << (current_write_count + 1) + << " (unique keys written: " << written_keys.size() << ") ---"; + + // Determine which key to use + std::string key; + + // Always use a new key, allowing unlimited key growth + // Use timestamp-based key index (incremented from initial timestamp) + key_index_++; + key = GenerateKey(key_index_.load()); + LOG(INFO) << "[Key Selection] Using key: " << key + << " (timestamp=" << key_index_.load() << ")"; + + // Simulate GET operation: check if key exists + LOG(INFO) << "[GET] Checking if key exists: " << key; + auto exist_result = client_.ExistKey(key); + bool exists = false; + if (exist_result.has_value()) { + exists = exist_result.value(); + LOG(INFO) << "[GET] Key " << key + << (exists ? " EXISTS" : " does NOT exist"); + } else { + LOG(WARNING) << "[GET] ExistKey failed for key=" << key + << ", error=" + << static_cast(exist_result.error()); + } + + if (!exists) { + // Key doesn't exist, simulate PUT operation + LOG(INFO) << "[PUT] Starting PUT operation for new key: " << key + << ", value_size=" << FLAGS_value_size; + + // Step 1: PutStart + std::vector slice_lengths = { + static_cast(FLAGS_value_size)}; + LOG(INFO) << "[PUT] Step 1/2: Calling PutStart..."; + auto put_start_result = + client_.PutStart(key, slice_lengths, config_); + if (!put_start_result.has_value()) { + ErrorCode err = put_start_result.error(); + failure_count++; + if (err == ErrorCode::NO_AVAILABLE_HANDLE) { + LOG(WARNING) << "[PUT] PutStart FAILED: key=" << key + << ", error=NO_AVAILABLE_HANDLE (-200) " + << "(master_service may not have segments configured or is out of space)"; + } else { + LOG(ERROR) << "[PUT] PutStart FAILED: key=" << key + << ", error=" << static_cast(err); + } + write_count_++; + std::this_thread::sleep_for( + std::chrono::milliseconds(FLAGS_write_interval_ms)); + continue; + } + + LOG(INFO) << "[PUT] PutStart SUCCESS: key=" << key + << ", replicas=" << put_start_result.value().size(); + + // Step 2: PutEnd (complete the put operation) + LOG(INFO) << "[PUT] Step 2/2: Calling PutEnd..."; + auto put_end_result = + client_.PutEnd(key, ReplicaType::MEMORY); + if (put_end_result.has_value()) { + success_count++; + new_key_count++; + write_count_++; + written_keys.insert(key); + LOG(INFO) << "[PUT] PutEnd SUCCESS: key=" << key + << " (new key created)"; + LOG(INFO) << "[PUT] Operation COMPLETE: key=" << key + << " (total unique keys: " << written_keys.size() + << ", new keys: " << new_key_count << ")"; + } else { + failure_count++; + write_count_++; + LOG(ERROR) << "[PUT] PutEnd FAILED: key=" << key + << ", error=" + << static_cast(put_end_result.error()); + LOG(ERROR) << "[PUT] Operation INCOMPLETE: key=" << key + << " (PutStart succeeded but PutEnd failed)"; + LOG(ERROR) << "[PUT] This may indicate segment is full or " + "other resource issue"; + } + } else { + // Key exists, simulate PUT (update) operation + LOG(INFO) << "[PUT] Starting PUT operation for existing key: " + << key << ", value_size=" << FLAGS_value_size; + + // For update, we also use PutStart + PutEnd + std::vector slice_lengths = { + static_cast(FLAGS_value_size)}; + LOG(INFO) << "[PUT] Step 1/2: Calling PutStart (update)..."; + auto put_start_result = + client_.PutStart(key, slice_lengths, config_); + if (!put_start_result.has_value()) { + ErrorCode err = put_start_result.error(); + failure_count++; + if (err == ErrorCode::NO_AVAILABLE_HANDLE) { + LOG(WARNING) << "[PUT] PutStart (update) FAILED: key=" + << key + << ", error=NO_AVAILABLE_HANDLE (-200) " + << "(master_service may not have segments configured or is out of space)"; + LOG(WARNING) << "[PUT] Current unique keys written: " + << written_keys.size() + << ", this may indicate segment is full"; + } else { + LOG(ERROR) << "[PUT] PutStart (update) FAILED: key=" + << key << ", error=" + << static_cast(err); + } + write_count_++; + std::this_thread::sleep_for( + std::chrono::milliseconds(FLAGS_write_interval_ms)); + continue; + } + + LOG(INFO) << "[PUT] PutStart (update) SUCCESS: key=" << key + << ", replicas=" << put_start_result.value().size(); + + LOG(INFO) << "[PUT] Step 2/2: Calling PutEnd (update)..."; + auto put_end_result = + client_.PutEnd(key, ReplicaType::MEMORY); + if (put_end_result.has_value()) { + success_count++; + update_count++; + write_count_++; + // Note: We don't add to written_keys for updates, only for new keys + LOG(INFO) << "[PUT] PutEnd (update) SUCCESS: key=" << key; + LOG(INFO) << "[PUT] Operation COMPLETE: key=" << key + << " (key updated, total unique keys: " + << written_keys.size() << ", updates: " << update_count + << ")"; + } else { + failure_count++; + write_count_++; + LOG(ERROR) << "[PUT] PutEnd (update) FAILED: key=" << key + << ", error=" + << static_cast(put_end_result.error()); + LOG(ERROR) << "[PUT] Operation INCOMPLETE: key=" << key + << " (PutStart succeeded but PutEnd failed)"; + LOG(ERROR) << "[PUT] This may indicate segment is full or " + "other resource issue"; + } + } + + // Simulate DELETE operation periodically + if (FLAGS_delete_interval > 0 && + write_count_ % FLAGS_delete_interval == 0 && + write_count_ > 0 && !written_keys.empty()) { + delete_count++; + // Delete a random key from the written keys set + // Use a simple hash-based selection for deterministic behavior + size_t key_to_delete_index = (write_count_ / FLAGS_delete_interval) % written_keys.size(); + auto it = written_keys.begin(); + std::advance(it, key_to_delete_index); + std::string delete_key = *it; + LOG(INFO) << "[DELETE] Attempting to delete key: " << delete_key + << " (delete operation #" << delete_count << ")"; + + // First check if the key exists + auto exist_result = client_.ExistKey(delete_key); + bool key_exists = false; + if (exist_result.has_value()) { + key_exists = exist_result.value(); + LOG(INFO) << "[DELETE] Key " << delete_key + << (key_exists ? " EXISTS" : " does NOT exist"); + } else { + LOG(WARNING) << "[DELETE] Failed to check key existence: " + << delete_key + << ", error=" + << static_cast(exist_result.error()); + } + + if (!key_exists) { + LOG(INFO) << "[DELETE] Skipping delete: key=" << delete_key + << " does not exist (this is normal if the key " + "hasn't been written yet or was already deleted)"; + continue; + } + + // Key exists, proceed with deletion + auto remove_result = client_.Remove(delete_key); + if (remove_result.has_value()) { + delete_success_count++; + written_keys.erase(delete_key); // Remove from tracked keys + LOG(INFO) << "[DELETE] SUCCESS: key=" << delete_key + << " deleted"; + } else { + ErrorCode err = remove_result.error(); + if (err == ErrorCode::OBJECT_NOT_FOUND) { + // Key was deleted between ExistKey and Remove (race condition) + written_keys.erase(delete_key); // Remove from tracked keys anyway + delete_success_count++; // Treat as success (key is already gone) + LOG(INFO) << "[DELETE] Key " << delete_key + << " was already deleted (race condition)"; + } else if (err == ErrorCode::OBJECT_HAS_LEASE) { + // Object has active lease, cannot delete yet (this is normal) + // Note: ExistKey may have granted a lease, causing this error + LOG(INFO) << "[DELETE] Key " << delete_key + << " has active lease, skipping deletion " + << "(this is normal, the key will be deleted after lease expires)"; + // Don't count as failure, and don't remove from written_keys + // so it can be retried later + } else { + delete_failure_count++; + LOG(ERROR) << "[DELETE] FAILED: key=" << delete_key + << ", error=" << static_cast(err); + } + } + } + + // Print statistics periodically + int total_ops = success_count + failure_count; + if (total_ops > 0 && total_ops % stats_interval == 0) { + auto current_time = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast( + current_time - start_time).count(); + double success_rate = + (total_ops > 0) ? (100.0 * success_count / total_ops) : 0.0; + double ops_per_sec = (elapsed > 0) ? (total_ops / elapsed) : 0.0; + + LOG(INFO) << "=== Statistics (last " << stats_interval + << " operations) ==="; + LOG(INFO) << " Total operations: " << total_ops; + LOG(INFO) << " Successful: " << success_count + << " (" << success_rate << "%)"; + LOG(INFO) << " - New keys created: " << new_key_count; + LOG(INFO) << " - Keys updated: " << update_count; + LOG(INFO) << " Failed: " << failure_count; + LOG(INFO) << " Unique keys written: " << written_keys.size(); + LOG(INFO) << " Delete operations: " << delete_count + << " (success: " << delete_success_count + << ", failed: " << delete_failure_count << ")"; + LOG(INFO) << " Elapsed time: " << elapsed << " seconds"; + LOG(INFO) << " Operations/sec: " << ops_per_sec; + LOG(INFO) << "========================================"; + } + + // Sleep for the specified interval + LOG(INFO) << "[Sleep] Waiting " << FLAGS_write_interval_ms + << " ms before next operation..."; + std::this_thread::sleep_for( + std::chrono::milliseconds(FLAGS_write_interval_ms)); + } + + // Print final statistics + auto end_time = std::chrono::steady_clock::now(); + auto total_elapsed = std::chrono::duration_cast( + end_time - start_time).count(); + int total_ops = success_count + failure_count; + double final_success_rate = + (total_ops > 0) ? (100.0 * success_count / total_ops) : 0.0; + double final_ops_per_sec = + (total_elapsed > 0) ? (total_ops / total_elapsed) : 0.0; + + LOG(INFO) << "=== Final Statistics ==="; + LOG(INFO) << " Total operations: " << total_ops; + LOG(INFO) << " Successful: " << success_count << " (" + << final_success_rate << "%)"; + LOG(INFO) << " - New keys created: " << new_key_count; + LOG(INFO) << " - Keys updated: " << update_count; + LOG(INFO) << " Failed: " << failure_count; + LOG(INFO) << " Unique keys written: " << written_keys.size(); + LOG(INFO) << " Delete operations: " << delete_count + << " (success: " << delete_success_count + << ", failed: " << delete_failure_count << ")"; + LOG(INFO) << " Total elapsed time: " << total_elapsed << " seconds"; + LOG(INFO) << " Average operations/sec: " << final_ops_per_sec; + LOG(INFO) << "========================="; + + // Stop ping thread + running_.store(false); + if (ping_thread.joinable()) { + ping_thread.join(); + } + } + + void Stop() { running_.store(false); } + + private: + std::string master_address_; + UUID client_id_; + MasterClient client_; + ReplicateConfig config_; + Segment mounted_segment_; + std::atomic write_count_; + std::atomic key_index_; + std::atomic running_; +}; + +} // namespace mooncake + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + if (FLAGS_master_address.empty()) { + LOG(FATAL) << "master_address must be specified"; + } + + mooncake::MockClientSimulator simulator(FLAGS_master_address); + + // Handle Ctrl+C gracefully + std::signal(SIGINT, [](int) { + LOG(INFO) << "Received SIGINT, stopping..."; + exit(0); + }); + + simulator.Run(); + + return 0; +} diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp new file mode 100644 index 0000000000..3f30307f45 --- /dev/null +++ b/mooncake-store/src/oplog_applier.cpp @@ -0,0 +1,584 @@ +#include "oplog_applier.h" + +#include + + +#include +#include + +#include "etcd_oplog_store.h" +#include "ha_metric_manager.h" +#include "metadata_store.h" +#include "oplog_manager.h" + +namespace mooncake { + +OpLogApplier::OpLogApplier(MetadataStore* metadata_store, + const std::string& cluster_id) + : metadata_store_(metadata_store), + cluster_id_(cluster_id), + expected_sequence_id_(1) { + if (metadata_store_ == nullptr) { + LOG(FATAL) << "OpLogApplier: metadata_store cannot be null"; + } + + // Validate cluster_id if provided (required for etcd operations). + // Normalize by stripping trailing slashes for validation. + std::string normalized = cluster_id_; + while (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); + } + if (!normalized.empty() && !IsValidClusterIdComponent(normalized)) { + LOG(FATAL) << "Invalid cluster_id for OpLogApplier: '" << cluster_id_ + << "' (normalized: '" << normalized + << "'). Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } +} + +EtcdOpLogStore* OpLogApplier::GetEtcdOpLogStore() const { +#ifdef STORE_USE_ETCD + if (cluster_id_.empty()) { + return nullptr; + } + + std::lock_guard lock(etcd_oplog_store_mutex_); + if (!etcd_oplog_store_) { + // Reader: do not start `/latest` batch update thread. + etcd_oplog_store_ = + std::make_unique(cluster_id_, /*enable_latest_seq_batch_update=*/false); + } + return etcd_oplog_store_.get(); +#else + return nullptr; +#endif +} + +bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { + // Basic DoS protection: validate key/payload sizes before parsing/applying. + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "OpLogApplier: entry size rejected, sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key << ", reason=" << size_reason; + return false; + } + + // Verify checksum to detect data corruption or tampering. + if (!OpLogManager::VerifyChecksum(entry)) { + LOG(ERROR) << "OpLogApplier: checksum mismatch, sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key + << ". Possible data corruption or tampering. Discarding entry."; + HAMetricManager::instance().inc_oplog_checksum_failures(); + return false; + } + + // Global ordering only. + // + // IMPORTANT: + // - Watch callbacks / retries may deliver duplicate or already-applied entries. + // - Those must be treated as no-op, not as "out-of-order pending", otherwise + // pending_entries_ can grow and the applier may appear stuck. + const uint64_t expected = expected_sequence_id_.load(); + if (IsSequenceOlder(entry.sequence_id, expected)) { + // Late arrival of a previously-skipped gap entry: apply only if it's a delete/revoke. + bool was_skipped = false; + { + std::lock_guard lock(pending_mutex_); + auto it = skipped_sequence_ids_.find(entry.sequence_id); + if (it != skipped_sequence_ids_.end()) { + was_skipped = true; + skipped_sequence_ids_.erase(it); + } + } + if (was_skipped) { + if (entry.op_type == OpType::REMOVE || entry.op_type == OpType::PUT_REVOKE) { + // Safe: ensure we don't keep stale metadata. + if (entry.op_type == OpType::REMOVE) { + ApplyRemove(entry); + } else { + ApplyPutRevoke(entry); + } + return true; + } + // PUT_END (or others): discard to avoid resurrecting stale state. + VLOG(1) << "OpLogApplier: discard late skipped entry, op_type=" + << static_cast(entry.op_type) + << ", sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key; + return true; + } + + VLOG(2) << "OpLogApplier: skip already-applied entry, sequence_id=" + << entry.sequence_id << ", expected=" << expected + << ", key=" << entry.object_key; + return true; // consumed (no-op) + } + if (IsSequenceNewer(entry.sequence_id, expected)) { + // Future entry - store into pending, wait for the gap to be filled. + std::lock_guard lock(pending_mutex_); + + if (pending_entries_.size() >= static_cast(kMaxPendingEntries)) { + LOG(ERROR) << "OpLogApplier: too many pending entries (" + << pending_entries_.size() << "), discarding entry sequence_id=" + << entry.sequence_id << ", key=" << entry.object_key; + return false; + } + + pending_entries_[entry.sequence_id] = entry; + VLOG(1) << "OpLogApplier: future entry buffered, sequence_id=" + << entry.sequence_id << ", expected=" << expected + << ", key=" << entry.object_key + << ", pending_entries=" << pending_entries_.size(); + return false; + } + + // Apply the operation based on type + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + default: + LOG(ERROR) << "OpLogApplier: unsupported op_type=" + << static_cast(entry.op_type) + << ", sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key; + return false; + } + + // Update expected sequence ID + expected_sequence_id_.store(entry.sequence_id + 1); + + // Update metrics + HAMetricManager::instance().inc_oplog_applied_entries(); + HAMetricManager::instance().set_oplog_applied_sequence_id( + static_cast(entry.sequence_id)); + + // Try to process pending entries + ProcessPendingEntries(); + + return true; +} + +size_t OpLogApplier::ApplyOpLogEntries(const std::vector& entries) { + size_t applied_count = 0; + for (const auto& entry : entries) { + if (ApplyOpLogEntry(entry)) { + applied_count++; + } + } + return applied_count; +} + +uint64_t OpLogApplier::GetKeySequenceId(const std::string& key) const { + // Global sequence_id is used for ordering. + (void)key; // Suppress unused parameter warning + return 0; +} + +uint64_t OpLogApplier::GetExpectedSequenceId() const { + return expected_sequence_id_.load(); +} + +void OpLogApplier::Recover(uint64_t last_applied_sequence_id) { + expected_sequence_id_.store(last_applied_sequence_id + 1); + LOG(INFO) << "OpLogApplier: recovered from sequence_id=" + << last_applied_sequence_id + << ", expected_sequence_id set to=" << expected_sequence_id_.load(); +} + +size_t OpLogApplier::ProcessPendingEntries() { + // Check for missing sequence IDs, possibly skip after timeout, and/or request them. + uint64_t missing_seq_to_request = 0; + uint64_t skipped_count = 0; + { + std::lock_guard lock(pending_mutex_); + auto now = std::chrono::steady_clock::now(); + for (;;) { + if (pending_entries_.empty()) { + break; + } + const uint64_t first_pending_seq = pending_entries_.begin()->first; + const uint64_t expected = expected_sequence_id_.load(); + if (IsSequenceOlderOrEqual(first_pending_seq, expected)) { + break; + } + + // There's a gap: expected is missing. + const uint64_t missing_seq = expected; + auto it = missing_sequence_ids_.find(missing_seq); + if (it == missing_sequence_ids_.end()) { + missing_sequence_ids_[missing_seq] = now; + ScheduleWaitForMissingEntries(missing_seq); + break; + } + + const auto waited = std::chrono::duration_cast(now - it->second); + + // Skip after timeout to avoid global stall (user requested behavior). + if (waited.count() >= kMissingEntrySkipSeconds) { + skipped_sequence_ids_[missing_seq] = now; + missing_sequence_ids_.erase(missing_seq); + expected_sequence_id_.store(missing_seq + 1); + skipped_count++; + HAMetricManager::instance().inc_oplog_skipped_entries(); + LOG(WARNING) << "OpLogApplier: skipped missing entry seq=" << missing_seq + << " after " << waited.count() << "s timeout"; + continue; // may skip multiple consecutive gaps + } + + // Best-effort request from etcd (before skip triggers). + if (waited.count() >= kMissingEntryRequestSeconds) { + missing_seq_to_request = missing_seq; + break; + } + break; + } + } + + // Request missing OpLog if needed (outside the lock to avoid deadlock) + bool retrieved_missing = false; + if (missing_seq_to_request > 0) { + retrieved_missing = RequestMissingOpLog(missing_seq_to_request); + if (retrieved_missing) { + std::lock_guard lock(pending_mutex_); + missing_sequence_ids_.erase(missing_seq_to_request); + } + } + + size_t processed_count = 0; + for (;;) { + OpLogEntry entry_copy; + bool has_entry = false; + + { + std::lock_guard lock(pending_mutex_); + if (pending_entries_.empty()) { + break; + } + + auto it = pending_entries_.begin(); + const uint64_t expected = expected_sequence_id_.load(); + if (!IsSequenceEqual(it->first, expected)) { + break; // still waiting for earlier sequence_id + } + + entry_copy = it->second; + pending_entries_.erase(it); + has_entry = true; + } + + if (!has_entry) { + break; + } + + // Apply outside lock. + switch (entry_copy.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry_copy); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry_copy); + break; + case OpType::REMOVE: + ApplyRemove(entry_copy); + break; + default: + LOG(ERROR) << "OpLogApplier: unsupported op_type in pending entry"; + break; + } + + expected_sequence_id_.store(entry_copy.sequence_id + 1); + + { + std::lock_guard lock(pending_mutex_); + missing_sequence_ids_.erase(entry_copy.sequence_id); + } + + processed_count++; + } + + // Clean up old missing sequence IDs (older than 1 minute) + { + std::lock_guard lock(pending_mutex_); + auto now = std::chrono::steady_clock::now(); + for (auto it = missing_sequence_ids_.begin(); it != missing_sequence_ids_.end();) { + auto age = std::chrono::duration_cast(now - it->second); + if (age.count() > 60) { + LOG(WARNING) << "OpLogApplier: giving up on missing sequence_id=" + << it->first << " after " << age.count() << " seconds"; + it = missing_sequence_ids_.erase(it); + } else { + ++it; + } + } + + // Clean up old skipped sequence IDs too (avoid unbounded growth). + for (auto it = skipped_sequence_ids_.begin(); it != skipped_sequence_ids_.end();) { + auto age = std::chrono::duration_cast(now - it->second); + if (age.count() > 60) { + it = skipped_sequence_ids_.erase(it); + } else { + ++it; + } + } + } + + if (skipped_count > 0) { + LOG(WARNING) << "OpLogApplier: skipped " << skipped_count + << " missing sequence_id(s) after timeout, expected_sequence_id now=" + << expected_sequence_id_.load(); + } + + if (processed_count > 0) { + LOG(INFO) << "OpLogApplier: processed " << processed_count + << " pending entries, expected_sequence_id now=" + << expected_sequence_id_.load(); + } + + // Update pending entries metric + { + std::lock_guard lock(pending_mutex_); + HAMetricManager::instance().set_oplog_pending_entries( + static_cast(pending_entries_.size())); + } + + return processed_count; +} + +OpLogApplier::GapResolveResult OpLogApplier::TryResolveGapsOnceForPromotion( + size_t max_ids) { + GapResolveResult r; +#ifdef STORE_USE_ETCD + EtcdOpLogStore* store = GetEtcdOpLogStore(); + if (store == nullptr) { + return r; + } + + std::vector gap_ids; + gap_ids.reserve(max_ids); + { + std::lock_guard lock(pending_mutex_); + for (const auto& kv : missing_sequence_ids_) { + if (gap_ids.size() >= max_ids) break; + gap_ids.push_back(kv.first); + } + for (const auto& kv : skipped_sequence_ids_) { + if (gap_ids.size() >= max_ids) break; + gap_ids.push_back(kv.first); + } + } + + if (gap_ids.empty()) { + return r; + } + + std::sort(gap_ids.begin(), gap_ids.end()); + gap_ids.erase(std::unique(gap_ids.begin(), gap_ids.end()), gap_ids.end()); + + r.attempted = gap_ids.size(); + std::vector successfully_processed; + for (uint64_t seq : gap_ids) { + OpLogEntry e; + ErrorCode err = store->ReadOpLog(seq, e); + if (err != ErrorCode::OK) { + // Log failed gap for monitoring, but don't clear it so it can be retried later. + LOG(WARNING) << "Promotion gap resolve: failed to fetch seq=" << seq + << ", err=" << static_cast(err); + continue; + } + r.fetched++; + + // Apply policy: only delete/revoke; drop PUT_END. + if (e.op_type == OpType::REMOVE) { + ApplyRemove(e); + r.applied_deletes++; + successfully_processed.push_back(seq); + } else if (e.op_type == OpType::PUT_REVOKE) { + ApplyPutRevoke(e); + r.applied_deletes++; + successfully_processed.push_back(seq); + } else { + // PUT_END or others: mark as processed (dropped) so we don't retry. + successfully_processed.push_back(seq); + } + } + + // Only clear gaps we successfully fetched and processed. + // Failed gaps remain in missing_sequence_ids_/skipped_sequence_ids_ for potential + // retry or monitoring. + if (!successfully_processed.empty()) { + std::lock_guard lock(pending_mutex_); + for (uint64_t seq : successfully_processed) { + missing_sequence_ids_.erase(seq); + skipped_sequence_ids_.erase(seq); + } + } + return r; +#else + (void)max_ids; + return r; +#endif +} + +bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { + // Only check global sequence order. + // Use IsSequenceEqual for wrap-around safety (though equality check doesn't + // need special handling, we use it for consistency). + return IsSequenceEqual(entry.sequence_id, expected_sequence_id_.load()); +} + +void OpLogApplier::ApplyPutEnd(const OpLogEntry& entry) { + // Payload contains serialized metadata (replicas, size, etc.) in JSON format. + // Deserialize the payload immediately and store structured metadata. + // This allows Standby to serve requests immediately after promotion. + + if (entry.payload.empty()) { + // No payload - create empty metadata (legacy compatibility) + LOG(WARNING) << "OpLogApplier: PUT_END without payload, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + StandbyObjectMetadata empty_metadata; + empty_metadata.last_sequence_id = entry.sequence_id; + if (!metadata_store_->PutMetadata(entry.object_key, empty_metadata)) { + LOG(ERROR) << "OpLogApplier: failed to PutMetadata key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } + return; + } + + // Deserialize payload using struct_pack (msgpack binary format) + MetadataPayload payload; + bool parse_success = false; + auto result = struct_pack::deserialize_to(payload, entry.payload); + if (result == struct_pack::errc::ok) { + parse_success = true; + } else { + LOG(ERROR) << "OpLogApplier: failed to deserialize payload for key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << ", payload_size=" << entry.payload.size() + << ", error_code=" << static_cast(result); + } + + if (!parse_success) { + // Fallback to empty metadata if parsing fails + StandbyObjectMetadata empty_metadata; + empty_metadata.last_sequence_id = entry.sequence_id; + metadata_store_->PutMetadata(entry.object_key, empty_metadata); + return; + } + + // Convert to StandbyObjectMetadata and store + StandbyObjectMetadata metadata = payload.ToStandbyMetadata(entry.sequence_id); + + if (!metadata_store_->PutMetadata(entry.object_key, metadata)) { + LOG(ERROR) << "OpLogApplier: failed to PutMetadata key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } else { + VLOG(1) << "OpLogApplier: applied PUT_END, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << ", replicas=" << metadata.replicas.size() + << ", size=" << metadata.size; + } +} + +void OpLogApplier::ApplyPutRevoke(const OpLogEntry& entry) { + // PUT_REVOKE means the object should be removed from metadata store + // (but the key itself may still exist if there are other replicas). + // Current implementation removes the entire key; if we later support + // partial replica revocation this logic will need to be refined. + if (!metadata_store_->Remove(entry.object_key)) { + LOG(WARNING) << "OpLogApplier: failed to Remove key=" << entry.object_key + << " in PUT_REVOKE, sequence_id=" << entry.sequence_id + << " (key may not exist)"; + } else { + VLOG(1) << "OpLogApplier: applied PUT_REVOKE, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } +} + +void OpLogApplier::ApplyRemove(const OpLogEntry& entry) { + if (!metadata_store_->Remove(entry.object_key)) { + LOG(WARNING) << "OpLogApplier: failed to Remove key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << " (key may not exist)"; + } else { + VLOG(1) << "OpLogApplier: applied REMOVE, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } +} + +bool OpLogApplier::RequestMissingOpLog(uint64_t missing_seq_id) { +#ifdef STORE_USE_ETCD + HAMetricManager::instance().inc_oplog_gap_resolve_attempts(); + + EtcdOpLogStore* oplog_store = GetEtcdOpLogStore(); + if (oplog_store == nullptr) { + LOG(WARNING) << "OpLogApplier: cannot request missing OpLog, cluster_id not set"; + return false; + } + + OpLogEntry entry; + ErrorCode err = oplog_store->ReadOpLog(missing_seq_id, entry); + if (err == ErrorCode::ETCD_KEY_NOT_EXIST) { + LOG(INFO) << "OpLogApplier: missing OpLog entry not found in etcd, sequence_id=" + << missing_seq_id; + return false; + } + if (err != ErrorCode::OK) { + LOG(ERROR) << "OpLogApplier: failed to read missing OpLog from etcd, sequence_id=" + << missing_seq_id << ", error=" << static_cast(err); + return false; + } + + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "OpLogApplier: missing entry size rejected, sequence_id=" + << missing_seq_id << ", key=" << entry.object_key + << ", reason=" << size_reason; + return false; + } + + // Verify checksum before adding to pending entries. + if (!OpLogManager::VerifyChecksum(entry)) { + LOG(ERROR) << "OpLogApplier: checksum mismatch for retrieved missing entry, sequence_id=" + << missing_seq_id << ", key=" << entry.object_key + << ". Possible data corruption. Discarding entry."; + HAMetricManager::instance().inc_oplog_checksum_failures(); + return false; + } + + // Successfully retrieved the missing OpLog entry + LOG(INFO) << "OpLogApplier: retrieved missing OpLog entry, sequence_id=" + << missing_seq_id << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; + HAMetricManager::instance().inc_oplog_gap_resolve_success(); + + // Add to pending entries + // Note: We don't call ProcessPendingEntries() here to avoid potential recursion. + // The caller (ProcessPendingEntries itself) will process the entry in the next loop. + { + std::lock_guard lock(pending_mutex_); + pending_entries_[entry.sequence_id] = entry; + } + + return true; +#else + LOG(WARNING) << "OpLogApplier: STORE_USE_ETCD not enabled, cannot request missing OpLog"; + return false; +#endif +} + +void OpLogApplier::ScheduleWaitForMissingEntries(uint64_t missing_seq_id) { + // This method is called when we first detect a missing sequence_id. + // The actual waiting and requesting is handled in ProcessPendingEntries(). + // We just log it here for tracking. + VLOG(1) << "OpLogApplier: scheduling wait for missing sequence_id=" << missing_seq_id + << ", will request after " << kMissingEntryRequestSeconds << " seconds"; +} + +} // namespace mooncake + diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp new file mode 100644 index 0000000000..e21f041796 --- /dev/null +++ b/mooncake-store/src/oplog_manager.cpp @@ -0,0 +1,221 @@ +#include "oplog_manager.h" + +#include +#include +#include +#include + +#include "etcd_oplog_store.h" + +namespace mooncake { + +OpLogManager::OpLogManager() = default; + +void OpLogManager::SetEtcdOpLogStore( + std::shared_ptr etcd_oplog_store) { + std::unique_lock lock(mutex_); + etcd_oplog_store_ = etcd_oplog_store; +} + +uint64_t OpLogManager::Append(OpType type, const std::string& key, + const std::string& payload) { + OpLogEntry entry; + entry.op_type = type; + entry.object_key = key; + entry.payload = payload; + entry.timestamp_ms = NowMs(); + entry.checksum = ComputeChecksum(entry.payload); + entry.prefix_hash = ComputePrefixHash(entry.object_key); + + std::unique_lock lock(mutex_); + entry.sequence_id = ++last_seq_id_; + + if (buffer_.size() >= kMaxBufferEntries_) { + buffer_.pop_front(); + ++first_seq_id_; + } + + buffer_.emplace_back(entry); // Copy entry to buffer + + // Write to etcd if EtcdOpLogStore is set + if (etcd_oplog_store_) { + // Release lock before writing to etcd to avoid blocking + // We use the original entry (before it was copied to buffer) + lock.unlock(); + + // Strategy 2+: PUT_END is Async, REMOVE (and others) are Sync + bool sync = (entry.op_type != OpType::PUT_END); + ErrorCode err = etcd_oplog_store_->WriteOpLog(entry, sync); + if (err != ErrorCode::OK) { + // Log error but don't fail the operation + // The entry is already in the memory buffer + LOG(WARNING) << "Failed to write OpLog to etcd, sequence_id=" + << entry.sequence_id + << ", but entry is in memory buffer"; + } + } + + return last_seq_id_; +} + +OpLogEntry OpLogManager::AllocateEntry(OpType type, const std::string& key, + const std::string& payload) { + OpLogEntry entry; + entry.op_type = type; + entry.object_key = key; + entry.payload = payload; + entry.timestamp_ms = NowMs(); + entry.checksum = ComputeChecksum(entry.payload); + entry.prefix_hash = ComputePrefixHash(entry.object_key); + + std::unique_lock lock(mutex_); + entry.sequence_id = ++last_seq_id_; + + if (buffer_.size() >= kMaxBufferEntries_) { + buffer_.pop_front(); + ++first_seq_id_; + } + buffer_.emplace_back(entry); + return entry; +} + +ErrorCode OpLogManager::PersistEntryToEtcd(const OpLogEntry& entry) const { + std::shared_lock lock(mutex_); + auto store = etcd_oplog_store_; + lock.unlock(); + if (!store) { + return ErrorCode::ETCD_OPERATION_ERROR; + } + // Strategy 2+: PUT_END is Async, REMOVE (and others) are Sync + bool sync = (entry.op_type != OpType::PUT_END); + return store->WriteOpLog(entry, sync); +} + +tl::expected OpLogManager::AppendAndPersist( + OpType type, const std::string& key, const std::string& payload) { + // Seq pre-allocation semantics: allocate first, then persist. + OpLogEntry entry = AllocateEntry(type, key, payload); + ErrorCode err = PersistEntryToEtcd(entry); + if (err != ErrorCode::OK) { + return tl::make_unexpected(err); + } + return entry.sequence_id; +} + +uint64_t OpLogManager::GetLastSequenceId() const { + std::shared_lock lock(mutex_); + return last_seq_id_; +} + +void OpLogManager::SetInitialSequenceId(uint64_t sequence_id) { + std::unique_lock lock(mutex_); + if (last_seq_id_ == 0 && buffer_.empty()) { + // Only allow setting initial sequence_id if OpLogManager is empty + last_seq_id_ = sequence_id; + first_seq_id_ = sequence_id + 1; // first_seq_id_ should be > last_seq_id_ when empty + LOG(INFO) << "OpLogManager initial sequence_id set to " << sequence_id; + } else { + LOG(WARNING) << "Cannot set initial sequence_id: OpLogManager is not empty " + << "(last_seq_id_=" << last_seq_id_ << ", buffer_size=" << buffer_.size() << ")"; + } +} + +size_t OpLogManager::GetEntryCount() const { + std::shared_lock lock(mutex_); + return buffer_.size(); +} + +size_t OpLogManager::CleanupBefore(uint64_t before_sequence_id) { + size_t memory_cleaned = 0; + + // 1. Cleanup from etcd first (if configured) + std::shared_ptr store; + { + std::shared_lock lock(mutex_); + store = etcd_oplog_store_; + } + + if (store) { + ErrorCode err = store->CleanupOpLogBefore(before_sequence_id); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to cleanup OpLog from etcd before sequence_id=" + << before_sequence_id << ", error=" << static_cast(err); + // Continue to cleanup memory buffer anyway + } else { + LOG(INFO) << "Cleaned up OpLog from etcd before sequence_id=" << before_sequence_id; + } + } + + // 2. Cleanup from memory buffer + { + std::unique_lock lock(mutex_); + + // Remove entries from the front of the buffer while their sequence_id < before_sequence_id + while (!buffer_.empty() && buffer_.front().sequence_id < before_sequence_id) { + buffer_.pop_front(); + memory_cleaned++; + } + + // Update first_seq_id_ if we removed entries + if (memory_cleaned > 0) { + first_seq_id_ = buffer_.empty() ? (last_seq_id_ + 1) : buffer_.front().sequence_id; + } + } + + if (memory_cleaned > 0) { + LOG(INFO) << "Cleaned up " << memory_cleaned << " OpLog entries from memory buffer" + << ", before_sequence_id=" << before_sequence_id; + } + + return memory_cleaned; +} + +uint64_t OpLogManager::NowMs() { + using namespace std::chrono; + return duration_cast(steady_clock::now().time_since_epoch()) + .count(); +} + +uint32_t OpLogManager::ComputeChecksum(const std::string& data) { + // Use xxHash XXH32 for a fast, deterministic 32-bit checksum. + // Requires linking against xxHash (e.g., libxxhash) and including . + return static_cast(XXH32(data.data(), data.size(), 0)); +} + +uint32_t OpLogManager::ComputePrefixHash(const std::string& key) { + if (key.empty()) { + return 0; + } + // Use XXH32 for consistency with ComputeChecksum and better performance. + // XXH32 provides faster hashing and lower collision rate than std::hash. + // Computing hash for the entire key ensures better distribution and fewer collisions. + return static_cast(XXH32(key.data(), key.size(), 0)); +} + +bool OpLogManager::VerifyChecksum(const OpLogEntry& entry) { + uint32_t computed = ComputeChecksum(entry.payload); + return computed == entry.checksum; +} + +bool OpLogManager::ValidateEntrySize(const OpLogEntry& entry, + std::string* reason) { + if (entry.object_key.size() > kMaxObjectKeySize) { + if (reason) { + *reason = "object_key too large: size=" + + std::to_string(entry.object_key.size()); + } + return false; + } + if (entry.payload.size() > kMaxPayloadSize) { + if (reason) { + *reason = + "payload too large: size=" + std::to_string(entry.payload.size()); + } + return false; + } + return true; +} + +} // namespace mooncake + + diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp new file mode 100644 index 0000000000..3ce96ed3bc --- /dev/null +++ b/mooncake-store/src/oplog_watcher.cpp @@ -0,0 +1,610 @@ +#include "oplog_watcher.h" + +#include +#include +#include +#include +#include + +#ifdef STORE_USE_ETCD +#include "etcd_helper.h" +#include "etcd_oplog_store.h" +#include "ha_metric_manager.h" +#include "oplog_applier.h" +#include "oplog_manager.h" + +#if __has_include() +#include // Ubuntu +#else +#include // CentOS +#endif + +namespace mooncake { + +namespace { + +// Base64 decoding for binary payload (must match encoding in etcd_oplog_store.cpp) +std::string Base64Decode(const std::string& encoded) { + static const unsigned char decode_table[256] = { + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 64, 64, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 64, 64, 64, + 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 64, + 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 + }; + + std::string result; + result.reserve((encoded.size() * 3) / 4); + + size_t i = 0; + while (i < encoded.size()) { + // Skip whitespace and invalid chars + while (i < encoded.size() && (encoded[i] == ' ' || encoded[i] == '\n' || encoded[i] == '\r' || encoded[i] == '\t')) { + i++; + } + if (i >= encoded.size()) break; + + uint32_t sextet_a = decode_table[static_cast(encoded[i++])]; + if (i >= encoded.size() || sextet_a == 64) break; + + uint32_t sextet_b = decode_table[static_cast(encoded[i++])]; + if (sextet_b == 64) break; + + uint32_t sextet_c = (i < encoded.size()) ? decode_table[static_cast(encoded[i++])] : 64; + uint32_t sextet_d = (i < encoded.size()) ? decode_table[static_cast(encoded[i++])] : 64; + + uint32_t triple = (sextet_a << 18) | (sextet_b << 12) | + ((sextet_c != 64) ? (sextet_c << 6) : 0) | + ((sextet_d != 64) ? sextet_d : 0); + + result.push_back(static_cast((triple >> 16) & 0xFF)); + if (sextet_c != 64) { + result.push_back(static_cast((triple >> 8) & 0xFF)); + } + if (sextet_d != 64) { + result.push_back(static_cast(triple & 0xFF)); + } + } + + return result; +} + +} // namespace + +OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier) + : etcd_endpoints_(etcd_endpoints), cluster_id_(cluster_id), applier_(applier) { + if (applier_ == nullptr) { + LOG(FATAL) << "OpLogApplier cannot be null"; + } + // Normalize cluster_id to avoid double slashes in watch prefix. + while (!cluster_id_.empty() && cluster_id_.back() == '/') { + cluster_id_.pop_back(); + } + if (!cluster_id_.empty() && !IsValidClusterIdComponent(cluster_id_)) { + LOG(FATAL) << "Invalid cluster_id for OpLogWatcher: '" << cluster_id_ + << "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } +} + +OpLogWatcher::~OpLogWatcher() { + Stop(); +} + +void OpLogWatcher::Start() { + // Backward-compatible: start from the last processed sequence id. + (void)StartFromSequenceId(last_processed_sequence_id_.load()); +} + +bool OpLogWatcher::StartFromSequenceId(uint64_t start_seq_id) { + if (running_.load()) { + LOG(WARNING) << "OpLogWatcher is already running"; + return true; + } + +#ifdef STORE_USE_ETCD + uint64_t read_seq_id = start_seq_id; + EtcdRevisionId last_read_rev = 0; + size_t total_applied = 0; + + for (;;) { + std::vector batch; + EtcdRevisionId rev = 0; + if (!ReadOpLogSince(read_seq_id, batch, rev)) { + last_read_rev = 0; + break; + } + last_read_rev = rev; + if (!batch.empty()) { + for (const auto& e : batch) { + if (applier_->ApplyOpLogEntry(e)) { + last_processed_sequence_id_.store(e.sequence_id); + read_seq_id = e.sequence_id; + total_applied++; + } + } + } + if (batch.size() < kSyncBatchSize) { + break; + } + } + + if (last_read_rev > 0) { + next_watch_revision_.store(static_cast(last_read_rev + 1)); + } else { + next_watch_revision_.store(0); + } + + LOG(INFO) << "OpLogWatcher initial sync done: applied=" << total_applied + << ", last_seq=" << last_processed_sequence_id_.load() + << ", next_watch_revision=" << next_watch_revision_.load(); +#endif + + running_.store(true); + watch_thread_ = std::thread(&OpLogWatcher::WatchOpLog, this); + LOG(INFO) << "OpLogWatcher started for cluster_id=" << cluster_id_; + return true; +} + +void OpLogWatcher::Stop() { + if (!running_.load()) { + return; + } + + running_.store(false); + +#ifdef STORE_USE_ETCD + // Wait for watch thread to finish first. This ensures that the watch thread + // has exited before we cancel the watch, reducing the chance of race conditions. + if (watch_thread_.joinable()) { + watch_thread_.join(); + } + + // Now cancel the watch. This will trigger the Go goroutine to exit. + // The watch thread has already stopped, so we won't have race conditions + // with it trying to access the watcher object. + std::string watch_prefix = "/oplog/" + cluster_id_ + "/"; + ErrorCode err = EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to cancel watch for prefix " << watch_prefix + << ", error=" << static_cast(err); + } + + // Wait for Go watch goroutine to fully exit (no more callbacks). + // This avoids callback-after-free without relying on sleeps. + (void)EtcdHelper::WaitWatchWithPrefixStopped(watch_prefix.c_str(), + watch_prefix.size(), + /*timeout_ms=*/5000); +#endif + + LOG(INFO) << "OpLogWatcher stopped"; +} + +bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries, + EtcdRevisionId& revision_id) { +#ifdef STORE_USE_ETCD + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + ErrorCode err = oplog_store.ReadOpLogSinceWithRevision( + start_seq_id, kSyncBatchSize, entries, revision_id); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to read OpLog since sequence_id=" << start_seq_id + << ", error=" << static_cast(err); + return false; + } + return true; +#else + (void)start_seq_id; + (void)entries; + (void)revision_id; + return false; +#endif +} + +uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { + return last_processed_sequence_id_.load(); +} + +void OpLogWatcher::WatchCallback(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, + int event_type, int64_t mod_revision) { + // Use try-catch to prevent crashes if object is destroyed + try { + OpLogWatcher* watcher = static_cast(context); + if (watcher == nullptr) { + // Context is null, ignore callback + return; + } + + // Early exit check: First, try to read running_ flag with minimal object access. + // If object is destroyed, this access might cause SIGSEGV, which will be caught + // by signal handler or cause immediate crash (better than accessing more members). + // We use memory_order_acquire for consistency, but if object is destroyed, + // even this access can fail. + // + // Note: There's no perfect way to check if a C++ object is still valid without + // potentially accessing invalid memory. The best we can do is: + // 1. Check quickly and exit early if stopped + // 2. Use try-catch for C++ exceptions (won't catch SIGSEGV) + // 3. Ensure Stop() waits long enough for all callbacks to complete + bool is_running = false; + try { + is_running = watcher->running_.load(std::memory_order_acquire); + } catch (...) { + // Object may be destroyed, ignore callback + return; + } + + if (!is_running) { + // Watcher is being stopped, ignore callback + return; + } + + std::string key_str; + if (key != nullptr && key_size > 0) { + key_str.assign(key, key_size); + } + std::string value_str; + if (value != nullptr && value_size > 0) { + value_str = std::string(value, value_size); + } + watcher->HandleWatchEvent(key_str, value_str, event_type, mod_revision); + } catch (const std::exception& e) { + // C++ object may have been destroyed, ignore the exception + LOG(WARNING) << "Exception in WatchCallback (likely object destroyed): " << e.what(); + } catch (...) { + // Catch all other exceptions (including access violations) + LOG(WARNING) << "Unknown exception in WatchCallback (likely object destroyed)"; + } +} + +void OpLogWatcher::WatchOpLog() { +#ifdef STORE_USE_ETCD + LOG(INFO) << "OpLog watch thread started for cluster_id=" << cluster_id_; + + std::string watch_prefix = "/oplog/" + cluster_id_ + "/"; + + while (running_.load()) { + // Cancel any existing watch before starting a new one + // This prevents "prefix already being watched" errors + (void)EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + (void)EtcdHelper::WaitWatchWithPrefixStopped(watch_prefix.c_str(), + watch_prefix.size(), + /*timeout_ms=*/5000); + + // Start watching - pass static callback function and this pointer as context + EtcdRevisionId start_rev = + static_cast(next_watch_revision_.load()); + // Use watcher with mod_revision so we can update next_watch_revision_ precisely. + ErrorCode err = EtcdHelper::WatchWithPrefixFromRevision( + watch_prefix.c_str(), watch_prefix.size(), start_rev, this, WatchCallback); + + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to start watch for prefix " << watch_prefix + << ", error=" << static_cast(err); + watch_healthy_.store(false); + NotifyStateEvent(StandbyEvent::WATCH_BROKEN); + + // Wait a bit longer before retrying, to ensure old goroutines have time to exit + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + + // Try to reconnect + TryReconnect(); + continue; + } + + LOG(INFO) << "Watch started for prefix " << watch_prefix; + watch_healthy_.store(true); + consecutive_errors_.store(0); + NotifyStateEvent(StandbyEvent::WATCH_HEALTHY); + + // The watch is now running in the background (via Go goroutine) + // We just need to keep the thread alive until Stop() is called or watch fails + while (running_.load() && watch_healthy_.load()) { + // Drive pending/missing handling even when no new watch events arrive. + // Without this, a single out-of-order arrival could park entries in + // pending_entries_ forever if the missing entry isn't delivered via watch + // (but exists in etcd and could be fetched). + (void)applier_->ProcessPendingEntries(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Periodically check watch health + if (consecutive_errors_.load() >= kMaxConsecutiveErrors) { + LOG(WARNING) << "Too many consecutive errors (" << consecutive_errors_.load() + << "), reconnecting watch..."; + watch_healthy_.store(false); + NotifyStateEvent(StandbyEvent::MAX_ERRORS_REACHED); + break; + } + } + + if (running_.load() && !watch_healthy_.load()) { + // Cancel current watch before reconnecting + (void)EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + (void)EtcdHelper::WaitWatchWithPrefixStopped(watch_prefix.c_str(), + watch_prefix.size(), + /*timeout_ms=*/5000); + NotifyStateEvent(StandbyEvent::WATCH_BROKEN); + TryReconnect(); + } + } + + LOG(INFO) << "OpLog watch thread stopped"; +#else + LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot watch OpLog from etcd"; + running_.store(false); +#endif +} + +void OpLogWatcher::TryReconnect() { + if (!running_.load()) { + return; + } + + int reconnect_attempt = reconnect_count_.fetch_add(1) + 1; + + // Calculate delay with exponential backoff + int delay_ms = std::min(kReconnectDelayMs * reconnect_attempt, kMaxReconnectDelayMs); + + LOG(INFO) << "Attempting to reconnect watch (attempt #" << reconnect_attempt + << "), waiting " << delay_ms << "ms..."; + + std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); + + // Sync any missed entries before resuming watch + if (SyncMissedEntries()) { + LOG(INFO) << "Successfully synced missed OpLog entries"; + NotifyStateEvent(StandbyEvent::RECOVERY_SUCCESS); + } else { + LOG(WARNING) << "Failed to sync missed OpLog entries, continuing anyway"; + NotifyStateEvent(StandbyEvent::RECOVERY_FAILED); + } +} + +bool OpLogWatcher::SyncMissedEntries() { +#ifdef STORE_USE_ETCD + uint64_t last_seq = last_processed_sequence_id_.load(); + if (last_seq == 0) { + // No entries processed yet, nothing to sync + return true; + } + + LOG(INFO) << "Syncing missed OpLog entries since sequence_id=" << last_seq; + + std::vector entries; + EtcdRevisionId rev = 0; + if (!ReadOpLogSince(last_seq, entries, rev)) { + LOG(ERROR) << "Failed to read missed OpLog entries"; + return false; + } + if (rev > 0) { + next_watch_revision_.store(static_cast(rev + 1)); + } + + if (entries.empty()) { + LOG(INFO) << "No missed OpLog entries to sync"; + return true; + } + + LOG(INFO) << "Syncing " << entries.size() << " missed OpLog entries"; + + for (const auto& entry : entries) { + if (applier_->ApplyOpLogEntry(entry)) { + last_processed_sequence_id_.store(entry.sequence_id); + } else { + LOG(WARNING) << "Failed to apply missed OpLog entry, sequence_id=" + << entry.sequence_id; + } + } + + return true; +#else + return false; +#endif +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type) { + HandleWatchEvent(key, value, event_type, /*mod_revision=*/0); +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision) { + // event_type: + // 0 = PUT, 1 = DELETE, 2 = WATCH_BROKEN (Go watcher terminated; should reconnect) + if (event_type == 2) { + LOG(WARNING) << "OpLog watch broken, will reconnect. cluster_id=" << cluster_id_ + << ", next_watch_revision=" << next_watch_revision_.load() + << ", last_seq=" << last_processed_sequence_id_.load(); + watch_healthy_.store(false); + consecutive_errors_.fetch_add(1); + return; + } + + if (mod_revision > 0) { + // Keep next_watch_revision_ monotonic: next = max(next, modRev+1) + int64_t candidate = mod_revision + 1; + int64_t cur = next_watch_revision_.load(); + while (candidate > cur && + !next_watch_revision_.compare_exchange_weak(cur, candidate)) { + // retry + } + } + // event_type: 0 = PUT, 1 = DELETE + if (event_type == 1) { + // DELETE event - OpLog entry was cleaned up + VLOG(1) << "OpLog entry deleted: " << key; + consecutive_errors_.store(0); // Watch is working + return; + } + + if (event_type != 0) { + LOG(WARNING) << "Unknown event type: " << event_type << " for key: " << key; + consecutive_errors_.fetch_add(1); + return; + } + + // Skip the "latest" key and snapshot keys + if (key.find("/latest") != std::string::npos || + key.find("/snapshot/") != std::string::npos) { + return; + } + + // Parse the OpLog entry from JSON + OpLogEntry entry; + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry from key: " << key; + consecutive_errors_.fetch_add(1); + return; + } + + // Basic DoS protection: validate key/payload sizes before further processing. + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "OpLog entry size rejected: sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key << ", reason=" << size_reason; + consecutive_errors_.fetch_add(1); + return; + } + + // Verify checksum to detect data corruption or tampering. + if (!OpLogManager::VerifyChecksum(entry)) { + LOG(ERROR) << "OpLog entry checksum mismatch: sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key + << ". Possible data corruption or tampering. Discarding entry."; + consecutive_errors_.fetch_add(1); + HAMetricManager::instance().inc_oplog_checksum_failures(); + return; + } + + // Apply the OpLog entry + if (applier_->ApplyOpLogEntry(entry)) { + // last_processed_sequence_id_ must be monotonic. We may "consume" duplicate + // / already-applied entries (entry.sequence_id < expected) as no-ops, so + // never regress this counter. + uint64_t cur = last_processed_sequence_id_.load(); + while (IsSequenceNewer(entry.sequence_id, cur) && + !last_processed_sequence_id_.compare_exchange_weak(cur, entry.sequence_id)) { + // retry + } + consecutive_errors_.store(0); // Reset error counter on success + reconnect_count_.store(0); // Reset reconnect counter on success + VLOG(2) << "Applied OpLog entry: sequence_id=" << entry.sequence_id + << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; + } else { + // ApplyOpLogEntry returns false for out-of-order entries, + // which is expected behavior, not an error + VLOG(1) << "OpLog entry not applied (may be out of order): sequence_id=" + << entry.sequence_id; + } +} + +bool OpLogWatcher::DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) { + Json::Value root; + Json::CharReaderBuilder reader; + std::string errs; + std::istringstream s(json_str); + + if (!Json::parseFromStream(reader, s, &root, &errs)) { + LOG(ERROR) << "Failed to parse OpLogEntry JSON: " << errs; + return false; + } + + entry.sequence_id = root.get("sequence_id", 0).asUInt64(); + entry.timestamp_ms = root.get("timestamp_ms", 0).asUInt64(); + entry.op_type = static_cast(root.get("op_type", 0).asInt()); + entry.object_key = root.get("object_key", "").asString(); + + // CRITICAL: Base64 decode payload to restore binary data + std::string encoded_payload = root.get("payload", "").asString(); + entry.payload = Base64Decode(encoded_payload); + + entry.checksum = root.get("checksum", 0).asUInt(); + entry.prefix_hash = root.get("prefix_hash", 0).asUInt(); + return true; +} + +} // namespace mooncake + +#else // STORE_USE_ETCD not defined + +namespace mooncake { + +OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier) + : etcd_endpoints_(etcd_endpoints), cluster_id_(cluster_id), applier_(applier) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +OpLogWatcher::~OpLogWatcher() { + Stop(); +} + +void OpLogWatcher::Start() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +bool OpLogWatcher::StartFromSequenceId(uint64_t /*start_seq_id*/) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +void OpLogWatcher::Stop() { + // No-op when STORE_USE_ETCD is not enabled +} + +bool OpLogWatcher::ReadOpLogSince(uint64_t /*start_seq_id*/, + std::vector& /*entries*/, + EtcdRevisionId& /*revision_id*/) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { + return last_processed_sequence_id_.load(); +} + +void OpLogWatcher::WatchOpLog() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision) { + (void)key; + (void)value; + (void)event_type; + (void)mod_revision; + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::TryReconnect() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +bool OpLogWatcher::SyncMissedEntries() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +} // namespace mooncake + +#endif // STORE_USE_ETCD + diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index db1cec0cad..7b33698814 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -13,12 +13,14 @@ #include #include +#include "ha_metric_manager.h" #include "master_metric_manager.h" #include "master_service.h" #include "rpc_helper.h" #include "types.h" #include "utils/scoped_vlog_timer.h" #include "version.h" +// replication_service.h removed - using etcd-based OpLog sync instead namespace mooncake { @@ -31,12 +33,24 @@ WrappedMasterService::WrappedMasterService( metric_report_running_(config.enable_metric_reporting) { init_http_server(); + // ReplicationService removed - using etcd-based OpLog sync instead + // TODO: In Phase 1, initialize EtcdOpLogStore and integrate with + // OpLogManager + if (config.enable_ha) { + LOG(INFO) << "HA mode enabled - etcd-based OpLog sync will be " + "implemented in Phase 1"; + } + if (config.enable_metric_reporting) { metric_report_thread_ = std::thread([this]() { while (metric_report_running_) { std::string metrics_summary = MasterMetricManager::instance().get_summary_string(); LOG(INFO) << "Master Metrics: " << metrics_summary; + // Log HA metrics summary + std::string ha_summary = + HAMetricManager::instance().get_summary_string(); + LOG(INFO) << ha_summary; std::this_thread::sleep_for( std::chrono::seconds(kMetricReportIntervalSeconds)); } @@ -49,9 +63,19 @@ WrappedMasterService::~WrappedMasterService() { if (metric_report_thread_.joinable()) { metric_report_thread_.join(); } + + // ReplicationService removed - using etcd-based OpLog sync instead + http_server_.stop(); } +void WrappedMasterService::RestoreFromStandby( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id) { + master_service_.RestoreFromStandbySnapshot(snapshot, + initial_oplog_sequence_id); +} + void WrappedMasterService::init_http_server() { using namespace coro_http; @@ -59,6 +83,8 @@ void WrappedMasterService::init_http_server() { "/metrics", [](coro_http_request& req, coro_http_response& resp) { std::string metrics = MasterMetricManager::instance().serialize_metrics(); + // Append HA metrics + metrics += HAMetricManager::instance().serialize_metrics(); resp.add_header("Content-Type", "text/plain; version=0.0.4"); resp.set_status_and_content(status_type::ok, std::move(metrics)); }); @@ -68,10 +94,21 @@ void WrappedMasterService::init_http_server() { [](coro_http_request& req, coro_http_response& resp) { std::string summary = MasterMetricManager::instance().get_summary_string(); + summary += "\n"; + summary += HAMetricManager::instance().get_summary_string(); resp.add_header("Content-Type", "text/plain; version=0.0.4"); resp.set_status_and_content(status_type::ok, std::move(summary)); }); + // Dedicated HA metrics endpoint + http_server_.set_http_handler( + "/metrics/ha", [](coro_http_request& req, coro_http_response& resp) { + std::string metrics = + HAMetricManager::instance().serialize_metrics(); + resp.add_header("Content-Type", "text/plain; version=0.0.4"); + resp.set_status_and_content(status_type::ok, std::move(metrics)); + }); + http_server_.set_http_handler( "/query_key", [&](coro_http_request& req, coro_http_response& resp) { auto key = req.get_query_value("key"); diff --git a/mooncake-store/src/standby_state_machine.cpp b/mooncake-store/src/standby_state_machine.cpp new file mode 100644 index 0000000000..616649a454 --- /dev/null +++ b/mooncake-store/src/standby_state_machine.cpp @@ -0,0 +1,283 @@ +#include "standby_state_machine.h" + +#include + +namespace mooncake { + +StandbyStateMachine::StandbyStateMachine() + : state_enter_time_(std::chrono::steady_clock::now()) {} + +StateTransitionResult StandbyStateMachine::ValidateTransition(StandbyState from, + StandbyEvent event) const { + StateTransitionResult result; + result.allowed = false; + result.old_state = from; + result.new_state = from; + + // State transition table + switch (from) { + case StandbyState::STOPPED: + if (event == StandbyEvent::START) { + result.allowed = true; + result.new_state = StandbyState::CONNECTING; + } + break; + + case StandbyState::CONNECTING: + switch (event) { + case StandbyEvent::CONNECTED: + result.allowed = true; + result.new_state = StandbyState::SYNCING; + break; + case StandbyEvent::CONNECTION_FAILED: + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + default: + break; + } + break; + + case StandbyState::SYNCING: + switch (event) { + case StandbyEvent::SYNC_COMPLETE: + result.allowed = true; + result.new_state = StandbyState::WATCHING; + break; + case StandbyEvent::SYNC_FAILED: + case StandbyEvent::DISCONNECTED: + result.allowed = true; + result.new_state = StandbyState::RECONNECTING; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + default: + break; + } + break; + + case StandbyState::WATCHING: + switch (event) { + case StandbyEvent::WATCH_BROKEN: + case StandbyEvent::DISCONNECTED: + result.allowed = true; + result.new_state = StandbyState::RECONNECTING; + break; + case StandbyEvent::MAX_ERRORS_REACHED: + result.allowed = true; + result.new_state = StandbyState::RECOVERING; + break; + case StandbyEvent::PROMOTE: + result.allowed = true; + result.new_state = StandbyState::PROMOTING; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + // WATCH_HEALTHY in WATCHING state is a no-op (stay in WATCHING) + case StandbyEvent::WATCH_HEALTHY: + result.allowed = true; + result.new_state = StandbyState::WATCHING; + break; + default: + break; + } + break; + + case StandbyState::RECOVERING: + switch (event) { + case StandbyEvent::RECOVERY_SUCCESS: + result.allowed = true; + result.new_state = StandbyState::WATCHING; + break; + case StandbyEvent::RECOVERY_FAILED: + case StandbyEvent::DISCONNECTED: + result.allowed = true; + result.new_state = StandbyState::RECONNECTING; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + default: + break; + } + break; + + case StandbyState::RECONNECTING: + switch (event) { + case StandbyEvent::CONNECTED: + result.allowed = true; + result.new_state = StandbyState::SYNCING; + break; + case StandbyEvent::MAX_ERRORS_REACHED: + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + default: + break; + } + break; + + case StandbyState::PROMOTING: + switch (event) { + case StandbyEvent::PROMOTION_SUCCESS: + result.allowed = true; + result.new_state = StandbyState::PROMOTED; + break; + case StandbyEvent::PROMOTION_FAILED: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + default: + break; + } + break; + + case StandbyState::PROMOTED: + if (event == StandbyEvent::STOP) { + result.allowed = true; + result.new_state = StandbyState::STOPPED; + } + break; + + case StandbyState::FAILED: + if (event == StandbyEvent::STOP) { + result.allowed = true; + result.new_state = StandbyState::STOPPED; + } else if (event == StandbyEvent::START) { + // Allow restart from FAILED state + result.allowed = true; + result.new_state = StandbyState::CONNECTING; + } + break; + } + + if (!result.allowed) { + result.reason = std::string("Invalid transition from ") + StandbyStateToString(from) + + " on event " + StandbyEventToString(event); + } + + return result; +} + +StateTransitionResult StandbyStateMachine::ProcessEvent(StandbyEvent event) { + StandbyState old_state = current_state_.load(std::memory_order_acquire); + StateTransitionResult result = ValidateTransition(old_state, event); + + if (result.allowed && result.new_state != old_state) { + std::lock_guard lock(mutex_); + + // Double-check state hasn't changed (compare-and-swap pattern) + StandbyState current = current_state_.load(std::memory_order_acquire); + if (current != old_state) { + // State changed by another thread, re-validate + result = ValidateTransition(current, event); + old_state = current; + result.old_state = current; + if (!result.allowed || result.new_state == old_state) { + return result; + } + } + + // Record transition + TransitionRecord record; + record.timestamp = std::chrono::steady_clock::now(); + record.from_state = old_state; + record.to_state = result.new_state; + record.event = event; + + transition_history_.push_back(record); + if (transition_history_.size() > kMaxHistorySize) { + transition_history_.erase(transition_history_.begin()); + } + + // Update state + current_state_.store(result.new_state, std::memory_order_release); + state_enter_time_ = record.timestamp; + + LOG(INFO) << "Standby state transition: " << StandbyStateToString(old_state) << " -> " + << StandbyStateToString(result.new_state) + << " (event: " << StandbyEventToString(event) << ")"; + + // Make a copy of callbacks to release the lock before calling them + std::vector callbacks_copy = callbacks_; + + // Release lock by ending scope, then notify callbacks + // Note: We need to unlock before calling callbacks to avoid deadlock + // So we copy callbacks and call after the lock_guard scope ends + for (const auto& callback : callbacks_copy) { + if (callback) { + callback(old_state, result.new_state, event); + } + } + } else if (!result.allowed) { + VLOG(1) << "Standby state transition rejected: " << result.reason; + } + + return result; +} + +void StandbyStateMachine::RegisterCallback(StateChangeCallback callback) { + std::lock_guard lock(mutex_); + callbacks_.push_back(std::move(callback)); +} + +std::vector StandbyStateMachine::GetTransitionHistory( + size_t max_records) const { + std::lock_guard lock(mutex_); + + if (transition_history_.size() <= max_records) { + return transition_history_; + } + + return std::vector(transition_history_.end() - max_records, + transition_history_.end()); +} + +std::chrono::milliseconds StandbyStateMachine::GetTimeInCurrentState() const { + auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(mutex_); + return std::chrono::duration_cast(now - state_enter_time_); +} + +int StandbyStateMachine::IncrementErrors() { + int new_count = consecutive_errors_.fetch_add(1) + 1; + if (new_count >= kMaxConsecutiveErrors) { + // Trigger MAX_ERRORS_REACHED event + ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + } + return new_count; +} + +} // namespace mooncake + diff --git a/mooncake-store/tests/CMakeLists.txt b/mooncake-store/tests/CMakeLists.txt index 15c838e190..46587e88b8 100644 --- a/mooncake-store/tests/CMakeLists.txt +++ b/mooncake-store/tests/CMakeLists.txt @@ -1,5 +1,10 @@ function(add_store_test name) add_executable(${name} ${ARGN}) + # Set include directories for tests + target_include_directories(${name} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/../include + ${CMAKE_CURRENT_BINARY_DIR}/../include + ) target_link_libraries(${name} PUBLIC mooncake_store cachelib_memory_allocator @@ -35,6 +40,17 @@ add_store_test(non_ha_reconnect_test non_ha_reconnect_test.cpp) add_store_test(storage_backend_test storage_backend_test.cpp) add_store_test(mutex_test mutex_test.cpp) add_store_test(file_storage_test file_storage_test.cpp) + +# Hot Standby Unit Tests +add_store_test(standby_state_machine_test hot_standby_ut/standby_state_machine_test.cpp) +add_store_test(oplog_manager_test hot_standby_ut/oplog_manager_test.cpp) +add_store_test(etcd_oplog_store_test hot_standby_ut/etcd_oplog_store_test.cpp) +add_store_test(oplog_applier_test hot_standby_ut/oplog_applier_test.cpp) +add_store_test(oplog_watcher_test hot_standby_ut/oplog_watcher_test.cpp) +add_store_test(hot_standby_service_test hot_standby_ut/hot_standby_service_test.cpp) +add_store_test(ha_metric_manager_test hot_standby_ut/ha_metric_manager_test.cpp) +add_store_test(hot_standby_integration_test hot_standby_integration_test.cpp) + add_subdirectory(e2e) add_executable(high_availability_test high_availability_test.cpp) diff --git a/mooncake-store/tests/e2e/CMakeLists.txt b/mooncake-store/tests/e2e/CMakeLists.txt index e06e7d568d..c8f143d7bf 100644 --- a/mooncake-store/tests/e2e/CMakeLists.txt +++ b/mooncake-store/tests/e2e/CMakeLists.txt @@ -58,4 +58,13 @@ target_link_libraries(chaos_rand_test PUBLIC gtest pthread ${ETCD_WRAPPER_LIB} +) + +add_executable(client_put_test client_put_test.cpp ${MOONCAKE_E2E_TEST_SOURCES}) +target_link_libraries(client_put_test PUBLIC + mooncake_store + cachelib_memory_allocator + glog + pthread + ${ETCD_WRAPPER_LIB} ) \ No newline at end of file diff --git a/mooncake-store/tests/e2e/client_put_test.cpp b/mooncake-store/tests/e2e/client_put_test.cpp new file mode 100644 index 0000000000..f4c4616545 --- /dev/null +++ b/mooncake-store/tests/e2e/client_put_test.cpp @@ -0,0 +1,261 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "client_wrapper.h" +#include "types.h" + +// Command line flags +DEFINE_string(hostname, "localhost:9001", "Local hostname for the client"); +DEFINE_string(metadata_server, "P2PHANDSHAKE", + "Metadata server connection string. Use 'P2PHANDSHAKE' for P2P mode " + "(no HTTP metadata server required), or 'http://IP:PORT/metadata' for HTTP mode"); +DEFINE_string(protocol, "tcp", "Transfer protocol: rdma|tcp"); +DEFINE_string(device_name, "", "Device name to use, valid if protocol=rdma"); +DEFINE_string(master_server_entry, "127.0.0.1:50051", + "Master server entry (IP:Port)"); +DEFINE_int32(segment_size_mb, 128, "Segment size in MB"); +DEFINE_int32(value_size, 1024, "Value size in bytes (default: 1KB)"); +DEFINE_int32(operation_interval_microsec, 1, + "Interval between operations (GET+PUT) in microseconds"); +DEFINE_int32(max_operations, 0, + "Maximum number of operations (0 = unlimited)"); +DEFINE_int32(key_space_size, 1000, + "Key space size (number of unique keys to choose from). " + "Smaller values lead to higher hit rates as more keys are written"); +DEFINE_int32(stats_report_interval, 10, + "Report statistics every N operations (0 = disable periodic reports)"); + +namespace mooncake { +namespace testing { + +class ClientPutTester { + public: + ClientPutTester() + : gen_(std::random_device{}()), + key_dist_(0, 999), // Will be reinitialized in Initialize() + total_requests_(0), + get_hits_(0), + get_misses_(0), + put_count_(0), + put_success_(0), + put_failures_(0) {} + + bool Initialize() { + // Reinitialize key distribution with the actual key space size + // (after gflags parsing) + key_dist_ = std::uniform_int_distribution<>(0, FLAGS_key_space_size - 1); + + // Create client wrapper + auto client_opt = ClientTestWrapper::CreateClientWrapper( + FLAGS_hostname, FLAGS_metadata_server, FLAGS_protocol, + FLAGS_device_name, FLAGS_master_server_entry); + + if (!client_opt.has_value()) { + LOG(ERROR) << "Failed to create client wrapper"; + return false; + } + + client_ = client_opt.value(); + LOG(INFO) << "Successfully created client wrapper"; + + // Mount a segment + size_t segment_size = static_cast(FLAGS_segment_size_mb) * 1024 * 1024; + void* buffer = nullptr; + ErrorCode mount_err = client_->Mount(segment_size, buffer); + if (mount_err != ErrorCode::OK) { + LOG(ERROR) << "Failed to mount segment: " << toString(mount_err); + return false; + } + + segment_buffer_ = buffer; + LOG(INFO) << "Successfully mounted segment: size=" << segment_size + << " bytes (" << FLAGS_segment_size_mb << " MB)"; + + return true; + } + + void Run() { + LOG(INFO) << "Starting inference simulation test:"; + LOG(INFO) << " Operation interval: " << FLAGS_operation_interval_microsec + << " us"; + LOG(INFO) << " Value size: " << FLAGS_value_size << " bytes"; + LOG(INFO) << " Key space size: " << FLAGS_key_space_size + << " (smaller = higher hit rate as data accumulates)"; + LOG(INFO) << " Max operations: " + << (FLAGS_max_operations > 0 ? std::to_string(FLAGS_max_operations) + : "unlimited"); + LOG(INFO) << " Stats report interval: " + << (FLAGS_stats_report_interval > 0 + ? std::to_string(FLAGS_stats_report_interval) + : "disabled"); + + auto start_time = std::chrono::steady_clock::now(); + + while (true) { + // Generate a random key from the key space + std::string key = GenerateRandomKey(); + + // Step 1: Try GET first (simulate inference query) + std::string retrieved_value; + ErrorCode get_err = client_->Get(key, retrieved_value); + + total_requests_++; + + if (get_err == ErrorCode::OK) { + // GET hit: key exists, no need to PUT + get_hits_++; + VLOG(1) << "[REQUEST #" << total_requests_ << "] GET HIT: key=" << key + << ", value_size=" << retrieved_value.size(); + } else if (get_err == ErrorCode::OBJECT_NOT_FOUND) { + // GET miss: key doesn't exist, need to PUT + get_misses_++; + std::string new_value = GenerateRandomValue(); + + // Step 2: Perform PUT operation + ErrorCode put_err = client_->Put(key, new_value); + put_count_++; + + if (put_err != ErrorCode::OK) { + put_failures_++; + LOG(WARNING) << "[REQUEST #" << total_requests_ + << "] GET MISS -> PUT FAILED: key=" << key + << ", value_size=" << new_value.size() + << ", error=" << toString(put_err); + } else { + put_success_++; + VLOG(1) << "[REQUEST #" << total_requests_ + << "] GET MISS -> PUT SUCCESS: key=" << key + << ", value_size=" << new_value.size(); + } + } else { + // GET error (other than NOT_FOUND) + LOG(WARNING) << "[REQUEST #" << total_requests_ + << "] GET ERROR: key=" << key + << ", error=" << toString(get_err); + } + + // Report statistics periodically + if (FLAGS_stats_report_interval > 0 && + total_requests_ % FLAGS_stats_report_interval == 0) { + ReportStatistics(start_time); + } + + // Check if we've reached the maximum number of operations + if (FLAGS_max_operations > 0 && + total_requests_ >= FLAGS_max_operations) { + LOG(INFO) << "Reached maximum operations limit: " << total_requests_; + break; + } + + // Sleep for the specified interval + std::this_thread::sleep_for( + std::chrono::microseconds(FLAGS_operation_interval_microsec)); + } + + // Final statistics report + auto end_time = std::chrono::steady_clock::now(); + auto total_elapsed = std::chrono::duration_cast( + end_time - start_time); + LOG(INFO) << "========================================"; + LOG(INFO) << "Test completed:"; + LOG(INFO) << " Total time: " << total_elapsed.count() << "s"; + ReportStatistics(start_time); + } + + private: + void ReportStatistics(const std::chrono::steady_clock::time_point& start_time) { + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast(now - start_time); + + double hit_rate = 0.0; + if (total_requests_ > 0) { + hit_rate = (static_cast(get_hits_) / total_requests_) * 100.0; + } + + double put_success_rate = 0.0; + if (put_count_ > 0) { + put_success_rate = (static_cast(put_success_) / put_count_) * 100.0; + } + + LOG(INFO) << "========================================"; + LOG(INFO) << "Statistics (after " << total_requests_ << " requests, " + << elapsed.count() << "s elapsed):"; + LOG(INFO) << " Total requests: " << total_requests_; + LOG(INFO) << " GET hits: " << get_hits_ << " (" + << std::fixed << std::setprecision(2) << hit_rate << "%)"; + LOG(INFO) << " GET misses: " << get_misses_; + LOG(INFO) << " PUT attempts: " << put_count_; + LOG(INFO) << " PUT success: " << put_success_ << " (" + << std::fixed << std::setprecision(2) << put_success_rate << "%)"; + LOG(INFO) << " PUT failures: " << put_failures_; + LOG(INFO) << " Unique keys written: ~" << put_success_ + << " (out of " << FLAGS_key_space_size << " key space)"; + LOG(INFO) << "========================================"; + } + + std::string GenerateRandomKey() { + // Generate a random key from the key space: test_key_<0 to key_space_size-1> + // This ensures that as more keys are written, the hit rate increases + int key_index = key_dist_(gen_); + return "test_key_" + std::to_string(key_index); + } + + std::string GenerateRandomValue() { + // Generate a random value of the specified size + std::string value; + value.reserve(FLAGS_value_size); + + // Fill with random alphanumeric characters + const char charset[] = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + const size_t charset_size = sizeof(charset) - 1; + + std::uniform_int_distribution<> char_dist(0, charset_size - 1); + for (int i = 0; i < FLAGS_value_size; ++i) { + value += charset[char_dist(gen_)]; + } + + return value; + } + + std::shared_ptr client_; + void* segment_buffer_{nullptr}; + std::mt19937 gen_; + std::uniform_int_distribution<> key_dist_; + + // Statistics + std::atomic total_requests_; + std::atomic get_hits_; + std::atomic get_misses_; + std::atomic put_count_; + std::atomic put_success_; + std::atomic put_failures_; +}; + +} // namespace testing +} // namespace mooncake + +int main(int argc, char* argv[]) { + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); + FLAGS_logtostderr = 1; + + mooncake::testing::ClientPutTester tester; + + if (!tester.Initialize()) { + LOG(ERROR) << "Failed to initialize client tester"; + return -1; + } + + tester.Run(); + + return 0; +} + diff --git a/mooncake-store/tests/e2e/client_wrapper.cpp b/mooncake-store/tests/e2e/client_wrapper.cpp index 74e666370e..7f97aca6d2 100644 --- a/mooncake-store/tests/e2e/client_wrapper.cpp +++ b/mooncake-store/tests/e2e/client_wrapper.cpp @@ -25,9 +25,16 @@ ClientTestWrapper::CreateClientWrapper(const std::string& hostname, const std::string& device_name, const std::string& master_server_entry, size_t local_buffer_size) { - auto client_opt = - Client::Create(hostname, // Local hostname - metadata_connstring, protocol, master_server_entry); + // Fix: Correctly pass device_names and master_server_entry parameters + // Client::Create signature: + // (hostname, metadata_connstring, protocol, device_names, master_server_entry, ...) + auto client_opt = Client::Create( + hostname, // Local hostname + metadata_connstring, + protocol, + device_name.empty() ? std::nullopt : std::make_optional(device_name), // device_names + master_server_entry // master_server_entry (supports etcd:// protocol for HA mode) + ); if (!client_opt.has_value()) { return std::nullopt; diff --git a/mooncake-store/tests/hot_standby_integration_test.cpp b/mooncake-store/tests/hot_standby_integration_test.cpp new file mode 100644 index 0000000000..5bbaf31784 --- /dev/null +++ b/mooncake-store/tests/hot_standby_integration_test.cpp @@ -0,0 +1,912 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "etcd_helper.h" +#include "etcd_oplog_store.h" +#include "ha_helper.h" +#include "ha_metric_manager.h" +#include "hot_standby_service.h" +#include "master_service.h" +#include "metadata_store.h" +#include "oplog_manager.h" +#include "standby_state_machine.h" + +namespace mooncake { +namespace testing { + +DEFINE_string(hs_etcd_endpoints, "", + "Etcd endpoints for hot-standby integration tests (required for integration tests)"); +DEFINE_string(hs_cluster_id, "hs_integration_cluster", + "Cluster ID prefix for hot-standby integration tests"); + +// RAII helper to ensure HotStandbyService is always stopped +class StandbyServiceGuard { + public: + explicit StandbyServiceGuard(HotStandbyService* service) : service_(service) {} + ~StandbyServiceGuard() { + if (service_) { + service_->Stop(); + // Give etcd watch goroutines time to clean up and exit. + // Increased delay to ensure all pending callbacks are processed + // and Go goroutines have time to exit completely. + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); + } + } + // Disable copy and move + StandbyServiceGuard(const StandbyServiceGuard&) = delete; + StandbyServiceGuard& operator=(const StandbyServiceGuard&) = delete; + StandbyServiceGuard(StandbyServiceGuard&&) = delete; + StandbyServiceGuard& operator=(StandbyServiceGuard&&) = delete; + + private: + HotStandbyService* service_; +}; + +class HotStandbyIntegrationTest : public ::testing::Test { + protected: + // Static mutex to ensure tests run serially and prevent etcd resource conflicts + static std::mutex test_mutex_; + // Per-test lock guard to ensure lock is released even if test fails + std::unique_ptr> test_lock_; + + static void SetUpTestSuite() { +#ifdef STORE_USE_ETCD + // Initialize glog + google::InitGoogleLogging("HotStandbyIntegrationTest"); + google::SetVLOGLevel("*", 1); + FLAGS_logtostderr = 1; + + // Check if etcd endpoints are provided + if (FLAGS_hs_etcd_endpoints.empty()) { + GTEST_SKIP() << "hs_etcd_endpoints not provided. " + "Set --hs_etcd_endpoints= to run integration tests."; + } + + // Initialize etcd client + ASSERT_EQ(ErrorCode::OK, + EtcdHelper::ConnectToEtcdStoreClient(FLAGS_hs_etcd_endpoints)) + << "Failed to connect to etcd at " << FLAGS_hs_etcd_endpoints; +#else + GTEST_SKIP() << "STORE_USE_ETCD is not enabled; " + "hot-standby integration tests require etcd."; +#endif + } + + static void TearDownTestSuite() { +#ifdef STORE_USE_ETCD + google::ShutdownGoogleLogging(); +#endif + } + + void SetUp() override { +#ifdef STORE_USE_ETCD + // Acquire lock to ensure tests run serially + // Use lock_guard to ensure lock is released even if test fails + test_lock_ = std::make_unique>(test_mutex_); + // Clean up test data before each test + CleanupTestData(); +#endif + } + + void TearDown() override { +#ifdef STORE_USE_ETCD + // Clean up test data after each test + CleanupTestData(); + // Give etcd watch goroutines time to clean up before next test + // Increased delay to allow watch streams and gRPC connections to fully close + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); + // Release lock (lock_guard will automatically unlock in destructor) + test_lock_.reset(); +#endif + } + + private: + // Helper function to construct end_key for DeleteRange to delete a single key + static std::string MakeEndKeyForSingleKey(const std::string& key) { + std::string end_key = key; + for (int i = static_cast(end_key.size()) - 1; i >= 0; --i) { + unsigned char c = static_cast(end_key[i]); + if (c < 0xFF) { + end_key[i] = static_cast(c + 1); + end_key.resize(i + 1); + return end_key; + } + } + // If all characters are 0xFF, append '\0' + return key + std::string(1, '\0'); + } + + void CleanupTestData() { +#ifdef STORE_USE_ETCD + // Delete all keys under /oplog/{cluster_id}/ prefix + std::string prefix = std::string("/oplog/") + FLAGS_hs_cluster_id + "/"; + + auto prefix_end = [](std::string p) -> std::string { + for (int i = static_cast(p.size()) - 1; i >= 0; --i) { + unsigned char c = static_cast(p[i]); + if (c < 0xFF) { + p[i] = static_cast(c + 1); + p.resize(i + 1); + return p; + } + } + return std::string(1, '\0'); + }; + std::string end_key = prefix_end(prefix); + + (void)EtcdHelper::DeleteRange(prefix.c_str(), prefix.size(), end_key.c_str(), + end_key.size()); + + // Also delete /latest key using DeleteRange + std::string latest_key = std::string("/oplog/") + FLAGS_hs_cluster_id + "/latest"; + std::string latest_end_key = MakeEndKeyForSingleKey(latest_key); + (void)EtcdHelper::DeleteRange(latest_key.c_str(), latest_key.size(), + latest_end_key.c_str(), latest_end_key.size()); +#endif + } +}; + +// ========== 8.1.1 End-to-end tests ========== + +TEST_F(HotStandbyIntegrationTest, TestPrimaryStandbySync) { +#ifndef STORE_USE_ETCD + GTEST_SKIP() << "STORE_USE_ETCD is not enabled."; +#else + if (FLAGS_hs_etcd_endpoints.empty()) { + GTEST_SKIP() << "hs_etcd_endpoints not provided."; + } + + // 1. Simulate primary writing OpLog entries to etcd + OpLogManager primary_oplog; + primary_oplog.SetEtcdOpLogStore( + std::make_shared(FLAGS_hs_cluster_id, true)); + + // Write several test entries + std::vector test_keys; + std::vector test_seq_ids; + for (int i = 0; i < 10; ++i) { + std::string key = "test_key_" + std::to_string(i); + std::string payload = + R"({"client_id_first":1,"client_id_second":2,"size":1024,"replicas":[]})"; + + uint64_t seq_id = primary_oplog.Append(OpType::PUT_END, key, payload); + test_keys.push_back(key); + test_seq_ids.push_back(seq_id); + + LOG(INFO) << "Primary wrote OpLog entry: seq_id=" << seq_id + << ", key=" << key; + } + + uint64_t last_seq_id = primary_oplog.GetLastSequenceId(); + LOG(INFO) << "Primary wrote " << last_seq_id << " OpLog entries"; + + // 2. Start the standby HotStandbyService + HotStandbyConfig hs_config; + hs_config.enable_verification = false; + hs_config.max_replication_lag_entries = 1000; + + HotStandbyService standby(hs_config); + StandbyServiceGuard guard(&standby); // RAII: ensure cleanup even on failure + + ErrorCode start_err = standby.Start( + /*primary_address_unused=*/"", FLAGS_hs_etcd_endpoints, + FLAGS_hs_cluster_id); + + ASSERT_EQ(ErrorCode::OK, start_err) + << "Failed to start HotStandbyService"; + + // 3. Wait for the standby to finish initial sync + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + bool synced = false; + + while (std::chrono::steady_clock::now() < deadline) { + auto status = standby.GetSyncStatus(); + + LOG(INFO) << "Standby status: state=" + << StandbyStateToString(status.state) + << ", applied_seq_id=" << status.applied_seq_id + << ", primary_seq_id=" << status.primary_seq_id + << ", lag_entries=" << status.lag_entries + << ", is_connected=" << status.is_connected; + + if (status.state == StandbyState::WATCHING && + status.lag_entries == 0 && + status.applied_seq_id >= last_seq_id) { + synced = true; + break; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + + ASSERT_TRUE(synced) << "Standby failed to sync within timeout"; + + // 4. Verify standby metadata snapshot + std::vector> snapshot; + ASSERT_TRUE(standby.ExportMetadataSnapshot(snapshot)); + + LOG(INFO) << "Standby metadata snapshot size: " << snapshot.size(); + + // Verify all written keys exist in the snapshot + std::set snapshot_keys; + for (const auto& kv : snapshot) { + snapshot_keys.insert(kv.first); + } + + for (const auto& key : test_keys) { + EXPECT_NE(snapshot_keys.end(), snapshot_keys.find(key)) + << "Key " << key << " not found in Standby snapshot"; + } + + EXPECT_GE(snapshot.size(), test_keys.size()) + << "Standby snapshot should contain at least the test keys"; + + // 5. Verify sequence IDs + uint64_t latest_applied = standby.GetLatestAppliedSequenceId(); + EXPECT_GE(latest_applied, last_seq_id) + << "Standby should have applied at least the last test sequence ID"; + + // guard will automatically stop the service +#endif +} + +TEST_F(HotStandbyIntegrationTest, TestStandbyPromotion) { +#ifndef STORE_USE_ETCD + GTEST_SKIP() << "STORE_USE_ETCD is not enabled."; +#else + if (FLAGS_hs_etcd_endpoints.empty()) { + GTEST_SKIP() << "hs_etcd_endpoints not provided."; + } + + // 1. Write several OpLog entries + OpLogManager primary_oplog; + primary_oplog.SetEtcdOpLogStore( + std::make_shared(FLAGS_hs_cluster_id, true)); + + std::vector test_keys; + for (int i = 0; i < 5; ++i) { + std::string key = "promote_test_key_" + std::to_string(i); + std::string payload = + R"({"client_id_first":1,"client_id_second":2,"size":1024,"replicas":[]})"; + primary_oplog.Append(OpType::PUT_END, key, payload); + test_keys.push_back(key); + } + + uint64_t last_seq_id = primary_oplog.GetLastSequenceId(); + + // 2. Start the standby and wait for sync + HotStandbyConfig hs_config; + hs_config.enable_verification = false; + HotStandbyService standby(hs_config); + StandbyServiceGuard guard(&standby); // RAII: ensure cleanup + + ASSERT_EQ(ErrorCode::OK, standby.Start("", FLAGS_hs_etcd_endpoints, + FLAGS_hs_cluster_id)); + + // Wait until sync finishes + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (std::chrono::steady_clock::now() < deadline) { + auto status = standby.GetSyncStatus(); + if (status.state == StandbyState::WATCHING && + status.applied_seq_id >= last_seq_id && + status.lag_entries == 0) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + // 3. Verify standby is ready for promotion + ASSERT_TRUE(standby.IsReadyForPromotion()) + << "Standby should be ready for promotion"; + + // 4. Perform promotion + auto master = standby.Promote(); + // Note: Promote() returns nullptr as MasterService creation is handled externally + EXPECT_EQ(nullptr, master); + + // 5. Verify sequence ID after promotion + uint64_t promoted_seq_id = standby.GetLatestAppliedSequenceId(); + EXPECT_GE(promoted_seq_id, last_seq_id) + << "Promoted sequence ID should be at least the last written sequence ID"; + + // 6. Verify metadata snapshot is still valid after promotion + std::vector> snapshot; + ASSERT_TRUE(standby.ExportMetadataSnapshot(snapshot)); + + std::set snapshot_keys; + for (const auto& kv : snapshot) { + snapshot_keys.insert(kv.first); + } + + for (const auto& key : test_keys) { + EXPECT_NE(snapshot_keys.end(), snapshot_keys.find(key)) + << "Key " << key << " should be in snapshot after promotion"; + } + + // guard will automatically stop the service +#endif +} + +TEST_F(HotStandbyIntegrationTest, TestFailoverScenario) { +#ifndef STORE_USE_ETCD + GTEST_SKIP() << "STORE_USE_ETCD is not enabled."; +#else + if (FLAGS_hs_etcd_endpoints.empty()) { + GTEST_SKIP() << "hs_etcd_endpoints not provided."; + } + + // 1. Simulate primary writing data + OpLogManager primary_oplog; + primary_oplog.SetEtcdOpLogStore( + std::make_shared(FLAGS_hs_cluster_id, true)); + + std::vector test_keys; + for (int i = 0; i < 10; ++i) { + std::string key = "failover_key_" + std::to_string(i); + std::string payload = + R"({"client_id_first":1,"client_id_second":2,"size":1024,"replicas":[]})"; + primary_oplog.Append(OpType::PUT_END, key, payload); + test_keys.push_back(key); + } + + uint64_t last_seq_id = primary_oplog.GetLastSequenceId(); + + // 2. Start the standby + HotStandbyConfig hs_config; + hs_config.enable_verification = false; + HotStandbyService standby(hs_config); + StandbyServiceGuard guard(&standby); // RAII: ensure cleanup + + ASSERT_EQ(ErrorCode::OK, standby.Start("", FLAGS_hs_etcd_endpoints, + FLAGS_hs_cluster_id)); + + // Wait for sync + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (std::chrono::steady_clock::now() < deadline) { + auto status = standby.GetSyncStatus(); + if (status.state == StandbyState::WATCHING && + status.applied_seq_id >= last_seq_id) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + // 3. Simulate primary failure (stop writing new OpLogs) + // In real scenarios this could be a primary crash or a network partition + + // 4. Verify standby data integrity + std::vector> snapshot; + ASSERT_TRUE(standby.ExportMetadataSnapshot(snapshot)); + + std::set snapshot_keys; + for (const auto& kv : snapshot) { + snapshot_keys.insert(kv.first); + } + + for (const auto& key : test_keys) { + EXPECT_NE(snapshot_keys.end(), snapshot_keys.find(key)) + << "Key " << key << " should be in Standby after Primary failure"; + } + + // 5. Perform promotion (simulate failover) + ASSERT_TRUE(standby.IsReadyForPromotion()); + auto master = standby.Promote(); + EXPECT_EQ(nullptr, master); + + // 6. Verify state after promotion + uint64_t promoted_seq_id = standby.GetLatestAppliedSequenceId(); + EXPECT_GE(promoted_seq_id, last_seq_id); + + // guard will automatically stop the service +#endif +} + +TEST_F(HotStandbyIntegrationTest, TestDataConsistency) { +#ifndef STORE_USE_ETCD + GTEST_SKIP() << "STORE_USE_ETCD is not enabled."; +#else + if (FLAGS_hs_etcd_endpoints.empty()) { + GTEST_SKIP() << "hs_etcd_endpoints not provided."; + } + + // 1. Simulate primary writing mixed operations + OpLogManager primary_oplog; + primary_oplog.SetEtcdOpLogStore( + std::make_shared(FLAGS_hs_cluster_id, true)); + + std::map expected_keys; // key -> should_exist + + // PUT operations + for (int i = 0; i < 5; ++i) { + std::string key = "put_key_" + std::to_string(i); + std::string payload = + R"({"client_id_first":1,"client_id_second":2,"size":1024,"replicas":[]})"; + primary_oplog.Append(OpType::PUT_END, key, payload); + expected_keys[key] = true; + } + + // REMOVE operations + for (int i = 0; i < 2; ++i) { + std::string key = "put_key_" + std::to_string(i); + primary_oplog.Append(OpType::REMOVE, key, ""); + expected_keys[key] = false; // Deleted + } + + // Then PUT some more keys + for (int i = 5; i < 8; ++i) { + std::string key = "put_key_" + std::to_string(i); + std::string payload = + R"({"client_id_first":1,"client_id_second":2,"size":2048,"replicas":[]})"; + primary_oplog.Append(OpType::PUT_END, key, payload); + expected_keys[key] = true; + } + + uint64_t last_seq_id = primary_oplog.GetLastSequenceId(); + LOG(INFO) << "Primary wrote " << last_seq_id << " OpLog entries"; + + // 2. Start the standby + HotStandbyConfig hs_config; + hs_config.enable_verification = false; + HotStandbyService standby(hs_config); + StandbyServiceGuard guard(&standby); // RAII: ensure cleanup + + ASSERT_EQ(ErrorCode::OK, standby.Start("", FLAGS_hs_etcd_endpoints, + FLAGS_hs_cluster_id)); + + // 3. Wait for sync + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (std::chrono::steady_clock::now() < deadline) { + auto status = standby.GetSyncStatus(); + if (status.state == StandbyState::WATCHING && + status.applied_seq_id >= last_seq_id && + status.lag_entries == 0) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + // 4. Verify consistency + std::vector> snapshot; + ASSERT_TRUE(standby.ExportMetadataSnapshot(snapshot)); + + std::set actual_keys; + for (const auto& kv : snapshot) { + actual_keys.insert(kv.first); + } + + // Verify all keys expected to exist are present + for (const auto& kv : expected_keys) { + if (kv.second) { + EXPECT_NE(actual_keys.end(), actual_keys.find(kv.first)) + << "Key " << kv.first << " should exist but not found"; + } else { + EXPECT_EQ(actual_keys.end(), actual_keys.find(kv.first)) + << "Key " << kv.first << " should be removed but still exists"; + } + } + + // Verify metadata entry count + size_t expected_count = 0; + for (const auto& kv : expected_keys) { + if (kv.second) expected_count++; + } + EXPECT_EQ(expected_count, actual_keys.size()) + << "Standby metadata count mismatch. Expected: " << expected_count + << ", Actual: " << actual_keys.size(); + + // guard will automatically stop the service +#endif +} + +// ========== 8.1.2 Multi-node tests ========== + +TEST_F(HotStandbyIntegrationTest, TestMultipleStandbys) { +#ifndef STORE_USE_ETCD + GTEST_SKIP() << "STORE_USE_ETCD is not enabled."; +#else + if (FLAGS_hs_etcd_endpoints.empty()) { + GTEST_SKIP() << "hs_etcd_endpoints not provided."; + } + + // 1. Simulate primary writing data + OpLogManager primary_oplog; + primary_oplog.SetEtcdOpLogStore( + std::make_shared(FLAGS_hs_cluster_id, true)); + + std::vector test_keys; + for (int i = 0; i < 10; ++i) { + std::string key = "multi_standby_key_" + std::to_string(i); + std::string payload = + R"({"client_id_first":1,"client_id_second":2,"size":1024,"replicas":[]})"; + primary_oplog.Append(OpType::PUT_END, key, payload); + test_keys.push_back(key); + } + + uint64_t last_seq_id = primary_oplog.GetLastSequenceId(); + + // 2. Start two standby instances + HotStandbyConfig hs_config; + hs_config.enable_verification = false; + + HotStandbyService standby1(hs_config); + HotStandbyService standby2(hs_config); + StandbyServiceGuard guard1(&standby1); // RAII: ensure cleanup + StandbyServiceGuard guard2(&standby2); // RAII: ensure cleanup + + ASSERT_EQ(ErrorCode::OK, standby1.Start("", FLAGS_hs_etcd_endpoints, + FLAGS_hs_cluster_id)); + ASSERT_EQ(ErrorCode::OK, standby2.Start("", FLAGS_hs_etcd_endpoints, + FLAGS_hs_cluster_id)); + + // 3. Wait for both standbys to finish syncing + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + bool both_synced = false; + + while (std::chrono::steady_clock::now() < deadline) { + auto status1 = standby1.GetSyncStatus(); + auto status2 = standby2.GetSyncStatus(); + + if (status1.state == StandbyState::WATCHING && + status2.state == StandbyState::WATCHING && + status1.applied_seq_id >= last_seq_id && + status2.applied_seq_id >= last_seq_id && + status1.lag_entries == 0 && status2.lag_entries == 0) { + both_synced = true; + break; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + ASSERT_TRUE(both_synced) << "Both standbys failed to sync within timeout"; + + // 4. Verify metadata snapshots of both standbys are identical + std::vector> snapshot1; + std::vector> snapshot2; + + ASSERT_TRUE(standby1.ExportMetadataSnapshot(snapshot1)); + ASSERT_TRUE(standby2.ExportMetadataSnapshot(snapshot2)); + + EXPECT_EQ(snapshot1.size(), snapshot2.size()) + << "Both standbys should have the same number of metadata entries"; + + std::set keys1, keys2; + for (const auto& kv : snapshot1) { + keys1.insert(kv.first); + } + for (const auto& kv : snapshot2) { + keys2.insert(kv.first); + } + + EXPECT_EQ(keys1, keys2) << "Both standbys should have identical key sets"; + + // 5. Verify all test keys are present in both standbys + for (const auto& key : test_keys) { + EXPECT_NE(keys1.end(), keys1.find(key)) + << "Key " << key << " not found in standby1"; + EXPECT_NE(keys2.end(), keys2.find(key)) + << "Key " << key << " not found in standby2"; + } + + // guards will automatically stop the services +#endif +} + +TEST_F(HotStandbyIntegrationTest, TestLeaderElection) { +#ifndef STORE_USE_ETCD + GTEST_SKIP() << "STORE_USE_ETCD is not enabled."; +#else + if (FLAGS_hs_etcd_endpoints.empty()) { + GTEST_SKIP() << "hs_etcd_endpoints not provided."; + } + + // 1. Clean up master view (delete any existing key) + MasterViewHelper mv_helper(FLAGS_hs_cluster_id); + mv_helper.ConnectToEtcd(FLAGS_hs_etcd_endpoints); + std::string master_view_key = mv_helper.GetMasterViewKey(); + // Use DeleteRange to delete a single key + std::string master_view_end_key = master_view_key; + for (int i = static_cast(master_view_end_key.size()) - 1; i >= 0; --i) { + unsigned char c = static_cast(master_view_end_key[i]); + if (c < 0xFF) { + master_view_end_key[i] = static_cast(c + 1); + master_view_end_key.resize(i + 1); + break; + } + } + if (master_view_end_key == master_view_key) { + master_view_end_key = master_view_key + std::string(1, '\0'); + } + (void)EtcdHelper::DeleteRange(master_view_key.c_str(), master_view_key.size(), + master_view_end_key.c_str(), master_view_end_key.size()); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + + // 2. First node election (run in background thread because ElectLeader blocks) + std::string master1_address = "10.0.0.1:8888"; + ViewVersionId version1 = 0; + EtcdLeaseId lease1 = 0; + bool election1_done = false; + + std::thread election1_thread([&]() { + mv_helper.ElectLeader(master1_address, version1, lease1); + election1_done = true; + }); + + // Wait for the election to complete + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!election1_done && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + ASSERT_TRUE(election1_done) << "Master1 election should complete"; + ASSERT_NE(0, lease1) << "Master1 should successfully elect as leader"; + + // Verify master view is set + std::string current_master; + ViewVersionId current_version = 0; + ASSERT_EQ(ErrorCode::OK, + mv_helper.GetMasterView(current_master, current_version)); + EXPECT_EQ(master1_address, current_master); + + // 3. Cancel the first node's lease (simulate failure) + EtcdHelper::CancelKeepAlive(lease1); + std::this_thread::sleep_for(std::chrono::seconds(2)); + + // 4. Second node should now be able to win the election + MasterViewHelper mv_helper2(FLAGS_hs_cluster_id); + mv_helper2.ConnectToEtcd(FLAGS_hs_etcd_endpoints); + std::string master2_address = "10.0.0.2:8888"; + ViewVersionId version2 = 0; + EtcdLeaseId lease2 = 0; + bool election2_done = false; + + std::thread election2_thread([&]() { + mv_helper2.ElectLeader(master2_address, version2, lease2); + election2_done = true; + }); + + deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!election2_done && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + ASSERT_TRUE(election2_done) << "Master2 election should complete"; + ASSERT_NE(0, lease2) << "Master2 should successfully elect after master1 fails"; + + // Verify master view switches to the second node + ASSERT_EQ(ErrorCode::OK, + mv_helper2.GetMasterView(current_master, current_version)); + EXPECT_EQ(master2_address, current_master) + << "Master view should switch to master2"; + + // Cleanup + if (election1_thread.joinable()) { + election1_thread.join(); + } + if (election2_thread.joinable()) { + election2_thread.join(); + } + if (lease2 != 0) { + EtcdHelper::CancelKeepAlive(lease2); + } +#endif +} + +// ========== 8.1.3 Stress tests ========== + +TEST_F(HotStandbyIntegrationTest, TestHighThroughputSync) { +#ifndef STORE_USE_ETCD + GTEST_SKIP() << "STORE_USE_ETCD is not enabled."; +#else + if (FLAGS_hs_etcd_endpoints.empty()) { + GTEST_SKIP() << "hs_etcd_endpoints not provided."; + } + + // 1. Start the standby + HotStandbyConfig hs_config; + hs_config.enable_verification = false; + HotStandbyService standby(hs_config); + StandbyServiceGuard guard(&standby); // RAII: ensure cleanup + + ASSERT_EQ(ErrorCode::OK, standby.Start("", FLAGS_hs_etcd_endpoints, + FLAGS_hs_cluster_id)); + + // Wait for standby to enter WATCHING state + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (std::chrono::steady_clock::now() < deadline) { + auto status = standby.GetSyncStatus(); + if (status.state == StandbyState::WATCHING) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + // 2. Simulate high-throughput writes on the primary + OpLogManager primary_oplog; + primary_oplog.SetEtcdOpLogStore( + std::make_shared(FLAGS_hs_cluster_id, true)); + + const int num_writes = 100; + std::string payload = + R"({"client_id_first":1,"client_id_second":2,"size":1024,"replicas":[]})"; + + auto write_start = std::chrono::steady_clock::now(); + for (int i = 0; i < num_writes; ++i) { + std::string key = "throughput_key_" + std::to_string(i); + primary_oplog.Append(OpType::PUT_END, key, payload); + } + auto write_end = std::chrono::steady_clock::now(); + auto write_duration = std::chrono::duration_cast( + write_end - write_start); + + uint64_t last_seq_id = primary_oplog.GetLastSequenceId(); + LOG(INFO) << "Wrote " << num_writes << " entries in " + << write_duration.count() << "ms, last_seq_id=" << last_seq_id; + + // 3. Monitor standby lag + deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + uint64_t max_lag = 0; + + while (std::chrono::steady_clock::now() < deadline) { + auto status = standby.GetSyncStatus(); + if (status.lag_entries > max_lag) { + max_lag = status.lag_entries; + } + + LOG(INFO) << "Standby lag: " << status.lag_entries + << " entries, applied_seq_id=" << status.applied_seq_id + << ", primary_seq_id=" << status.primary_seq_id; + + if (status.applied_seq_id >= last_seq_id && status.lag_entries == 0) { + break; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + // 4. Verify final sync result + auto final_status = standby.GetSyncStatus(); + EXPECT_GE(final_status.applied_seq_id, last_seq_id) + << "Standby should have applied all entries"; + EXPECT_EQ(0u, final_status.lag_entries) + << "Standby lag should be zero after sync"; + + LOG(INFO) << "Max lag observed: " << max_lag << " entries"; + + // guard will automatically stop the service +#endif +} + +TEST_F(HotStandbyIntegrationTest, TestLargePayloadSync) { +#ifndef STORE_USE_ETCD + GTEST_SKIP() << "STORE_USE_ETCD is not enabled."; +#else + if (FLAGS_hs_etcd_endpoints.empty()) { + GTEST_SKIP() << "hs_etcd_endpoints not provided."; + } + + // 1. Start the standby + HotStandbyConfig hs_config; + hs_config.enable_verification = false; + HotStandbyService standby(hs_config); + StandbyServiceGuard guard(&standby); // RAII: ensure cleanup + + ASSERT_EQ(ErrorCode::OK, standby.Start("", FLAGS_hs_etcd_endpoints, + FLAGS_hs_cluster_id)); + + // Wait for standby to enter WATCHING state + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (std::chrono::steady_clock::now() < deadline) { + auto status = standby.GetSyncStatus(); + if (status.state == StandbyState::WATCHING) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + // 2. Write an entry with payload close to the maximum size + OpLogManager primary_oplog; + primary_oplog.SetEtcdOpLogStore( + std::make_shared(FLAGS_hs_cluster_id, true)); + + // Create a JSON payload close to but not exceeding the logical max payload size. + // Note: etcd has a default max request size limit for the entire request + // (including key, value, and metadata). The actual limit may be smaller than 2MB. + // We use 1MB to stay safely under typical etcd limits without requiring etcd config changes. + // kMaxPayloadSize = 10MB, but for integration tests we use 1MB to work with default etcd settings. + const size_t large_payload_size = 1024 * 1024; // 1MB (safe for typical etcd limits) + std::string large_payload = R"({"client_id_first":1,"client_id_second":2,"size":1024,"replicas":[])"; + size_t padding_size = large_payload_size - large_payload.size() - 1; + if (padding_size > 0) { + large_payload += std::string(padding_size, 'X'); + } + large_payload += "}"; + + std::string key = "large_payload_key"; + // Use AppendAndPersist to check if write succeeds (may fail if etcd limit is too small) + auto result = primary_oplog.AppendAndPersist(OpType::PUT_END, key, large_payload); + if (!result.has_value()) { + // Write failed (likely due to etcd size limit), skip this test + GTEST_SKIP() << "Failed to write large payload to etcd (error=" + << static_cast(result.error()) + << "). This may indicate etcd --max-request-bytes is too small. " + << "Payload size was " << large_payload.size() << " bytes."; + } + uint64_t seq_id = result.value(); + + LOG(INFO) << "Wrote large payload entry: seq_id=" << seq_id + << ", payload_size=" << large_payload.size(); + + // 3. Wait for standby to sync + deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + bool synced = false; + + while (std::chrono::steady_clock::now() < deadline) { + auto status = standby.GetSyncStatus(); + if (status.applied_seq_id >= seq_id && status.lag_entries == 0) { + synced = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + ASSERT_TRUE(synced) << "Standby failed to sync large payload entry"; + + // 4. Verify standby metadata snapshot contains the key + std::vector> snapshot; + ASSERT_TRUE(standby.ExportMetadataSnapshot(snapshot)); + + bool found = false; + for (const auto& kv : snapshot) { + if (kv.first == key) { + found = true; + break; + } + } + + EXPECT_TRUE(found) << "Large payload key should be in Standby snapshot"; + + // 5. Verify that entries exceeding the maximum payload size are rejected + std::string oversized_payload(OpLogManager::kMaxPayloadSize + 1, 'X'); + // Note: OpLogManager::Append does not directly reject oversize payloads, but + // EtcdOpLogStore or OpLogApplier will validate them when applying. + // Here we verify that ValidateEntrySize rejects such an entry. + OpLogEntry test_entry; + test_entry.object_key = "oversized_key"; + test_entry.payload = oversized_payload; + + std::string reason; + bool is_valid = OpLogManager::ValidateEntrySize(test_entry, &reason); + EXPECT_FALSE(is_valid) << "Oversized payload should be rejected: " << reason; + + // guard will automatically stop the service +#endif +} + +} // namespace testing +} // namespace mooncake + +// Define static member +namespace mooncake { +namespace testing { +std::mutex HotStandbyIntegrationTest::test_mutex_; +} // namespace testing +} // namespace mooncake + +int main(int argc, char** argv) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + diff --git a/mooncake-store/tests/hot_standby_ut/etcd_oplog_store_test.cpp b/mooncake-store/tests/hot_standby_ut/etcd_oplog_store_test.cpp new file mode 100644 index 0000000000..bec40f3bb1 --- /dev/null +++ b/mooncake-store/tests/hot_standby_ut/etcd_oplog_store_test.cpp @@ -0,0 +1,390 @@ +#include "etcd_oplog_store.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include "etcd_helper.h" + +DEFINE_string(etcd_endpoints, "0.0.0.0:2379", "Etcd endpoints for EtcdOpLogStoreTest"); + +namespace mooncake::test { + +class EtcdOpLogStoreTest : public ::testing::Test { + protected: + static void SetUpTestSuite() { +#ifdef STORE_USE_ETCD + google::InitGoogleLogging("EtcdOpLogStoreTest"); + FLAGS_logtostderr = 1; + + ASSERT_EQ(ErrorCode::OK, + EtcdHelper::ConnectToEtcdStoreClient(FLAGS_etcd_endpoints)) + << "Failed to connect to etcd at " << FLAGS_etcd_endpoints; +#endif + } + + static void TearDownTestSuite() { +#ifdef STORE_USE_ETCD + google::ShutdownGoogleLogging(); +#endif + } + + void SetUp() override { +#ifndef STORE_USE_ETCD + GTEST_SKIP() << "STORE_USE_ETCD is disabled, skipping EtcdOpLogStore tests."; +#else + cluster_id_ = "test_cluster_etcd_oplog_store"; + store_ = std::make_unique(cluster_id_, + /*enable_latest_seq_batch_update=*/false); + CleanupTestData(); +#endif + } + + void TearDown() override { +#ifdef STORE_USE_ETCD + CleanupTestData(); + store_.reset(); +#endif + } + + std::string cluster_id_; + std::unique_ptr store_; + + void CleanupTestData() { +#ifdef STORE_USE_ETCD + // Delete all keys under /oplog/{cluster_id_}/ prefix + std::string prefix = std::string("/oplog/") + cluster_id_ + "/"; + + auto prefix_end = [](std::string p) -> std::string { + for (int i = static_cast(p.size()) - 1; i >= 0; --i) { + unsigned char c = static_cast(p[i]); + if (c < 0xFF) { + p[i] = static_cast(c + 1); + p.resize(i + 1); + return p; + } + } + return std::string(1, '\0'); + }; + std::string end_key = prefix_end(prefix); + + (void)EtcdHelper::DeleteRange(prefix.c_str(), prefix.size(), end_key.c_str(), + end_key.size()); +#endif + } + + static OpLogEntry MakeEntry(uint64_t seq, OpType type, + const std::string& key, + const std::string& payload) { + OpLogEntry e; + e.sequence_id = seq; + e.timestamp_ms = 123456; + e.op_type = type; + e.object_key = key; + e.payload = payload; + e.checksum = 0; + e.prefix_hash = 0; + return e; + } +}; + +// ========== 3.1.1 Basic CRUD tests ========== + +TEST_F(EtcdOpLogStoreTest, TestWriteOpLog) { + OpLogEntry e = MakeEntry(1, OpType::PUT_END, "key1", "value1"); + + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e)); + + // Latest sequence ID should be updated to 1 + uint64_t latest = 0; + ASSERT_EQ(ErrorCode::OK, store_->GetLatestSequenceId(latest)); + EXPECT_EQ(1u, latest); +} + +TEST_F(EtcdOpLogStoreTest, TestReadOpLog) { + OpLogEntry e = MakeEntry(2, OpType::PUT_END, "key2", "value2"); + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e)); + + OpLogEntry out; + ASSERT_EQ(ErrorCode::OK, store_->ReadOpLog(2, out)); + + EXPECT_EQ(2u, out.sequence_id); + EXPECT_EQ(OpType::PUT_END, out.op_type); + EXPECT_EQ("key2", out.object_key); + EXPECT_EQ("value2", out.payload); +} + +TEST_F(EtcdOpLogStoreTest, TestReadOpLogSince) { + // Write multiple entries + for (uint64_t i = 10; i < 15; ++i) { + OpLogEntry e = MakeEntry(i, OpType::PUT_END, + "key_" + std::to_string(i), + "value_" + std::to_string(i)); + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e)); + } + + std::vector entries; + ASSERT_EQ(ErrorCode::OK, store_->ReadOpLogSince(11, 10, entries)); + + // Expect entries with seq > 11: 12,13,14 + ASSERT_EQ(3u, entries.size()); + EXPECT_EQ(12u, entries[0].sequence_id); + EXPECT_EQ(13u, entries[1].sequence_id); + EXPECT_EQ(14u, entries[2].sequence_id); +} + +TEST_F(EtcdOpLogStoreTest, TestReadOpLogSince_Empty) { + std::vector entries; + ASSERT_EQ(ErrorCode::OK, store_->ReadOpLogSince(1000, 10, entries)); + EXPECT_TRUE(entries.empty()); +} + +TEST_F(EtcdOpLogStoreTest, TestReadOpLogSince_Limit) { + for (uint64_t i = 1; i <= 5; ++i) { + OpLogEntry e = MakeEntry(i, OpType::PUT_END, + "key_" + std::to_string(i), + "value_" + std::to_string(i)); + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e)); + } + + std::vector entries; + ASSERT_EQ(ErrorCode::OK, store_->ReadOpLogSince(0, 3, entries)); + ASSERT_EQ(3u, entries.size()); + EXPECT_EQ(1u, entries[0].sequence_id); + EXPECT_EQ(2u, entries[1].sequence_id); + EXPECT_EQ(3u, entries[2].sequence_id); +} + +// ========== 3.1.2 Serialization tests ========== + +TEST_F(EtcdOpLogStoreTest, TestSerializeDeserializeRoundTrip) { + OpLogEntry in = MakeEntry(42, OpType::PUT_END, "roundtrip-key", "roundtrip-value"); + + // Indirectly verify serialization / deserialization via WriteOpLog + ReadOpLog + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(in)); + + OpLogEntry out; + ASSERT_EQ(ErrorCode::OK, store_->ReadOpLog(42, out)); + + EXPECT_EQ(in.sequence_id, out.sequence_id); + EXPECT_EQ(in.op_type, out.op_type); + EXPECT_EQ(in.object_key, out.object_key); + EXPECT_EQ(in.payload, out.payload); +} + +TEST_F(EtcdOpLogStoreTest, TestDeserializeInvalidJson) { + // Write invalid JSON directly into etcd; subsequent ReadOpLog should return INTERNAL_ERROR + std::string key = "/oplog/" + cluster_id_ + "/00000000000000000077"; + std::string bad_json = "{ this is not valid json }"; + ASSERT_EQ(ErrorCode::OK, + EtcdHelper::Put(key.c_str(), key.size(), + bad_json.c_str(), bad_json.size())); + + OpLogEntry out; + ASSERT_EQ(ErrorCode::INTERNAL_ERROR, store_->ReadOpLog(77, out)); +} + +// ========== 3.1.3 Fencing tests ========== + +TEST_F(EtcdOpLogStoreTest, TestWriteOpLog_Fencing) { + OpLogEntry e1 = MakeEntry(100, OpType::PUT_END, "key_fence", "value1"); + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e1)); + + // Same seq, same content => idempotent (OK) + OpLogEntry e2 = e1; + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e2)); + + // Same seq, different content => conflict (ETCD_OPERATION_ERROR) + OpLogEntry e3 = e1; + e3.payload = "value2"; + EXPECT_EQ(ErrorCode::ETCD_OPERATION_ERROR, store_->WriteOpLog(e3)); +} + +TEST_F(EtcdOpLogStoreTest, TestWriteOpLog_Idempotent) { + OpLogEntry e = MakeEntry(200, OpType::PUT_END, "key_idem", "v"); + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e)); + + // Repeatedly writing the exact same entry should return OK (idempotent) + // even if the underlying Create operation reports a transaction failure. + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e)); +} + +// ========== 3.1.4 Sequence ID management tests ========== + +TEST_F(EtcdOpLogStoreTest, TestGetLatestSequenceId) { + OpLogEntry e1 = MakeEntry(1, OpType::PUT_END, "k1", "v1"); + OpLogEntry e2 = MakeEntry(2, OpType::PUT_END, "k2", "v2"); + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e1)); + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e2)); + + uint64_t latest = 0; + ASSERT_EQ(ErrorCode::OK, store_->GetLatestSequenceId(latest)); + EXPECT_EQ(2u, latest); +} + +TEST_F(EtcdOpLogStoreTest, TestGetMaxSequenceIdAndEmpty) { + uint64_t max_seq = 0; + + // Empty cluster: after cleanup, GetMaxSequenceId should return ETCD_KEY_NOT_EXIST + CleanupTestData(); + EXPECT_EQ(ErrorCode::ETCD_KEY_NOT_EXIST, store_->GetMaxSequenceId(max_seq)); + + // After writing several entries, MaxSequenceId should equal the last entry's seq + for (uint64_t i = 10; i <= 15; ++i) { + OpLogEntry e = MakeEntry(i, OpType::PUT_END, + "key_" + std::to_string(i), + "value_" + std::to_string(i)); + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e)); + } + + ASSERT_EQ(ErrorCode::OK, store_->GetMaxSequenceId(max_seq)); + EXPECT_EQ(15u, max_seq); +} + +TEST_F(EtcdOpLogStoreTest, TestUpdateLatestSequenceId) { + // Directly call UpdateLatestSequenceId, then GetLatestSequenceId should match + ASSERT_EQ(ErrorCode::OK, store_->UpdateLatestSequenceId(12345)); + + uint64_t latest = 0; + ASSERT_EQ(ErrorCode::OK, store_->GetLatestSequenceId(latest)); + EXPECT_EQ(12345u, latest); +} + +// ========== 3.1.5 Batch update tests ========== + +TEST_F(EtcdOpLogStoreTest, TestBatchUpdate_EnabledAndThreshold) { + // Use a store with batch enabled, then verify /latest is updated to the max seq + EtcdOpLogStore writer(cluster_id_, + /*enable_latest_seq_batch_update=*/true); + + const uint64_t base_seq = 1000; + const int kEntries = 5; + for (int i = 0; i < kEntries; ++i) { + OpLogEntry e = MakeEntry(base_seq + i, OpType::PUT_END, + "batch_key_" + std::to_string(i), + "batch_val_" + std::to_string(i)); + ASSERT_EQ(ErrorCode::OK, writer.WriteOpLog(e)); + } + + // Wait a short period to give the batch thread a chance to flush `/latest` + std::this_thread::sleep_for(std::chrono::milliseconds(2 * 1000)); + + uint64_t latest = 0; + ASSERT_EQ(ErrorCode::OK, store_->GetLatestSequenceId(latest)); + EXPECT_EQ(base_seq + kEntries - 1, latest); +} + +TEST_F(EtcdOpLogStoreTest, TestBatchUpdate_FailurePlaceholder) { + GTEST_SKIP() << "Batch failure scenarios are better tested with a fault-injection etcd wrapper."; +} + +// ========== 3.1.6 Cleanup tests ========== + +TEST_F(EtcdOpLogStoreTest, TestCleanupOpLogBeforeAndBoundary) { + // Write seq 1..5 + for (uint64_t i = 1; i <= 5; ++i) { + OpLogEntry e = MakeEntry(i, OpType::PUT_END, + "cleanup_key_" + std::to_string(i), + "cleanup_val_" + std::to_string(i)); + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e)); + } + + // Cleanup seq < 3 => 1,2 should be deleted; 3,4,5 should remain + ASSERT_EQ(ErrorCode::OK, store_->CleanupOpLogBefore(3)); + + OpLogEntry out; + EXPECT_EQ(ErrorCode::ETCD_KEY_NOT_EXIST, store_->ReadOpLog(1, out)); + EXPECT_EQ(ErrorCode::ETCD_KEY_NOT_EXIST, store_->ReadOpLog(2, out)); + + ASSERT_EQ(ErrorCode::OK, store_->ReadOpLog(3, out)); + EXPECT_EQ(3u, out.sequence_id); + + ASSERT_EQ(ErrorCode::OK, store_->ReadOpLog(5, out)); + EXPECT_EQ(5u, out.sequence_id); +} + +TEST_F(EtcdOpLogStoreTest, TestCleanupOpLogBefore_Empty) { + // Cleanup on an empty cluster should return OK + CleanupTestData(); + EXPECT_EQ(ErrorCode::OK, store_->CleanupOpLogBefore(100)); +} + +// ========== 3.1.7 Cluster ID validation tests ========== + +TEST_F(EtcdOpLogStoreTest, TestInvalidClusterId_Rejected) { + // Invalid cluster_id (containing slashes) should trigger LOG(FATAL) and terminate + EXPECT_DEATH( + { + EtcdOpLogStore bad_store("invalid/cluster", false); + (void)bad_store; + }, + "Invalid cluster_id"); +} + +TEST_F(EtcdOpLogStoreTest, TestClusterIdNormalization) { + // Trailing slashes should be normalized away from the cluster_id + std::string raw_cluster = cluster_id_ + "///"; + EtcdOpLogStore normalized_store(raw_cluster, false); + + OpLogEntry e = MakeEntry(999, OpType::PUT_END, "norm-key", "norm-val"); + ASSERT_EQ(ErrorCode::OK, normalized_store.WriteOpLog(e)); + + // Read the same seq via the current store_ to confirm the normalized cluster_id is used + OpLogEntry out; + ASSERT_EQ(ErrorCode::OK, store_->ReadOpLog(999, out)); + EXPECT_EQ("norm-key", out.object_key); + EXPECT_EQ("norm-val", out.payload); +} + +// ========== 3.1.8 Pagination tests ========== + +TEST_F(EtcdOpLogStoreTest, TestReadOpLogSince_Pagination) { + // Write 20 entries and verify pagination via limit + for (uint64_t i = 1; i <= 20; ++i) { + OpLogEntry e = MakeEntry(i, OpType::PUT_END, + "page_key_" + std::to_string(i), + "page_val_" + std::to_string(i)); + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e)); + } + + std::vector entries; + ASSERT_EQ(ErrorCode::OK, store_->ReadOpLogSince(0, 20, entries)); + ASSERT_EQ(20u, entries.size()); + for (uint64_t i = 0; i < entries.size(); ++i) { + EXPECT_EQ(i + 1, entries[i].sequence_id); + } +} + +TEST_F(EtcdOpLogStoreTest, TestReadOpLogSince_LargeDataset) { + // Write a larger number of entries to verify ReadOpLogSince returns the first N correctly + const uint64_t total = 200; + const uint64_t limit = 150; + CleanupTestData(); + for (uint64_t i = 1; i <= total; ++i) { + OpLogEntry e = MakeEntry(i, OpType::PUT_END, + "large_key_" + std::to_string(i), + "large_val_" + std::to_string(i)); + ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e)); + } + + std::vector entries; + ASSERT_EQ(ErrorCode::OK, store_->ReadOpLogSince(0, limit, entries)); + ASSERT_EQ(limit, entries.size()); + for (uint64_t i = 0; i < limit; ++i) { + EXPECT_EQ(i + 1, entries[i].sequence_id); + } +} + +} // namespace mooncake::test + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + diff --git a/mooncake-store/tests/hot_standby_ut/ha_metric_manager_test.cpp b/mooncake-store/tests/hot_standby_ut/ha_metric_manager_test.cpp new file mode 100644 index 0000000000..41e1b6005a --- /dev/null +++ b/mooncake-store/tests/hot_standby_ut/ha_metric_manager_test.cpp @@ -0,0 +1,183 @@ +#include "ha_metric_manager.h" + +#include +#include + +#include +#include +#include +#include + +namespace mooncake::test { + +class HAMetricManagerTest : public ::testing::Test { + protected: + void SetUp() override { + google::InitGoogleLogging("HAMetricManagerTest"); + FLAGS_logtostderr = 1; + } + + void TearDown() override { google::ShutdownGoogleLogging(); } + + HAMetricManager& M() { return HAMetricManager::instance(); } +}; + +// ========== 7.1.1 Metric update tests ========== + +TEST_F(HAMetricManagerTest, TestSetOpLogLastSequenceId) { + M().set_oplog_last_sequence_id(123); + EXPECT_EQ(123, M().get_oplog_last_sequence_id()); +} + +TEST_F(HAMetricManagerTest, TestSetOpLogAppliedSequenceId) { + M().set_oplog_applied_sequence_id(456); + EXPECT_EQ(456, M().get_oplog_applied_sequence_id()); +} + +TEST_F(HAMetricManagerTest, TestSetOpLogStandbyLag) { + M().set_oplog_standby_lag(10); + EXPECT_EQ(10, M().get_oplog_standby_lag()); +} + +TEST_F(HAMetricManagerTest, TestSetOpLogPendingEntries) { + M().set_oplog_pending_entries(7); + EXPECT_EQ(7, M().get_oplog_pending_entries()); +} + +TEST_F(HAMetricManagerTest, TestSetPendingMutationQueueSize) { + M().set_pending_mutation_queue_size(5); + EXPECT_EQ(5, M().get_pending_mutation_queue_size()); +} + +TEST_F(HAMetricManagerTest, TestIncOpLogSkippedEntries) { + auto before = M().get_oplog_skipped_entries_total(); + M().inc_oplog_skipped_entries(); + EXPECT_EQ(before + 1, M().get_oplog_skipped_entries_total()); +} + +TEST_F(HAMetricManagerTest, TestIncOpLogChecksumFailures) { + auto before = M().get_oplog_checksum_failures_total(); + M().inc_oplog_checksum_failures(2); + EXPECT_EQ(before + 2, M().get_oplog_checksum_failures_total()); +} + +TEST_F(HAMetricManagerTest, TestIncGapResolveCounters) { + auto before_attempts = M().get_oplog_gap_resolve_attempts_total(); + auto before_success = M().get_oplog_gap_resolve_success_total(); + + M().inc_oplog_gap_resolve_attempts(3); + M().inc_oplog_gap_resolve_success(1); + + EXPECT_EQ(before_attempts + 3, M().get_oplog_gap_resolve_attempts_total()); + EXPECT_EQ(before_success + 1, M().get_oplog_gap_resolve_success_total()); +} + +TEST_F(HAMetricManagerTest, TestIncOpLogEtcdWriteFailuresAndRetries) { + auto before_failures = M().get_oplog_etcd_write_failures_total(); + auto before_retries = M().get_oplog_etcd_write_retries_total(); + + M().inc_oplog_etcd_write_failures(4); + M().inc_oplog_etcd_write_retries(5); + + EXPECT_EQ(before_failures + 4, M().get_oplog_etcd_write_failures_total()); + EXPECT_EQ(before_retries + 5, M().get_oplog_etcd_write_retries_total()); +} + +TEST_F(HAMetricManagerTest, TestIncWatchDisconnectionsAndAppliedEntries) { + auto before_disc = M().get_oplog_watch_disconnections_total(); + auto before_applied = M().get_oplog_applied_entries_total(); + + M().inc_oplog_watch_disconnections(2); + M().inc_oplog_applied_entries(10); + + EXPECT_EQ(before_disc + 2, M().get_oplog_watch_disconnections_total()); + EXPECT_EQ(before_applied + 10, M().get_oplog_applied_entries_total()); +} + +TEST_F(HAMetricManagerTest, TestRecordOpLogEtcdWriteLatency) { + // Call histogram observe functions, mainly to ensure they do not crash + M().observe_oplog_etcd_write_latency_us(100); + M().observe_oplog_etcd_write_latency_us(5000); + SUCCEED(); +} + +TEST_F(HAMetricManagerTest, TestRecordOpLogApplyLatency) { + M().observe_oplog_apply_latency_us(50); + M().observe_oplog_apply_latency_us(1000); + SUCCEED(); +} + +// ========== 7.1.2 Metric serialization tests ========== + +TEST_F(HAMetricManagerTest, TestSerializeMetrics) { + M().set_oplog_last_sequence_id(1); + M().set_oplog_applied_sequence_id(1); + M().set_oplog_standby_lag(0); + + std::string text = M().serialize_metrics(); + EXPECT_FALSE(text.empty()); + + // Basic fields should appear in the Prometheus text output + EXPECT_NE(std::string::npos, text.find("ha_oplog_last_sequence_id")); + EXPECT_NE(std::string::npos, text.find("ha_oplog_applied_sequence_id")); + EXPECT_NE(std::string::npos, text.find("ha_oplog_standby_lag")); +} + +TEST_F(HAMetricManagerTest, TestGetSummaryString) { + M().set_oplog_last_sequence_id(100); + M().set_oplog_applied_sequence_id(95); + M().set_oplog_standby_lag(5); + + std::string summary = M().get_summary_string(); + EXPECT_FALSE(summary.empty()); + // Summary string should contain key fields + EXPECT_NE(std::string::npos, summary.find("last_seq")); + EXPECT_NE(std::string::npos, summary.find("applied_seq")); +} + +// ========== 7.1.3 Singleton tests ========== + +TEST_F(HAMetricManagerTest, TestSingletonInstance) { + HAMetricManager& a = HAMetricManager::instance(); + HAMetricManager& b = HAMetricManager::instance(); + + EXPECT_EQ(&a, &b); + + a.set_oplog_last_sequence_id(1234); + EXPECT_EQ(1234, b.get_oplog_last_sequence_id()); +} + +TEST_F(HAMetricManagerTest, TestConcurrentAccess) { + constexpr int kThreads = 8; + constexpr int kIncrementsPerThread = 1000; + + auto& mgr = HAMetricManager::instance(); + auto before = mgr.get_oplog_applied_entries_total(); + + std::vector threads; + threads.reserve(kThreads); + + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&mgr]() { + for (int j = 0; j < kIncrementsPerThread; ++j) { + mgr.inc_oplog_applied_entries(); + } + }); + } + + for (auto& t : threads) { + t.join(); + } + + auto after = mgr.get_oplog_applied_entries_total(); + EXPECT_EQ(before + kThreads * kIncrementsPerThread, after); +} + +} // namespace mooncake::test + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + diff --git a/mooncake-store/tests/hot_standby_ut/hot_standby_service_test.cpp b/mooncake-store/tests/hot_standby_ut/hot_standby_service_test.cpp new file mode 100644 index 0000000000..8f2f5ba2fe --- /dev/null +++ b/mooncake-store/tests/hot_standby_ut/hot_standby_service_test.cpp @@ -0,0 +1,326 @@ +#include "hot_standby_service.h" + +#include +#include + +#include +#include +#include +#include + +#include "master_service.h" + +namespace mooncake::test { + +class HotStandbyServiceTest : public ::testing::Test { + protected: + void SetUp() override { + google::InitGoogleLogging("HotStandbyServiceTest"); + FLAGS_logtostderr = 1; + + config_.enable_verification = false; + config_.max_replication_lag_entries = 1000; + + service_ = std::make_unique(config_); + etcd_endpoints_ = "http://localhost:2379"; + cluster_id_ = "test_cluster_001"; + } + + void TearDown() override { + if (service_) { + service_->Stop(); + } + google::ShutdownGoogleLogging(); + } + + HotStandbyConfig config_; + std::unique_ptr service_; + std::string etcd_endpoints_; + std::string cluster_id_; +}; + +// ========== 6.1.1 Start/Stop tests ========== + +TEST_F(HotStandbyServiceTest, TestStart) { +#ifdef STORE_USE_ETCD + // Requires a real etcd cluster and valid cluster configuration; acts as an integration placeholder + GTEST_SKIP() << "Requires real etcd connection, run in integration environment."; +#else + ErrorCode err = service_->Start("primary_unused", etcd_endpoints_, cluster_id_); + EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err); + EXPECT_EQ(StandbyState::FAILED, service_->GetState()); +#endif +} + +TEST_F(HotStandbyServiceTest, TestStart_AlreadyRunning) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Requires real etcd connection to verify double start semantics."; +#else + // After the first Start fails and state becomes FAILED, the second Start should still return INTERNAL_ERROR + ErrorCode err1 = service_->Start("primary_unused", etcd_endpoints_, cluster_id_); + EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err1); + ErrorCode err2 = service_->Start("primary_unused", etcd_endpoints_, cluster_id_); + EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err2); +#endif +} + +TEST_F(HotStandbyServiceTest, TestStart_InvalidEtcdEndpoints) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Requires real etcd to simulate invalid endpoints."; +#else + std::string invalid_endpoints = "invalid_endpoint"; + ErrorCode err = + service_->Start("primary_unused", invalid_endpoints, cluster_id_); + EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err); +#endif +} + +TEST_F(HotStandbyServiceTest, TestStop) { + // Stop should be safe and idempotent even if Start was never called + service_->Stop(); + SUCCEED(); +} + +TEST_F(HotStandbyServiceTest, TestStop_WhenNotRunning) { + // Multiple Stop calls should be idempotent + service_->Stop(); + service_->Stop(); + SUCCEED(); +} + +// ========== 6.1.2 State transition tests ========== + +TEST_F(HotStandbyServiceTest, TestStateTransition_StartToWatching) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Requires real etcd to drive full state transition to WATCHING."; +#else + // In non-STORE_USE_ETCD builds, Start will set the state machine directly to FAILED + EXPECT_EQ(StandbyState::STOPPED, service_->GetState()); + ErrorCode err = service_->Start("primary_unused", etcd_endpoints_, cluster_id_); + EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err); + EXPECT_EQ(StandbyState::FAILED, service_->GetState()); +#endif +} + +TEST_F(HotStandbyServiceTest, TestStateTransition_ConnectionFailed) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Connection failure requires real etcd and invalid endpoints."; +#else + // In non-etcd mode we cannot distinguish detailed connection errors; only verify it doesn't crash + ErrorCode err = service_->Start("primary_unused", "bad_endpoint", cluster_id_); + EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err); +#endif +} + +TEST_F(HotStandbyServiceTest, TestStateTransition_SyncFailed) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Sync failure requires real etcd and OpLog watcher behavior."; +#else + // In non-etcd mode, the sync phase is not actually executed; just ensure the call is safe + ErrorCode err = service_->Start("primary_unused", etcd_endpoints_, cluster_id_); + EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err); +#endif +} + +// ========== 6.1.3 Sync status tests ========== + +TEST_F(HotStandbyServiceTest, TestGetSyncStatus_InitialState) { + StandbySyncStatus status = service_->GetSyncStatus(); + EXPECT_EQ(0u, status.applied_seq_id); + EXPECT_EQ(0u, status.primary_seq_id); + EXPECT_EQ(0u, status.lag_entries); + EXPECT_FALSE(status.is_syncing); + EXPECT_FALSE(status.is_connected); + EXPECT_EQ(StandbyState::STOPPED, status.state); +} + +TEST_F(HotStandbyServiceTest, TestGetSyncStatus_AfterSync) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires real etcd and OpLog activity to change sync status."; +#else + // In non-etcd mode, calling Start will not change applied/primary, but the state machine enters FAILED + (void)service_->Start("primary_unused", etcd_endpoints_, cluster_id_); + StandbySyncStatus status = service_->GetSyncStatus(); + EXPECT_EQ(StandbyState::FAILED, status.state); +#endif +} + +TEST_F(HotStandbyServiceTest, TestGetSyncStatus) { + // Basic coverage: multiple calls should return consistent values and not crash + StandbySyncStatus s1 = service_->GetSyncStatus(); + StandbySyncStatus s2 = service_->GetSyncStatus(); + EXPECT_EQ(s1.state, s2.state); +} + +// ========== 6.1.4 Promotion tests ========== + +TEST_F(HotStandbyServiceTest, TestPromote_WhenNotReady) { + // In the initial state promotion preconditions are not met, so it should return nullptr + auto master = service_->Promote(); + EXPECT_EQ(nullptr, master); +} + +TEST_F(HotStandbyServiceTest, TestPromote_WhenReady) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires real etcd and full replication pipeline to reach ready state."; +#else + // Even in non-etcd mode, Promote should safely return nullptr + auto master = service_->Promote(); + EXPECT_EQ(nullptr, master); +#endif +} + +TEST_F(HotStandbyServiceTest, TestPromote_FinalCatchUp) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires real etcd and OpLog data to exercise final catch-up logic."; +#else + auto master = service_->Promote(); + EXPECT_EQ(nullptr, master); +#endif +} + +TEST_F(HotStandbyServiceTest, TestPromote_WithGaps) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires real etcd and gaps in OpLog to validate gap resolution."; +#else + auto master = service_->Promote(); + EXPECT_EQ(nullptr, master); +#endif +} + +TEST_F(HotStandbyServiceTest, TestPromote_Timeout) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires real etcd and slow reads to trigger catch-up timeout."; +#else + auto master = service_->Promote(); + EXPECT_EQ(nullptr, master); +#endif +} + +TEST_F(HotStandbyServiceTest, TestPromote_BatchLimit) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires real etcd and large OpLog to hit batch limit."; +#else + auto master = service_->Promote(); + EXPECT_EQ(nullptr, master); +#endif +} + +// ========== 6.1.5 Warm start tests ========== + +TEST_F(HotStandbyServiceTest, TestWarmStart_WithLocalState) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires real etcd and pre-populated local metadata to test warm start."; +#else + // In non-etcd mode, only verify that Start is safe to call + (void)service_->Start("primary_unused", etcd_endpoints_, cluster_id_); + SUCCEED(); +#endif +} + +TEST_F(HotStandbyServiceTest, TestWarmStart_WithoutLocalState) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires real etcd and snapshot provider configuration."; +#else + (void)service_->Start("primary_unused", etcd_endpoints_, cluster_id_); + SUCCEED(); +#endif +} + +TEST_F(HotStandbyServiceTest, TestWarmStart_WithSnapshot) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires snapshot provider and real etcd to exercise snapshot bootstrap."; +#else + config_.enable_snapshot_bootstrap = true; + // Recreate service to apply the new configuration + service_.reset(new HotStandbyService(config_)); + (void)service_->Start("primary_unused", etcd_endpoints_, cluster_id_); + SUCCEED(); +#endif +} + +// ========== 6.1.6 Metadata operation tests ========== + +TEST_F(HotStandbyServiceTest, TestGetMetadataCount) { + EXPECT_EQ(0u, service_->GetMetadataCount()); +} + +TEST_F(HotStandbyServiceTest, TestExportMetadataSnapshot) { + std::vector> snapshot; + EXPECT_TRUE(service_->ExportMetadataSnapshot(snapshot)); + EXPECT_TRUE(snapshot.empty()); +} + +TEST_F(HotStandbyServiceTest, TestGetLatestAppliedSequenceId) { + uint64_t seq = service_->GetLatestAppliedSequenceId(); + EXPECT_EQ(0u, seq); +} + +// ========== 6.1.7 Replication loop tests ========== + +TEST_F(HotStandbyServiceTest, TestReplicationLoop_UpdatesMetrics) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires real etcd and running replication loop to update metrics."; +#else + // In non-etcd mode, ReplicationLoop is never started, but calling Stop should be safe + service_->Stop(); + SUCCEED(); +#endif +} + +TEST_F(HotStandbyServiceTest, TestReplicationLoop_HandlesDisconnect) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires real etcd and watcher disconnect to exercise disconnect path."; +#else + // Calling DisconnectFromPrimary is currently safe (does not depend on etcd) + service_->DisconnectFromPrimary(); + SUCCEED(); +#endif +} + +// ========== 6.1.8 Verification loop tests ========== + +TEST_F(HotStandbyServiceTest, TestVerificationLoop_WhenEnabled) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires real etcd and running verification loop to observe behavior."; +#else + config_.enable_verification = true; + service_.reset(new HotStandbyService(config_)); + (void)service_->Start("primary_unused", etcd_endpoints_, cluster_id_); + service_->Stop(); + SUCCEED(); +#endif +} + +TEST_F(HotStandbyServiceTest, TestVerificationLoop_WhenDisabled) { + // By default config_.enable_verification = false, so Start should not spawn a verification thread +#ifdef STORE_USE_ETCD + GTEST_SKIP() + << "Requires real etcd connection to start service."; +#else + (void)service_->Start("primary_unused", etcd_endpoints_, cluster_id_); + service_->Stop(); + SUCCEED(); +#endif +} + +} // namespace mooncake::test + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + diff --git a/mooncake-store/tests/hot_standby_ut/oplog_applier_test.cpp b/mooncake-store/tests/hot_standby_ut/oplog_applier_test.cpp new file mode 100644 index 0000000000..59bf05dce5 --- /dev/null +++ b/mooncake-store/tests/hot_standby_ut/oplog_applier_test.cpp @@ -0,0 +1,613 @@ +#include "oplog_applier.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include "etcd_oplog_store.h" +#include "metadata_store.h" +#include "oplog_manager.h" +#include "types.h" + +namespace mooncake::test { + +// Mock MetadataStore for testing OpLogApplier +class MockMetadataStore : public MetadataStore { + public: + MockMetadataStore() = default; + ~MockMetadataStore() override = default; + + bool PutMetadata(const std::string& key, + const StandbyObjectMetadata& metadata) override { + metadata_map_[key] = metadata; + return true; + } + + bool Put(const std::string& key, const std::string& payload) override { + // For testing, we can use PutMetadata with empty metadata + StandbyObjectMetadata meta; + metadata_map_[key] = meta; + return true; + } + + const StandbyObjectMetadata* GetMetadata(const std::string& key) const override { + auto it = metadata_map_.find(key); + if (it != metadata_map_.end()) { + return &it->second; + } + return nullptr; + } + + bool Remove(const std::string& key) override { + auto it = metadata_map_.find(key); + if (it != metadata_map_.end()) { + metadata_map_.erase(it); + return true; + } + return false; + } + + bool Exists(const std::string& key) const override { + return metadata_map_.find(key) != metadata_map_.end(); + } + + size_t GetKeyCount() const override { return metadata_map_.size(); } + + // Test helper methods + void Clear() { metadata_map_.clear(); } + + size_t Size() const { return metadata_map_.size(); } + + private: + std::map metadata_map_; +}; + +// Helper function to create a valid OpLogEntry with checksum +// Uses the same checksum algorithm as OpLogManager (XXH32) +OpLogEntry MakeEntry(uint64_t seq, OpType type, const std::string& key, + const std::string& payload) { + OpLogEntry e; + e.sequence_id = seq; + e.timestamp_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + e.op_type = type; + e.object_key = key; + e.payload = payload; + // Compute checksum and prefix_hash using the same algorithm as OpLogManager + e.checksum = static_cast(XXH32(payload.data(), payload.size(), 0)); + e.prefix_hash = key.empty() ? 0 + : static_cast( + XXH32(key.data(), key.size(), 0)); + return e; +} + +// Helper function to create a valid JSON payload for PUT_END +std::string MakeValidJsonPayload(uint64_t client_id_first = 1, + uint64_t client_id_second = 2, + uint64_t size = 1024) { + return R"({"client_id_first":)" + std::to_string(client_id_first) + + R"(,"client_id_second":)" + std::to_string(client_id_second) + + R"(,"size":)" + std::to_string(size) + R"(,"replicas":[]})"; +} + +class OpLogApplierTest : public ::testing::Test { + protected: + void SetUp() override { + google::InitGoogleLogging("OpLogApplierTest"); + FLAGS_logtostderr = 1; + mock_metadata_store_ = std::make_unique(); + cluster_id_ = "test_cluster_001"; + applier_ = std::make_unique(mock_metadata_store_.get(), + cluster_id_); + } + + void TearDown() override { google::ShutdownGoogleLogging(); } + + std::unique_ptr mock_metadata_store_; + std::unique_ptr applier_; + std::string cluster_id_; +}; + +// ========== 4.1.1 Basic apply tests ========== + +TEST_F(OpLogApplierTest, TestApplyPutEnd) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload); + + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry)); + // After applying seq=1, expected_sequence_id becomes 2 + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + EXPECT_TRUE(mock_metadata_store_->Exists("key1")); + EXPECT_EQ(1u, mock_metadata_store_->GetKeyCount()); +} + +TEST_F(OpLogApplierTest, TestApplyPutRevoke) { + // First add a key + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload); + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1)); + EXPECT_TRUE(mock_metadata_store_->Exists("key1")); + + // Then revoke it + OpLogEntry entry2 = MakeEntry(2, OpType::PUT_REVOKE, "key1", ""); + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry2)); + EXPECT_EQ(3u, applier_->GetExpectedSequenceId()); + EXPECT_FALSE(mock_metadata_store_->Exists("key1")); +} + +TEST_F(OpLogApplierTest, TestApplyRemove) { + // First add a key + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload); + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1)); + EXPECT_TRUE(mock_metadata_store_->Exists("key1")); + + // Then remove it + OpLogEntry entry2 = MakeEntry(2, OpType::REMOVE, "key1", ""); + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry2)); + EXPECT_EQ(3u, applier_->GetExpectedSequenceId()); + EXPECT_FALSE(mock_metadata_store_->Exists("key1")); +} + +TEST_F(OpLogApplierTest, TestApplyOpLogEntry_InvalidOpType) { + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", MakeValidJsonPayload()); + // Manually set an invalid op_type (assuming OpType is an enum) + // Since we can't directly set invalid enum, we test with valid types + // and verify that unsupported types in ProcessPendingEntries are handled + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry)); +} + +// ========== 4.1.2 Sequence ordering tests ========== + +TEST_F(OpLogApplierTest, TestApplyInOrder) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload); + OpLogEntry entry2 = MakeEntry(2, OpType::PUT_END, "key2", payload); + OpLogEntry entry3 = MakeEntry(3, OpType::PUT_END, "key3", payload); + + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry2)); + EXPECT_EQ(3u, applier_->GetExpectedSequenceId()); + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry3)); + EXPECT_EQ(4u, applier_->GetExpectedSequenceId()); + + EXPECT_EQ(3u, mock_metadata_store_->GetKeyCount()); + EXPECT_TRUE(mock_metadata_store_->Exists("key1")); + EXPECT_TRUE(mock_metadata_store_->Exists("key2")); + EXPECT_TRUE(mock_metadata_store_->Exists("key3")); +} + +TEST_F(OpLogApplierTest, TestApplyOutOfOrder) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload); + OpLogEntry entry3 = MakeEntry(3, OpType::PUT_END, "key3", payload); + OpLogEntry entry2 = MakeEntry(2, OpType::PUT_END, "key2", payload); + + // Apply entry1 (seq=1) - should succeed + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + + // Apply entry3 (seq=3) - should be cached (out of order) + EXPECT_FALSE(applier_->ApplyOpLogEntry(entry3)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); // Still waiting for seq=2 + EXPECT_FALSE(mock_metadata_store_->Exists("key3")); + + // Apply entry2 (seq=2) - should succeed and trigger processing of entry3 + // ApplyOpLogEntry internally calls ProcessPendingEntries(), so entry3 should be processed + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry2)); + EXPECT_EQ(4u, applier_->GetExpectedSequenceId()); // Now at seq=4 + + // entry3 should already be processed by ApplyOpLogEntry + EXPECT_TRUE(mock_metadata_store_->Exists("key1")); + EXPECT_TRUE(mock_metadata_store_->Exists("key2")); + EXPECT_TRUE(mock_metadata_store_->Exists("key3")); + + // ProcessPendingEntries may return 0 if entry3 was already processed + (void)applier_->ProcessPendingEntries(); + EXPECT_EQ(4u, applier_->GetExpectedSequenceId()); +} + +TEST_F(OpLogApplierTest, TestApplyWithGap) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload); + OpLogEntry entry4 = MakeEntry(4, OpType::PUT_END, "key4", payload); + + // Apply entry1 (seq=1) + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + + // Apply entry4 (seq=4) - gap at seq=2,3 + EXPECT_FALSE(applier_->ApplyOpLogEntry(entry4)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); // Still waiting for seq=2 + + // Process pending entries - should detect gap and schedule wait + (void)applier_->ProcessPendingEntries(); + // May process 0 entries if gap resolution is still waiting + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); +} + +TEST_F(OpLogApplierTest, TestApplyDuplicateSequenceId) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload); + OpLogEntry entry1_dup = MakeEntry(1, OpType::PUT_END, "key1_dup", payload); + + // Apply entry1 + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + + // Try to apply duplicate sequence_id (older than expected) + // Should be treated as no-op (already applied) and return true + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1_dup)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + // key1_dup should not be added (treated as no-op) + EXPECT_FALSE(mock_metadata_store_->Exists("key1_dup")); +} + +// ========== 4.1.3 Gap resolution tests ========== + +TEST_F(OpLogApplierTest, TestRequestMissingOpLog_Success) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Requires real EtcdOpLogStore, skipping integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogApplierTest, TestRequestMissingOpLog_Failure) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Requires real EtcdOpLogStore, skipping integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogApplierTest, TestRequestMissingOpLog_Timeout) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Requires real EtcdOpLogStore, skipping integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogApplierTest, TestTryResolveGapsOnceForPromotion) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Requires real EtcdOpLogStore, skipping integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogApplierTest, TestGapResolution_Retry) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Requires real EtcdOpLogStore, skipping integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +// ========== 4.1.4 Checksum tests ========== + +TEST_F(OpLogApplierTest, TestApplyOpLogEntry_ValidChecksum) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload); + + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + EXPECT_TRUE(mock_metadata_store_->Exists("key1")); +} + +TEST_F(OpLogApplierTest, TestApplyOpLogEntry_InvalidChecksum) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload); + + // Tamper with the checksum + entry.checksum = entry.checksum + 1; + + EXPECT_FALSE(applier_->ApplyOpLogEntry(entry)); + EXPECT_EQ(1u, applier_->GetExpectedSequenceId()); // Should not advance + EXPECT_FALSE(mock_metadata_store_->Exists("key1")); +} + +TEST_F(OpLogApplierTest, TestChecksumFailureMetric) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload); + + // Tamper with the checksum + entry.checksum = entry.checksum + 1; + + EXPECT_FALSE(applier_->ApplyOpLogEntry(entry)); + // Metric increment is tested implicitly by the failure +} + +// ========== 4.1.5 Size validation tests ========== + +TEST_F(OpLogApplierTest, TestApplyOpLogEntry_ValidSize) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload); + + EXPECT_TRUE(OpLogManager::ValidateEntrySize(entry)); + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry)); +} + +TEST_F(OpLogApplierTest, TestApplyOpLogEntry_InvalidSize) { + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", ""); + + // Make key too large + entry.object_key.assign(OpLogManager::kMaxObjectKeySize + 1, 'k'); + + EXPECT_FALSE(OpLogManager::ValidateEntrySize(entry)); + EXPECT_FALSE(applier_->ApplyOpLogEntry(entry)); + EXPECT_EQ(1u, applier_->GetExpectedSequenceId()); // Should not advance + EXPECT_FALSE(mock_metadata_store_->Exists("key1")); +} + +TEST_F(OpLogApplierTest, TestApplyOpLogEntry_PayloadTooLarge) { + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", ""); + + // Make payload too large + entry.payload.assign(OpLogManager::kMaxPayloadSize + 1, 'p'); + + EXPECT_FALSE(OpLogManager::ValidateEntrySize(entry)); + EXPECT_FALSE(applier_->ApplyOpLogEntry(entry)); + EXPECT_EQ(1u, applier_->GetExpectedSequenceId()); // Should not advance +} + +// ========== 4.1.6 Recovery tests ========== + +TEST_F(OpLogApplierTest, TestRecover) { + // Set initial state: last applied sequence_id = 10 + applier_->Recover(10); + EXPECT_EQ(11u, applier_->GetExpectedSequenceId()); + + // Apply entry with seq=11 should succeed + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry = MakeEntry(11, OpType::PUT_END, "key1", payload); + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry)); + EXPECT_EQ(12u, applier_->GetExpectedSequenceId()); +} + +TEST_F(OpLogApplierTest, TestRecover_ZeroSequenceId) { + // Recover from sequence_id 0 + applier_->Recover(0); + EXPECT_EQ(1u, applier_->GetExpectedSequenceId()); + + // Apply entry with seq=1 should succeed + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload); + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); +} + +TEST_F(OpLogApplierTest, TestRecover_AfterGap) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload); + OpLogEntry entry3 = MakeEntry(3, OpType::PUT_END, "key3", payload); + + // Apply entry1 + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + + // Apply entry3 (creates gap) + EXPECT_FALSE(applier_->ApplyOpLogEntry(entry3)); + + // Recover from seq=3 (skip the gap) + applier_->Recover(3); + EXPECT_EQ(4u, applier_->GetExpectedSequenceId()); + + // Now entry3 should be processable + (void)applier_->ProcessPendingEntries(); + // entry3 should be in pending, but expected_seq is now 4, so it won't be processed + // This tests that recovery resets the expected sequence +} + +// ========== 4.1.7 Pending entries tests ========== + +TEST_F(OpLogApplierTest, TestProcessPendingEntries) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload); + OpLogEntry entry3 = MakeEntry(3, OpType::PUT_END, "key3", payload); + OpLogEntry entry2 = MakeEntry(2, OpType::PUT_END, "key2", payload); + + // Apply entry1 + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + + // Apply entry3 (out of order) + EXPECT_FALSE(applier_->ApplyOpLogEntry(entry3)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + + // Process pending - should not process entry3 yet (waiting for seq=2) + size_t processed1 = applier_->ProcessPendingEntries(); + EXPECT_EQ(0u, processed1); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + + // Apply entry2 - this will internally call ProcessPendingEntries() and process entry3 + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry2)); + EXPECT_EQ(4u, applier_->GetExpectedSequenceId()); + + // entry3 should already be processed by ApplyOpLogEntry + EXPECT_TRUE(mock_metadata_store_->Exists("key1")); + EXPECT_TRUE(mock_metadata_store_->Exists("key2")); + EXPECT_TRUE(mock_metadata_store_->Exists("key3")); + + // ProcessPendingEntries may return 0 if entry3 was already processed + (void)applier_->ProcessPendingEntries(); + EXPECT_EQ(4u, applier_->GetExpectedSequenceId()); +} + +TEST_F(OpLogApplierTest, TestPendingEntriesTimeout) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload); + OpLogEntry entry3 = MakeEntry(3, OpType::PUT_END, "key3", payload); + + // Apply entry1 + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + + // Apply entry3 (creates gap at seq=2) + EXPECT_FALSE(applier_->ApplyOpLogEntry(entry3)); + + // Process pending entries multiple times to trigger timeout + // After kMissingEntrySkipSeconds (3s), the gap should be skipped + size_t processed = 0; + for (int i = 0; i < 10; ++i) { + processed = applier_->ProcessPendingEntries(); + if (processed > 0 || applier_->GetExpectedSequenceId() > 2) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + + // After timeout, gap should be skipped and entry3 should be processed + // Note: This test may be flaky due to timing, but it tests the timeout logic + EXPECT_GE(applier_->GetExpectedSequenceId(), 2u); +} + +TEST_F(OpLogApplierTest, TestPendingEntriesSkip) { + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload); + OpLogEntry entry4 = MakeEntry(4, OpType::PUT_END, "key4", payload); + + // Apply entry1 + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + + // Apply entry4 (creates gap at seq=2,3) + EXPECT_FALSE(applier_->ApplyOpLogEntry(entry4)); + + // Process pending entries to trigger skip logic + // After timeout (3 seconds), gaps should be skipped + for (int i = 0; i < 10; ++i) { + applier_->ProcessPendingEntries(); + uint64_t expected = applier_->GetExpectedSequenceId(); + if (expected >= 3) { // Gap at seq=2 is skipped, expected becomes 3 + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + + // After skip, expected_seq should advance to 3 (gap at seq=2 is skipped) + // entry4 is still pending, waiting for seq=3 + EXPECT_GE(applier_->GetExpectedSequenceId(), 3u); + // entry4 should still be pending (not applied yet) + EXPECT_FALSE(mock_metadata_store_->Exists("key4")); +} + +// ========== 4.1.8 JSON parsing tests ========== + +TEST_F(OpLogApplierTest, TestApplyPutEnd_ValidJson) { + std::string payload = MakeValidJsonPayload(1, 2, 2048); + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload); + + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + EXPECT_TRUE(mock_metadata_store_->Exists("key1")); + + const StandbyObjectMetadata* meta = mock_metadata_store_->GetMetadata("key1"); + ASSERT_NE(nullptr, meta); + EXPECT_EQ(1u, meta->client_id.first); + EXPECT_EQ(2u, meta->client_id.second); + EXPECT_EQ(2048u, meta->size); +} + +TEST_F(OpLogApplierTest, TestApplyPutEnd_InvalidJson) { + std::string invalid_json = "{invalid json}"; + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", invalid_json); + + // Should still succeed (fallback to empty metadata) + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + EXPECT_TRUE(mock_metadata_store_->Exists("key1")); + + // Metadata should exist but with default values + const StandbyObjectMetadata* meta = mock_metadata_store_->GetMetadata("key1"); + ASSERT_NE(nullptr, meta); + EXPECT_EQ(0u, meta->client_id.first); + EXPECT_EQ(0u, meta->client_id.second); + EXPECT_EQ(0u, meta->size); +} + +TEST_F(OpLogApplierTest, TestApplyPutEnd_EmptyPayload) { + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", ""); + + // Should succeed with empty payload (creates empty metadata) + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); + EXPECT_TRUE(mock_metadata_store_->Exists("key1")); + + const StandbyObjectMetadata* meta = mock_metadata_store_->GetMetadata("key1"); + ASSERT_NE(nullptr, meta); + EXPECT_EQ(0u, meta->client_id.first); + EXPECT_EQ(0u, meta->client_id.second); + EXPECT_EQ(0u, meta->size); +} + +// ========== Additional Edge Case Tests ========== + +TEST_F(OpLogApplierTest, TestApplyOpLogEntries_Batch) { + std::string payload = MakeValidJsonPayload(); + std::vector entries; + entries.push_back(MakeEntry(1, OpType::PUT_END, "key1", payload)); + entries.push_back(MakeEntry(2, OpType::PUT_END, "key2", payload)); + entries.push_back(MakeEntry(3, OpType::PUT_END, "key3", payload)); + + size_t applied = applier_->ApplyOpLogEntries(entries); + EXPECT_EQ(3u, applied); + EXPECT_EQ(4u, applier_->GetExpectedSequenceId()); + EXPECT_EQ(3u, mock_metadata_store_->GetKeyCount()); +} + +TEST_F(OpLogApplierTest, TestApplyOpLogEntries_WithGaps) { + std::string payload = MakeValidJsonPayload(); + std::vector entries; + entries.push_back(MakeEntry(1, OpType::PUT_END, "key1", payload)); + entries.push_back(MakeEntry(3, OpType::PUT_END, "key3", payload)); // Gap at seq=2 + entries.push_back(MakeEntry(2, OpType::PUT_END, "key2", payload)); + + size_t applied = applier_->ApplyOpLogEntries(entries); + // entry1 should be applied, entry3 should be pending, entry2 should be applied and trigger processing of entry3 + EXPECT_GE(applied, 2u); // entry1 and entry2 are applied + EXPECT_LE(applied, 3u); + + // entry2's ApplyOpLogEntry internally calls ProcessPendingEntries(), so entry3 should be processed + EXPECT_GE(applier_->GetExpectedSequenceId(), 4u); + EXPECT_TRUE(mock_metadata_store_->Exists("key1")); + EXPECT_TRUE(mock_metadata_store_->Exists("key2")); + EXPECT_TRUE(mock_metadata_store_->Exists("key3")); + + // ProcessPendingEntries may return 0 if entry3 was already processed + (void)applier_->ProcessPendingEntries(); + EXPECT_GE(applier_->GetExpectedSequenceId(), 4u); +} + +TEST_F(OpLogApplierTest, TestGetExpectedSequenceId) { + EXPECT_EQ(1u, applier_->GetExpectedSequenceId()); + + std::string payload = MakeValidJsonPayload(); + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload); + EXPECT_TRUE(applier_->ApplyOpLogEntry(entry)); + EXPECT_EQ(2u, applier_->GetExpectedSequenceId()); +} + +TEST_F(OpLogApplierTest, TestGetKeySequenceId_Deprecated) { + // This method is deprecated and always returns 0 + EXPECT_EQ(0u, applier_->GetKeySequenceId("any_key")); +} + +} // namespace mooncake::test + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/mooncake-store/tests/hot_standby_ut/oplog_manager_test.cpp b/mooncake-store/tests/hot_standby_ut/oplog_manager_test.cpp new file mode 100644 index 0000000000..828373e51a --- /dev/null +++ b/mooncake-store/tests/hot_standby_ut/oplog_manager_test.cpp @@ -0,0 +1,282 @@ +#include "oplog_manager.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace mooncake::test { + +class OpLogManagerTest : public ::testing::Test { + protected: + void SetUp() override { + google::InitGoogleLogging("OpLogManagerTest"); + FLAGS_logtostderr = 1; + manager_ = std::make_unique(); + } + + void TearDown() override { google::ShutdownGoogleLogging(); } + + OpLogManager& M() { return *manager_; } + + std::unique_ptr manager_; +}; + +// ========== 2.1.1 Basic functionality tests ========== + +TEST_F(OpLogManagerTest, TestAppendEntry) { + uint64_t id = M().Append(OpType::PUT_END, "key1", "value1"); + + EXPECT_LT(0u, id); + EXPECT_EQ(id, M().GetLastSequenceId()); + EXPECT_EQ(1u, M().GetEntryCount()); +} + +TEST_F(OpLogManagerTest, TestSequenceIdIncrement) { + uint64_t id1 = M().Append(OpType::PUT_END, "key1", "value1"); + uint64_t id2 = M().Append(OpType::PUT_END, "key2", "value2"); + uint64_t id3 = M().Append(OpType::REMOVE, "key3", ""); + + EXPECT_LT(0u, id1); + EXPECT_EQ(id1 + 1, id2); + EXPECT_EQ(id2 + 1, id3); + EXPECT_EQ(id3, M().GetLastSequenceId()); + EXPECT_EQ(3u, M().GetEntryCount()); +} + +TEST_F(OpLogManagerTest, TestAllocateEntry) { + OpLogEntry e1 = M().AllocateEntry(OpType::PUT_END, "key1", "value1"); + OpLogEntry e2 = M().AllocateEntry(OpType::PUT_END, "key2", "value2"); + + EXPECT_LT(0u, e1.sequence_id); + EXPECT_EQ(e1.sequence_id + 1, e2.sequence_id); + EXPECT_EQ(e2.sequence_id, M().GetLastSequenceId()); + EXPECT_EQ(2u, M().GetEntryCount()); + + // Basic field validation + EXPECT_EQ(OpType::PUT_END, e1.op_type); + EXPECT_EQ("key1", e1.object_key); + EXPECT_EQ("value1", e1.payload); + EXPECT_NE(0u, e1.timestamp_ms); + EXPECT_NE(0u, e1.checksum); + EXPECT_NE(0u, e1.prefix_hash); +} + +TEST_F(OpLogManagerTest, TestPersistEntryToEtcd) { + // Without an EtcdOpLogStore configured, PersistEntryToEtcd should return an error + OpLogEntry entry = + M().AllocateEntry(OpType::PUT_END, "key", "payload-data"); + + ErrorCode err = M().PersistEntryToEtcd(entry); + EXPECT_EQ(ErrorCode::ETCD_OPERATION_ERROR, err); +} + +TEST_F(OpLogManagerTest, TestAppendAndPersist) { + // Without an EtcdOpLogStore configured, AppendAndPersist should return an error + auto res = M().AppendAndPersist(OpType::REMOVE, "key", ""); + ASSERT_FALSE(res.has_value()); + EXPECT_EQ(ErrorCode::ETCD_OPERATION_ERROR, res.error()); +} + +// ========== Initial Sequence Id Tests ========== + +TEST_F(OpLogManagerTest, SetInitialSequenceIdOnEmptyManager) { + EXPECT_EQ(0u, M().GetLastSequenceId()); + EXPECT_EQ(0u, M().GetEntryCount()); + + M().SetInitialSequenceId(100); + EXPECT_EQ(100u, M().GetLastSequenceId()); + + // The first appended entry should have sequence_id 101 + uint64_t id = M().Append(OpType::PUT_END, "key", "value"); + EXPECT_EQ(101u, id); +} + +TEST_F(OpLogManagerTest, SetInitialSequenceIdIgnoredWhenNotEmpty) { + uint64_t id1 = M().Append(OpType::PUT_END, "key1", "value1"); + EXPECT_EQ(id1, M().GetLastSequenceId()); + + // Setting initial sequence id on a non-empty manager should be ignored + M().SetInitialSequenceId(500); + EXPECT_EQ(id1, M().GetLastSequenceId()); +} + +// ========== 2.1.2 Checksum tests ========== + +TEST_F(OpLogManagerTest, TestChecksumComputation) { + // Same payload => same checksum; different payload => different checksum + OpLogEntry e1 = M().AllocateEntry(OpType::PUT_END, "k1", "payload-X"); + OpLogEntry e2 = M().AllocateEntry(OpType::PUT_END, "k2", "payload-X"); + OpLogEntry e3 = M().AllocateEntry(OpType::PUT_END, "k3", "payload-Y"); + + EXPECT_EQ(e1.checksum, e2.checksum); + EXPECT_NE(e1.checksum, e3.checksum); +} + +TEST_F(OpLogManagerTest, TestPrefixHashComputation) { + // Same key => same prefix_hash; different key => (with high probability) different prefix_hash + OpLogEntry e1 = M().AllocateEntry(OpType::PUT_END, "same-key", "v1"); + OpLogEntry e2 = M().AllocateEntry(OpType::PUT_END, "same-key", "v2"); + OpLogEntry e3 = M().AllocateEntry(OpType::PUT_END, "other-key", "v3"); + + EXPECT_EQ(e1.prefix_hash, e2.prefix_hash); + EXPECT_NE(e1.prefix_hash, e3.prefix_hash); +} + +TEST_F(OpLogManagerTest, TestVerifyChecksum) { + OpLogEntry entry = + M().AllocateEntry(OpType::PUT_END, "key", "payload-data"); + EXPECT_TRUE(OpLogManager::VerifyChecksum(entry)); + + // Verification should fail after tampering with the payload + entry.payload = "tampered"; + EXPECT_FALSE(OpLogManager::VerifyChecksum(entry)); +} + +// ========== 2.1.3 Size validation tests ========== + +TEST_F(OpLogManagerTest, TestValidateEntrySize_Valid) { + OpLogEntry entry; + entry.object_key = "normal-key"; + entry.payload = "small-payload"; + + std::string reason; + EXPECT_TRUE(OpLogManager::ValidateEntrySize(entry, &reason)); + EXPECT_TRUE(reason.empty()); +} + +TEST_F(OpLogManagerTest, TestValidateEntrySize_KeyTooLarge) { + OpLogEntry entry; + entry.object_key.assign(OpLogManager::kMaxObjectKeySize + 1, 'k'); + entry.payload = "payload"; + + std::string reason; + EXPECT_FALSE(OpLogManager::ValidateEntrySize(entry, &reason)); + EXPECT_FALSE(reason.empty()); +} + +TEST_F(OpLogManagerTest, TestValidateEntrySize_PayloadTooLarge) { + OpLogEntry entry; + entry.object_key = "key"; + entry.payload.assign(OpLogManager::kMaxPayloadSize + 1, 'p'); + + std::string reason; + EXPECT_FALSE(OpLogManager::ValidateEntrySize(entry, &reason)); + EXPECT_FALSE(reason.empty()); +} + +TEST_F(OpLogManagerTest, TestValidateEntrySize_EmptyKey) { + // Current implementation only enforces upper bounds; empty keys are accepted + OpLogEntry entry; + entry.object_key = ""; + entry.payload = "payload"; + + std::string reason; + EXPECT_TRUE(OpLogManager::ValidateEntrySize(entry, &reason)); +} + +// ========== 2.1.4 Etcd integration tests (placeholder) ========== + +TEST_F(OpLogManagerTest, TestWriteToEtcd_Success) { +#if defined(STORE_USE_ETCD) + GTEST_SKIP() << "TODO: requires real EtcdOpLogStore and running etcd cluster."; +#else + GTEST_SKIP() << "STORE_USE_ETCD is disabled."; +#endif +} + +TEST_F(OpLogManagerTest, TestWriteToEtcd_Failure) { + OpLogEntry entry = + M().AllocateEntry(OpType::PUT_END, "key", "payload-data"); + ErrorCode err = M().PersistEntryToEtcd(entry); + EXPECT_EQ(ErrorCode::ETCD_OPERATION_ERROR, err); +} + +TEST_F(OpLogManagerTest, TestWriteToEtcd_Retry) { + GTEST_SKIP() << "TODO: retry / idempotent semantics are tested at EtcdOpLogStore level."; +} + +TEST_F(OpLogManagerTest, TestIdempotentWrite) { + GTEST_SKIP() << "TODO: idempotent write belongs to EtcdOpLogStore::WriteOpLog tests."; +} + +// ========== 2.1.5 Boundary condition tests ========== + +TEST_F(OpLogManagerTest, TestSequenceIdWrapAround) { + // Theoretical wrap-around test: set initial value near UINT64_MAX and verify wrap-around semantics + uint64_t near_max = std::numeric_limits::max() - 2; + M().SetInitialSequenceId(near_max); + + std::vector ids; + ids.push_back(M().Append(OpType::PUT_END, "k1", "v1")); // max-1 + ids.push_back(M().Append(OpType::PUT_END, "k2", "v2")); // max + ids.push_back(M().Append(OpType::PUT_END, "k3", "v3")); // 0 (wrap) + + ASSERT_EQ(3u, ids.size()); + // Use wrap-around-safe comparison helpers to verify monotonic increase + EXPECT_TRUE(IsSequenceNewer(ids[1], ids[0])); + EXPECT_TRUE(IsSequenceNewer(ids[2], ids[1])); // 0 is considered newer than UINT64_MAX +} + +TEST_F(OpLogManagerTest, TestConcurrentAppend) { + constexpr int kThreads = 8; + constexpr int kPerThread = 1000; + + std::vector ids; + ids.reserve(kThreads * kPerThread); + std::mutex m; + + auto worker = [&]() { + for (int i = 0; i < kPerThread; ++i) { + uint64_t id = M().Append(OpType::PUT_END, "key", "value"); + std::lock_guard lock(m); + ids.push_back(id); + } + }; + + std::vector threads; + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back(worker); + } + for (auto& t : threads) { + t.join(); + } + + EXPECT_EQ(static_cast(kThreads * kPerThread), ids.size()); + + std::sort(ids.begin(), ids.end()); + // Ensure there are no duplicates and sequence IDs are strictly increasing + for (size_t i = 1; i < ids.size(); ++i) { + EXPECT_GT(ids[i], ids[i - 1]); + } +} + +TEST_F(OpLogManagerTest, TestLargePayload) { + // Construct a payload close to the upper limit and verify it passes validation and appends successfully + std::string key = "large-payload-key"; + std::string payload(OpLogManager::kMaxPayloadSize - 1, 'x'); + + OpLogEntry entry; + entry.object_key = key; + entry.payload = payload; + + std::string reason; + EXPECT_TRUE(OpLogManager::ValidateEntrySize(entry, &reason)); + + uint64_t id = M().Append(OpType::PUT_END, key, payload); + EXPECT_LT(0u, id); + EXPECT_EQ(1u, M().GetEntryCount()); +} + +} // namespace mooncake::test + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/mooncake-store/tests/hot_standby_ut/oplog_watcher_test.cpp b/mooncake-store/tests/hot_standby_ut/oplog_watcher_test.cpp new file mode 100644 index 0000000000..b7f6122f4f --- /dev/null +++ b/mooncake-store/tests/hot_standby_ut/oplog_watcher_test.cpp @@ -0,0 +1,421 @@ +#include "oplog_watcher.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#if __has_include() +#include +#else +#include +#endif + +#include "metadata_store.h" +#include "oplog_applier.h" +#include "oplog_manager.h" +#include "standby_state_machine.h" +#include "types.h" +#include "etcd_oplog_store.h" + +namespace mooncake::test { + +// Minimal MetadataStore implementation for OpLogApplier +class MinimalMockMetadataStore : public MetadataStore { + public: + bool PutMetadata(const std::string&, const StandbyObjectMetadata&) override { + return true; + } + bool Put(const std::string&, const std::string&) override { return true; } + const StandbyObjectMetadata* GetMetadata(const std::string&) const override { + return nullptr; + } + bool Remove(const std::string&) override { return true; } + bool Exists(const std::string&) const override { return false; } + size_t GetKeyCount() const override { return 0; } +}; + +// Simple wrapper around OpLogApplier for testing OpLogWatcher. +// We don't override any methods since OpLogApplier's methods are not virtual; +// we just provide a valid instance to OpLogWatcher. +class MockOpLogApplier : public OpLogApplier { + public: + MockOpLogApplier() + : OpLogApplier(&metadata_store_, "test_cluster") {} + + private: + MinimalMockMetadataStore metadata_store_; +}; + +// Helper function to create a valid OpLogEntry with checksum +OpLogEntry MakeEntry(uint64_t seq, OpType type, const std::string& key, + const std::string& payload) { + OpLogEntry e; + e.sequence_id = seq; + e.timestamp_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + e.op_type = type; + e.object_key = key; + e.payload = payload; + e.checksum = static_cast(XXH32(payload.data(), payload.size(), 0)); + e.prefix_hash = key.empty() ? 0 + : static_cast(XXH32(key.data(), key.size(), 0)); + return e; +} + +// Helper function to serialize OpLogEntry to JSON (same as EtcdOpLogStore) +std::string SerializeOpLogEntry(const OpLogEntry& entry) { + Json::Value root; + root["sequence_id"] = static_cast(entry.sequence_id); + root["timestamp_ms"] = static_cast(entry.timestamp_ms); + root["op_type"] = static_cast(entry.op_type); + root["object_key"] = entry.object_key; + root["payload"] = entry.payload; + root["checksum"] = static_cast(entry.checksum); + root["prefix_hash"] = static_cast(entry.prefix_hash); + + Json::StreamWriterBuilder builder; + builder["indentation"] = ""; // Compact format + std::unique_ptr writer(builder.newStreamWriter()); + std::ostringstream oss; + writer->write(root, &oss); + return oss.str(); +} + +class OpLogWatcherTest : public ::testing::Test { + protected: + void SetUp() override { + google::InitGoogleLogging("OpLogWatcherTest"); + FLAGS_logtostderr = 1; + etcd_endpoints_ = "http://localhost:2379"; + cluster_id_ = "test_cluster_001"; + mock_applier_ = std::make_unique(); + watcher_ = std::make_unique(etcd_endpoints_, cluster_id_, + mock_applier_.get()); + } + + void TearDown() override { + if (watcher_) { + watcher_->Stop(); + } + google::ShutdownGoogleLogging(); + } + + std::string etcd_endpoints_; + std::string cluster_id_; + std::unique_ptr mock_applier_; + std::unique_ptr watcher_; +}; + +// ========== 5.1.1 Start/Stop tests ========== + +TEST_F(OpLogWatcherTest, TestStart) { +#ifdef STORE_USE_ETCD + // This test requires a real etcd connection + GTEST_SKIP() << "Requires real etcd connection, skipping integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogWatcherTest, TestStartFromSequenceId) { +#ifdef STORE_USE_ETCD + // This test requires a real etcd connection + GTEST_SKIP() << "Requires real etcd connection, skipping integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogWatcherTest, TestStop) { +#ifdef STORE_USE_ETCD + // Stop without starting should be safe + watcher_->Stop(); + EXPECT_FALSE(watcher_->IsWatchHealthy()); +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogWatcherTest, TestStartWhenAlreadyRunning) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Requires real etcd connection, skipping integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +// ========== 5.1.2 Watch event handling tests ========== + +TEST_F(OpLogWatcherTest, TestHandleWatchEvent_Put) { +#ifdef STORE_USE_ETCD + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "payload1"); + std::string json_value = SerializeOpLogEntry(entry); + std::string key = "/oplog/" + cluster_id_ + "/00000000000000000001"; + + // Use reflection to call HandleWatchEvent (it's private, so we test via public interface) + // For unit testing, we can't directly call HandleWatchEvent, so we skip this test + // and rely on integration tests + GTEST_SKIP() << "HandleWatchEvent is private, requires integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogWatcherTest, TestHandleWatchEvent_Delete) { +#ifdef STORE_USE_ETCD + // DELETE events are handled but don't apply entries + GTEST_SKIP() << "HandleWatchEvent is private, requires integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogWatcherTest, TestHandleWatchEvent_InvalidEntry) { +#ifdef STORE_USE_ETCD + // Invalid JSON should be rejected + std::string invalid_json = "{invalid json}"; + std::string key = "/oplog/" + cluster_id_ + "/00000000000000000001"; + + // Can't directly test HandleWatchEvent, requires integration test + GTEST_SKIP() << "HandleWatchEvent is private, requires integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogWatcherTest, TestHandleWatchEvent_OutOfOrder) { +#ifdef STORE_USE_ETCD + // Out-of-order entries should be handled by OpLogApplier + GTEST_SKIP() << "HandleWatchEvent is private, requires integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +// ========== 5.1.3 Reconnection tests ========== + +TEST_F(OpLogWatcherTest, TestReconnectAfterDisconnect) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Requires real etcd connection and watch failure simulation"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogWatcherTest, TestReconnectWithStateCallback) { +#ifdef STORE_USE_ETCD + std::atomic callback_called{false}; + StandbyEvent received_event = StandbyEvent::START; + + watcher_->SetStateCallback([&](StandbyEvent event) { + callback_called.store(true); + received_event = event; + }); + + // State callbacks are triggered during watch operations + // Requires integration test with real etcd + GTEST_SKIP() << "Requires real etcd connection for state callback testing"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogWatcherTest, TestReconnectResumeFromLastSequence) { +#ifdef STORE_USE_ETCD + GTEST_SKIP() << "Requires real etcd connection and reconnection simulation"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +// ========== 5.1.4 Checksum verification tests ========== + +TEST_F(OpLogWatcherTest, TestHandleWatchEvent_ValidChecksum) { +#ifdef STORE_USE_ETCD + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "payload1"); + std::string json_value = SerializeOpLogEntry(entry); + + // Valid checksum should pass validation + // Can't directly test HandleWatchEvent, requires integration test + GTEST_SKIP() << "HandleWatchEvent is private, requires integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogWatcherTest, TestHandleWatchEvent_InvalidChecksum) { +#ifdef STORE_USE_ETCD + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "payload1"); + std::string json_value = SerializeOpLogEntry(entry); + + // Tamper with checksum in JSON + // Replace checksum value in JSON string + size_t pos = json_value.find("\"checksum\":"); + if (pos != std::string::npos) { + size_t start = pos + 10; // length of "checksum": + size_t end = json_value.find_first_of(",}", start); + if (end != std::string::npos) { + json_value.replace(start, end - start, "999999"); + } + } + + // Invalid checksum should be rejected + // Can't directly test HandleWatchEvent, requires integration test + GTEST_SKIP() << "HandleWatchEvent is private, requires integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +// ========== 5.1.5 Size validation tests ========== + +TEST_F(OpLogWatcherTest, TestHandleWatchEvent_ValidSize) { +#ifdef STORE_USE_ETCD + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "payload1"); + std::string json_value = SerializeOpLogEntry(entry); + + // Valid size should pass validation + EXPECT_TRUE(OpLogManager::ValidateEntrySize(entry)); + GTEST_SKIP() << "HandleWatchEvent is private, requires integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogWatcherTest, TestHandleWatchEvent_InvalidSize) { +#ifdef STORE_USE_ETCD + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "payload1"); + // Make key too large + entry.object_key.assign(OpLogManager::kMaxObjectKeySize + 1, 'k'); + std::string json_value = SerializeOpLogEntry(entry); + + // Invalid size should be rejected + EXPECT_FALSE(OpLogManager::ValidateEntrySize(entry)); + GTEST_SKIP() << "HandleWatchEvent is private, requires integration test"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +// ========== 5.1.6 State callback tests ========== + +TEST_F(OpLogWatcherTest, TestStateCallback_WatchHealthy) { +#ifdef STORE_USE_ETCD + std::atomic callback_called{false}; + StandbyEvent received_event = StandbyEvent::START; + + watcher_->SetStateCallback([&](StandbyEvent event) { + callback_called.store(true); + received_event = event; + }); + + // Watch healthy events are triggered during normal watch operations + // Requires integration test with real etcd + GTEST_SKIP() << "Requires real etcd connection for state callback testing"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogWatcherTest, TestStateCallback_WatchBroken) { +#ifdef STORE_USE_ETCD + std::atomic callback_called{false}; + StandbyEvent received_event = StandbyEvent::START; + + watcher_->SetStateCallback([&](StandbyEvent event) { + callback_called.store(true); + received_event = event; + }); + + // Watch broken events are triggered when watch fails + // Requires integration test with real etcd and watch failure + GTEST_SKIP() << "Requires real etcd connection and watch failure simulation"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +// ========== 5.1.7 Cluster ID validation tests ========== + +TEST_F(OpLogWatcherTest, TestInvalidClusterId_Rejected) { + // Invalid cluster_id should cause LOG(FATAL) in constructor + // We can't test this directly as it would terminate the process + // But we can verify that valid cluster_id works + std::string valid_cluster_id = "test_cluster_001"; + std::unique_ptr mock_applier = + std::make_unique(); + std::unique_ptr watcher = std::make_unique( + etcd_endpoints_, valid_cluster_id, mock_applier.get()); + EXPECT_NE(nullptr, watcher); + watcher->Stop(); +} + +TEST_F(OpLogWatcherTest, TestGetLastProcessedSequenceId) { +#ifdef STORE_USE_ETCD + // Initially should be 0 + EXPECT_EQ(0u, watcher_->GetLastProcessedSequenceId()); + + // After processing entries, should be updated + // This requires integration test with real etcd + GTEST_SKIP() << "Requires real etcd connection for sequence ID tracking"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +TEST_F(OpLogWatcherTest, TestIsWatchHealthy) { +#ifdef STORE_USE_ETCD + // Initially should be false (not started) + EXPECT_FALSE(watcher_->IsWatchHealthy()); + + // After starting and successful watch, should be true + // This requires integration test with real etcd + GTEST_SKIP() << "Requires real etcd connection for watch health testing"; +#else + GTEST_SKIP() << "STORE_USE_ETCD not enabled"; +#endif +} + +// ========== Additional Helper Tests ========== + +TEST_F(OpLogWatcherTest, TestSerializeOpLogEntry) { + OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "payload1"); + std::string json = SerializeOpLogEntry(entry); + + // Verify JSON contains expected fields + EXPECT_NE(std::string::npos, json.find("sequence_id")); + EXPECT_NE(std::string::npos, json.find("op_type")); + EXPECT_NE(std::string::npos, json.find("object_key")); + EXPECT_NE(std::string::npos, json.find("payload")); + EXPECT_NE(std::string::npos, json.find("checksum")); + EXPECT_NE(std::string::npos, json.find("prefix_hash")); +} + +TEST_F(OpLogWatcherTest, TestMakeEntry) { + OpLogEntry entry = MakeEntry(100, OpType::REMOVE, "test_key", "test_payload"); + + EXPECT_EQ(100u, entry.sequence_id); + EXPECT_EQ(OpType::REMOVE, entry.op_type); + EXPECT_EQ("test_key", entry.object_key); + EXPECT_EQ("test_payload", entry.payload); + EXPECT_NE(0u, entry.checksum); + EXPECT_NE(0u, entry.prefix_hash); + EXPECT_NE(0u, entry.timestamp_ms); +} + +} // namespace mooncake::test + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp b/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp new file mode 100644 index 0000000000..9b7bb2ac64 --- /dev/null +++ b/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp @@ -0,0 +1,729 @@ +#include "standby_state_machine.h" + +#include +#include + +#include +#include +#include +#include + +namespace mooncake::test { + +class StandbyStateMachineTest : public ::testing::Test { + protected: + void SetUp() override { + google::InitGoogleLogging("StandbyStateMachineTest"); + FLAGS_logtostderr = true; + machine_ = std::make_unique(); + } + + void TearDown() override { google::ShutdownGoogleLogging(); } + + std::unique_ptr machine_; + + // Helper function to reach WATCHING state + void ReachWatchingState() { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); + } + + // Helper function to reach SYNCING state + void ReachSyncingState() { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + } +}; + +// ========== Initial State Tests ========== + +TEST_F(StandbyStateMachineTest, TestInitialState) { + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsConnected()); + EXPECT_FALSE(machine_->IsWatchHealthy()); + EXPECT_FALSE(machine_->IsReadyForPromotion()); + EXPECT_EQ(0, machine_->GetConsecutiveErrors()); + EXPECT_EQ(0, machine_->GetReconnectCount()); +} + +// ========== Basic State Transition Tests ========== + +TEST_F(StandbyStateMachineTest, TestStartTransition) { + auto result = machine_->ProcessEvent(StandbyEvent::START); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::STOPPED, result.old_state); + EXPECT_EQ(StandbyState::CONNECTING, result.new_state); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + // In CONNECTING state, syncing has not actually started yet, so IsRunning/IsConnected should both be false + EXPECT_FALSE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsConnected()); +} + +TEST_F(StandbyStateMachineTest, TestConnectedTransition) { + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::CONNECTING, result.old_state); + EXPECT_EQ(StandbyState::SYNCING, result.new_state); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_TRUE(machine_->IsConnected()); +} + +TEST_F(StandbyStateMachineTest, TestSyncCompleteTransition) { + ReachSyncingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::SYNCING, result.old_state); + EXPECT_EQ(StandbyState::WATCHING, result.new_state); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_TRUE(machine_->IsConnected()); + EXPECT_TRUE(machine_->IsWatchHealthy()); + EXPECT_TRUE(machine_->IsReadyForPromotion()); +} + +TEST_F(StandbyStateMachineTest, TestWatchHealthyNoOp) { + ReachWatchingState(); + + // WATCH_HEALTHY in WATCHING state is a no-op (stays in WATCHING) + auto result = machine_->ProcessEvent(StandbyEvent::WATCH_HEALTHY); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::WATCHING, result.new_state); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestWatchBrokenTransition) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsWatchHealthy()); + EXPECT_FALSE(machine_->IsReadyForPromotion()); +} + +TEST_F(StandbyStateMachineTest, TestDisconnectedFromWatching) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::DISCONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestPromoteTransition) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::PROMOTING, result.new_state); + EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_TRUE(machine_->IsConnected()); + EXPECT_FALSE(machine_->IsWatchHealthy()); + EXPECT_FALSE(machine_->IsReadyForPromotion()); +} + +TEST_F(StandbyStateMachineTest, TestPromotionSuccessTransition) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::PROMOTING, result.old_state); + EXPECT_EQ(StandbyState::PROMOTED, result.new_state); + EXPECT_EQ(StandbyState::PROMOTED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsConnected()); +} + +TEST_F(StandbyStateMachineTest, TestPromotionFailedTransition) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::PROMOTION_FAILED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::PROMOTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); +} + +TEST_F(StandbyStateMachineTest, TestStopTransition) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::STOPPED, result.new_state); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsConnected()); +} + +// ========== Error and Failure State Tests ========== + +TEST_F(StandbyStateMachineTest, TestConnectionFailedFromConnecting) { + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::CONNECTION_FAILED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::CONNECTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); +} + +TEST_F(StandbyStateMachineTest, TestFatalErrorFromConnecting) { + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::CONNECTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestSyncFailedFromSyncing) { + ReachSyncingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::SYNC_FAILED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::SYNCING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); +} + +TEST_F(StandbyStateMachineTest, TestDisconnectedFromSyncing) { + ReachSyncingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::DISCONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::SYNCING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestFatalErrorFromSyncing) { + ReachSyncingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::SYNCING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestFatalErrorFromWatching) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +// ========== Reconnecting State Tests ========== + +TEST_F(StandbyStateMachineTest, TestReconnectingToSyncing) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECONNECTING, result.old_state); + EXPECT_EQ(StandbyState::SYNCING, result.new_state); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestReconnectingToFailed) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECONNECTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestReconnectingMaxErrors) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECONNECTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +// ========== Recovering State Tests ========== + +TEST_F(StandbyStateMachineTest, TestRecoveringToWatching) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::RECOVERY_SUCCESS); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECOVERING, result.old_state); + EXPECT_EQ(StandbyState::WATCHING, result.new_state); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestRecoveringToReconnecting) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::RECOVERY_FAILED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECOVERING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestRecoveringDisconnected) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::DISCONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECOVERING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestRecoveringFatalError) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECOVERING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +// ========== Failed State Tests ========== + +TEST_F(StandbyStateMachineTest, TestFailedToStopped) { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::FAILED, result.old_state); + EXPECT_EQ(StandbyState::STOPPED, result.new_state); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestFailedToConnecting) { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); + + // Allow restart from FAILED state + auto result = machine_->ProcessEvent(StandbyEvent::START); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::FAILED, result.old_state); + EXPECT_EQ(StandbyState::CONNECTING, result.new_state); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); +} + +// ========== Promoted State Tests ========== + +TEST_F(StandbyStateMachineTest, TestPromotedToStopped) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::PROMOTE); + machine_->ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + EXPECT_EQ(StandbyState::PROMOTED, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::PROMOTED, result.old_state); + EXPECT_EQ(StandbyState::STOPPED, result.new_state); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); +} + +// ========== Invalid Transition Tests ========== + +TEST_F(StandbyStateMachineTest, TestInvalidTransitions) { + // Cannot transition from STOPPED directly to WATCHING + auto result1 = machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_FALSE(result1.allowed); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); + + // Cannot promote when not in WATCHING state + ReachSyncingState(); + auto result2 = machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_FALSE(result2.allowed); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + + // Cannot transition from STOPPED to CONNECTED + machine_->ProcessEvent(StandbyEvent::STOP); + auto result3 = machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_FALSE(result3.allowed); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); +} + +// ========== Error Handling Tests ========== + +TEST_F(StandbyStateMachineTest, TestConsecutiveErrors) { + ReachWatchingState(); + + // Simulate multiple errors + for (int i = 0; i < 5; ++i) { + machine_->IncrementErrors(); + } + EXPECT_EQ(5, machine_->GetConsecutiveErrors()); + + // Reset errors + machine_->ResetErrors(); + EXPECT_EQ(0, machine_->GetConsecutiveErrors()); +} + +TEST_F(StandbyStateMachineTest, TestMaxErrorsReachedAutoTransition) { + ReachWatchingState(); + + // IncrementErrors() automatically triggers MAX_ERRORS_REACHED when threshold is reached + for (int i = 0; i < StandbyStateMachine::kMaxConsecutiveErrors; ++i) { + machine_->IncrementErrors(); + } + + // Should have transitioned to RECOVERING (from WATCHING on MAX_ERRORS_REACHED) + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + EXPECT_EQ(StandbyStateMachine::kMaxConsecutiveErrors, + machine_->GetConsecutiveErrors()); +} + +TEST_F(StandbyStateMachineTest, TestMaxErrorsReachedManual) { + ReachWatchingState(); + + // Manually trigger MAX_ERRORS_REACHED + auto result = machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::RECOVERING, result.new_state); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestReconnectCount) { + EXPECT_EQ(0, machine_->GetReconnectCount()); + + machine_->IncrementReconnectCount(); + EXPECT_EQ(1, machine_->GetReconnectCount()); + + machine_->IncrementReconnectCount(); + EXPECT_EQ(2, machine_->GetReconnectCount()); + + machine_->ResetReconnectCount(); + EXPECT_EQ(0, machine_->GetReconnectCount()); +} + +// ========== Callback Tests ========== + +TEST_F(StandbyStateMachineTest, TestStateChangeCallback) { + std::vector state_history; + std::vector event_history; + + machine_->RegisterCallback( + [&](StandbyState old_state, StandbyState new_state, StandbyEvent event) { + state_history.push_back(new_state); + event_history.push_back(event); + }); + + // Trigger state transitions + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + + // Verify callbacks were called + EXPECT_EQ(3, state_history.size()); + EXPECT_EQ(StandbyState::CONNECTING, state_history[0]); + EXPECT_EQ(StandbyState::SYNCING, state_history[1]); + EXPECT_EQ(StandbyState::WATCHING, state_history[2]); + EXPECT_EQ(StandbyEvent::START, event_history[0]); + EXPECT_EQ(StandbyEvent::CONNECTED, event_history[1]); + EXPECT_EQ(StandbyEvent::SYNC_COMPLETE, event_history[2]); +} + +TEST_F(StandbyStateMachineTest, TestMultipleCallbacks) { + int callback1_count = 0; + int callback2_count = 0; + + machine_->RegisterCallback([&](StandbyState, StandbyState, StandbyEvent) { + callback1_count++; + }); + machine_->RegisterCallback([&](StandbyState, StandbyState, StandbyEvent) { + callback2_count++; + }); + + // Trigger state transitions + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + + // Both callbacks should be called + EXPECT_EQ(2, callback1_count); + EXPECT_EQ(2, callback2_count); +} + +TEST_F(StandbyStateMachineTest, TestCallbackExceptionHandling) { + bool callback_called = false; + + machine_->RegisterCallback([&](StandbyState, StandbyState, StandbyEvent) { + callback_called = true; + }); + + // Callback should be invoked and must not interfere with state transitions + auto result = machine_->ProcessEvent(StandbyEvent::START); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + EXPECT_TRUE(callback_called); +} + +// ========== History Tests ========== + +TEST_F(StandbyStateMachineTest, TestTransitionHistory) { + // Perform several transitions + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + + auto history = machine_->GetTransitionHistory(10); + EXPECT_EQ(3, history.size()); + EXPECT_EQ(StandbyState::STOPPED, history[0].from_state); + EXPECT_EQ(StandbyState::CONNECTING, history[0].to_state); + EXPECT_EQ(StandbyEvent::START, history[0].event); + + EXPECT_EQ(StandbyState::CONNECTING, history[1].from_state); + EXPECT_EQ(StandbyState::SYNCING, history[1].to_state); + EXPECT_EQ(StandbyEvent::CONNECTED, history[1].event); + + EXPECT_EQ(StandbyState::SYNCING, history[2].from_state); + EXPECT_EQ(StandbyState::WATCHING, history[2].to_state); + EXPECT_EQ(StandbyEvent::SYNC_COMPLETE, history[2].event); +} + +TEST_F(StandbyStateMachineTest, TestTransitionHistoryLimit) { + // Perform many transitions to test history limit + for (int i = 0; i < 20; ++i) { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::STOP); + } + + // Request limited history + auto history = machine_->GetTransitionHistory(5); + EXPECT_LE(history.size(), 5); +} + +TEST_F(StandbyStateMachineTest, TestTimeInState) { + machine_->ProcessEvent(StandbyEvent::START); + + // Wait a bit + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + auto time_in_state = machine_->GetTimeInCurrentState(); + EXPECT_GE(time_in_state.count(), 100); + EXPECT_LE(time_in_state.count(), 200); // Allow some margin for test execution +} + +// ========== Concurrent Tests ========== + +TEST_F(StandbyStateMachineTest, TestConcurrentStateQueries) { + ReachWatchingState(); + + // Multiple threads querying state concurrently + std::vector threads; + std::atomic success_count{0}; + + for (int i = 0; i < 10; ++i) { + threads.emplace_back([&]() { + for (int j = 0; j < 100; ++j) { + StandbyState state = machine_->GetState(); + if (state == StandbyState::WATCHING) { + success_count++; + } + } + }); + } + + for (auto& t : threads) { + t.join(); + } + + EXPECT_EQ(1000, success_count.load()); +} + +TEST_F(StandbyStateMachineTest, TestConcurrentEventProcessing) { + ReachWatchingState(); + + // Multiple threads trying to process events concurrently + // Only one should succeed (state machine should serialize) + std::vector threads; + std::atomic success_count{0}; + std::atomic failure_count{0}; + + for (int i = 0; i < 10; ++i) { + threads.emplace_back([&]() { + auto result = machine_->ProcessEvent(StandbyEvent::STOP); + if (result.allowed) { + success_count++; + } else { + failure_count++; + } + }); + } + + for (auto& t : threads) { + t.join(); + } + + // Only one STOP should succeed (transition to STOPPED) + EXPECT_EQ(1, success_count.load()); + EXPECT_EQ(9, failure_count.load()); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); +} + +// ========== State Query Tests ========== + +TEST_F(StandbyStateMachineTest, TestIsRunning) { + EXPECT_FALSE(machine_->IsRunning()); // STOPPED + + machine_->ProcessEvent(StandbyEvent::START); + // CONNECTING only means establishing connection; sync has not started, so it is not considered "running" + EXPECT_FALSE(machine_->IsRunning()); // CONNECTING + + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_TRUE(machine_->IsRunning()); // SYNCING + + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_TRUE(machine_->IsRunning()); // WATCHING + + machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_FALSE(machine_->IsRunning()); // STOPPED +} + +TEST_F(StandbyStateMachineTest, TestIsConnected) { + EXPECT_FALSE(machine_->IsConnected()); // STOPPED + + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_FALSE(machine_->IsConnected()); // CONNECTING + + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_TRUE(machine_->IsConnected()); // SYNCING + + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_TRUE(machine_->IsConnected()); // WATCHING + + machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_FALSE(machine_->IsConnected()); // STOPPED +} + +TEST_F(StandbyStateMachineTest, TestIsWatchHealthy) { + EXPECT_FALSE(machine_->IsWatchHealthy()); // STOPPED + + ReachWatchingState(); + EXPECT_TRUE(machine_->IsWatchHealthy()); // WATCHING + + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_FALSE(machine_->IsWatchHealthy()); // RECONNECTING +} + +TEST_F(StandbyStateMachineTest, TestIsReadyForPromotion) { + EXPECT_FALSE(machine_->IsReadyForPromotion()); // STOPPED + + ReachWatchingState(); + EXPECT_TRUE(machine_->IsReadyForPromotion()); // WATCHING + + machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_FALSE(machine_->IsReadyForPromotion()); // PROMOTING +} + +// ========== Complete State Machine Flow Tests ========== + +TEST_F(StandbyStateMachineTest, TestCompleteNormalFlow) { + // Complete flow: STOPPED -> CONNECTING -> SYNCING -> WATCHING + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); + EXPECT_TRUE(machine_->IsReadyForPromotion()); +} + +TEST_F(StandbyStateMachineTest, TestCompletePromotionFlow) { + // Complete promotion flow + ReachWatchingState(); + + machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + EXPECT_EQ(StandbyState::PROMOTED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestCompleteReconnectFlow) { + // Complete reconnect flow: WATCHING -> RECONNECTING -> SYNCING -> WATCHING + ReachWatchingState(); + + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestCompleteRecoveryFlow) { + // Complete recovery flow: WATCHING -> RECOVERING -> WATCHING + ReachWatchingState(); + + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::RECOVERY_SUCCESS); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); +} + +} // namespace mooncake::test + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} +