-
Notifications
You must be signed in to change notification settings - Fork 286
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(satp-hermes): add crash recovery & rollback protocol
Signed-off-by: Yogesh01000100 <[email protected]> chore(satp-hermes): improve DB management Signed-off-by: Rafael Belchior <[email protected]> chore(satp-hermes): crash recovery architecture Signed-off-by: Rafael Belchior <[email protected]> fix(recovery): enhance crash recovery and rollback implementation Signed-off-by: Yogesh01000100 <[email protected]> refactor(recovery): consolidate logic and improve SATP message handling Signed-off-by: Yogesh01000100 <[email protected]> feat(recovery): add rollback implementations Signed-off-by: Yogesh01000100 <[email protected]> fix: correct return types and inits Signed-off-by: Yogesh01000100 <[email protected]> Co-authored-by: Rafael Belchior <[email protected]> fix: add unit tests and resolve rollbackstate Signed-off-by: Yogesh01000100 <[email protected]> feat: add function processing logs from g2 Signed-off-by: Yogesh01000100 <[email protected]> feat: add cron schedule for periodic crash checks Signed-off-by: Yogesh01000100 <[email protected]> fix: resolve rollback condition and add tests Signed-off-by: Yogesh01000100 <[email protected]> feat: add orchestrator communication layer using connect-RPC Signed-off-by: Yogesh01000100 <[email protected]> feat: add rollback protocol rpc Signed-off-by: Yogesh01000100 <[email protected]> fix: handle server log synchronization Signed-off-by: Yogesh01000100 <[email protected]> fix: resolve gol errors, add unit tests Signed-off-by: Yogesh01000100 <[email protected]> fix: handle server-side rollback Signed-off-by: Yogesh01000100 <[email protected]> fix: resolve networkId in rollback strategies Signed-off-by: Yogesh01000100 <[email protected]>
- Loading branch information
1 parent
7f99648
commit d73a5eb
Showing
32 changed files
with
4,535 additions
and
29 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
119 changes: 119 additions & 0 deletions
119
...ges/cactus-plugin-satp-hermes/src/main/typescript/core/crash-management/client-service.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| import { | ||
| RecoverMessage, | ||
| RecoverMessageSchema, | ||
| RecoverSuccessMessage, | ||
| RecoverSuccessMessageSchema, | ||
| RollbackMessage, | ||
| RollbackMessageSchema, | ||
| RollbackState, | ||
| } from "../../../typescript/generated/proto/cacti/satp/v02/crash_recovery_pb"; | ||
| import { Logger } from "@hyperledger/cactus-common"; | ||
| import { SATPSession } from "../satp-session"; | ||
| import { create } from "@bufbuild/protobuf"; | ||
| import { SATPLogger } from "../../logging"; | ||
| import { stringify as safeStableStringify } from "safe-stable-stringify"; | ||
|
|
||
| export class CrashRecoveryClientService { | ||
| private readonly log: Logger; | ||
|
|
||
| constructor( | ||
| private readonly dbLogger: SATPLogger, | ||
| log: Logger, | ||
| ) { | ||
| this.log = log; | ||
| this.log.trace(`Initialized ${CrashRecoveryClientService.name}`); | ||
| } | ||
|
|
||
| public async createRecoverMessage( | ||
| session: SATPSession, | ||
| ): Promise<RecoverMessage> { | ||
| const fnTag = `${CrashRecoveryClientService.name}#createRecoverMessage`; | ||
| this.log.debug( | ||
| `${fnTag} - Creating RecoverMessage for sessionId: ${session.getSessionId()}`, | ||
| ); | ||
|
|
||
| const sessionData = session.getClientSessionData(); | ||
|
|
||
| const recoverMessage = create(RecoverMessageSchema, { | ||
| sessionId: session.getSessionId(), | ||
| messageType: "urn:ietf:SATP-2pc:msgtype:recover-msg", | ||
| satpPhase: "", | ||
| sequenceNumber: Number(sessionData.lastSequenceNumber), | ||
| isBackup: false, | ||
| newIdentityPublicKey: "", | ||
| lastEntryTimestamp: BigInt(sessionData.lastMessageReceivedTimestamp), | ||
| senderSignature: "", | ||
| }); | ||
|
|
||
| await this.dbLogger.persistLogEntry({ | ||
| sessionID: recoverMessage.sessionId, | ||
| type: "urn:ietf:SATP-2pc:msgtype:recover-msg", | ||
| operation: "done", | ||
| data: safeStableStringify(sessionData), | ||
| sequenceNumber: Number(sessionData.lastSequenceNumber), | ||
| }); | ||
| this.log.debug(`${fnTag} - RecoverMessage created:`, recoverMessage); | ||
|
|
||
| return recoverMessage; | ||
| } | ||
|
|
||
| public async createRecoverSuccessMessage( | ||
| session: SATPSession, | ||
| ): Promise<RecoverSuccessMessage> { | ||
| const fnTag = `${CrashRecoveryClientService.name}#createRecoverSuccessMessage`; | ||
| this.log.debug( | ||
| `${fnTag} - Creating RecoverSuccessMessage for sessionId: ${session.getSessionId()}`, | ||
| ); | ||
| const sessionData = session.getClientSessionData(); | ||
| const recoverSuccessMessage = create(RecoverSuccessMessageSchema, { | ||
| sessionId: session.getSessionId(), | ||
| messageType: "urn:ietf:SATP-2pc:msgtype:recover-success-msg", | ||
| hashRecoverUpdateMessage: "", | ||
| success: true, | ||
| entriesChanged: [], | ||
| senderSignature: "", | ||
| }); | ||
| await this.dbLogger.persistLogEntry({ | ||
| sessionID: recoverSuccessMessage.sessionId, | ||
| type: "urn:ietf:SATP-2pc:msgtype:recover-success-msg", | ||
| operation: "done", | ||
| data: safeStableStringify(sessionData), | ||
| sequenceNumber: Number(sessionData.lastSequenceNumber), | ||
| }); | ||
| this.log.debug( | ||
| `${fnTag} - RecoverSuccessMessage created:`, | ||
| recoverSuccessMessage, | ||
| ); | ||
|
|
||
| return recoverSuccessMessage; | ||
| } | ||
|
|
||
| public async createRollbackMessage( | ||
| session: SATPSession, | ||
| rollbackState: RollbackState, | ||
| ): Promise<RollbackMessage> { | ||
| const fnTag = `${CrashRecoveryClientService.name}#createRollbackMessage`; | ||
| this.log.debug( | ||
| `${fnTag} - Creating RollbackMessage for sessionId: ${session.getSessionId()}`, | ||
| ); | ||
| const sessionData = session.getClientSessionData(); | ||
| const rollbackMessage = create(RollbackMessageSchema, { | ||
| sessionId: session.getSessionId(), | ||
| messageType: "urn:ietf:SATP-2pc:msgtype:rollback-msg", | ||
| success: rollbackState.status === "completed", | ||
| actionsPerformed: [], | ||
| proofs: [], | ||
| senderSignature: "", | ||
| }); | ||
| await this.dbLogger.persistLogEntry({ | ||
| sessionID: rollbackMessage.sessionId, | ||
| type: "urn:ietf:SATP-2pc:msgtype:rollback-msg", | ||
| operation: "done", | ||
| data: safeStableStringify(sessionData), | ||
| sequenceNumber: Number(sessionData.lastSequenceNumber), | ||
| }); | ||
| this.log.debug(`${fnTag} - RollbackMessage created:`, rollbackMessage); | ||
|
|
||
| return rollbackMessage; | ||
| } | ||
| } |
146 changes: 146 additions & 0 deletions
146
...ages/cactus-plugin-satp-hermes/src/main/typescript/core/crash-management/crash-handler.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,146 @@ | ||
| import { ConnectRouter } from "@connectrpc/connect"; | ||
| import { Logger } from "@hyperledger/cactus-common"; | ||
| import { | ||
| CrashRecovery, | ||
| RecoverSuccessMessageResponse, | ||
| } from "../../generated/proto/cacti/satp/v02/crash_recovery_pb"; | ||
| import { CrashRecoveryServerService } from "./server-service"; | ||
| import { CrashRecoveryClientService } from "./client-service"; | ||
| import { SATPSession } from "../satp-session"; | ||
| import { | ||
| RecoverMessage, | ||
| RecoverUpdateMessage, | ||
| RecoverSuccessMessage, | ||
| RollbackMessage, | ||
| RollbackAckMessage, | ||
| RollbackState, | ||
| } from "../../generated/proto/cacti/satp/v02/crash_recovery_pb"; | ||
| import { SATPHandler, SATPHandlerType } from "../../types/satp-protocol"; | ||
|
|
||
| export class CrashRecoveryHandler implements SATPHandler { | ||
| private readonly log: Logger; | ||
|
|
||
| constructor( | ||
| private readonly serverService: CrashRecoveryServerService, | ||
| private readonly clientService: CrashRecoveryClientService, | ||
| log: Logger, | ||
| ) { | ||
| this.log = log; | ||
| this.log.trace(`Initialized ${CrashRecoveryHandler.name}`); | ||
| } | ||
|
|
||
| public getHandlerIdentifier(): SATPHandlerType { | ||
| return SATPHandlerType.CRASH; | ||
| } | ||
|
|
||
| public getHandlerSessions(): string[] { | ||
| return []; | ||
| } | ||
|
|
||
| public getStage(): string { | ||
| return "crash"; | ||
| } | ||
|
|
||
| // Server-side | ||
|
|
||
| private async recoverV2MessageImplementation( | ||
| req: RecoverMessage, | ||
| ): Promise<RecoverUpdateMessage> { | ||
| const fnTag = `${CrashRecoveryHandler.name}#recoverV2MessageImplementation`; | ||
| this.log.debug(`${fnTag} - Handling RecoverMessage: ${req}`); | ||
| try { | ||
| return await this.serverService.handleRecover(req); | ||
| } catch (error) { | ||
| this.log.error(`${fnTag} - Error:`, error); | ||
| throw error; | ||
| } | ||
| } | ||
|
|
||
| private async recoverV2SuccessMessageImplementation( | ||
| req: RecoverSuccessMessage, | ||
| ): Promise<RecoverSuccessMessageResponse> { | ||
| const fnTag = `${CrashRecoveryHandler.name}#recoverV2SuccessMessageImplementation`; | ||
| this.log.debug(`${fnTag} - Handling RecoverSuccessMessage:${req}`); | ||
| try { | ||
| return await this.serverService.handleRecoverSuccess(req); | ||
| } catch (error) { | ||
| this.log.error(`${fnTag} - Error:`, error); | ||
| throw error; | ||
| } | ||
| } | ||
|
|
||
| private async rollbackV2MessageImplementation( | ||
| req: RollbackMessage, | ||
| ): Promise<RollbackAckMessage> { | ||
| const fnTag = `${CrashRecoveryHandler.name}#rollbackV2MessageImplementation`; | ||
| this.log.debug(`${fnTag} - Handling RollbackMessage: ${req}`); | ||
| try { | ||
| return await this.serverService.handleRollback(req); | ||
| } catch (error) { | ||
| this.log.error(`${fnTag} - Error:`, error); | ||
| throw error; | ||
| } | ||
| } | ||
|
|
||
| public setupRouter(router: ConnectRouter): void { | ||
| // eslint-disable-next-line @typescript-eslint/no-this-alias | ||
| const that = this; | ||
| router.service(CrashRecovery, { | ||
| async recoverV2Message(req) { | ||
| return await that.recoverV2MessageImplementation(req); | ||
| }, | ||
| async recoverV2SuccessMessage(req) { | ||
| return await that.recoverV2SuccessMessageImplementation(req); | ||
| }, | ||
| async rollbackV2Message(req) { | ||
| return await that.rollbackV2MessageImplementation(req); | ||
| }, | ||
| }); | ||
|
|
||
| this.log.info("Router setup completed for CrashRecoveryHandler"); | ||
| } | ||
|
|
||
| // Client-side | ||
|
|
||
| public async createRecoverMessage( | ||
| session: SATPSession, | ||
| ): Promise<RecoverMessage> { | ||
| const fnTag = `${this.constructor.name}#createRecoverMessage`; | ||
| try { | ||
| return this.clientService.createRecoverMessage(session); | ||
| } catch (error) { | ||
| this.log.error(`${fnTag} - Failed to create RecoverMessage: ${error}`); | ||
| throw new Error(`Error in createRecoverMessage: ${error}`); | ||
| } | ||
| } | ||
|
|
||
| public async createRecoverSuccessMessage( | ||
| session: SATPSession, | ||
| ): Promise<RecoverSuccessMessage> { | ||
| const fnTag = `${this.constructor.name}#createRecoverSuccessMessage`; | ||
| try { | ||
| return await this.clientService.createRecoverSuccessMessage(session); | ||
| } catch (error) { | ||
| this.log.error( | ||
| `${fnTag} - Failed to create RecoverSuccessMessage: ${error}`, | ||
| ); | ||
| throw new Error(`Error in createRecoverSuccessMessage: ${error}`); | ||
| } | ||
| } | ||
|
|
||
| public async createRollbackMessage( | ||
| session: SATPSession, | ||
| rollbackState: RollbackState, | ||
| ): Promise<RollbackMessage> { | ||
| const fnTag = `${this.constructor.name}#createRollbackMessage`; | ||
| try { | ||
| return await this.clientService.createRollbackMessage( | ||
| session, | ||
| rollbackState, | ||
| ); | ||
| } catch (error) { | ||
| this.log.error(`${fnTag} - Failed to create RollbackMessage: ${error}`); | ||
| throw new Error(`Error in createRollbackMessage: ${error}`); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.