From 6586ab589598cc703cc0a8d363bc1ade850158a7 Mon Sep 17 00:00:00 2001 From: Sepehr Ganji Date: Fri, 16 Feb 2024 22:29:18 -0700 Subject: [PATCH 1/4] Setup healthcheck skel --- README.md | 6 ++++++ src/health-check.ts | 10 ++++++++++ src/index.ts | 18 ++++++++++++------ 3 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 src/health-check.ts diff --git a/README.md b/README.md index 320358a..31da1f8 100644 --- a/README.md +++ b/README.md @@ -56,3 +56,9 @@ npm run test ``` npm run lint ``` + +### HealthCheck + +``` +/.well-known/apollo/server-health +``` \ No newline at end of file diff --git a/src/health-check.ts b/src/health-check.ts new file mode 100644 index 0000000..a36a6bc --- /dev/null +++ b/src/health-check.ts @@ -0,0 +1,10 @@ +export const checkHealth = () => { + const checks = {}; + + const healthy = Object.values(checks).every(() => true); + + return { + msg: "Custom message", + healthy + } +}; diff --git a/src/index.ts b/src/index.ts index 0616c2d..f4a82b9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,7 +11,7 @@ import responseCachePlugin from "apollo-server-plugin-response-cache"; import { GraphQLSchema } from "graphql"; import depthLimit from "graphql-depth-limit"; import { blockWatcher } from "./block-watcher"; -import { redisClient } from "./caching"; +// import { redisClient } from "./caching"; import { DEFAULT_MAX_QUERY_DEPTH, MAX_CACHE_AGE } from "./consts"; import { DatabaseContext } from "./context/database-context"; import { initializeDataSource } from "./data-source"; @@ -40,9 +40,9 @@ async function startServer(schema: GraphQLSchema, dataContext: DatabaseContext) defaultMaxAge: MAX_CACHE_AGE, calculateHttpHeaders: true }), - responseCachePlugin({ - cache: new BaseRedisCache({ client: redisClient }) - }), + // responseCachePlugin({ + // cache: new BaseRedisCache({ client: redisClient }) + // }), ApolloServerPluginLandingPageGraphQLPlayground() ], validationRules: [ @@ -55,14 +55,20 @@ async function startServer(schema: GraphQLSchema, dataContext: DatabaseContext) } } ) - ] + ], + onHealthCheck: () => { + return new Promise((resolve, reject) => { + if(1) resolve("Something should go here"); + else reject(); + }) + } }); const { url } = await server.listen({ port: 3000 }); console.log(`🚀 Server ready at ${url}`); } async function startBlockWatcher(dataContext: DatabaseContext) { - blockWatcher.start(dataContext.headers).onNewBlock(() => redisClient.flushdb()); + blockWatcher.start(dataContext.headers); console.log("🚀 Block watcher started"); } From 11d04de18c72dbb0cd4cfd76da045155f74bae29 Mon Sep 17 00:00:00 2001 From: SepehrGanji Date: Thu, 22 Feb 2024 16:44:56 -0700 Subject: [PATCH 2/4] Add db HC --- src/context/database-context.ts | 6 ++++++ src/health-check.ts | 22 +++++++++++++++------- src/index.ts | 18 +++++++----------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/context/database-context.ts b/src/context/database-context.ts index 8bda6da..9b8d78a 100644 --- a/src/context/database-context.ts +++ b/src/context/database-context.ts @@ -14,6 +14,7 @@ import { TokenRepository } from "./token-repository"; import { BlocksRepository } from "./blocks-repository"; export class DatabaseContext { + private dataSource: DataSource; public readonly transactions!: TransactionRepository; public readonly blockInfo!: BlocksRepository; public readonly boxes!: BoxRepository; @@ -28,6 +29,7 @@ export class DatabaseContext { public readonly unconfirmedInputs!: UnconfirmedInputRepository; constructor(dataSource: DataSource) { + this.dataSource = dataSource; const context: RepositoryDataContext = { dataSource, graphQLDataLoader: new GraphQLDatabaseLoader(dataSource, { disableCache: true }) @@ -48,4 +50,8 @@ export class DatabaseContext { this.dataInputs = new BaseRepository(DataInputEntity, "dti", { context, defaults }); this.inputs = new BaseRepository(InputEntity, "input", { context, defaults }); } + + checkConnection = () => { + return this.dataSource.isInitialized; + }; } diff --git a/src/health-check.ts b/src/health-check.ts index a36a6bc..211ec86 100644 --- a/src/health-check.ts +++ b/src/health-check.ts @@ -1,10 +1,18 @@ -export const checkHealth = () => { - const checks = {}; +import { DatabaseContext } from "./context/database-context"; - const healthy = Object.values(checks).every(() => true); +export const checkHealth = async(dataContext: DatabaseContext): Promise => { + /** + * Things to check: + * - Database connection + * - Redis connection + * - Node connection + * - Block watcher + */ + const checks = { + db: dataContext.checkConnection + }; - return { - msg: "Custom message", - healthy - } + // const healthy = Object.values(checks).every(() => true); + // There's no feature in apollo-server-express to return custom error messages, so we leave it empty + throw new Error(""); }; diff --git a/src/index.ts b/src/index.ts index f4a82b9..b947679 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,12 +11,13 @@ import responseCachePlugin from "apollo-server-plugin-response-cache"; import { GraphQLSchema } from "graphql"; import depthLimit from "graphql-depth-limit"; import { blockWatcher } from "./block-watcher"; -// import { redisClient } from "./caching"; +import { redisClient } from "./caching"; import { DEFAULT_MAX_QUERY_DEPTH, MAX_CACHE_AGE } from "./consts"; import { DatabaseContext } from "./context/database-context"; import { initializeDataSource } from "./data-source"; import { generateSchema } from "./graphql/schema"; import { nodeService } from "./services"; +import { checkHealth } from "./health-check"; const { TS_NODE_DEV, MAX_QUERY_DEPTH } = process.env; @@ -40,9 +41,9 @@ async function startServer(schema: GraphQLSchema, dataContext: DatabaseContext) defaultMaxAge: MAX_CACHE_AGE, calculateHttpHeaders: true }), - // responseCachePlugin({ - // cache: new BaseRedisCache({ client: redisClient }) - // }), + responseCachePlugin({ + cache: new BaseRedisCache({ client: redisClient }) + }), ApolloServerPluginLandingPageGraphQLPlayground() ], validationRules: [ @@ -56,19 +57,14 @@ async function startServer(schema: GraphQLSchema, dataContext: DatabaseContext) } ) ], - onHealthCheck: () => { - return new Promise((resolve, reject) => { - if(1) resolve("Something should go here"); - else reject(); - }) - } + onHealthCheck: checkHealth.bind(null, dataContext) }); const { url } = await server.listen({ port: 3000 }); console.log(`🚀 Server ready at ${url}`); } async function startBlockWatcher(dataContext: DatabaseContext) { - blockWatcher.start(dataContext.headers); + blockWatcher.start(dataContext.headers).onNewBlock(() => redisClient.flushdb()); console.log("🚀 Block watcher started"); } From 1d23b62a9a3567ca6ca39ecc649dafab706a7fc0 Mon Sep 17 00:00:00 2001 From: SepehrGanji Date: Sat, 24 Feb 2024 17:49:09 -0700 Subject: [PATCH 3/4] add redis healthcheck --- src/health-check.ts | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/health-check.ts b/src/health-check.ts index 211ec86..940986e 100644 --- a/src/health-check.ts +++ b/src/health-check.ts @@ -1,18 +1,35 @@ +import { redisClient } from "./caching"; import { DatabaseContext } from "./context/database-context"; -export const checkHealth = async(dataContext: DatabaseContext): Promise => { +export const checkHealth = async(dataContext: DatabaseContext): Promise => { /** * Things to check: - * - Database connection - * - Redis connection + * - Database connection (Done) + * - Redis connection (Done) * - Node connection * - Block watcher */ + const checks = { - db: dataContext.checkConnection + db: () => dataContext.checkConnection, + redis: async() => (await redisClient.mget("test")).length === 1, }; - // const healthy = Object.values(checks).every(() => true); - // There's no feature in apollo-server-express to return custom error messages, so we leave it empty - throw new Error(""); + const results = await Promise.all( + Object.entries(checks).map(async ([key, func]) => ({ + [key]: await func(), + })) + ); + + let isAnyFailed = false; + for(const result of results) { + for(const [key, value] of Object.entries(result)) { + if(!value) { + console.error(`🚫 Health check failed for ${key}`); + isAnyFailed = true; + } + } + } + + if(isAnyFailed) throw new Error("Health check failed"); }; From 325730df6eb62cc3f968b629885a03211873671d Mon Sep 17 00:00:00 2001 From: SepehrGanji Date: Tue, 27 Feb 2024 17:44:04 -0700 Subject: [PATCH 4/4] finialize healthChecks --- src/block-watcher.ts | 9 +++++++++ src/health-check.ts | 30 +++++++++++++----------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/block-watcher.ts b/src/block-watcher.ts index 3ef62e6..bdd9392 100644 --- a/src/block-watcher.ts +++ b/src/block-watcher.ts @@ -6,9 +6,12 @@ class BlockWatcher { private _lastBlockHeight!: number; private _timer!: NodeJS.Timeout; + private _healthy: boolean; + constructor() { this._callbacks = []; this._lastBlockHeight = 0; + this._healthy = true; } public onNewBlock(callback: (height?: number) => void): BlockWatcher { @@ -45,6 +48,7 @@ class BlockWatcher { repository .getMaxHeight() .then((height) => { + this._healthy = true; if (!height || this._lastBlockHeight >= height) { return; } @@ -57,10 +61,15 @@ class BlockWatcher { this._notify(height); }) .catch((e) => { + this._healthy = true; this._notify(); console.error(e); }); } + + public isHealthy(): boolean { + return this._healthy; + } } export const blockWatcher = new BlockWatcher(); diff --git a/src/health-check.ts b/src/health-check.ts index 940986e..8cb5609 100644 --- a/src/health-check.ts +++ b/src/health-check.ts @@ -1,35 +1,31 @@ +import { blockWatcher } from "./block-watcher"; import { redisClient } from "./caching"; import { DatabaseContext } from "./context/database-context"; +import { nodeService } from "./services"; -export const checkHealth = async(dataContext: DatabaseContext): Promise => { - /** - * Things to check: - * - Database connection (Done) - * - Redis connection (Done) - * - Node connection - * - Block watcher - */ - +export const checkHealth = async (dataContext: DatabaseContext): Promise => { const checks = { - db: () => dataContext.checkConnection, - redis: async() => (await redisClient.mget("test")).length === 1, + db: () => () => dataContext.checkConnection, + redis: async () => (await redisClient.mget("test")).length === 1, + node: async () => (await nodeService.getNodeInfo()).status === 200, + blockWatcher: () => blockWatcher.isHealthy() }; const results = await Promise.all( Object.entries(checks).map(async ([key, func]) => ({ - [key]: await func(), + [key]: await func() })) ); let isAnyFailed = false; - for(const result of results) { - for(const [key, value] of Object.entries(result)) { - if(!value) { - console.error(`🚫 Health check failed for ${key}`); + for (const result of results) { + for (const [key, value] of Object.entries(result)) { + if (!value) { + console.error(`🚫 ${key} is not healthy.`); isAnyFailed = true; } } } - if(isAnyFailed) throw new Error("Health check failed"); + if (isAnyFailed) throw new Error("Service is unhealthy."); };