From 3e6ecd41abc92966bc1df3d9eebd577a3b879e5a Mon Sep 17 00:00:00 2001 From: narayana Date: Wed, 10 Jan 2024 13:14:21 +0530 Subject: [PATCH 01/11] health check --- requirements.txt | 2 +- sharq_server/server.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 504e982..39b1d1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ itsdangerous==0.24 msgpack==0.5.6 ujson==2.0.0 uWSGI==2.0.21 -SharQ==1.3.0 +git@github.com:plivo/sharq.git@worker-status \ No newline at end of file diff --git a/sharq_server/server.py b/sharq_server/server.py index 3a2e825..0295d66 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -56,6 +56,9 @@ def __init__(self, config_path): self.app.add_url_rule( '/deepstatus/', view_func=self._view_deep_status, methods=['GET']) + self.app.add_url_rule( + '/workerhealthstatus/', + view_func=self._view_worker_health_status, methods=['GET']) def requeue(self): """Loop endlessly and requeue expired jobs.""" @@ -263,6 +266,21 @@ def _view_deep_status(self): for line in traceback.format_exc().splitlines(): print(line) raise Exception + def _view_worker_health_status(self): + """Checks worker health status""" + try: + key = self.config.get('region', 'msg-worker-nps:us-west-1') + self.sq.worker_health_status(key) + response = { + 'status': "success" + } + return jsonify(**response) + except Exception as e: + print(e) + import traceback + for line in traceback.format_exc().splitlines(): + print(line) + raise Exception def _view_clear_queue(self, queue_type, queue_id): """remove queue from SharQ based on the queue_type and queue_id.""" From 5dd008933ed3dd25a1e941550c6445159641cbc2 Mon Sep 17 00:00:00 2001 From: narayana Date: Wed, 10 Jan 2024 13:58:12 +0530 Subject: [PATCH 02/11] update name --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 39b1d1e..1609a84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ itsdangerous==0.24 msgpack==0.5.6 ujson==2.0.0 uWSGI==2.0.21 -git@github.com:plivo/sharq.git@worker-status \ No newline at end of file +git+https://github.com/plivo/sharq.git@worker-status \ No newline at end of file From 9df01030e86047257a37582a0bb524393e53e4a4 Mon Sep 17 00:00:00 2001 From: narayana Date: Wed, 10 Jan 2024 14:22:39 +0530 Subject: [PATCH 03/11] skip --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1056106..929109d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,5 +4,6 @@ deliveryPipeline ([ buildContainer: 'plivo/jenkins-ci/python/2.7.14/ci-base/ubuntu/trusty:18.02.01.139', - disableQAStages: true + disableQAStages: true, + trivyBypass: true ]) From 4de6484f7e687c0049e7b47207744bd8b16ef025 Mon Sep 17 00:00:00 2001 From: narayana Date: Wed, 10 Jan 2024 15:03:26 +0530 Subject: [PATCH 04/11] worker health check basic auth fix --- src/config/nginx-sharq.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/config/nginx-sharq.conf b/src/config/nginx-sharq.conf index f4d81dd..2e72892 100644 --- a/src/config/nginx-sharq.conf +++ b/src/config/nginx-sharq.conf @@ -15,6 +15,11 @@ server { uwsgi_pass unix:///var/run/sharq/sharq.sock; include uwsgi_params; } + location /workerhealthstatus/ { + log_not_found off; + uwsgi_pass unix:///var/run/sharq/sharq.sock; + include uwsgi_params; + } location / { # Not needed because it's all in the VPC From 635bc8c2d86b3a1ddc6062054e1a1b0353d96009 Mon Sep 17 00:00:00 2001 From: narayana Date: Wed, 10 Jan 2024 15:45:38 +0530 Subject: [PATCH 05/11] health check key added --- sharq.conf | 3 ++- src/config/sharq.conf | 3 ++- src/config/sharq.conf.ctmpl | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sharq.conf b/sharq.conf index 9588f84..f2f2114 100644 --- a/sharq.conf +++ b/sharq.conf @@ -18,4 +18,5 @@ unix_socket_path : /tmp/redis.sock ;; tcp connection settings port : 6379 host : 127.0.0.1 -clustered : true +clustered : true +worker_health_key : msg-worker-nps:us-east-1 diff --git a/src/config/sharq.conf b/src/config/sharq.conf index 22a0ed3..4f29881 100644 --- a/src/config/sharq.conf +++ b/src/config/sharq.conf @@ -11,4 +11,5 @@ unix_socket_path = /tmp/redis.sock port = 6379 host = 127.0.0.1 clustered = false -password = hello \ No newline at end of file +password = hello +worker_health_key = msg-worker-nps:us-east-1 \ No newline at end of file diff --git a/src/config/sharq.conf.ctmpl b/src/config/sharq.conf.ctmpl index 2d39fc5..9885140 100644 --- a/src/config/sharq.conf.ctmpl +++ b/src/config/sharq.conf.ctmpl @@ -22,3 +22,4 @@ port : {{ printf "%s/%s/%s/%s/config/redis/port" $team $appenv $ host : {{ printf "%s/%s/%s/%s/config/redis/host" $team $appenv $sharq_type $region | key }} clustered : {{ printf "%s/%s/%s/%s/config/redis/clustered" $team $appenv $sharq_type $region | key }} password : {{ printf "%s/%s/%s/%s/config/redis/password" $team $appenv $sharq_type $region | key }} +worker_health_key : {{ printf "%s/%s/%s/%s/config/redis/worker_health_key" $team $appenv $sharq_type $region | key }} \ No newline at end of file From afb879eac069676d5ca6245ca28fa4f16addfcea Mon Sep 17 00:00:00 2001 From: narayana Date: Wed, 10 Jan 2024 15:49:49 +0530 Subject: [PATCH 06/11] fix --- docs/configuration.rst | 1 + sharq_server/server.py | 2 +- src/config/sharq.conf | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 5f9ff7b..2d86e88 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -119,3 +119,4 @@ A sample configuration file looks like this. You can also get this configuration ;; tcp connection settings port : 6379 host : 127.0.0.1 + worker_health_key : msg-worker-nps:us-east-1 \ No newline at end of file diff --git a/sharq_server/server.py b/sharq_server/server.py index 0295d66..ee87a9e 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -269,7 +269,7 @@ def _view_deep_status(self): def _view_worker_health_status(self): """Checks worker health status""" try: - key = self.config.get('region', 'msg-worker-nps:us-west-1') + key = self.config.get('redis', 'worker_health_key') self.sq.worker_health_status(key) response = { 'status': "success" diff --git a/src/config/sharq.conf b/src/config/sharq.conf index 4f29881..7bbc3a9 100644 --- a/src/config/sharq.conf +++ b/src/config/sharq.conf @@ -12,4 +12,4 @@ port = 6379 host = 127.0.0.1 clustered = false password = hello -worker_health_key = msg-worker-nps:us-east-1 \ No newline at end of file +worker_health_key = msg-worker-nps:us-east-1 \ No newline at end of file From d8ae6b64b84bc08cd0f17600dcf9013a9c3823e6 Mon Sep 17 00:00:00 2001 From: narayana Date: Wed, 10 Jan 2024 16:29:44 +0530 Subject: [PATCH 07/11] value check --- sharq_server/server.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sharq_server/server.py b/sharq_server/server.py index ee87a9e..c1ebfa1 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -270,7 +270,12 @@ def _view_worker_health_status(self): """Checks worker health status""" try: key = self.config.get('redis', 'worker_health_key') - self.sq.worker_health_status(key) + value = self.sq.worker_health_status(key) + if value is None: + response = { + 'status': "failure", + } + return jsonify(**response), 500 response = { 'status': "success" } From 3385b73f96105df8c1ebbd5b40050ea012584956 Mon Sep 17 00:00:00 2001 From: narayana-plivo Date: Mon, 29 Jan 2024 17:37:44 +0530 Subject: [PATCH 08/11] health check fix --- sharq_server/server.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sharq_server/server.py b/sharq_server/server.py index c1ebfa1..f2c4f86 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -269,16 +269,16 @@ def _view_deep_status(self): def _view_worker_health_status(self): """Checks worker health status""" try: + response = {} key = self.config.get('redis', 'worker_health_key') - value = self.sq.worker_health_status(key) - if value is None: - response = { - 'status': "failure", - } - return jsonify(**response), 500 - response = { - 'status': "success" - } + with self.sq.redis_client().lock('worker-health-lock-key', timeout=2): + value = self.sq.worker_health_status(key) + if value is None: + return jsonify(**response), 500 + return jsonify(**response) + except LockError: + # the lock wasn't acquired within specified time + print("worker-health-lock-key lock not acquired", key) return jsonify(**response) except Exception as e: print(e) From 35b62ba9d592459bab4324e3d2d1d55a0717049f Mon Sep 17 00:00:00 2001 From: narayana-plivo Date: Mon, 29 Jan 2024 21:39:32 +0530 Subject: [PATCH 09/11] debug log --- sharq_server/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sharq_server/server.py b/sharq_server/server.py index f2c4f86..b9bf41c 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -273,6 +273,7 @@ def _view_worker_health_status(self): key = self.config.get('redis', 'worker_health_key') with self.sq.redis_client().lock('worker-health-lock-key', timeout=2): value = self.sq.worker_health_status(key) + print(key, value) if value is None: return jsonify(**response), 500 return jsonify(**response) From 8b954062eaacd8a3dc7e36ac3b9ba9a066902906 Mon Sep 17 00:00:00 2001 From: narayana-plivo Date: Mon, 29 Jan 2024 22:01:36 +0530 Subject: [PATCH 10/11] debug log --- sharq_server/server.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sharq_server/server.py b/sharq_server/server.py index b9bf41c..11a6c66 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -271,16 +271,16 @@ def _view_worker_health_status(self): try: response = {} key = self.config.get('redis', 'worker_health_key') - with self.sq.redis_client().lock('worker-health-lock-key', timeout=2): - value = self.sq.worker_health_status(key) - print(key, value) - if value is None: - return jsonify(**response), 500 - return jsonify(**response) - except LockError: - # the lock wasn't acquired within specified time - print("worker-health-lock-key lock not acquired", key) + # with self.sq.redis_client().lock('worker-health-lock-key', timeout=2): + value = self.sq.worker_health_status(key) + print(key, value) + if value is None: + return jsonify(**response), 500 return jsonify(**response) + # except LockError: + # the lock wasn't acquired within specified time + # print("worker-health-lock-key lock not acquired", key) + # return jsonify(**response) except Exception as e: print(e) import traceback From 89a5bf821fa7b09a3aba969f1fc3b7f68cd06ece Mon Sep 17 00:00:00 2001 From: narayana-plivo Date: Tue, 30 Jan 2024 11:18:19 +0530 Subject: [PATCH 11/11] fix --- sharq_server/server.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sharq_server/server.py b/sharq_server/server.py index 11a6c66..508662f 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -271,16 +271,16 @@ def _view_worker_health_status(self): try: response = {} key = self.config.get('redis', 'worker_health_key') - # with self.sq.redis_client().lock('worker-health-lock-key', timeout=2): - value = self.sq.worker_health_status(key) - print(key, value) - if value is None: - return jsonify(**response), 500 - return jsonify(**response) - # except LockError: + with self.sq.redis_client().lock('worker-health-lock-key', timeout=2): # will be changed later + value = self.sq.worker_health_status(key) + print(key, value) + if value is None: + return jsonify(**response), 500 + return jsonify(**response) + except LockError: # the lock wasn't acquired within specified time - # print("worker-health-lock-key lock not acquired", key) - # return jsonify(**response) + print("worker-health-lock-key lock not acquired", key) + return jsonify(**response) except Exception as e: print(e) import traceback