diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 5a713d8..9da8ba4 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.23.0
+current_version = 1.23.1
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)([-](?P<release>(pre|rc))(?P<build>\d+))?
diff --git a/servicelayer/__init__.py b/servicelayer/__init__.py
index 41ea992..0254b45 100644
--- a/servicelayer/__init__.py
+++ b/servicelayer/__init__.py
@@ -1,6 +1,6 @@
 import logging
 
-__version__ = "1.23.0"
+__version__ = "1.23.1"
 
 logging.getLogger("boto3").setLevel(logging.WARNING)
 logging.getLogger("botocore").setLevel(logging.WARNING)
diff --git a/servicelayer/taskqueue.py b/servicelayer/taskqueue.py
index af0a654..d85a473 100644
--- a/servicelayer/taskqueue.py
+++ b/servicelayer/taskqueue.py
@@ -63,6 +63,9 @@ def increment_retry_count(self, conn):
         conn.incr(self.retry_key)
         conn.expire(self.retry_key, settings.REDIS_EXPIRE)
 
+    def clear_retry_count(self, conn):
+        conn.delete(self.retry_key)
+
     def get_dataset(self, conn):
         dataset = Dataset(
             conn=conn, name=dataset_from_collection_id(self.collection_id)
@@ -317,9 +320,6 @@ def mark_for_retry(self, task):
         pipe.srem(make_key(PREFIX, "qds", self.name, stage, "running"), task_id)
         pipe.srem(make_key(PREFIX, "qds", self.name, stage), task_id)
 
-        # delete the retry key for the task
-        pipe.delete(task.retry_key)
-
         pipe.set(self.last_update_key, pack_now())
 
         pipe.execute()
@@ -595,6 +595,7 @@ def handle(self, task: Task, channel) -> Tuple[bool, bool]:
             log.exception(
                 f"Task {task.task_id} permanently failed and will be discarded."
             )
+            task.clear_retry_count(self.conn)
             success = False
             retry = False
         except Exception:
diff --git a/setup.py b/setup.py
index 8ba8069..25e490e 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name="servicelayer",
-    version="1.23.0",
+    version="1.23.1",
     description="Basic remote service functions for alephdata components",
     classifiers=[
         "Development Status :: 3 - Alpha",
diff --git a/tests/test_taskqueue.py b/tests/test_taskqueue.py
index b493272..b5be595 100644
--- a/tests/test_taskqueue.py
+++ b/tests/test_taskqueue.py
@@ -113,8 +113,9 @@ def test_task_queue(self):
                         delivery_tag=1, multiple=False, requeue=False
                     )
             assert "Max retries reached for task test-task. Aborting." in ctx.output[0]
-            # Assert that retry count stays the same
-            assert task.get_retry_count(conn) == 1
+            # Assert that the retry count is 0 and the key is gone
+            assert task.get_retry_count(conn) == 0
+            assert conn.get("tq:qdj:2:taskretry:test-task") is None
 
         worker.ack_message(worker.test_task, channel)
         status = dataset.get_status()