diff --git a/src/isolate/logger.py b/src/isolate/logger.py
index 3065b9e..424514e 100644
--- a/src/isolate/logger.py
+++ b/src/isolate/logger.py
@@ -9,6 +9,8 @@
 # but it handling `source` would be not trivial, so we are better off
 # just keeping it simple for now.
 class IsolateLogger:
+    extra_labels: Dict[str, str] = {}
+
     def __init__(self, log_labels: Dict[str, str]):
         self.log_labels = log_labels
 
@@ -18,6 +20,7 @@ def log(self, level: LogLevel, message: str, source: LogSource) -> None:
             "level": level.name,
             "message": message,
             **self.log_labels,
+            **self.extra_labels,
         }
         print(json.dumps(record))
 
diff --git a/src/isolate/server/server.py b/src/isolate/server/server.py
index d06cb52..6e171e0 100644
--- a/src/isolate/server/server.py
+++ b/src/isolate/server/server.py
@@ -367,9 +367,7 @@ def SetMetadata(
 
         task = self.background_tasks[request.task_id]
 
-        task.logger = IsolateLogger.with_env_expanded(
-            dict(request.metadata.logger_labels)
-        )
+        task.logger.extra_labels = dict(request.metadata.logger_labels)
 
         return definitions.SetMetadataResponse()