diff --git a/pdns/dnsdist.cc b/pdns/dnsdist.cc
index cf3f6129e947c..67382a04bbfa8 100644
--- a/pdns/dnsdist.cc
+++ b/pdns/dnsdist.cc
@@ -868,7 +868,7 @@ void responderThread(std::shared_ptr<DownstreamState> dss)
           continue;
         }
 
-        if (fd != ids->backendFD) {
+        if (!ids->isXSK() && fd != ids->backendFD) {
           dss->restoreState(queryId, std::move(*ids));
           continue;
         }
@@ -1992,8 +1992,6 @@ bool XskProcessQuery(ClientState& cs, LocalHolders& holders, XskPacket& packet)
       return false;
     }
     else {
-      const auto& xskInfo = ss->pickWorkerForSending();
-      ids.backendFD = xskInfo->workerWaker;
       assignOutgoingUDPQueryToBackend(ss, dh->id, dq, query, false);
       auto sourceAddr = ss->pickSourceAddressForSending();
       packet.setAddr(sourceAddr, ss->d_config.sourceMACAddr, ss->d_config.remote, ss->d_config.destMACAddr);
diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh
index 52019d56e2851..4dba2db925f80 100644
--- a/pdns/dnsdist.hh
+++ b/pdns/dnsdist.hh
@@ -995,7 +995,6 @@ public:
 #ifdef HAVE_XSK
   void registerXsk(std::vector<std::shared_ptr<XskSocket>>& xsks);
   [[nodiscard]] ComboAddress pickSourceAddressForSending();
-  [[nodiscard]] const std::shared_ptr<XskWorker>& pickWorkerForSending();
 #endif /* HAVE_XSK */
 
   dnsdist::Protocol getProtocol() const
diff --git a/pdns/dnsdistdist/dnsdist-backend.cc b/pdns/dnsdistdist/dnsdist-backend.cc
index e2dcd6b9fa95f..0da27de984ce5 100644
--- a/pdns/dnsdistdist/dnsdist-backend.cc
+++ b/pdns/dnsdistdist/dnsdist-backend.cc
@@ -887,16 +887,6 @@ void DownstreamState::submitHealthCheckResult(bool initial, bool newResult)
   return (*addresses)[idx % numberOfAddresses];
 }
 
-[[nodiscard]] const std::shared_ptr<XskWorker>& DownstreamState::pickWorkerForSending()
-{
-  auto numberOfWorkers = d_xskInfos.size();
-  if (numberOfWorkers == 0) {
-    throw std::runtime_error("No XSK worker available for sending XSK data to backend " + getNameWithAddr());
-  }
-  size_t idx = dnsdist::getRandomValue(numberOfWorkers);
-  return d_xskInfos[idx % numberOfWorkers];
-}
-
 void DownstreamState::registerXsk(std::vector<std::shared_ptr<XskSocket>>& xsks)
 {
   d_xskSockets = xsks;
diff --git a/pdns/dnsdistdist/dnsdist-xsk.cc b/pdns/dnsdistdist/dnsdist-xsk.cc
index fc77706aa0209..058e381908da4 100644
--- a/pdns/dnsdistdist/dnsdist-xsk.cc
+++ b/pdns/dnsdistdist/dnsdist-xsk.cc
@@ -42,7 +42,6 @@ void XskResponderThread(std::shared_ptr<DownstreamState> dss, std::shared_ptr<Xs
     auto localRespRuleActions = g_respruleactions.getLocal();
     auto localCacheInsertedRespRuleActions = g_cacheInsertedRespRuleActions.getLocal();
     auto pollfds = getPollFdsForWorker(*xskInfo);
-    const auto xskFd = xskInfo->workerWaker.getHandle();
     while (!dss->isStopped()) {
       poll(pollfds.data(), pollfds.size(), -1);
       bool needNotify = false;
@@ -63,7 +62,6 @@ void XskResponderThread(std::shared_ptr<DownstreamState> dss, std::shared_ptr<Xs
           auto ids = dss->getState(queryId);
           if (ids) {
             if (!ids->isXSK()) {
-              // if (xskFd != ids->backendFD || !ids->isXSK()) {
               dss->restoreState(queryId, std::move(*ids));
               ids = std::nullopt;
             }