diff --git a/continuous_integration/condarc b/continuous_integration/condarc
index 258ac5fa5d..ee8a8d2539 100644
--- a/continuous_integration/condarc
+++ b/continuous_integration/condarc
@@ -1,6 +1,5 @@
 channels:
   - conda-forge
-  - defaults
 channel_priority: true
 auto_activate_base: false
 remote_backoff_factor: 20
diff --git a/continuous_integration/environment-3.10.yaml b/continuous_integration/environment-3.10.yaml
index a3209963dc..d06b4bd212 100644
--- a/continuous_integration/environment-3.10.yaml
+++ b/continuous_integration/environment-3.10.yaml
@@ -1,7 +1,6 @@
 name: dask-distributed
 channels:
   - conda-forge
-  - defaults
 dependencies:
   - python=3.10
   - packaging
@@ -38,7 +37,6 @@ dependencies:
   - pytest-rerunfailures
   - pytest-timeout
   - python-snappy  # Only tested here
-  - pytorch  # Only tested here
   - requests
   - s3fs  # overridden by git tip below
   - scikit-learn
@@ -46,7 +44,6 @@ dependencies:
   - sortedcollections
   - tblib
   - toolz
-  - torchvision  # Only tested here
   - tornado
   - zict  # overridden by git tip below
   - zstandard
@@ -61,3 +58,5 @@ dependencies:
       # - git+https://github.com/dask/s3fs
       # - git+https://github.com/fsspec/filesystem_spec
       - keras
+      - torch  # Only tested here (using pip due to lack of Windows conda-forge builds)
+      - torchvision  # Only tested here (using pip due to lack of Windows conda-forge builds)
diff --git a/continuous_integration/environment-3.11.yaml b/continuous_integration/environment-3.11.yaml
index dfc2a56be7..994b70a3e9 100644
--- a/continuous_integration/environment-3.11.yaml
+++ b/continuous_integration/environment-3.11.yaml
@@ -1,7 +1,6 @@
 name: dask-distributed
 channels:
   - conda-forge
-  - defaults
 dependencies:
   - python=3.11
   - packaging
diff --git a/continuous_integration/environment-3.12.yaml b/continuous_integration/environment-3.12.yaml
index 39a313d5ef..bc2213bc85 100644
--- a/continuous_integration/environment-3.12.yaml
+++ b/continuous_integration/environment-3.12.yaml
@@ -1,7 +1,6 @@
 name: dask-distributed
 channels:
   - conda-forge
-  - defaults
 dependencies:
   - python=3.12
   - packaging