diff --git a/BUILD b/BUILD
index 097665c738..e9a0d19546 100644
--- a/BUILD
+++ b/BUILD
@@ -20,5 +20,6 @@ sh_binary(
         "required_packages.py",
         "setup.py",
         "//tensorflow_probability",
+        "//tensorflow_probability/substrates",
     ],
 )
diff --git a/conftest.py b/conftest.py
index abc9bec2cc..56206d8ea8 100644
--- a/conftest.py
+++ b/conftest.py
@@ -20,7 +20,6 @@
 collect_ignore = [
     "discussion/",
     "setup.py",
-    "tensorflow_probability/python/experimental/substrates/"
 ]
 
 
diff --git a/tensorflow_probability/python/BUILD b/tensorflow_probability/python/BUILD
index 3a11d8d473..1431949e91 100644
--- a/tensorflow_probability/python/BUILD
+++ b/tensorflow_probability/python/BUILD
@@ -41,6 +41,7 @@ py_library(
         "//tensorflow_probability/python/glm",
         "//tensorflow_probability/python/internal",
         "//tensorflow_probability/python/internal:all_util",
+        "//tensorflow_probability/python/internal:lazy_loader",
         "//tensorflow_probability/python/layers",
         "//tensorflow_probability/python/math",
         "//tensorflow_probability/python/mcmc",
diff --git a/tensorflow_probability/python/experimental/BUILD b/tensorflow_probability/python/experimental/BUILD
index 6ae164fef5..f955ce118a 100644
--- a/tensorflow_probability/python/experimental/BUILD
+++ b/tensorflow_probability/python/experimental/BUILD
@@ -41,7 +41,6 @@ multi_substrate_py_library(
         "//tensorflow_probability/python/experimental/auto_batching",
         "//tensorflow_probability/python/experimental/marginalize",
         "//tensorflow_probability/python/experimental/nn",
-        "//tensorflow_probability/python/experimental/substrates",
         "//tensorflow_probability/python/experimental/timeseries",
         "//tensorflow_probability/python/internal:auto_composite_tensor",
         "//tensorflow_probability/python/experimental/util:composite_tensor",
@@ -64,7 +63,6 @@ multi_substrate_py_library(
         "//tensorflow_probability/python/experimental/sequential",
         "//tensorflow_probability/python/experimental/stats",
         "//tensorflow_probability/python/experimental/sts_gibbs",
-        "//tensorflow_probability/python/experimental/substrates",
         "//tensorflow_probability/python/experimental/tangent_spaces",
         "//tensorflow_probability/python/experimental/timeseries",
         "//tensorflow_probability/python/experimental/util",
diff --git a/tensorflow_probability/python/experimental/__init__.py b/tensorflow_probability/python/experimental/__init__.py
index 58a72d64ae..d316a44d90 100644
--- a/tensorflow_probability/python/experimental/__init__.py
+++ b/tensorflow_probability/python/experimental/__init__.py
@@ -43,7 +43,6 @@
 from tensorflow_probability.python.experimental import sequential
 from tensorflow_probability.python.experimental import stats
 from tensorflow_probability.python.experimental import sts_gibbs
-from tensorflow_probability.python.experimental import substrates
 from tensorflow_probability.python.experimental import tangent_spaces
 from tensorflow_probability.python.experimental import timeseries
 from tensorflow_probability.python.experimental import util
@@ -76,7 +75,6 @@
     'sequential',
     'sts_gibbs',
     'stats',
-    'substrates',
     'tangent_spaces',
     'timeseries',
     'unnest',
diff --git a/tensorflow_probability/python/experimental/substrates/BUILD b/tensorflow_probability/python/experimental/substrates/BUILD
deleted file mode 100644
index c7f3bd836d..0000000000
--- a/tensorflow_probability/python/experimental/substrates/BUILD
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright 2019 The TensorFlow Probability Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-# Description:
-#   API-unstable code that is part of the TFP package.
-
-# Placeholder: py_library
-
-package(
-    # default_applicable_licenses
-    default_visibility = [
-        "//tensorflow_probability:__subpackages__",
-    ],
-)
-
-licenses(["notice"])
-
-py_library(
-    name = "substrates",
-    srcs = ["__init__.py"],
-    deps = [
-        "//tensorflow_probability/python/internal:all_util",
-        "//tensorflow_probability/python/internal:lazy_loader",
-        "//tensorflow_probability/substrates",
-    ],
-)
diff --git a/tensorflow_probability/python/experimental/substrates/__init__.py b/tensorflow_probability/python/experimental/substrates/__init__.py
deleted file mode 100644
index 6cc75af087..0000000000
--- a/tensorflow_probability/python/experimental/substrates/__init__.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# Copyright 2019 The TensorFlow Probability Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""TensorFlow Probability alternative substrates."""
-
-from tensorflow_probability.python.internal import all_util
-from tensorflow_probability.python.internal import lazy_loader
-
-jax = lazy_loader.LazyLoader(
-    'jax', globals(),
-    'tensorflow_probability.substrates.jax')
-numpy = lazy_loader.LazyLoader(
-    'numpy', globals(),
-    'tensorflow_probability.substrates.numpy')
-
-
-_allowed_symbols = [
-    'jax',
-    'numpy',
-]
-
-all_util.remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow_probability/python/internal/lazy_loader.py b/tensorflow_probability/python/internal/lazy_loader.py
index 9cac7eceac..c42f0cca5d 100644
--- a/tensorflow_probability/python/internal/lazy_loader.py
+++ b/tensorflow_probability/python/internal/lazy_loader.py
@@ -19,6 +19,9 @@
 import types
 
 
+__all__ = ['LazyLoader']
+
+
 class LazyLoader(types.ModuleType):
   """Lazily import a module to avoid pulling in large deps, defer checks."""
 
diff --git a/tensorflow_probability/substrates/BUILD b/tensorflow_probability/substrates/BUILD
index 3f99a48001..5a186f4cfa 100644
--- a/tensorflow_probability/substrates/BUILD
+++ b/tensorflow_probability/substrates/BUILD
@@ -32,6 +32,8 @@ py_library(
     srcs = [
         "__init__.py",
     ],
+    # :substrates needs to be visible to the external :pip_pkg target.
+    visibility = ["//visibility:public"],  # EnableOnExport
     deps = [
         ":jax",
         ":numpy",
diff --git a/tensorflow_probability/substrates/meta/rewrite.py b/tensorflow_probability/substrates/meta/rewrite.py
index fd6dea9444..f856bdaf1a 100644
--- a/tensorflow_probability/substrates/meta/rewrite.py
+++ b/tensorflow_probability/substrates/meta/rewrite.py
@@ -258,6 +258,9 @@ def disable_all(name):
   if FLAGS.numpy_to_jax:
     contents = contents.replace('tfp.substrates.numpy', 'tfp.substrates.jax')
     contents = contents.replace('substrates.numpy', 'substrates.jax')
+    contents = contents.replace(
+        'tensorflow_probability.substrates import numpy',
+        'tensorflow_probability.substrates import jax')
     contents = contents.replace('backend.numpy', 'backend.jax')
     contents = contents.replace('backend import numpy as tf',
                                 'backend import jax as tf')
diff --git a/tensorflow_probability/tools/build_docs.py b/tensorflow_probability/tools/build_docs.py
index d832748ca3..2adb3b21a6 100644
--- a/tensorflow_probability/tools/build_docs.py
+++ b/tensorflow_probability/tools/build_docs.py
@@ -46,12 +46,7 @@
 
 FLAGS = flags.FLAGS
 
-DO_NOT_GENERATE_DOCS_FOR = [
-    tfp.experimental.substrates.jax.tf2jax,
-    tfp.experimental.substrates.jax.experimental,
-    tfp.experimental.substrates.numpy.tf2numpy,
-    tfp.experimental.substrates.numpy.experimental,
-]
+DO_NOT_GENERATE_DOCS_FOR = []
 
 
 def internal_filter(path, parent, children):