diff --git a/src/gluonnlp/__init__.py b/src/gluonnlp/__init__.py
index bf6102f7ae..489854a7d4 100644
--- a/src/gluonnlp/__init__.py
+++ b/src/gluonnlp/__init__.py
@@ -1,14 +1,5 @@
 __version__ = '1.0.0.dev'
 from . import base
 from . import data
-from . import models
-from . import utils
-from . import attention_cell
-from . import initializer as init
-from . import layers
-from . import loss
-from . import lr_scheduler
-from . import op
-from . import torch
-from . import sequence_sampler
 from . import embedding
+from . import utils
diff --git a/src/gluonnlp/data/__init__.py b/src/gluonnlp/data/__init__.py
index d26d7c2534..ed053bf27c 100644
--- a/src/gluonnlp/data/__init__.py
+++ b/src/gluonnlp/data/__init__.py
@@ -3,4 +3,4 @@
 from . import batchify
 from .vocab import *
 
-__all__ = ['batchify', 'tokenizers'] + vocab.__all__
+__all__ = ['batchify', 'tokenizers', 'vocab'] + vocab.__all__
diff --git a/src/gluonnlp/mx/__init__.py b/src/gluonnlp/mx/__init__.py
new file mode 100644
index 0000000000..3ecb8d1d65
--- /dev/null
+++ b/src/gluonnlp/mx/__init__.py
@@ -0,0 +1,17 @@
+from ..utils.lazy_imports import try_import_mxnet
+try_import_mxnet()
+
+
+from . import data
+from . import models
+from . import utils
+from . import attention_cell
+from . import initializer as init
+from . import layers
+from . import loss
+from . import lr_scheduler
+from . import op
+from . import sequence_sampler
+
+__all__ = ['data', ' models', 'utils', 'attention_cell',
+           'init', 'layers', 'loss', 'lr_scheduler', 'op', 'sequence_sampler']
diff --git a/src/gluonnlp/attention_cell.py b/src/gluonnlp/mx/attention_cell.py
similarity index 100%
rename from src/gluonnlp/attention_cell.py
rename to src/gluonnlp/mx/attention_cell.py
diff --git a/src/gluonnlp/mx/data/__init__.py b/src/gluonnlp/mx/data/__init__.py
new file mode 100644
index 0000000000..5e66b4718b
--- /dev/null
+++ b/src/gluonnlp/mx/data/__init__.py
@@ -0,0 +1,4 @@
+from . import batchify
+from . import loading
+
+__all__ = ['batchify', 'loading']
diff --git a/src/gluonnlp/data/batchify.py b/src/gluonnlp/mx/data/batchify.py
similarity index 100%
rename from src/gluonnlp/data/batchify.py
rename to src/gluonnlp/mx/data/batchify.py
diff --git a/src/gluonnlp/data/loading.py b/src/gluonnlp/mx/data/loading.py
similarity index 100%
rename from src/gluonnlp/data/loading.py
rename to src/gluonnlp/mx/data/loading.py
diff --git a/src/gluonnlp/initializer.py b/src/gluonnlp/mx/initializer.py
similarity index 100%
rename from src/gluonnlp/initializer.py
rename to src/gluonnlp/mx/initializer.py
diff --git a/src/gluonnlp/layers.py b/src/gluonnlp/mx/layers.py
similarity index 100%
rename from src/gluonnlp/layers.py
rename to src/gluonnlp/mx/layers.py
diff --git a/src/gluonnlp/loss.py b/src/gluonnlp/mx/loss.py
similarity index 100%
rename from src/gluonnlp/loss.py
rename to src/gluonnlp/mx/loss.py
diff --git a/src/gluonnlp/lr_scheduler.py b/src/gluonnlp/mx/lr_scheduler.py
similarity index 100%
rename from src/gluonnlp/lr_scheduler.py
rename to src/gluonnlp/mx/lr_scheduler.py
diff --git a/src/gluonnlp/models/__init__.py b/src/gluonnlp/mx/models/__init__.py
similarity index 100%
rename from src/gluonnlp/models/__init__.py
rename to src/gluonnlp/mx/models/__init__.py
diff --git a/src/gluonnlp/models/albert.py b/src/gluonnlp/mx/models/albert.py
similarity index 100%
rename from src/gluonnlp/models/albert.py
rename to src/gluonnlp/mx/models/albert.py
diff --git a/src/gluonnlp/models/bart.py b/src/gluonnlp/mx/models/bart.py
similarity index 100%
rename from src/gluonnlp/models/bart.py
rename to src/gluonnlp/mx/models/bart.py
diff --git a/src/gluonnlp/models/base.py b/src/gluonnlp/mx/models/base.py
similarity index 100%
rename from src/gluonnlp/models/base.py
rename to src/gluonnlp/mx/models/base.py
diff --git a/src/gluonnlp/models/bert.py b/src/gluonnlp/mx/models/bert.py
similarity index 100%
rename from src/gluonnlp/models/bert.py
rename to src/gluonnlp/mx/models/bert.py
diff --git a/src/gluonnlp/models/electra.py b/src/gluonnlp/mx/models/electra.py
similarity index 100%
rename from src/gluonnlp/models/electra.py
rename to src/gluonnlp/mx/models/electra.py
diff --git a/src/gluonnlp/models/gpt2.py b/src/gluonnlp/mx/models/gpt2.py
similarity index 100%
rename from src/gluonnlp/models/gpt2.py
rename to src/gluonnlp/mx/models/gpt2.py
diff --git a/src/gluonnlp/models/mobilebert.py b/src/gluonnlp/mx/models/mobilebert.py
similarity index 100%
rename from src/gluonnlp/models/mobilebert.py
rename to src/gluonnlp/mx/models/mobilebert.py
diff --git a/src/gluonnlp/models/model_zoo_checksums/albert.txt b/src/gluonnlp/mx/models/model_zoo_checksums/albert.txt
similarity index 100%
rename from src/gluonnlp/models/model_zoo_checksums/albert.txt
rename to src/gluonnlp/mx/models/model_zoo_checksums/albert.txt
diff --git a/src/gluonnlp/models/model_zoo_checksums/bart.txt b/src/gluonnlp/mx/models/model_zoo_checksums/bart.txt
similarity index 100%
rename from src/gluonnlp/models/model_zoo_checksums/bart.txt
rename to src/gluonnlp/mx/models/model_zoo_checksums/bart.txt
diff --git a/src/gluonnlp/models/model_zoo_checksums/bert.txt b/src/gluonnlp/mx/models/model_zoo_checksums/bert.txt
similarity index 100%
rename from src/gluonnlp/models/model_zoo_checksums/bert.txt
rename to src/gluonnlp/mx/models/model_zoo_checksums/bert.txt
diff --git a/src/gluonnlp/models/model_zoo_checksums/electra.txt b/src/gluonnlp/mx/models/model_zoo_checksums/electra.txt
similarity index 100%
rename from src/gluonnlp/models/model_zoo_checksums/electra.txt
rename to src/gluonnlp/mx/models/model_zoo_checksums/electra.txt
diff --git a/src/gluonnlp/models/model_zoo_checksums/gpt2.txt b/src/gluonnlp/mx/models/model_zoo_checksums/gpt2.txt
similarity index 100%
rename from src/gluonnlp/models/model_zoo_checksums/gpt2.txt
rename to src/gluonnlp/mx/models/model_zoo_checksums/gpt2.txt
diff --git a/src/gluonnlp/models/model_zoo_checksums/mobilebert.txt b/src/gluonnlp/mx/models/model_zoo_checksums/mobilebert.txt
similarity index 100%
rename from src/gluonnlp/models/model_zoo_checksums/mobilebert.txt
rename to src/gluonnlp/mx/models/model_zoo_checksums/mobilebert.txt
diff --git a/src/gluonnlp/models/model_zoo_checksums/mt5.txt b/src/gluonnlp/mx/models/model_zoo_checksums/mt5.txt
similarity index 100%
rename from src/gluonnlp/models/model_zoo_checksums/mt5.txt
rename to src/gluonnlp/mx/models/model_zoo_checksums/mt5.txt
diff --git a/src/gluonnlp/models/model_zoo_checksums/roberta.txt b/src/gluonnlp/mx/models/model_zoo_checksums/roberta.txt
similarity index 100%
rename from src/gluonnlp/models/model_zoo_checksums/roberta.txt
rename to src/gluonnlp/mx/models/model_zoo_checksums/roberta.txt
diff --git a/src/gluonnlp/models/model_zoo_checksums/t5.txt b/src/gluonnlp/mx/models/model_zoo_checksums/t5.txt
similarity index 100%
rename from src/gluonnlp/models/model_zoo_checksums/t5.txt
rename to src/gluonnlp/mx/models/model_zoo_checksums/t5.txt
diff --git a/src/gluonnlp/models/model_zoo_checksums/xlmr.txt b/src/gluonnlp/mx/models/model_zoo_checksums/xlmr.txt
similarity index 100%
rename from src/gluonnlp/models/model_zoo_checksums/xlmr.txt
rename to src/gluonnlp/mx/models/model_zoo_checksums/xlmr.txt
diff --git a/src/gluonnlp/models/mt5.py b/src/gluonnlp/mx/models/mt5.py
similarity index 100%
rename from src/gluonnlp/models/mt5.py
rename to src/gluonnlp/mx/models/mt5.py
diff --git a/src/gluonnlp/models/roberta.py b/src/gluonnlp/mx/models/roberta.py
similarity index 100%
rename from src/gluonnlp/models/roberta.py
rename to src/gluonnlp/mx/models/roberta.py
diff --git a/src/gluonnlp/models/t5.py b/src/gluonnlp/mx/models/t5.py
similarity index 100%
rename from src/gluonnlp/models/t5.py
rename to src/gluonnlp/mx/models/t5.py
diff --git a/src/gluonnlp/models/transformer.py b/src/gluonnlp/mx/models/transformer.py
similarity index 100%
rename from src/gluonnlp/models/transformer.py
rename to src/gluonnlp/mx/models/transformer.py
diff --git a/src/gluonnlp/models/transformer_xl.py b/src/gluonnlp/mx/models/transformer_xl.py
similarity index 100%
rename from src/gluonnlp/models/transformer_xl.py
rename to src/gluonnlp/mx/models/transformer_xl.py
diff --git a/src/gluonnlp/models/xlmr.py b/src/gluonnlp/mx/models/xlmr.py
similarity index 100%
rename from src/gluonnlp/models/xlmr.py
rename to src/gluonnlp/mx/models/xlmr.py
diff --git a/src/gluonnlp/op.py b/src/gluonnlp/mx/op.py
similarity index 100%
rename from src/gluonnlp/op.py
rename to src/gluonnlp/mx/op.py
diff --git a/src/gluonnlp/sequence_sampler.py b/src/gluonnlp/mx/sequence_sampler.py
similarity index 100%
rename from src/gluonnlp/sequence_sampler.py
rename to src/gluonnlp/mx/sequence_sampler.py
diff --git a/src/gluonnlp/mx/utils/__init__.py b/src/gluonnlp/mx/utils/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/src/gluonnlp/utils/parameter.py b/src/gluonnlp/mx/utils/parameter.py
similarity index 100%
rename from src/gluonnlp/utils/parameter.py
rename to src/gluonnlp/mx/utils/parameter.py
diff --git a/src/gluonnlp/utils/testing.py b/src/gluonnlp/mx/utils/testing.py
similarity index 100%
rename from src/gluonnlp/utils/testing.py
rename to src/gluonnlp/mx/utils/testing.py
diff --git a/src/gluonnlp/torch/__init__.py b/src/gluonnlp/torch/__init__.py
index 8dc913ba65..d7b4c00480 100644
--- a/src/gluonnlp/torch/__init__.py
+++ b/src/gluonnlp/torch/__init__.py
@@ -1,4 +1,9 @@
+from ..utils.lazy_imports import try_import_torch
+try_import_torch()
+
 from . import attention_cell
 from . import layers
 from . import models
 from . import utils
+
+__all__ = ['attention_cell', 'layers', 'models', 'utils']
diff --git a/src/gluonnlp/utils/lazy_imports.py b/src/gluonnlp/utils/lazy_imports.py
index e2555697a3..62415897f0 100644
--- a/src/gluonnlp/utils/lazy_imports.py
+++ b/src/gluonnlp/utils/lazy_imports.py
@@ -27,7 +27,9 @@
            'try_import_boto3',
            'try_import_jieba',
            'try_import_tvm',
-           'try_import_wikiextractor']
+           'try_import_wikiextractor',
+           'try_import_torch',
+           'try_import_mxnet']
 
 
 def try_import_sentencepiece():
@@ -180,3 +182,19 @@ def try_import_wikiextractor():
                           'refer to the official guide in '
                           'https://github.com/attardi/wikiextractor.')
     return wikiextractor
+
+
+def try_import_torch():
+    try:
+        import torch
+    except ImportError:
+        raise ImportError('PyTorch is not installed. You may install PyTorch by referring to the '
+                          'official guide in https://pytorch.org/.')
+
+
+def try_import_mxnet():
+    try:
+        import mxnet
+    except ImportError:
+        raise ImportError('MXNet is not installed. You may install MXNet by referring to the '
+                          'official guide in https://mxnet.apache.org/.')
diff --git a/tests/test_attention_cell.py b/tests/mxnet/test_attention_cell.py
similarity index 100%
rename from tests/test_attention_cell.py
rename to tests/mxnet/test_attention_cell.py