!885 sync commit to master

Merge pull request !885 from Lin/update-master

configs/README.md

@@ -121,6 +121,14 @@ configs统一在run_xxx.yaml中，排序按照修改频率的顺序和一般的
     - model_config: 模型参数配置
         - type: 模型参数配置类
         - checkpoint_name_or_path: 评估时不指定权重，模型默认加载的权重名
+
+          *\# 以下配置针对大规模语言模型推理*
+        - top_k: 从概率最大的top_k个tokens中采样
+        - top_p: 从概率最大且概率累计不超过top_p的tokens中采样
+        - do_sample: 使能top_k或top_p采样，为False时top_k和top_p均重置为1
+        - use_past: 使能增量推理，为True时为增量推理，否则为自回归推理，使用时请参考[模型支持列表](https://gitee.com/mindspore/mindformers/tree/dev/docs#text-generator)
+        - max_decode_length: 文本生成最大长度（输入长度统计在内）
+        - repetition_penalty: 重复文本惩罚系数，该值不小于1，等于1时不惩罚
 - lr_schedule: 学习率配置
     - type: 学习率类
 - layer_scale: 是否开启层衰减

docs/README.md

@@ -48,9 +48,9 @@ swr.cn-central-221.ovaijisuan.com/mindformers/mindformers_dev_mindspore_2_0:mind
 
 ### AICC镜像
 
-**详情请参考[MindFormers AICC使用教程](https://gitee.com/mindspore/mindformers/blob/dev/docs/aicc_cards/aicc_tutorial.md)**
+**详情请参考[MindFormers AICC使用教程](https://gitee.com/mindspore/mindformers/blob/dev/docs/aicc_cards/README.md)**
 
-我们在[镜像仓库网 (hqcases.com)](https://gitee.com/link?target=http%3A%2F%2Fai.hqcases.com%2Fmirrors.html)上发布了一些经过验证的**标准镜像版本**，可以通过几行简单的docker命令的形式，直接使用验证过的标准镜像拉起MindFormers套件的训练任务，而无需进行较为繁琐的自定义镜像并上传的步骤。
+我们在[镜像仓库网 (hqcases.com)](http://mirrors.cn-central-221.ovaijisuan.com/)上发布了一些经过验证的**标准镜像版本**，可以通过几行简单的docker命令的形式，直接使用验证过的标准镜像拉起MindFormers套件的训练任务，而无需进行较为繁琐的自定义镜像并上传的步骤。
 
 - 镜像列表
 

docs/aicc_cards/README.md

@@ -47,9 +47,9 @@ git clone https://gitee.com/mindspore/mindformers.git
 
 ModelArts上的所提供的预置训练镜像，通常MindSpore的版本较为老旧，不满足MindFormers套件的运行需求，所以通常需要自定义镜像，安装合适版本的MindSpore包以运行套件
 
-我们在[镜像仓库网 (hqcases.com)](http://ai.hqcases.com/mirrors.html)上发布了一些经过验证的**标准镜像版本**，可以通过几行简单的docker命令的形式，直接使用验证过的标准镜像拉起MindFormers套件的训练任务，而无需进行较为繁琐的自定义镜像并上传的步骤
+我们在[镜像仓库网 (hqcases.com)](http://mirrors.cn-central-221.ovaijisuan.com/)上发布了一些经过验证的**标准镜像版本**，可以通过几行简单的docker命令的形式，直接使用验证过的标准镜像拉起MindFormers套件的训练任务，而无需进行较为繁琐的自定义镜像并上传的步骤
 
-- 在镜像仓库网上找到当前版本的[MindFormers镜像](http://ai.hqcases.com/detail/78.html)，打开显示如下
+- 在镜像仓库网上找到当前版本的[MindFormers镜像](http://mirrors.cn-central-221.ovaijisuan.com/detail/78.html)，打开显示如下
 
 ![aihub](asserts/aihub.png)
 

mindformers/dataset/dataloader/flickr8k_dataloader.py

@@ -95,7 +95,7 @@ def __new__(cls, dataset_dir: str,
 
         kwargs.pop("None", None)
         flick8k_dataset = Flickr8kDataSet(dataset_dir, annotation_dir, stage)
-        return GeneratorDataset(flick8k_dataset, column_names)
+        return GeneratorDataset(flick8k_dataset, column_names, **kwargs)
 
 class Flickr8kDataSet:
     """Flickr8k DataSet"""

mindformers/models/llama/llama_config.py

@@ -33,7 +33,7 @@ class LlamaConfig(BaseConfig):
     _support_list = MindFormerBook.get_config_support_list()['llama']
 
     def __init__(self,
-                 batch_size: int = None,
+                 batch_size: int = 1,
                  seq_length: int = 2048,
                  hidden_size: int = 4096,
                  num_layers: int = 32,

mindformers/models/text_generator.py

@@ -389,7 +389,8 @@ def generate(self,
                  pad_token_id: Optional[int] = None,
                  repetition_penalty: Optional[float] = None,
                  max_length: Optional[int] = None,
-                 streamer: Optional[BaseStreamer] = None):
+                 streamer: Optional[BaseStreamer] = None,
+                 seed: Optional[int] = None):
         """
         Generate the words according to the given the input ids.
 
@@ -413,6 +414,7 @@ def generate(self,
             max_length: The maximum length of the generated words. If set None, it follows the setting in the
                 configureation in the model. Default None.
             streamer: The streamer that generator uses.
+            seed: Random seed used in sample.
 
 
         Examples:
@@ -450,6 +452,8 @@ def generate(self,
         eos_token_id = config.eos_token_id if eos_token_id is None else eos_token_id
         pad_token_id = config.pad_token_id if pad_token_id is None else pad_token_id
         do_sample = config.do_sample if do_sample is None else do_sample
+        seed = 0 if seed is None else seed
+        np.random.seed(seed)
 
         if not do_sample:
             top_p = 1

mindformers/pipeline/text_generation_pipeline.py

@@ -102,7 +102,7 @@ def _sanitize_parameters(self, **pipeline_parameters):
 
         postprocess_params = {}
 
-        forward_key_name = ['top_k', 'top_p', 'do_sample', 'eos_token_id', 'repetition_penalty', 'max_length']
+        forward_key_name = ['top_k', 'top_p', 'do_sample', 'eos_token_id', 'repetition_penalty', 'max_length', 'seed']
         forward_kwargs = {}
         for item in forward_key_name:
             if item in pipeline_parameters:

mindformers/tools/transform_ckpt.py

@@ -78,7 +78,7 @@ def get_strategy(startegy_path, rank_id=None):
     dst_ckpt_dir = args.dst_ckpt_dir
     prefix = args.prefix
 
-    assert os.path.exists(args.src_ckpt), f'{args.src_ckpt} not found!'
+    assert os.path.exists(args.src_ckpt_dir), f'{args.src_ckpt_dir} not found!'
 
     print(f"src_ckpt_strategy: {src_ckpt_strategy}")
     print(f"dst_ckpt_strategy: {dst_ckpt_strategy}")

mindformers/trainer/base_trainer.py

@@ -13,14 +13,15 @@
 # limitations under the License.
 # ============================================================================
 """Base Trainer."""
+import math
 import os
 import shutil
 from functools import partial
 from typing import Optional, Union, List
 import numpy as np
 from PIL.Image import Image
 import mindspore as ms
-from mindspore import Tensor
+from mindspore import Tensor, DatasetHelper
 from mindspore.train.model import Model
 from mindspore.train import Callback
 from mindspore.dataset import GeneratorDataset
@@ -516,6 +517,12 @@ def training_process(
             dataset = self.create_train_dataset()
         self.set_train_dataset(dataset)
         check_runner_config(config, dataset)
+        if config.runner_config.sink_mode:
+            epoch_num = math.ceil((config.runner_config.epochs - config.runner_config.initial_epoch)
+                                  * config.runner_config.sink_size / dataset.get_dataset_size())
+            # pylint: disable=W0212
+            dataset._dataset_helper = DatasetHelper(dataset, config.runner_config.sink_mode,
+                                                    config.runner_config.sink_size, epoch_num)
 
         # build network
         logger.info(".........Build Net For Train..........")
@@ -571,7 +578,6 @@ def training_process(
                 logger.info(".............Start resume training from checkpoint..................")
                 transform_and_load_checkpoint(config, model, network, dataset, optimizer=optimizer)
             else:
-                logger.info(".............Start load checkpoint from checkpoint..................")
                 if config.load_checkpoint in SUPPORT_MODEL_NAMES:
                     config.load_checkpoint = \
                         AutoModel.from_pretrained(config.load_checkpoint).default_checkpoint_download_path
@@ -586,7 +592,7 @@ def training_process(
                 "dataset_size": config.data_size,
                 "micro_batch_interleave_num": config.micro_batch_interleave_num,
                 "micro_batch_num": config.parallel_config.micro_batch_num,
-                "initial_epoch": config.runner_config.initial_epoch if config.runner_config.initial_epoch else 0})
+                "initial_epoch": config.runner_config.initial_epoch})
 
         # build evaluate in training
         if config.do_eval:
@@ -612,7 +618,7 @@ def training_process(
                     callbacks=callbacks,
                     dataset_sink_mode=config.runner_config.sink_mode,
                     sink_size=config.runner_config.sink_size,
-                    initial_epoch=config.runner_config.initial_epoch if config.runner_config.initial_epoch else 0)
+                    initial_epoch=config.runner_config.initial_epoch)
         logger.info(".........Training Over!.............")
 
     def evaluate_process(

mindformers/trainer/utils.py

@@ -111,6 +111,8 @@ def check_runner_config(config, dataset):
     data_size = dataset.get_dataset_size()
     new_epochs = config.runner_config.epochs
     config.runner_config.origin_epochs = new_epochs
+    if config.runner_config.initial_epoch is None:
+        config.runner_config.initial_epoch = 0
     if config.runner_config.sink_mode:
         if config.runner_config.sink_size != -1:
             if config.runner_config.sink_size <= 0:
@@ -277,8 +279,8 @@ def transform_and_load_checkpoint(config, model, network, dataset, optimizer=Non
     3. load checkpoint params into dict
     4. load params into net
     """
-    if not os.path.realpath(config.load_checkpoint) or \
-            not os.path.exists(config.load_checkpoint):
+    if not config.only_save_strategy and (not os.path.realpath(config.load_checkpoint) or
+                                          not os.path.exists(config.load_checkpoint)):
         raise FileNotFoundError(f"The load_checkpoint must be correct, "
                                 f"but get {config.load_checkpoint}")
 
@@ -301,7 +303,11 @@ def transform_and_load_checkpoint(config, model, network, dataset, optimizer=Non
                         sink_size=config.runner_config.sink_size)
 
         if config.only_save_strategy:
-            raise SystemExit("Only_save_strategy is True, model.build() finished, system exit! ")
+            raise SystemExit("Only_save_strategy is True, model.compile() finished, system exit! ")
+
+    elif config.only_save_strategy:
+        raise SystemExit("only_save_strategy is True, "
+                         "but stand_alone and data_parallel mode do not have strategy file, system exit! ")
 
     # 2. transform checkpoint if needed
 
@@ -310,6 +316,8 @@ def transform_and_load_checkpoint(config, model, network, dataset, optimizer=Non
             not os.path.exists(config.load_checkpoint):
         raise FileNotFoundError(f"The load_checkpoint must be correct, "
                                 f"but get {config.load_checkpoint}")
+
+    logger.info(".............Start load checkpoint from checkpoint..................")
     if os.path.isdir(config.load_checkpoint):
         checkpoint_dict = load_distributed_checkpoint(config)
     else:

tests/st/test_model/test_bert_model/test_autoclass.py

@@ -18,7 +18,6 @@
 pytest tests/st/test_model/test_bert_model/test_auto_class.py
 """
 import os
-import pytest
 
 import mindspore as ms
 
@@ -28,18 +27,13 @@
 ms.set_context(mode=0)
 
 
-@pytest.mark.level0
-@pytest.mark.platform_x86_ascend_training
-@pytest.mark.platform_arm_ascend_training
-@pytest.mark.env_onecard
 class TestBertAutoClassMethod:
     """A test class for testing Model classes"""
     def setup_method(self):
         """setup method."""
-        self.save_directory = os.path.join(MindFormerBook.get_project_path(), 'checkpoint_save')
+        self.save_directory = MindFormerBook.get_default_checkpoint_save_folder()
         self.test_list = ['bert_base_uncased']
 
-    @pytest.mark.run(order=1)
     def test_auto_model(self):
         """
         Feature: AutoModel.
@@ -54,7 +48,6 @@ def test_auto_model(self):
                 save_directory=os.path.join(self.save_directory, model_type),
                 save_name=model_type + '_model')
 
-    @pytest.mark.run(order=2)
     def test_auto_config(self):
         """
         Feature: AutoConfig.
@@ -69,7 +62,6 @@ def test_auto_config(self):
                 save_directory=os.path.join(self.save_directory, config_type),
                 save_name=config_type + '_config')
 
-    @pytest.mark.run(order=3)
     def test_auto_processor(self):
         """
         Feature: AutoProcessor.
@@ -84,7 +76,6 @@ def test_auto_processor(self):
                 save_directory=os.path.join(self.save_directory, processor_type),
                 save_name=processor_type + '_processor')
 
-    @pytest.mark.run(order=4)
     def test_auto_tokenizer(self):
         """
         Feature: AutoTokenizer, input config.

tests/st/test_model/test_bert_model/test_pipeline.py

@@ -17,18 +17,13 @@
 How to run this:
 pytest tests/st/test_model/test_bert_model/test_pipeline.py
 """
-import pytest
 import mindspore as ms
 
 from mindformers import pipeline
 
 ms.set_context(mode=0)
 
 
-@pytest.mark.level0
-@pytest.mark.platform_x86_ascend_training
-@pytest.mark.platform_arm_ascend_training
-@pytest.mark.env_onecard
 class TestBertPipelineMethod:
     """A test class for testing pipeline."""
     def setup_method(self):

tests/st/test_model/test_bloom_model/test_autoclass.py

@@ -19,7 +19,6 @@
 pytest tests/st/test_model/test_bloom_model/test_autoclass.py
 """
 import os
-import pytest
 
 import mindspore as ms
 
@@ -29,19 +28,14 @@
 ms.set_context(mode=0)
 
 
-@pytest.mark.level0
-@pytest.mark.platform_x86_ascend_training
-@pytest.mark.platform_arm_ascend_training
-@pytest.mark.env_onecard
 class TestBloomAutoClassMethod:
     """A test class for testing Model classes"""
 
     def setup_method(self):
         """setup method."""
-        self.save_directory = os.path.join(MindFormerBook.get_project_path(), 'checkpoint_save')
+        self.save_directory = MindFormerBook.get_default_checkpoint_save_folder()
         self.test_llm_list = ['bloom_560m']
 
-    @pytest.mark.run(order=1)
     def test_llm_model(self):
         """
         Feature: AutoModel.
@@ -56,7 +50,6 @@ def test_llm_model(self):
                 save_directory=os.path.join(self.save_directory, model_type),
                 save_name=model_type + '_model')
 
-    @pytest.mark.run(order=2)
     def test_llm_config(self):
         """
         Feature: AutoConfig.
@@ -71,7 +64,6 @@ def test_llm_config(self):
                 save_directory=os.path.join(self.save_directory, config_type),
                 save_name=config_type + '_config')
 
-    @pytest.mark.run(order=3)
     def test_llm_processor(self):
         """
         Feature: AutoConfig.
@@ -86,7 +78,6 @@ def test_llm_processor(self):
                 save_directory=os.path.join(self.save_directory, processor_type),
                 save_name=processor_type + '_processor')
 
-    @pytest.mark.run(order=4)
     def test_llm_tokenizer(self):
         """
         Feature: AutoTokenizer, input config.

tests/st/test_model/test_bloom_model/test_pipeline.py

@@ -17,19 +17,13 @@
 How to run this:
 pytest tests/st/test_model/test_bloom_model/test_pipeline.py
 """
-import pytest
-
 import mindspore as ms
 
 from mindformers import pipeline
 
 ms.set_context(mode=0)
 
 
-@pytest.mark.level0
-@pytest.mark.platform_x86_ascend_training
-@pytest.mark.platform_arm_ascend_training
-@pytest.mark.env_onecard
 class TestBloomPipelineMethod:
     """A test class for testing pipeline."""
     def setup_method(self):

tests/st/test_model/test_clip_model/test_clip_model.py

@@ -22,9 +22,8 @@
 """
 import os
 import time
-import pytest
 
-import  mindspore as ms
+import mindspore as ms
 
 from mindformers import MindFormerBook, AutoConfig, AutoModel
 from mindformers.models import CLIPModel, BaseModel
@@ -33,10 +32,6 @@
 ms.set_context(mode=0)
 
 
-@pytest.mark.level0
-@pytest.mark.platform_x86_ascend_training
-@pytest.mark.platform_arm_ascend_training
-@pytest.mark.env_onecard
 class TestCLIPModelMethod:
     """A test class for testing Model classes"""
     def setup_method(self):

tests/st/test_model/test_clip_model/test_clip_tokenizer.py

@@ -36,9 +36,6 @@ def generate_fake_vocab(output_path):
             fp.write(item + '\n')
 
 
-@pytest.mark.level0
-@pytest.mark.platform_x86_cpu
-@pytest.mark.env_onecard
 class TestAutoTokenizerMethod:
     """A test class for testing the AutoTokenizer"""
     def setup_method(self):
@@ -74,9 +71,6 @@ def test_load_from_yaml(self):
         assert res == ['hello</w>', 'world</w>', '?</w>']
 
 
-@pytest.mark.level0
-@pytest.mark.platform_x86_cpu
-@pytest.mark.env_onecard
 class TestPretrainedTokenizerMethod:
     """A test class for testing the PretrainedTokenizer"""
     def generate_fake_vocab(self):
@@ -105,9 +99,6 @@ def test_from_pretrained_tokenizer(self):
             tokenizer("hello world")
 
 
-@pytest.mark.level0
-@pytest.mark.platform_x86_cpu
-@pytest.mark.env_onecard
 class TestClipTokenizerMethod:
     """Test the basic usage of the CLIPTokenizer"""
     def test_padding(self):