bugfix

src/llamafactory/data/preprocess.py

@@ -19,7 +19,7 @@
 from .processors.pairwise import preprocess_pairwise_dataset, print_pairwise_dataset_example
 from .processors.pretrain import preprocess_pretrain_dataset
 from .processors.supervised import (
-    preprocess_packed_supervised_dataset,
+    preprocess_packed_supervised_dataset,preprocess_packed_supervised_dataset_fullDataGroup,
     preprocess_supervised_dataset,
     print_supervised_dataset_example,
 )
@@ -64,7 +64,8 @@ def __init__(self, data, **kwargs):
 
                 OptimizedTypedSequence.__init__ = __init__
             preprocess_func = partial(
-                preprocess_packed_supervised_dataset,
+                #preprocess_packed_supervised_dataset,
+                preprocess_packed_supervised_dataset_fullDataGroup,
                 template=template,
                 tokenizer=tokenizer,
                 processor=processor,

src/llamafactory/data/processors/supervised.py

@@ -180,7 +180,7 @@ def preprocess_packed_supervised_dataset(
         packed_input_ids, packed_attention_masks, packed_labels = [], [], []
         packed_images, packed_videos = [], []
         for i, length in enumerate(knapsack):
-            index = length2indexes[length].pop()
+            index = length2indexes[length].pop() ## this place is losing data samples, very difficult to fix when use length as key ,so introduce preprocess_packed_supervised_dataset_fullDataGroup
             packed_input_ids += batch_input_ids[index]
             packed_labels += batch_labels[index]
             packed_images += batch_images[index]
@@ -217,3 +217,138 @@ def print_supervised_dataset_example(example: Dict[str, List[int]], tokenizer: "
     print("inputs:\n{}".format(tokenizer.decode(example["input_ids"], skip_special_tokens=False)))
     print("label_ids:\n{}".format(example["labels"]))
     print(f"labels:\n{tokenizer.decode(valid_labels, skip_special_tokens=False)}")
+
+
+
+
+def pack_data_points_by_length(
+    lengths: List[int], max_length: int, max_size: int = -1
+) -> List[List[int]]:
+    """given lengths of data points, we merge consecutive data points into a new data point, as long as the concatenated length is less than max_length
+    Args:
+        lengths (List[int]): List of lengths of data points
+        max_length (int): the concatenated length must be less than or equal max_length
+        max_size: if != -1; the maximum number of consecutive items being merged; max_size: -1 --> no limit for number of items being merged
+
+    max_size: the maximum number of data points being merged
+    For example, lengths=[1, 3, 2, 2, 6, 4, 2, 6, 5]; max_length=10
+    if max_size=-1 --> [[0,1,2,3], [4, 5], [6,7], [8]]
+    if max_size=3 --> [[0,1,2], [3,4], [5, 6], [7], [8]]
+
+    Returns:
+        _type_: groups of indices: [[index1, index2, ...], [], ...]
+    """
+    result = []
+    current_concatenated_length = 0
+    current_list = []
+    for i in range(len(lengths)):
+        cur_length = lengths[i]
+        if cur_length + current_concatenated_length <= max_length and (
+            max_size == -1 or len(current_list) < max_size
+        ):
+            current_concatenated_length += cur_length
+            current_list.append(i)
+        else:  # current_list is done, create a new one
+            if len(current_list) > 0:
+                result.append(current_list)
+            current_list = [i]
+            current_concatenated_length = cur_length
+
+    if len(current_list) > 0:
+        result.append(current_list)
+
+    # assert to make sure no indices were missing
+    assert sum([len(indices) for indices in result]) == len(lengths)
+    return result #
+def preprocess_packed_supervised_dataset_fullDataGroup(
+    examples: Dict[str, List[Any]],
+    template: "Template",
+    tokenizer: "PreTrainedTokenizer",
+    processor: Optional["ProcessorMixin"],
+    data_args: "DataArguments",
+) -> Dict[str, List[Any]]:
+    # TODO: use `position_ids` to achieve packing
+    # build inputs with format `<bos> X1 Y1 <eos> <bos> X2 Y2 <eos>`
+    # and labels with format `<ignore> ... <ignore> Y1 <eos> <ignore> ... <ignore> Y2 <eos>`
+    valid_num = 0
+    batch_input_ids, batch_labels, batch_images, batch_videos = [], [], [], []
+    lengths = []
+    length2indexes = defaultdict(list)
+    for i in range(len(examples["_prompt"])):
+        if len(examples["_prompt"][i]) % 2 != 1 or len(examples["_response"][i]) != 1:
+            logger.warning_rank0(
+                "Dropped invalid example: {}".format(examples["_prompt"][i] + examples["_response"][i])
+            )
+            continue
+
+        input_ids, labels = _encode_supervised_example(
+            prompt=examples["_prompt"][i],
+            response=examples["_response"][i],
+            system=examples["_system"][i],
+            tools=examples["_tools"][i],
+            images=examples["_images"][i] or [],
+            videos=examples["_videos"][i] or [],
+            template=template,
+            tokenizer=tokenizer,
+            processor=processor,
+            cutoff_len=data_args.cutoff_len - 1,  # reserved for the padding token
+            train_on_prompt=data_args.train_on_prompt,
+            mask_history=data_args.mask_history,
+        )
+        length = len(input_ids)
+        if length > data_args.cutoff_len:
+            logger.warning_rank0(f"Dropped lengthy example with length {length} > {data_args.cutoff_len}.")
+        else:
+            lengths.append(length)
+            length2indexes[length].append(valid_num)
+            batch_input_ids.append(input_ids)
+            batch_labels.append(labels)
+            batch_images.append(examples["_images"][i] or [])
+            batch_videos.append(examples["_videos"][i] or [])
+            valid_num += 1
+
+    model_inputs = defaultdict(list);
+
+    lengthll=lengths;
+    noDegenerateGroups=pack_data_points_by_length(lengthll,data_args.cutoff_len - 1,-1)
+    for group in noDegenerateGroups:### each group
+        packed_input_ids, packed_attention_masks, packed_labels = [], [], []
+        packed_images, packed_videos = [], []
+        packed_posid = []  ### 1group
+        for i,index in enumerate(group):## each sample
+
+            packed_input_ids += batch_input_ids[index];
+            if data_args.neat_packing:
+                batch_labels[index][0]=IGNORE_INDEX
+            packed_labels +=  batch_labels[index]
+            packed_images += batch_images[index]
+            packed_videos += batch_videos[index]
+            packed_posid += list(range(len(batch_input_ids[index])))
+            if data_args.neat_packing:
+                packed_attention_masks += [i + 1] * len(batch_input_ids[index])  # start from 1
+            else:
+                packed_attention_masks += [1] * len(batch_input_ids[index])
+
+        ### pad
+        if len(packed_input_ids) < data_args.cutoff_len:  ### pad
+            pad_length = data_args.cutoff_len - len(packed_input_ids)
+            packed_input_ids += [tokenizer.pad_token_id] * pad_length
+            packed_labels += [IGNORE_INDEX] * pad_length
+            packed_posid += [0] * pad_length
+            if data_args.neat_packing:
+                packed_attention_masks += [0] * pad_length
+            else:
+                packed_attention_masks += [1] * pad_length  # more efficient flash_attn
+
+        if len(packed_input_ids) != data_args.cutoff_len:
+            raise ValueError("The length of packed example should be identical to the cutoff length.")
+
+        model_inputs["input_ids"].append(packed_input_ids)
+        model_inputs["attention_mask"].append(packed_attention_masks)
+        model_inputs["labels"].append(packed_labels)
+        model_inputs["images"].append(packed_images or None)
+        model_inputs["videos"].append(packed_videos or None)
+        if posidflag:
+            model_inputs['position_ids'].append(packed_posid)
+
+    return model_inputs