From ad9e8510910549aa7fa3f16f1fefa5ef4bf9487e Mon Sep 17 00:00:00 2001 From: yunss-ML Date: Sat, 26 Aug 2023 22:20:58 +0100 Subject: [PATCH] still documentation net done yet --- README.md | 23 ++++++++++++++--------- core/Quantized.py | 20 +++++++++++++------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f8f1a54..1e4008f 100644 --- a/README.md +++ b/README.md @@ -10,26 +10,24 @@ -## Comparative Features of "loralib" and "loratorch" Implementations +### Features **Distinguishing the "loralib" and "loratorch" Approaches for Implementation** The implementations of "loralib" and "loratorch" exhibit distinct methodologies, particularly when using the example of `nn.Linear`. The underlying mathematical representations are as follows: -1. * **loralib** Approach +1. **LoRa** Approaches The computation is defined as: - \[ - h = x W_0^\top + \frac{\alpha}{r} x(BA)^\top, - \] + $h = x W_0^\top + \frac{\alpha}{r} x(BA)^\top,$ - where: + $where: - `x` is an input matrix of dimensions \(k \times n\), - `W_0` is a pre-trained weight matrix of dimensions \(m \times n\), - `r` is a predefined LoRA rank, - `B` and `A` are LoRA matrices of dimensions \(m \times r\) and \(r \times n\) respectively, - - `\alpha` is a hyper-parameter. + - `\alpha` is a hyper-parameter.$ 1. For ``loralib``, @@ -40,7 +38,8 @@ where $x\in\mathbb{R}^{k\times n}$ is the input matrix, $W_0\in\mathbb{R}^{m\tim 2. For ``loratorch``, $h = x (W_0 + \frac{\alpha}{r} BA)^\top.$ -``loralib`` computes $xW_0^\top$ and $x(BA)^\top$ respectively and then merges the results. While ``loratorch`` merges pre-trained weight $W_0$ and its LoRA weight $BA$ and then computes the results by simply using ``nn.Linear.forward()``. There is no difference between ``loralib`` and ``loratorch`` in the linear layers. But in some no-linear or complex layers, we are no sure whether this layer satisfies $L(x, W_0)+L(x, BA) = L(x, W_0+BA)$. Hence, it is difficult to extend LoRA to some complex layers by using ``loralib``. On the contrary, the idea of merging weights first in ``loratorch`` is more general and extensible. You just call ``merge_lora_param()`` in ``loratorch`` to merge weights and then call ``forward()`` in the original layer to compute the results. With the help of ``loratorch``, you can easily implement LoRA to any type of layer of ``torch.nn``. +``loralib`` computes $xW_0^\top$ and $x(BA)^\top$ respectively and then merges the results. +While ``loratorch`` merges pre-trained weight $W_0$ and its LoRA weight $BA$ and then computes the results by simply using ``nn.Linear.forward()``. There is no difference between ``loralib`` and ``loratorch`` in the linear layers. But in some no-linear or complex layers, we are no sure whether this layer satisfies $L(x, W_0)+L(x, BA) = L(x, W_0+BA)$. Hence, it is difficult to extend LoRA to some complex layers by using ``loralib``. On the contrary, the idea of merging weights first in ``loratorch`` is more general and extensible. You just call ``merge_lora_param()`` in ``loratorch`` to merge weights and then call ``forward()`` in the original layer to compute the results. With the help of ``loratorch``, you can easily implement LoRA to any type of layer of ``torch.nn``. ## Supported Layers @@ -55,7 +54,6 @@ where $x\in\mathbb{R}^{k\times n}$ is the input matrix, $W_0\in\mathbb{R}^{m\tim | ``MergedLinear`` | ✓ (Error) | ✓ | [mergedlinear.ipynb](https://github.com/Baijiong-Lin/LoRA-Torch/blob/main/examples/mergedlinear.ipynb) | | $\cdots$ | hard to extend | easy to extend | | -*We compare the results of ``loralib`` and ``loratorch`` in [examples](./examples) to demonstrate the correctness of the implementation in ``loratorch``.* ## Quick Start @@ -155,6 +153,13 @@ i am providing Tool that are ready-to-use for Quantize the model: Go over to the Transfomer-Based-specific directory that you are interested in, and open the ```README.md```. We have included details about the LLMs, followed by performance results on open-source datasets! +### Methods Supports Quantization +the supports method for Quantize the Transfomer-Based Models + +- [x] LoRa +- [x] LoRaTorch +- [x] QLoRA + ## Roadmap Our plan is to perform these experiments on all the Transformer-Based model below. To that end, this is a tentative roadmap of the LLMs that we aim to cover: diff --git a/core/Quantized.py b/core/Quantized.py index b7d6141..dbcfd25 100644 --- a/core/Quantized.py +++ b/core/Quantized.py @@ -25,25 +25,27 @@ def __init__(self, model: nn.Module,LoRa=None,BitSand=None, method: str, Rank: i self.LORA = LoRa self.BITSAND = BitSand self.model = model - self.layer = [] + self.Instance_Layer = [] + self.layertyep = [] if method in self.Adapters: self.method = self.Adapters[method] else: raise ValueError("Invalid method provided") - def add_layer(self, layer: str): + def add_layer_and_Instance_Layer(self,layertyep:str ,layer: str): """ Add a layer to the list of layers to be adapted. Args: layer (str): The name of the layer to add. - + layerTyep(str): The layer nn.Linear or nn.Embedding to Adjust Returns: list: The updated list of layers. """ - self.layer.append(layer) - return self.layer + self.Instance_Layer.append(layer) + self.layertyep.append(layertyep) + return self.layertyep , self.Instance_Layer def freeze_weights(self, weight_freeze=False): """ @@ -63,7 +65,7 @@ def freeze_weights(self, weight_freeze=False): self.model.gradient_checkpointing_enable() self.model.encoder, self.model.decoder = CastOutputToFloat(), CastOutputToFloat() - def reconstruct_model(self): + def reconstruct_model(self,verbose=False): """ Reconstruct the model using LoRA-adapted layers. @@ -73,10 +75,14 @@ def reconstruct_model(self): if not isinstance(self.model, nn.Module): return "Please make sure the model is based on Torch nn.Module" - if self.LORA: + if self.LORA is not None: + make_lora_replace(self.model, self.lora_layer, self.Rank, self.layer) + return "Model successfully reconstructed with LoRA-adapted layers" + if self.BITSAND is not None: make_lora_replace(self.model, self.lora_layer, self.Rank, self.layer) return "Model successfully reconstructed with LoRA-adapted layers" + def implement_lora(self,verbose=False): """ Implement LoRA adaptation on the model.