Merge branch 'main' of github.com:microsoft/mttl into coalesced_conta…

…iner_fix

.github/workflows/codeql.yml

@@ -1,88 +1,83 @@
-# # For most projects, this workflow file will not need changing; you simply need
-# # to commit it to your repository.
-# #
-# # You may wish to alter this file to override the set of languages analyzed,
-# # or to provide custom queries or build logic.
-# #
-# # ******** NOTE ********
-# # We have attempted to detect the languages in your repository. Please check
-# # the `language` matrix defined below to confirm you have the correct set of
-# # supported CodeQL languages.
-# #
-# name: "CodeQL"
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
 
-# on:
-#   push:
-#     branches: [ "main" ]
-#   pull_request:
-#     # The branches below must be a subset of the branches above
-#     branches: [ "main" ]
-#   schedule:
-#     - cron: '37 20 * * 3'
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "main" ]
+  schedule:
+    - cron: '37 20 * * 3'
 
-# jobs:
-#   analyze:
-#     name: Analyze
-#     runs-on: ubuntu-latest
-#     permissions:
-#       actions: read
-#       contents: read
-#       security-events: write
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
 
-#     strategy:
-#       fail-fast: false
-#       matrix:
-#         language: [ "python" ]
-#         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-#         # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ "python" ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
 
-#     steps:
-#     - name: Checkout repository
-#       uses: actions/checkout@v3
-#     # Install Python dependencies manually
-#     - name: Set up python
-#       uses: actions/setup-python@v4
-#       with:
-#         python-version: '3.8'
-#     - name: Install dependencies
-#       run: |
-#         python -m pip install --upgrade pip
-#         pip install -r requirements.txt
-#         # Set the `CODEQL-PYTHON` environment variable to the Python executable
-#         # that includes the dependencies
-#         echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV
-#     # Initializes the CodeQL tools for scanning.
-#     - name: Initialize CodeQL
-#       uses: github/codeql-action/init@v2
-#       with:
-#         languages: python
-#         # We manually installed dependencies above
-#         setup-python-dependencies: false
-#         # languages: ${{ matrix.language }}
-#         # If you wish to specify custom queries, you can do so here or in a config file.
-#         # By default, queries listed here will override any specified in a config file.
-#         # Prefix the list here with "+" to use these queries and those in the config file.
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    # Install Python dependencies manually
+    - name: Set up python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.8'
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        pip install -r requirements.txt
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: python
+        # languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
 
-#         # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-#         # queries: security-extended,security-and-quality
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
 
 
-#     # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
-#     # If this step fails, then you should remove it and run the build manually (see below)
-#     - name: Autobuild
-#       uses: github/codeql-action/autobuild@v2
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
 
-#     # ℹ️ Command-line programs to run using the OS shell.
-#     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
-#     #   If the Autobuild fails above, remove it and uncomment the following three lines. 
-#     #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
 
-#     # - run: |
-#     #   echo "Run, Build Application using script"
-#     #   ./location_of_script_within_repo/buildscript.sh
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
 
-#     - name: Perform CodeQL Analysis
-#       uses: github/codeql-action/analyze@v2
-#       with:
-#         category: "/language:${{matrix.language}}"
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/formatter.yml

@@ -0,0 +1,24 @@
+name: "Formatter"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: '37 20 * * 3'
+
+jobs:
+  black:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: psf/black@stable
+
+  isort:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: isort/isort-action@v1
+        with:
+            requirements-files: "requirements.txt"

.github/workflows/tests.yml

@@ -1,26 +1,33 @@
 name: "Tests"
 
 on:
+  push:
+    branches: [ "main" ]
   pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "main" ]
+  schedule:
+    - cron: '37 20 * * 3'
 
 jobs:
   tests:
     name: Test
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         language: [ "python" ]
+
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.11'
+          cache: 'pip'
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
           pip install -e '.[dev]'
       # - name: Lint with flake8
       #   run: |
@@ -30,4 +37,4 @@ jobs:
       #     flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
       - name: Test with pytest
         run: |
-          pytest -vv
+          pytest -n 4 -vv

.gitignore

@@ -13,6 +13,7 @@ output/
 data/
 **/data
 .vscode/
+**/modular_artifacts/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -143,9 +144,9 @@ venv.bak/
 .mypy_cache/
 .dmypy.json
 dmypy.json
-     
+
 # Pyre type checker
-.pyre/     
+.pyre/
 **/eval/alpaca_eval/*
 **/eval/alpaca_eval/**/*.csv
 **/eval/alpaca_eval/**/*.json

.pre-commit-config.yaml

@@ -1,5 +1,11 @@
 repos:
--   repo: https://github.com/ambv/black
-    rev: 23.9.1
-    hooks:
-    - id: black
+- repo: https://github.com/pycqa/isort
+  rev: 5.13.2
+  hooks:
+  - id: isort
+    args: ["--profile", "black", "--filter-files"]
+
+- repo: https://github.com/psf/black
+  rev: 24.4.2
+  hooks:
+  - id: black

README.md

@@ -3,17 +3,20 @@
 
 # MTTL - Multi-Task Transfer Learning
 
-MTTL is a repository focusing on multi-task Natural Language Processing (NLP) methods that emphasize model reusability, model recombination, and parameter-efficient fine-tuning (PEFT) techniques, particularly in the context of few-shot and zero-shot learning.
+MTTL is a repository focusing on building LLMs that focus on model reusability, model recombination, and parameter-efficient fine-tuning (PEFT) techniques, particularly in the context of few-shot and zero-shot learning.
 
 Check out our papers on ArXiv:
 
 - [Towards Modular LLMs by Building and Reusing a Library of LoRAs](https://arxiv.org/abs/2405.11157)
 - [Multi-Head Adapter Routing for Cross-Task Generalization](https://arxiv.org/abs/2211.03831)
 
+## Towards Modular LLMs by Building and Reusing a Library of LoRAs (aka Arrow & MBC)
 
-Refer to the [Expert Library README](projects/modular_llm/README.md) for more details on training and evaluating experts.
+For the code that accompanies the paper _Towards Modular LLMs by Building and Reusing a Library of LoRAs_, please refer to the [Expert Library README](projects/modular_llm/README.md). This contains details on training and evaluating experts with Arrow.
 
-For the code that accompanies the paper [Multi-Head Adapter Routing for Cross-Task Generalization](https://arxiv.org/abs/2211.03831), please refer to [MHR-camera-ready](https://github.com/microsoft/mttl/tree/mhr-camera-ready).
+## Multi-Head Adapter Routing for Cross-Task Generalization (aka MHR)
+
+For the code that accompanies the paper _Multi-Head Adapter Routing for Cross-Task Generalization_, please refer to [MHR-camera-ready](https://github.com/microsoft/mttl/tree/mhr-camera-ready).
 
 
 ## Intended uses
@@ -32,8 +35,6 @@ MTTL is built on top of existing language models and LoRAs. MTTL is likely to in
 
 Given that MTTL is used with LoRAs chosen or built by the user, it’s important for users to fully understand the behavior and safety of the LoRAs that they use. Users should verify both the accuracy and the safety for their specific configuration and scenario.
 
-
-
 ## Contributing
 
 This project welcomes contributions and suggestions.  Most contributions require you to agree to a

mttl/__init__.py

@@ -1 +1 @@
-#
+#

mttl/callbacks.py

@@ -1,18 +1,19 @@
-import sys, os
 import copy
-import torch
-import tqdm
+import os
 import shutil
+import sys
 
 import pytorch_lightning as pl
-from pytorch_lightning import LightningModule, Trainer, callbacks as cb
+import torch
+import tqdm
+from pytorch_lightning import LightningModule, Trainer
+from pytorch_lightning import callbacks as cb
 from pytorch_lightning.callbacks.progress.tqdm_progress import Tqdm
 from pytorch_lightning.utilities.rank_zero import rank_zero_only
 from torch.optim import Optimizer
 
-from mttl.utils import logger
 from mttl.models.utils import transfer_batch_to_device
-
+from mttl.utils import logger
 
 DEBUG = False
 
@@ -99,7 +100,7 @@ def on_log(self, trainer, pl_module, metric_name, metric_value, **kwargs):
         if metric_name != self.monitor:
             return
 
-        if trainer.global_step == 0: 
+        if trainer.global_step == 0:
             # skip checkpoint before training (deepspeed fails checkpointing if optimizer non-existent
             return
 
@@ -418,8 +419,8 @@ def log_metrics(self, metrics, pl_module: pl.LightningModule, on_step=True):
             )
 
     def eval_mmlu(self, pl_module):
-        from mttl.evaluators import MMLUEvaluator
         from mttl.datamodule.mmlu_data_module import MMLUDataConfig
+        from mttl.evaluators import MMLUEvaluator
 
         if DEBUG:
             self.eval_mmlu_count += 1