Merge pull request #1044 from Trusted-AI/dev_1.6.1

Update to ART 1.6.1

.github/actions/deepspeech-v2/Dockerfile

@@ -0,0 +1,44 @@
+# Get base from a pytorch image
+FROM pytorch/pytorch:1.5.1-cuda10.1-cudnn7-runtime
+
+# Set to install things in non-interactive mode
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install system wide softwares
+RUN apt-get update \
+     && apt-get install -y \
+        libgl1-mesa-glx \
+        libx11-xcb1 \
+        git \
+        gcc \
+        mono-mcs \
+        cmake \
+        libavcodec-extra \
+        ffmpeg \
+        curl \
+     && apt-get clean all \
+     && rm -r /var/lib/apt/lists/*
+
+RUN /opt/conda/bin/conda install --yes \
+    astropy \
+    matplotlib \
+    pandas \
+    scikit-learn \
+    scikit-image
+
+# Install necessary libraries for deepspeech v2
+RUN pip install torch
+RUN pip install tensorflow
+RUN pip install torchaudio==0.5.1
+
+RUN git clone https://github.com/SeanNaren/warp-ctc.git
+RUN cd warp-ctc && mkdir build && cd build && cmake .. && make
+RUN cd warp-ctc/pytorch_binding && python setup.py install
+
+RUN git clone https://github.com/SeanNaren/deepspeech.pytorch.git
+RUN cd deepspeech.pytorch && git checkout V2.1
+RUN cd deepspeech.pytorch && pip install -r requirements.txt
+RUN cd deepspeech.pytorch && pip install -e .
+
+RUN pip install numba==0.50.0
+RUN pip install pytest-cov

.github/actions/deepspeech-v2/action.yml

@@ -0,0 +1,7 @@
+name: 'Test DeepSpeech v2'
+description: 'Run tests for DeepSpeech v2'
+runs:
+  using: 'composite'
+  steps:
+    - run: $GITHUB_ACTION_PATH/run.sh
+      shell: bash

.github/actions/deepspeech-v2/run.sh

@@ -0,0 +1,10 @@
+#!/bin/sh -l
+
+exit_code=0
+
+pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/speech_recognition/test_pytorch_deep_speech.py --framework=pytorch --skip_travis=True --durations=0
+if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/speech_recognition/test_pytorch_deep_speech tests"; fi
+pytest --cov-report=xml --cov=art --cov-append -q -vv tests/attacks/evasion/test_imperceptible_asr_pytorch.py --framework=pytorch --skip_travis=True --durations=0
+if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/evasion/test_imperceptible_asr_pytorch tests"; fi
+
+exit ${exit_code}

.github/workflows/ci-deepspeech-v2.yml

@@ -0,0 +1,32 @@
+name: CI PyTorchDeepSpeech v2
+on:
+  # Run on manual trigger
+  workflow_dispatch:
+
+  # Run on pull requests
+  pull_request:
+    paths-ignore:
+      - '*.md'
+
+  # Run when pushing to main or dev branches
+  push:
+    branches:
+      - main
+      - dev*
+
+  # Run scheduled CI flow daily
+  schedule:
+    - cron: '0 8 * * 0'
+
+jobs:
+  test_deepspeech_v2:
+    name: PyTorchDeepSpeech v2
+    runs-on: ubuntu-latest
+    container: minhitbk/art_testing_envs:deepspeech_v2
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+      - name: Run Test Action
+        uses: ./.github/actions/deepspeech-v2
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1

.github/workflows/ci-lingvo.yml

@@ -1,4 +1,4 @@
-name: Continuous Integration
+name: CI Lingvo
 on:
   # Run on manual trigger
   workflow_dispatch:

.github/workflows/ci-mxnet.yml

@@ -1,4 +1,4 @@
-name: Continuous Integration
+name: CI MXNet
 on:
   # Run on manual trigger
   workflow_dispatch:
@@ -46,6 +46,15 @@ jobs:
             torchvision: 0.8.2+cpu
             torchaudio: 0.7.2
             scikit-learn: 0.23.2
+          - name: legacy (TensorFlow 2.4.1 Keras 2.4.3 PyTorch 1.7.1 scikit-learn 0.24.1 Python 3.7)
+            framework: legacy
+            python: 3.7
+            tensorflow: 2.4.1
+            keras: 2.4.3
+            torch: 1.7.1+cpu
+            torchvision: 0.8.2+cpu
+            torchaudio: 0.7.2
+            scikit-learn: 0.24.1
 
     name: Run ${{ matrix.name }} Tests
     steps:

.github/workflows/ci-pytorch-fasterrcnn.yml

@@ -0,0 +1,49 @@
+name: CI PyTorchFasterRCNN
+on:
+  # Run on manual trigger
+  workflow_dispatch:
+
+  # Run on pull requests
+  pull_request:
+    paths-ignore:
+      - '*.md'
+
+  # Run when pushing to main or dev branches
+  push:
+    branches:
+      - main
+      - dev*
+
+  # Run scheduled CI flow daily
+  schedule:
+    - cron: '0 8 * * 0'
+
+jobs:
+  test_pytorch_fasterrcnn:
+    name: PyTorchFasterRCNN
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get -y -q install ffmpeg libavcodec-extra
+          python -m pip install --upgrade pip setuptools wheel
+          pip3 install -q -r requirements.txt
+          pip list
+      - name: Pre-install torch
+        run: |
+          pip install torch==1.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
+          pip install torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
+          pip install torchaudio==0.6.0 -f https://download.pytorch.org/whl/torch_stable.html
+      - name: Run Test Action
+        run: pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/object_detection/test_pytorch_faster_rcnn.py --framework=pytorch --skip_travis=True --durations=0
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: Continuous Integration
+name: CI General
 on:
   # Run on manual trigger
   workflow_dispatch:
@@ -73,8 +73,12 @@ jobs:
             framework: scikitlearn
             scikit-learn: 0.23.2
             python: 3.8
+          - name: scikit-learn 0.24.1 (Python 3.8)
+            framework: scikitlearn
+            scikit-learn: 0.24.1
+            python: 3.8
 
-    name: Run ${{ matrix.name }} Tests
+    name: ${{ matrix.name }}
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
@@ -126,7 +130,7 @@ jobs:
           pip install tensorflow-addons==0.9.1
           pip install model-pruning-google-research==0.0.3
           pip list
-      - name: Run ${{ matrix.name }} Tests
+      - name: Run Tests
         run: ./run_tests.sh ${{ matrix.framework }}
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
@@ -144,19 +148,22 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get -y -q install ffmpeg libavcodec-extra
-          pip install tensorflow==2.2.2
-          pip install keras==2.3.1
+          pip install tensorflow==2.4.1
+          pip install keras==2.4.3
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel
           pip install -q pylint pycodestyle
           pip install -q -r requirements.txt
           pip list
       - name: pycodestyle
-        run: (pycodestyle --ignore=C0330,C0415,E203,E231,W503 --max-line-length=120 art || exit 0)
+        run: pycodestyle --ignore=C0330,C0415,E203,E231,W503 --max-line-length=120 art
       - name: pylint
-        run: (pylint --disable=C0330,C0415,E203,E1136 -rn art || exit 0)
-      - name: Check Types (mypy)
-        run: (mypy art || exit 0)
-      - name: pytest
-        run: (py.test --flake8 -m flake8 || exit 0)
+        if: ${{ always() }}
+        run: pylint --disable=C0330,C0415,E203,E1136,E0401,E1102 -rn art
+      - name: mypy
+        if: ${{ always() }}
+        run: mypy art
+      - name: pytest-flake8
+        if: ${{ always() }}
+        run: pytest --flake8 -v -m flake8

.github/workflows/codeql-analysis.yml

@@ -7,10 +7,10 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [main, dev_1.6.0, dev_1.6.1]
+    branches: [main, dev_*]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [main]
+    branches: [main, dev_*]
   schedule:
     - cron: '0 7 * * 0'
 

.pylintrc

@@ -189,7 +189,7 @@ score=yes
 [REFACTORING]
 
 # Maximum number of nested blocks for function / method body
-max-nested-blocks=5
+max-nested-blocks=7
 
 # Complete name of functions that never returns. When checking for
 # inconsistent-return-statements if a never returning function is called then
@@ -531,7 +531,7 @@ max-branches=12
 max-locals=15
 
 # Maximum number of parents for a class (see R0901).
-max-parents=7
+max-parents=11
 
 # Maximum number of public methods for a class (see R0904).
 max-public-methods=20

art/__init__.py

@@ -11,21 +11,33 @@
 from art import wrappers
 
 # Semantic Version
-__version__ = "1.6.0"
+__version__ = "1.6.1-dev"
 
 # pylint: disable=C0103
 
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
     "formatters": {
-        "std": {"format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s", "datefmt": "%Y-%m-%d %H:%M",}
+        "std": {
+            "format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+            "datefmt": "%Y-%m-%d %H:%M",
+        }
     },
     "handlers": {
-        "default": {"class": "logging.NullHandler",},
-        "test": {"class": "logging.StreamHandler", "formatter": "std", "level": logging.INFO,},
+        "default": {
+            "class": "logging.NullHandler",
+        },
+        "test": {
+            "class": "logging.StreamHandler",
+            "formatter": "std",
+            "level": logging.INFO,
+        },
+    },
+    "loggers": {
+        "art": {"handlers": ["default"]},
+        "tests": {"handlers": ["test"], "level": "INFO", "propagate": True},
     },
-    "loggers": {"art": {"handlers": ["default"]}, "tests": {"handlers": ["test"], "level": "INFO", "propagate": True},},
 }
 logging.config.dictConfig(LOGGING)
 logger = logging.getLogger(__name__)

art/attacks/attack.py

@@ -34,12 +34,13 @@
 logger = logging.getLogger(__name__)
 
 
-class input_filter(abc.ABCMeta):
+class InputFilter(abc.ABCMeta):
     """
     Metaclass to ensure that inputs are ndarray for all of the subclass generate and extract calls
     """
 
-    def __init__(cls, name, bases, clsdict):
+    # pylint: disable=W0613
+    def __init__(cls, name, bases, clsdict):  # pylint: disable=W0231
         """
         This function overrides any existing generate or extract methods with a new method that
         ensures the input is an `np.ndarray`. There is an assumption that the input object has implemented
@@ -84,7 +85,7 @@ def replacement_function(self, *args, **kwargs):
                 setattr(cls, item, new_function)
 
 
-class Attack(abc.ABC, metaclass=input_filter):
+class Attack(abc.ABC, metaclass=InputFilter):
     """
     Abstract base class for all attack abstract base classes.
     """
@@ -96,6 +97,8 @@ def __init__(self, estimator):
         """
         :param estimator: An estimator.
         """
+        super().__init__()
+
         if self.estimator_requirements is None:
             raise ValueError("Estimator requirements have not been defined in `_estimator_requirements`.")
 
@@ -106,10 +109,12 @@ def __init__(self, estimator):
 
     @property
     def estimator(self):
+        """The estimator."""
         return self._estimator
 
     @property
     def estimator_requirements(self):
+        """The estimator requirements."""
         return self._estimator_requirements
 
     def set_params(self, **kwargs) -> None:
@@ -193,7 +198,7 @@ class PoisoningAttackTransformer(PoisoningAttack):
     These attacks have an additional method, `poison_estimator`, that returns the poisoned classifier.
     """
 
-    def __init__(self, classifier: Optional["CLASSIFIER_TYPE"], **kwargs) -> None:
+    def __init__(self, classifier: Optional["CLASSIFIER_TYPE"]) -> None:
         """
         :param classifier: A trained classifier (or none if no classifier is needed)
         """

art/attacks/evasion/adversarial_asr.py

@@ -76,7 +76,7 @@ def __init__(
 
         # re-implement init such that inherited methods work
         EvasionAttack.__init__(self, estimator=estimator)  # pylint: disable=W0233
-        self.masker = None
+        self.masker = None  # type: ignore
         self.eps = eps
         self.learning_rate_1 = learning_rate
         self.max_iter_1 = max_iter