Merge pull request #77 from kaklise/update

Updates to address changes in pandas plus testing updates

.github/workflows/build_tests.yml

@@ -1,4 +1,4 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# This workflow will install Python dependencies, run tests and coverage
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
 name: build
@@ -12,13 +12,17 @@ on:
 jobs:
 
   build:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+        os: [windows-latest, macOS-latest, ubuntu-latest]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
-        python-version: 3.9
+        python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python --version
@@ -30,150 +34,82 @@ jobs:
         python setup.py bdist_wheel
         ls dist/*
     - name: Save wheel
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
-        name: wheel
-        path: dist/pecos*.whl
+        name: pecos_${{ matrix.python-version }}_${{ matrix.os }}.whl
+        path: dist/pecos*
 
   test:
-    name: Import test
+    name: Test install and usage of pecos
     needs: build
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ['3.8', '3.9', '3.10']
         os: [windows-latest, macOS-latest, ubuntu-latest]    
     steps:
     - name: Set up Python 
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Download wheel
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v3
       with:
-        name: wheel
+        name: pecos_${{ matrix.python-version }}_${{ matrix.os }}.whl
     - name: Install pecos
       run: |
         python -m pip install --upgrade pip
-        pip install wheel
-        pip install --find-links=. pecos
-    - name: Import test
+        pip install wheel numpy pandas jinja2 matplotlib pytest
+        pip install --no-index --pre --find-links=. pecos
+    - name: Usage of pecos
       run: |
         python -c "import pecos"
 
-  windows:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
-        os: [windows-latest]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python --version
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install coveralls
-        python setup.py develop
-    - name: Run Tests
-      run: |
-        coverage erase
-        coverage run --context=${{ matrix.os }}.py${{ matrix.python-version }} --source=pecos --omit="*/tests/*" -m nose -v --nologcapture --with-doctest --traverse-namespace pecos
-      env:
-        COVERAGE_FILE: .coverage.${{ matrix.python-version }}.${{ matrix.os }}
-    - name: Save coverage
-      uses: actions/upload-artifact@v2
-      # if: ${{ matrix.os != 'windows-latest' }}
-      with:
-        name: coverage
-        path: .coverage.${{ matrix.python-version }}.${{ matrix.os }}
-
-  macOS:
+  create_coverage_reports:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
-        os: [macOS-latest]
+        python-version: ['3.8', '3.9', '3.10']
+        os: [windows-latest, macOS-latest, ubuntu-latest]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python --version
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install coveralls
-        python setup.py develop
-    - name: Run Tests
-      run: |
-        coverage erase
-        coverage run --context=${{ matrix.os }}.py${{ matrix.python-version }} --source=pecos --omit="*/tests/*" -m nose -v --nologcapture --with-doctest --traverse-namespace --doctest-extension=.rst documentation/*.rst pecos
-      env:
-        COVERAGE_FILE: .coverage.${{ matrix.python-version }}.${{ matrix.os }}
-    - name: Save coverage
-      uses: actions/upload-artifact@v2
-      # if: ${{ matrix.os != 'windows-latest' }}
+      uses: actions/setup-python@v4
       with:
-        name: coverage
-        path: .coverage.${{ matrix.python-version }}.${{ matrix.os }}
-
-  linux:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
-        os: [ubuntu-latest]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup conda
-      uses: s-weigand/setup-conda@v1
-      with:
-        update-conda: true
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        conda config --set always_yes yes --set changeps1 no
         python --version
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install coveralls                                             
-        python setup.py develop
+        python -m pip install -e .
     - name: Run Tests
       run: |
         coverage erase
-        coverage run --context=${{ matrix.os }}.py${{ matrix.python-version }} --source=pecos --omit="*/tests/*" -m nose -v --nologcapture --with-doctest --traverse-namespace pecos
+        coverage run --context=${{ matrix.os }}.py${{ matrix.python-version }} --source=pecos --omit="*/tests/*" -m pytest --doctest-modules --doctest-glob="*.rst" pecos
+        coverage run --context=${{ matrix.os }}.py${{ matrix.python-version }} --source=pecos --omit="*/tests/*" --append -m pytest --doctest-glob="*.rst" documentation                                                                                                                                                                                                                                       
       env:
         COVERAGE_FILE: .coverage.${{ matrix.python-version }}.${{ matrix.os }}
     - name: Save coverage
       uses: actions/upload-artifact@v2
-      # if: ${{ matrix.os != 'windows-latest' }}
       with:
         name: coverage
         path: .coverage.${{ matrix.python-version }}.${{ matrix.os }}
 
-  coverage:
-    needs: [ windows, macOS, linux ]
-    runs-on: windows-latest
+  combine_reports:
+    needs: [ create_coverage_reports ]
+    runs-on: ubuntu-latest
     steps:
     - name: Set up Python 
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - uses: actions/checkout@v2
     - name: Install coverage
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install coveralls
-        python setup.py develop
+        python -m pip install -e .
     - name: Download coverage artifacts from test matrix
       uses: actions/download-artifact@v2
       with:
@@ -204,13 +140,13 @@ jobs:
         name: coverage
         path: htmlcov
 
-  coveralls:
-    needs: [ windows, macOS, linux ]
+  combine_reports_upload_coveralls:
+    needs: [ create_coverage_reports ]
     runs-on: ubuntu-latest
     continue-on-error: true
     steps:
     - name: Set up Python 
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - uses: actions/checkout@v2
@@ -219,7 +155,7 @@ jobs:
         python -m pip install --upgrade pip
         pip install coveralls
         pip install -r requirements.txt
-        python setup.py develop
+        python -m pip install -e .
     - name: Download coverage artifacts from test matrix
       uses: actions/download-artifact@v2
       with:

.github/workflows/quick_check.yml

@@ -17,15 +17,16 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - name: Install packages
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install --upgrade coverage nose
+        pip install --upgrade coverage pytest
         python setup.py develop
     - name: Run tests
       run: |
-        nosetests -v --nologcapture --with-doctest --with-coverage --cover-package=pecos --cover-min-percentage=70 pecos
+        coverage run --source=pecos -m pytest --doctest-modules --doctest-glob="*.rst" pecos        
+        coverage report --fail-under=70                                        

documentation/developers.rst

@@ -8,9 +8,9 @@ The following services are used for software quality assurance:
 * Test coverage statistics are collected using Coveralls at https://coveralls.io/github/sandialabs/pecos.
 * The current release is hosted on PyPI at https://pypi.python.org/pypi/pecos.
 
-Tests can be run locally using nosetests::
+Tests can be run locally using pytest::
 
-	nosetests -v --with-coverage --cover-package=pecos pecos
+	pytest pecos
 
 Software developers are expected to follow standard practices to document and test new code. 
 Pull requests will be reviewed by the core development team.

documentation/installation.rst

@@ -1,7 +1,7 @@
 Installation
 ======================================
 
-Pecos requires Python (tested on 3.6, 3.7, 3.8, and 3.9) along with several Python 
+Pecos requires Python (tested on 3.8, 3.9, and 3.10) along with several Python 
 package dependencies.  Information on installing and using Python can be found at 
 https://www.python.org/.  Python distributions, such as Anaconda,
 are recommended to manage the Python interface.  

documentation/results.rst

@@ -53,8 +53,8 @@ A subset of quality control test results from the :ref:`simple_example` are show
     >>> pd.set_option('display.expand_frame_repr', False)
     >>> from os.path import abspath, dirname, join
     >>> import pecos
-    >>> docdir = dirname(abspath(__file__))
-    >>> datadir = join(docdir,'..','pecos', 'tests', 'data')
+    >>> packagedir = dirname(abspath(pecos.__file__))
+    >>> datadir = join(packagedir, 'tests', 'data')
 	>>> pm = pecos.monitoring.PerformanceMonitoring()
     >>> test_results = pd.read_csv(join(datadir,'Simple_test_results.csv'), index_col=0) # This file is tested against output from the Simple example
     >>> pm.test_results = test_results.loc[0:7,:]	

documentation/whatsnew.rst

@@ -1,7 +1,7 @@
 Release Notes
 ================
 
-.. include:: whatsnew/v0.2.1.rst
+.. include:: whatsnew/v0.3.0.rst
 
 .. include:: whatsnew/v0.2.0.rst
 

documentation/whatsnew/v0.2.1.rst → documentation/whatsnew/v0.3.0.rst

@@ -1,10 +1,10 @@
-.. _whatsnew_021:
+.. _whatsnew_030:
 
-v0.2.1 (main)
+v0.3.0 (main)
 --------------------------
 
 * Bug fix in custom static and streaming quality control tests to use a specific column
 * Minor updates to address pandas and yaml depreciation warnings
-* Minor updates for testing and documentation
-* Added GitHub Actions and Python 3.9 tests
-
+* Updated test to use pytest instead of nose
+* Added GitHub Actions 
+* Added Python 3.9 and 3.10 tests, dropped 3.6 and 3.7 tests

pecos/__init__.py

@@ -6,7 +6,7 @@
 from pecos import utils
 from pecos import pv
 
-__version__ = '0.2.1'
+__version__ = '0.3.0'
 
 __copyright__ = """Copyright 2016 National Technology & Engineering 
     Solutions of Sandia, LLC (NTESS). Under the terms of Contract 

pecos/graphics.py

@@ -16,13 +16,8 @@
 import textwrap
 import os
 import logging
+import pytest
 
-try:
-    from nose.tools import nottest as _nottest
-except ImportError:
-    def _nottest(afunction):
-        return afunction
-
 NoneType = type(None)
 
 logger = logging.getLogger(__name__)
@@ -425,7 +420,7 @@ def plot_doy_heatmap(data, cmap='nipy_spectral', vmin=None, vmax=None,
     ax.set_ylabel("Time of day (minutes)")
     plt.tight_layout()
 
-@_nottest
+@pytest.mark.skip()
 def plot_test_results(data, test_results, tfilter=None, image_format='png', 
                       dpi=500, figsize=(7.0,3.0), date_formatter=None, 
                       filename_root='test_results'):
@@ -489,7 +484,7 @@ def plot_test_results(data, test_results, tfilter=None, image_format='png',
                           'Missing timestamp', 
                           'Nonmonotonic timestamp']
     test_results = test_results[-test_results['Error Flag'].isin(remove_error_flags)]
-    grouped = test_results.groupby(['Variable Name'])
+    grouped = test_results.groupby('Variable Name')
 
     for col_name, test_results_group in grouped:
         logger.info("Creating graphic for " + col_name)

pecos/io.py

@@ -21,12 +21,8 @@
     import minimalmodbus 
 except:
     pass
+import pytest
 
-try:
-    from nose.tools import nottest as _nottest
-except ImportError:
-    def _nottest(afunction):
-        return afunction
 
 logger = logging.getLogger(__name__)
 
@@ -56,11 +52,12 @@ def read_campbell_scientific(filename, index_col='TIMESTAMP', encoding=None):
 
     try:
         df = pd.read_csv(filename, skiprows=1, encoding=encoding, index_col=index_col, 
-                         parse_dates=True, dtype ='unicode', error_bad_lines=False) #, low_memory=False)
+                         parse_dates=True, dtype ='unicode', on_bad_lines="skip") #, low_memory=False)
         df = df[2:]
         index = pd.to_datetime(df.index)
         Unnamed = df.filter(regex='Unnamed')
-        df = df.drop(Unnamed.columns, 1)
+        if Unnamed.columns.shape[0] > 0:
+            df = df.drop(Unnamed.columns, 1)
         df = pd.DataFrame(data = df.values, index = index, columns = df.columns, dtype='float64')
     except:
         logger.warning("Cannot extract database, CSV file reader failed " + filename)
@@ -166,11 +163,11 @@ def write_metrics(metrics, filename='metrics.csv'):
     logger.info("Write metrics file")
 
     try:
-        previous_metrics = pd.read_csv(filename, index_col='TIMESTEP') #, parse_dates=True)
+        previous_metrics = pd.read_csv(filename, index_col='TIMESTEP', parse_dates=True)
     except:
         previous_metrics = pd.DataFrame()
 
-    metrics.index = metrics.index.to_native_types() # this is necessary when using time zones
+    #metrics.index = metrics.index.to_native_types() # no longer needed when using time zones
     metrics = metrics.combine_first(previous_metrics) 
 
     if os.path.dirname(filename) == '':
@@ -183,7 +180,7 @@ def write_metrics(metrics, filename='metrics.csv'):
 
     return full_filename
 
-@_nottest
+@pytest.mark.skip()
 def write_test_results(test_results, filename='test_results.csv'):
     """
     Write test results file.