Merge pull request #1135 from pypeit/staged

Merges develop into master (1.3.1 tag)

.github/workflows/ci_cron.yml

@@ -0,0 +1,34 @@
+name: Cron tests on develop branch
+
+on:
+  schedule:
+    # run every Monday at 8am UTC
+    - cron: '0 8 * * 1'
+
+env:
+  SETUP_XVFB: True  # avoid issues if something tries to open a GUI window
+
+jobs:
+  cron-tests:
+    name: Tox env ${{ matrix.python }}-${{ matrix.toxenv }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python: [3.8]
+        toxenv: [test-alldeps, test-numpydev, test-linetoolsdev, test-gingadev, test-astropydev, conda]
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+      with:
+        ref: develop
+    - name: Set up Python ${{ matrix.python }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install base dependencies
+      run: |
+        python -m pip install --upgrade pip tox
+    - name: Test with tox
+      run: |
+        tox -e ${{ matrix.toxenv }}

.github/workflows/ci_tests.yml

@@ -4,24 +4,25 @@ on:
   push:
     branches:
     - master
+    - develop
     tags:
   pull_request:
+  schedule:
+    # run every Monday at 6am UTC
+    - cron: '0 6 * * 1'
 
 env:
-  SETUP_XVFB: True  # avoid issues if mpl tries to open a GUI window
+  SETUP_XVFB: True  # avoid issues if something tries to open a GUI window
 
 jobs:
   ci-tests:
-    name: Python-${{ matrix.python }}, deps=${{ matrix.deps }}
+    name: Tox env ${{ matrix.python }}-${{ matrix.toxenv }}
     runs-on: ${{ matrix.os }}
-    if: "!(contains(github.event.head_commit.message, '[skip ci]') || contains(github.event.head_commit.message, '[ci skip]'))"
-
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python: [3.7, 3.8]
-        deps: [current, numpy118, astropydev, numpydev, astropydev-numpydev]
-
+        python: [3.8]
+        toxenv: [test, test-alldeps, test-linetoolsdev, test-gingadev, test-astropydev, conda]
     steps:
     - name: Check out repository
       uses: actions/checkout@v2
@@ -31,36 +32,37 @@ jobs:
         python-version: ${{ matrix.python }}
     - name: Install base dependencies
       run: |
-        python -m pip install --upgrade pip
-    - name: Test with numpy = 1.18
-      if: "contains(matrix.deps, 'numpy118')"
-      run: |
-        python -m pip install numpy==1.18
-    - name: Test with dev version of numpy
-      if: "contains(matrix.deps, 'numpydev')"
-      run: |
-        python -m pip install git+https://github.com/numpy/numpy.git#egg=numpy
-    - name: Test with dev version astropy
-      if: "contains(matrix.deps, 'astropydev')"
+        python -m pip install --upgrade pip tox
+    - name: Test with tox
       run: |
-        python -m pip install git+https://github.com/astropy/astropy.git#egg=astropy
-    - name: Install pypeit requirements
+        tox -e ${{ matrix.toxenv }}
+
+  os-tests:
+    name: Python ${{ matrix.python }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      # set this to false for now so windows failures don't prevent macos test from finishing
+      fail-fast: false
+      matrix:
+        os: [windows-latest, macos-latest]
+        python: [3.8]
+        toxenv: [test-alldeps]
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install base dependencies
       run: |
-        python -m pip install wheel scipy pyyaml matplotlib future numba scikit-learn configobj IPython
-        python -m pip install -r pypeit/requirements.txt
-    - name: Print Python, pip, astropy, numpy, and setuptools versions
+        python -m pip install --upgrade pip tox
+    - name: Test with tox
       run: |
-        python -c "import sys; print(f'Python {sys.version}')"
-        python -c "import pip; print(f'pip {pip.__version__}')"
-        python -c "import astropy; print(f'astropy {astropy.__version__}')"
-        python -c "import numpy; print(f'numpy {numpy.__version__}')"
-        python -c "import setuptools; print(f'setuptools {setuptools.__version__}')"
-    - name: Run tests
-      run: python setup.py test
+        tox -e ${{ matrix.toxenv }}
 
   codestyle:
     runs-on: ubuntu-latest
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
     steps:
     - uses: actions/checkout@v2
     - name: Python codestyle check

.gitignore

@@ -1,9 +1,11 @@
 # Folders#
 .idea/
 .vscode/
+.tox/
 build/*
 dist/*
-doc/build_
+doc/_build/*
+doc/build_/*
 
 # Compiled source #
 ###################
@@ -19,7 +21,7 @@ doc/build_
 *_rjc*
 *.nbi
 *.nbc
- 
+
 # Packages #
 ############
 # it's better to unpack these files and commit the raw source
@@ -32,13 +34,13 @@ doc/build_
 *.rar
 *.tar
 *.zip
- 
+
 # Logs and databases #
 ######################
 *.log
 *.sql
 *.sqlite
- 
+
 # OS generated files #
 ######################
 .DS_Store
@@ -49,6 +51,11 @@ doc/build_
 ehthumbs.db
 Thumbs.db
 
+# setuptools generated version files #
+######################################
+*/version.py
+*/cython_version.py
+
 # ADDED by KBW
 .eggs
 *egg-info*
@@ -61,3 +68,7 @@ pypeit/data/telluric/*.fits
 pypeit/data/telluric/*.fits.gz
 
 *.pckl
+
+# added by TEP
+doc/sphinx_warnings.out
+pypeit/tests/files/setup_files/*

CHANGES.rst

@@ -1,8 +1,31 @@
 
-1.3.0 Hotfixes
---------------
+1.3.1 (01 Feb 2021)
+-------------------
+
+- pypeit_chk_wavecalib script
+- Option to limit channels shown for pypeit_show_2dspec
+- sigdetect on in full_template
+- Added new ArI, ArII lines
+- Improved 1Dfit QA
+- Final wavelength template for DEIMOS 900ZD
+- Fix a bug in `pypeit/core/arc.py` and `pypeit/core/wavecal/autoid.py` due
+  to the padding to the arc frames
+- Added a new XeI line
+- Turn off sigma clipping for DEIMOS arc frames.
+- Refactor setup.py to use setup.cfg to define package configuration
+- Refactor version handling to use setuptools_scm to grab version info from git tags
+- Add support for testing within isolated environments via tox
+- Refactor CI to use tox to run tests
+- Add cron-scheduled tests to CI
+- Add tests to CI to cover macos, windows, and conda installations
+- Refactor wrapper scripts in bin/ to be entry_points defined in setup.cfg
+- Deprecate check_requirements now that dependencies are handled by the installation
+- Add `pypeit_obslog` script that simple compiles and prints metadata
+  from a set of fits files needed by pypeit to run.
+- Change `PypeItSetup.from_file_root` to *require* the output path to
+  write the vanilla pypeit file.  If no path is provided, the object is
+  instatiated without creating any output.
 
-- Added as-published JOSS citation.
 
 1.3.0 (13 Dec 2020)
 -------------------
@@ -81,6 +104,7 @@
 - DEIMOS object name and `maskdef_id` visible in ginga when running `pypeit_show_2d`
 - Fix sigma clipping bug!
 
+
 1.2.0 (15 Oct 2020)
 -------------------
 

MANIFEST.in

@@ -1,7 +1,7 @@
 include README.md
 include LICENSE.rst
 #include CHANGES.rst
-
+include pyproject.toml
 include setup.cfg
 
 recursive-include pypeit *.pyx *.c *.pxd *.h
@@ -11,15 +11,9 @@ recursive-include licenses *
 recursive-include cextern *
 recursive-include scripts *
 
-include pypeit/requirements.txt
-
 prune build
 prune docs/_build
 prune docs/api
 prune pypeit/deprecated
 
-#recursive-include astropy_helpers *
-#exclude astropy_helpers/.git
-#exclude astropy_helpers/.gitignore
-
 global-exclude *.pyc *.o *.so *.DS_Store

README.md

@@ -1,4 +1,11 @@
 # PypeIt
+[![PyPI](https://img.shields.io/pypi/v/pypeit?label=PyPI&logo=pypi&logoColor=white)](https://pypi.org/project/pypeit/)
+[![Conda Version](https://img.shields.io/conda/vn/conda-forge/pypeit?label=conda%20version)](https://anaconda.org/conda-forge/pypeit)
+[![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/pypeit?label=conda%20downloads)](https://anaconda.org/conda-forge/pypeit)
+
+[![CI Tests](https://github.com/pypeit/PypeIt/workflows/CI%20Tests/badge.svg)](https://github.com/pypeit/PypeIt/actions?query=workflow%3A"CI+Tests")
+[![Documentation Status](https://readthedocs.org/projects/pypeit/badge/?version=latest)](https://pypeit.readthedocs.io/en/latest/?badge=latest)
+[![astropy](http://img.shields.io/badge/powered%20by-AstroPy-orange.svg?style=flat)](http://www.astropy.org/)
 
 The Python Spectroscopic Data Reduction Pipeline.  For
 documentation visit:
@@ -9,7 +16,7 @@ and/or see our HOWTO:
 
 https://tinyurl.com/pypeit-howto
 
-and/or join our PypeIt Users Slack 
+and/or join our PypeIt Users Slack
 (the invite is recorded in this Issue:
 https://github.com/pypeit/PypeIt/issues/676)
 
@@ -94,13 +101,14 @@ Conduct](https://pypeit.readthedocs.io/en/latest/codeconduct.html).
 
 # Instruments Served
 * Gemini/GNIRS
-* Gemini/GMOS 
-* Gemini/FLAMINGOS 2 
-* Lick/Kast 
+* Gemini/GMOS
+* Gemini/FLAMINGOS 2
+* Lick/Kast
 * Magellan/MagE
 * Magellan/Fire
 * MMT/BinoSpec (270 and 600 tested)
 * MMT/MMIRS (HK_zJ, J_zJ, and K_K tested)
+* MMT/Blue Channel (300 tested)
 * MDM/OSMOS
 * Keck/DEIMOS (600ZD, 830G, 1200G)
 * Keck/KCWI (BM, BH2)
@@ -120,7 +128,7 @@ Conduct](https://pypeit.readthedocs.io/en/latest/codeconduct.html).
 
 # Requirements
 
-(see `pypeit/requirements.txt`)
+(see `setup.cfg` or `environment.yml`)
 
 * python
 * numpy

doc/Makefile

@@ -56,7 +56,7 @@ clean:
 	rm -rf $(LOCALFILES)
 
 apirst:
-	SPHINX_APIDOC_OPTIONS=$(SPHINXAPIOPT) $(SPHINXAPI) --separate -o ./api ../pypeit ../pypeit/tests/* ../pypeit/scripts/* ../pypeit/deprecated/*
+	SPHINX_APIDOC_OPTIONS=$(SPHINXAPIOPT) $(SPHINXAPI) --separate -o ./api ../pypeit ../pypeit/tests/* ../pypeit/scripts/* ../pypeit/deprecated/* ../pypeit/version.py
 	python3 ./scripts/build_par_rst.py
 	python3 ./scripts/build_specobj_rst.py
 	python3 ./scripts/build_calib_images_rst.py

doc/api/pypeit.rst

@@ -24,7 +24,6 @@ Submodules
    pypeit.biasframe
    pypeit.bitmask
    pypeit.calibrations
-   pypeit.check_requirements
    pypeit.coadd1d
    pypeit.coadd2d
    pypeit.compiler_version

doc/conf.py

@@ -12,8 +12,17 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-#import sys
-#import os
+import os
+import sys
+from pkg_resources import get_distribution
+
+# Get configuration information from setup.cfg
+from configparser import ConfigParser
+conf = ConfigParser()
+
+conf.read([os.path.join(os.path.dirname(__file__), '..', 'setup.cfg')])
+setup_cfg = dict(conf.items('metadata'))
+
 #
 ## If extensions (or modules to document with autodoc) are in another directory,
 ## add these directories to sys.path here. If the directory is relative to the
@@ -50,14 +59,14 @@
 #automodapi
 numpydoc_show_class_members = False
 
-# Napoleon settings 
-napoleon_numpy_docstring = True 
-napoleon_include_private_with_doc = False 
-napoleon_include_special_with_doc = True 
-napoleon_use_admonition_for_examples = False 
-napoleon_use_admonition_for_notes = False 
-napoleon_use_admonition_for_references = False 
-napoleon_use_ivar = False 
+# Napoleon settings
+napoleon_numpy_docstring = True
+napoleon_include_private_with_doc = False
+napoleon_include_special_with_doc = True
+napoleon_use_admonition_for_examples = False
+napoleon_use_admonition_for_notes = False
+napoleon_use_admonition_for_references = False
+napoleon_use_ivar = False
 napoleon_use_param = True
 napoleon_use_rtype = True
 
@@ -88,11 +97,12 @@
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
-#
-# The short X.Y version.
-version = '1.3.0'
+
 # The full version, including alpha/beta/rc tags.
-release = '1.3.0'
+release = get_distribution(setup_cfg['name']).version
+# The short X.Y.Z version.
+version = '.'.join(release.split('.')[:3])
+
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.