ci(release): track deprecations in release notes

.build_rtd_docs/conf.py

@@ -56,6 +56,15 @@
 # copy the file
 shutil.copy(src, dst)
 
+# -- copy deprecations markdown ---------------------------------------------
+print("Copy the deprecations table")
+dstdir = "_mf6run"
+fpth = "deprecations.md"
+src = os.path.join("..", "doc", "mf6io", "mf6ivar", "md", fpth)
+dst = os.path.join(dstdir, fpth)
+# copy the file
+shutil.copy(src, dst)
+
 # -- build the mf6io markdown files -----------------------------------------
 print("Build the mf6io markdown files")
 pth = os.path.join("..", "doc", "mf6io", "mf6ivar")

.build_rtd_docs/index.rst

@@ -14,4 +14,5 @@ Contents:
    MODFLOW 6 Source Code Documentation <https://modflow-usgs.github.io/modflow6/>
    mf6io
    _mf6run/run-time-comparison.md
+   _mf6run/deprecations.md
 

.github/workflows/docs.yml

@@ -105,21 +105,35 @@ jobs:
         run: python benchmark.py
         env:
           GITHUB_TOKEN: ${{ github.token }}
-
-      - name: Run sphinx
-        working-directory: modflow6/.build_rtd_docs
-        run: make html
-
+
       - name: Show benchmarks
         working-directory: modflow6/distribution
         run: cat run-time-comparison.md
-
+    
       - name: Upload benchmarks
         uses: actions/upload-artifact@v3
         with:
           name: run-time-comparison
           path: modflow6/distribution/run-time-comparison.md
 
+      - name: Collect deprecations
+        working-directory: modflow6/doc/mf6io/mf6ivar
+        run: python deprecations.py
+
+      - name: Show deprecations
+        working-directory: modflow6/doc/mf6io/mf6ivar/md
+        run: cat deprecations.md
+
+      - name: Upload deprecations
+        uses: actions/upload-artifact@v3
+        with:
+          name: deprecations
+          path: modflow6/doc/mf6io/mf6ivar/md/deprecations.md
+
+      - name: Run sphinx
+        working-directory: modflow6/.build_rtd_docs
+        run: make html
+
       - name: Upload results
         uses: actions/upload-artifact@v3
         with:

.github/workflows/release.yml

@@ -353,6 +353,20 @@ jobs:
           cd modflow6/doc/ReleaseNotes
           python mk_folder_struct.py -dp "${{ github.workspace }}/$distname"
 
+      - name: Collect deprecations
+        working-directory: modflow6/doc/mf6io/mf6ivar
+        run: python deprecations.py
+
+      - name: Show deprecations
+        working-directory: modflow6/doc/mf6io/mf6ivar/md
+        run: cat deprecations.md
+
+      - name: Upload deprecations
+        uses: actions/upload-artifact@v3
+        with:
+          name: deprecations
+          path: modflow6/doc/mf6io/mf6ivar/md/deprecations.md
+
       - name: Build documentation
         env:
           # need a GITHUB_TOKEN to download example doc PDF asset from modflow6-examples repo

DEVELOPER.md

@@ -55,6 +55,8 @@ To build and test a parallel version of the program, first read the instructions
     - [Squash](#squash)
     - [Rebase](#rebase)
     - [Cleanup](#cleanup)
+- [Deprecation policy](#deprecation-policy)
+  - [Finding deprecations](#finding-deprecations)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
@@ -576,3 +578,19 @@ git branch -d feat-xyz-backup
 ```
 
 This process can be repeated periodically to stay in sync with the develop branch and keep a clean commit history.
+
+## Deprecation policy
+
+To deprecate a MODFLOW 6 input/output option in a DFN file:
+
+- Add a new `deprecated x.y.z` attribute to the appropriate variable in the package DFN file, where `x.y.z` is the version the deprecation is introduced. Mention the deprecation prominently in the release notes.
+- If support for the deprecated option is removed (typically after at least 2 minor or major releases or 1 year), add a new `removed x.y.z` attribute to the variable in the DFN file, where `x.y.z` is the version in which support for the option was removed. The line containing `deprecated x.y.z` should not be deleted. Mention the removal prominently in the release notes.
+- Deprecated/removed attributes are not removed from DFN files but remain in perpetuity. The `doc/mf6io/mf6ivar/deprecations.py` script generates a markdown deprecation table which is converted to LaTeX by `doc/ReleaseNotes/mk_deprecations.py` for inclusion in the MODFLOW 6 release notes. Deprecations and removals should still be mentioned separately in the release notes, however.
+
+### Finding deprecations
+
+To search for deprecations and removals in DFN files on a system with `git` and standard Unix commands available:
+
+```shell
+git grep 'deprecated' -- '*.dfn' | awk '/^*.dfn:deprecated/'
+```

distribution/build_docs.py

@@ -30,7 +30,7 @@
 _examples_repo_path = _project_root_path.parent / "modflow6-examples"
 _release_notes_path = _project_root_path / "doc" / "ReleaseNotes"
 _distribution_path = _project_root_path / "distribution"
-_benchmarks_path = _project_root_path / "distribution" / ".benchmarks"
+_benchmarks_dir_path = _project_root_path / "distribution" / ".benchmarks"
 _docs_path = _project_root_path / "doc"
 _dev_dist_tex_paths = [
     _docs_path / "mf6io" / "mf6io.tex",
@@ -146,12 +146,12 @@ def test_download_benchmarks(tmp_path, github_user):
 def build_benchmark_tex(
     output_path: PathLike, overwrite: bool = False, repo_owner: str = "MODFLOW-USGS"
 ):
-    _benchmarks_path.mkdir(parents=True, exist_ok=True)
-    benchmarks_path = _benchmarks_path / "run-time-comparison.md"
+    _benchmarks_dir_path.mkdir(parents=True, exist_ok=True)
+    benchmarks_path = _benchmarks_dir_path / "run-time-comparison.md"
 
     # download benchmark artifacts if any exist on GitHub
     if not benchmarks_path.is_file():
-        benchmarks_path = download_benchmarks(_benchmarks_path, repo_owner=repo_owner)
+        benchmarks_path = download_benchmarks(_benchmarks_dir_path, repo_owner=repo_owner)
 
     # run benchmarks again if no benchmarks found on GitHub or overwrite requested
     if overwrite or not benchmarks_path.is_file():
@@ -172,15 +172,15 @@ def build_benchmark_tex(
         )
         assert not ret, out + err
         assert tex_path.is_file()
-
+    
     if (_distribution_path / f"{benchmarks_path.stem}.md").is_file():
         assert (_docs_path / "ReleaseNotes" / f"{benchmarks_path.stem}.tex").is_file()
 
 
 @flaky
 @requires_github
 def test_build_benchmark_tex(tmp_path):
-    benchmarks_path = _benchmarks_path / "run-time-comparison.md"
+    benchmarks_path = _benchmarks_dir_path / "run-time-comparison.md"
     tex_path = _distribution_path / f"{benchmarks_path.stem}.tex"
 
     try:
@@ -190,6 +190,22 @@ def test_build_benchmark_tex(tmp_path):
         tex_path.unlink(missing_ok=True)
 
 
+def build_deprecations_tex():
+    deprecations_path = _docs_path / "mf6io" / "mf6ivar" / "md" / "deprecations.md"
+
+    # convert markdown deprecations to LaTeX
+    with set_dir(_release_notes_path):
+        tex_path = Path("deprecations.tex")
+        tex_path.unlink(missing_ok=True)
+        out, err, ret = run_cmd(
+            sys.executable, "mk_deprecations.py", deprecations_path, verbose=True
+        )
+        assert not ret, out + err
+        assert tex_path.is_file()
+
+    assert (_docs_path / "ReleaseNotes" / f"{deprecations_path.stem}.tex").is_file()
+
+
 def build_mf6io_tex_from_dfn(overwrite: bool = False):
     if overwrite:
         clean_tex_files()
@@ -461,8 +477,8 @@ def build_documentation(
             example_model_path=example_path,
         )
 
-    # build LaTeX file describing distribution folder structure
-    # build_tex_folder_structure(overwrite=True)
+    # build deprecations table for insertion into LaTex release notes
+    build_deprecations_tex()
 
     if not full:
         # convert LaTeX to PDF

doc/ReleaseNotes/ReleaseNotes.tex

@@ -297,6 +297,10 @@ \section{MODFLOW~6 Documentation}
 
 \noindent Description of the MODFLOW~6 input and output is included in this distribution in the ``doc'' folder as mf6io.pdf.
 
+% -------------------------------------------------
+% if deprecation information exists, then include the deprecation table
+\IfFileExists{./deprecations.tex}{\input{./deprecations.tex}}{}
+
 % -------------------------------------------------
 % if runtime information exists, then include the run time comparison table
 \IfFileExists{./run-time-comparison.tex}{\input{./run-time-comparison.tex}}{}

doc/ReleaseNotes/mk_deprecations.py

@@ -0,0 +1,72 @@
+# This script converts the markdown deprecations table
+# into a latex table for inclusion in the release notes
+import argparse
+from pathlib import Path
+from warnings import warn
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path")
+    args = parser.parse_args()
+
+    header = r"""
+    \section{Deprecations}
+
+    Deprecated/removed options in the current version of MODFLOW 6. Deprecated options are not suggested for use and may (but need not) be removed in a future version of MODFLOW 6. Removed options are no longer available in the current version of MODFLOW 6.
+
+    \small
+    \begin{longtable}[!htbp]{p{5cm} p{3cm} p{3cm} p{1.5cm}}
+    \caption{List of deprecations and removals}
+    \label{table:deprecations}
+    \tabularnewline
+
+    \hline
+    \hline
+    \textbf{Model--Package} & \textbf{Option} & \textbf{Deprecated} & \textbf{Removed}   \\
+    \hline
+    \endfirsthead
+
+    \hline
+    \hline
+    \textbf{Model--Package} & \textbf{Option} & \textbf{Deprecated} & \textbf{Removed}   \\
+    \hline
+    \endhead
+
+    """
+
+    footer = r"""
+    \hline
+    \end{longtable}
+    \normalsize
+    """
+
+    fname = "deprecations"
+    fpath = Path(args.path).expanduser().absolute()
+    fnametex = Path(f"{fname}.tex").absolute()
+    fnametex.unlink(missing_ok=True)
+
+    # if the markdown file exists, convert it to latex
+    if fpath.is_file():
+        ftex = open(fnametex, 'w')
+        ftex.write(header)
+        skipline = True
+        with open(fpath) as fmd:
+            for line in fmd:
+                if not skipline:
+                    ll = line.strip().split('|')
+                    ll = ll[1:-1]
+                    linetex = "& ".join(ll)
+                    linetex = linetex.replace("\\", "/")
+                    linetex += '\\\\' + '\n'
+                    linetex = linetex.replace("%", "\\%")
+                    linetex = linetex.replace("_", "\\_")
+                    ftex.write(linetex)
+                    ftex.write("\\hline\n")
+                if ":-" in line:
+                    skipline = False
+        ftex.write(footer)
+        ftex.close()
+        print(f"Created LaTex file {fnametex} from markdown deprecations file {fpath}")
+    else:
+        warn(f"Deprecations not found: {fpath}")

doc/mf6io/mf6ivar/deprecations.py

@@ -0,0 +1,52 @@
+import os
+from typing import List, Tuple, Optional
+from packaging.version import Version
+from pathlib import Path
+
+
+def get_deprecations(dfndir) -> List[Tuple[Path, str, Version, Optional[Version]]]:
+    dfns = Path(dfndir).rglob("*.dfn")
+    deps = {}
+    for dfn in dfns:
+        with open(dfn, "r") as f:
+            name = None
+            for line in f:
+                if line.startswith("#"):
+                    continue
+                keys = ["deprecated", "removed"]
+                ikeys = {k: i for i, k in enumerate(keys)}
+                for key in keys:
+                    if line.startswith("name"):
+                        name = line.split()[1]
+                    if line.startswith(key):
+                        val = deps.get((dfn, key), [None, None])
+                        key, ver = line.split()
+                        ik = ikeys[key]
+                        val[ik] = val[ik] if val[ik] else Version(ver)
+                        deps[(dfn, name)] = val
+
+    return [(file, key, dep, rem) for (file, key), (dep, rem) in deps.items()]
+
+
+def create_deprecations_file(dfndir, mddir, verbose):
+    deprecations = get_deprecations(dfndir)
+    deps_path = (Path(mddir) / 'deprecations.md').absolute()
+    if verbose:
+        print(f"Found {len(deprecations)} deprecations, writing {deps_path}")
+    with open(deps_path, "w") as f:
+        s = "#### Deprecations\n\n"
+        s += "The following table lists deprecated options and the versions in which they were deprecated and (optionally) removed.\n\n"
+        if any(deprecations):
+            s += "| Model-Package | Option | Deprecated | Removed |\n"
+            s += "|:--------------|:-------|:-----------|:--------|\n"
+            for (file, option, deprecated, removed) in deprecations:
+                s += f"| {file.stem} | {option} | {deprecated} | {removed if removed else ''} |\n"
+            if len(s) > 0:
+                s += "\n"
+        f.write(s)
+
+
+if __name__ == '__main__':
+    dfndir = os.path.join('.', 'dfn')
+    mddir = os.path.join('.', 'md')
+    create_deprecations_file(dfndir, mddir, verbose=True)

doc/mf6io/mf6ivar/md/deprecations.md

@@ -0,0 +1,14 @@
+#### Deprecations
+
+The following table lists deprecated options and the versions in which they were deprecated and (optionally) removed.
+
+| Model-Package | Option | Deprecated | Removed |
+|:--------------|:-------|:-----------|:--------|
+| gwf-sfr | unit_conversion | 6.4.2 |  |
+| sln-ims | csv_output_filerecord | 6.1.1 |  |
+| sln-ims | csv_output | 6.1.1 |  |
+| sln-ims | csvfile | 6.1.1 |  |
+| sln-ims | outer_hclose | 6.1.1 |  |
+| sln-ims | outer_rclosebnd | 6.1.1 |  |
+| sln-ims | inner_hclose | 6.1.1 |  |
+