diff --git a/PYTOOLS/dmschlo/CMAQ_DMS_ChlorA.ipynb b/PYTOOLS/dmschlo/CMAQ_DMS_ChlorA.ipynb index 1e75f36f8..f6fb3e084 100644 --- a/PYTOOLS/dmschlo/CMAQ_DMS_ChlorA.ipynb +++ b/PYTOOLS/dmschlo/CMAQ_DMS_ChlorA.ipynb @@ -31,10 +31,19 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ + "import os.path\n", + "from pathlib import Path\n", + "\n", + "data_folder = Path(\"/fsx\")\n", + "\n", + "file_to_open = data_folder / \"12US1_surf.ncf\"\n", + "file_to_create = data_folder / \"OCEAN_%m_L3m_MC_CHL_chlor_a_12US1.nc\"\n", + "\n", + "\n", "# dom : str\n", "# Name of output domain. For example, 12US1, 36US3, 108NHEMI2. This is used\n", "# to name outputs and inputs.\n", @@ -46,12 +55,14 @@ "# function described at https://strftime.net/. For 2016-01-01, here are a\n", "# few examples: %F = 2016-01-01, %Y%j = 2016001, %m%d = 0101, %b = Jan,\n", "# %^b = JAN. e.g., OCEAN_%Y%b.nc = OCEAN_2016Jan.nc\n", - "ocnintmpl = f'/work/MOD3DATA/2016_{dom}/surface/{dom}_surf.ncf'\n", + "#ocnintmpl = file_to_open\n", + "ocnintmpl = f'/fsx/12US1_surf.ncf'\n", "\n", "# ocnouttmpl : str\n", "# strftime template to create a new file. The new file will have ocnintmpl\n", "# variables in addition to DMS and CHLO.\n", - "ocnouttmpl = f'output/{dom}/OCEAN_%m_L3m_MC_CHL_chlor_a_{dom}.nc'\n", + "#ocnouttmpl = file_to_create\n", + "ocnouttmpl = f'OCEAN_%m_L3m_MC_CHL_chlor_a_12US1.nc'\n", "\n", "# gdpath : str\n", "# Path to an IOAPI file using the domain (dom). Most of the time you can use\n", @@ -88,7 +99,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 30, "metadata": { "id": "kEMW45MRTS2b" }, @@ -97,9 +108,9 @@ "installprereq = True\n", "installcdo = True\n", "# Use a preinstalled version of cdo in the user path\n", - "cdopath = 'cdo'\n", + "#cdopath = 'cdo'\n", "# Or specify a specific path. For example, at EPA use :\n", - "# cdopath = '/work/ROMO/anaconda_envs/cdo-1.9.8/bin/cdo'\n", + "cdopath = '/shared/build/proj/libs4cdo-0.0.11/build/bin/cdo'\n", "# If you intall cdo with miniconda\n", "# cdopath = './miniconda/bin/cdo'" ] @@ -121,19 +132,19 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 31, "metadata": { "id": "_Rj0_8nn93gD" }, "outputs": [], "source": [ - "import os\n", - "if installcdo:\n", - " cdopath = './miniconda/bin/cdo'\n", - " if not os.path.exists(cdopath):\n", - " !wget -N -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh\n", - " !bash ./Miniconda3-latest-Linux-x86_64.sh -b -p ./miniconda &> log.miniconda\n", - " !./miniconda/bin/conda install -c conda-forge -q -y 'cdo~=1.9' &> log.cdo" + "#import os\n", + "#if installcdo:\n", + "# cdopath = './miniconda/bin/cdo'\n", + "# if not os.path.exists(cdopath):\n", + "# !wget -N -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh\n", + " # !bash ./Miniconda3-latest-Linux-x86_64.sh -b -p ./miniconda &> log.miniconda\n", + " # !./miniconda/bin/conda install -c conda-forge -q -y 'cdo~=1.9' &> log.cdo" ] }, { @@ -150,7 +161,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 32, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -164,7 +175,7 @@ " # If on Google Colab:\n", " # 1. Copy the requirements file to this folder,\n", " # 2. remove --user\n", - " !python -m pip install -q --user -r requirements.txt" + " !python3 -m pip install -q --user -r requirements.txt" ] }, { @@ -203,7 +214,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 33, "metadata": { "id": "1Za8quom-Jw5" }, @@ -246,7 +257,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 34, "metadata": { "id": "48wbKUlv_p-Q" }, @@ -256,7 +267,7 @@ "output_type": "stream", "text": [ "Automatic pdb calling has been turned OFF\n", - "CDI version 1.9.10\n" + "CDI version 1.9.8\n" ] } ], @@ -280,7 +291,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 35, "metadata": { "id": "PNJHfh9F_lqN" }, @@ -294,7 +305,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 36, "metadata": { "id": "MNbuxMR53G41" }, @@ -308,7 +319,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 37, "metadata": { "id": "weQTj7odsz43" }, @@ -370,19 +381,11 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 38, "metadata": { "id": "Dsxsj5-zliXT" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 100.0%" - ] - } - ], + "outputs": [], "source": [ "dmsurl = 'https://www.bodc.ac.uk/solas_integration/implementation_products/group1/dms/documents/dmsclimatology.zip'\n", "dmsdest = 'dmsclimatology/dmsclimatology.zip'\n", @@ -401,7 +404,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 39, "metadata": { "id": "1KUkmDBclsOt" }, @@ -422,11 +425,19 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 41, "metadata": { "id": "rbf5_7XOlwi9" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Keeping file dmsclimatology/dmsconcentration.nc\n" + ] + } + ], "source": [ "dmsncpath = 'dmsclimatology/dmsconcentration.nc'\n", "if overwrite or not os.path.exists(dmsncpath):\n", @@ -490,7 +501,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 46, "metadata": { "id": "t0AxBqGyuPBk" }, @@ -499,6 +510,7 @@ "# Choose one or neither\n", "usesetmisstonn = True\n", "usefillmiss = False\n", + "overwrite = True\n", "\n", "dmsoutpath = f'dmsclimatology/{dom}/dmsconcentration.{dom}.nc'\n", "if overwrite or not os.path.exists(dmsoutpath):\n", @@ -566,27 +578,28 @@ "\n", " urls = []\n", " webroot = 'https://oceandata.sci.gsfc.nasa.gov:443/opendap/MODISA/L3SMI/'\n", - " for prefix in ['2003/213', '2003/244', '2003/274', '2003/305', '2003/335', '2004/001', '2004/032', '2004/061', '2004/092', '2004/122', '2004/153', '2004/183']:\n", + " for prefix in [ '2002/0701', '2002/0801', '2002/0901', '2002/1001', '2002/1101', \n", + " '2002/1201', '2003/0101', '2003/0201', '2003/0301', '2003/0401', '2003/0501', '2003/0601' ]:\n", " htmlout = urlopen(webroot + prefix)\n", " htmltxt = htmlout.read().decode()\n", - " mostrecent = sorted(re.compile('(?<=>).+L3m_MC_CHL_chlor_a_9km.nc(?=).+.L3m_MC_CHL_chlor_a_9km.nc(?=\n", + "root group (NETCDF3_64BIT_OFFSET data model, file format NETCDF3):\n", + " CDI: Climate Data Interface version 1.9.8 (https://mpimet.mpg.de/cdi)\n", + " Conventions: CF-1.6\n", + " history: Thu Sep 14 19:14:47 2023: cdo -O -f nc -setmisstoc,0 -remapycon,cdogrids/12US1.cdo -setctomiss,-999. -setmisstonn dmsclimatology/dmsconcentration.nc dmsclimatology/12US1/dmsconcentration.12US1.nc\n", + " dimensions(sizes): time(0), COL(459), ROW(299)\n", + " variables(dimensions): float64 time(time), float64 COL(COL), float64 ROW(ROW), int32 Projection()\n", + " groups: \n" + ] + }, + { + "ename": "IndexError", + "evalue": "index exceeds dimension bounds", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mti\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmydate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdmsf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m \u001b[0mdmstimef\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdmsf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mslice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mti\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 27\u001b[0m \u001b[0mdmsinv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdmstimef\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariables\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'DMS'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0mchlf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpncopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchloutpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mformat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'netcdf'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.local/lib/python3.6/site-packages/PseudoNetCDF/core/_files.py\u001b[0m in \u001b[0;36msliceDimensions\u001b[0;34m(self, newdims, verbose, **dimslices)\u001b[0m\n\u001b[1;32m 1973\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1974\u001b[0m \u001b[0mdvar\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1975\u001b[0;31m \u001b[0mnewdl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdvar\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdimslices\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdk\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1976\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1977\u001b[0m \u001b[0mnewdl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32msrc/netCDF4/_netCDF4.pyx\u001b[0m in \u001b[0;36mnetCDF4._netCDF4.Variable.__getitem__\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32msrc/netCDF4/_netCDF4.pyx\u001b[0m in \u001b[0;36mnetCDF4._netCDF4.Variable._get\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mIndexError\u001b[0m: index exceeds dimension bounds" ] } ], "source": [ "dmsf = pnc.pncopen(f'dmsclimatology/{dom}/dmsconcentration.{dom}.nc', format='netcdf')\n", "ocnoutpaths = []\n", - "for chloutpath in sorted(glob(f'chlor_a/{dom}/A2*_{dom}.nc')):\n", - " mydate = datetime.strptime(os.path.basename(chloutpath)[1:8], '%Y%j')\n", + "print('before for loop')\n", + "for chloutpath in sorted(glob(f'chlor_a/{dom}/AQUA*.nc')):\n", + " mydate = datetime.strptime(os.path.basename(chloutpath)[11:19], '%Y%m%d')\n", " # seaicepath = mydate.strftime(f'output/{dom}/SEAICE_2020%m01_CLIM.nc')\n", - " ocnoutpath = mydate.strftime(ocnouttmpl)\n", + " print(ocnouttmpl)\n", + " ocnoutpath = mydate.strftime('%m')\n", + " print(ocnoutpath)\n", " ocnoutpaths.append(ocnoutpath)\n", + " print(ocnintmpl)\n", " ocnpath = mydate.strftime(ocnintmpl)\n", + " print('working')\n", + " print(ocnpath)\n", " ocnf = pnc.pncopen(ocnpath, format='ioapi')\n", " if not overwrite and os.path.exists(ocnoutpath):\n", " print('Keeping', ocnoutpath)\n", " continue\n", " if os.path.exists(ocnoutpath):\n", " os.remove(ocnoutpath)\n", - "\n", - " ti = mydate.month - 1\n", + " print(mydate.month)\n", + " #ti = mydate.month - 1\n", + " ti = mydate.month\n", " print(ti, mydate)\n", + " print(dmsf)\n", " dmstimef = dmsf.slice(time=ti)\n", " dmsinv = dmstimef.variables['DMS']\n", " chlf = pnc.pncopen(chloutpath, format='netcdf')\n", @@ -867,7 +906,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -875,21 +914,7 @@ "id": "eq5Kfd2m9uyK", "outputId": "cdae52d3-2bf3-4563-bdb5-9ca0bb7f4b50" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "; OCEAN file /work/MOD3DATA/2016_12US1/surface/12US1_surf.ncf\n", - "DMS added from SOLAS project after regridding and gap filling\n", - "(https://www.bodc.ac.uk/solas_integration/implementation_products/group1/dms/documents/dmsclimatology.zip)\n", - "CHLO added from MODIS after regridding and gap filling\n", - "chlor_a/12US1/A20041832020213.L3m_MC_CHL_chlor_a_12US1.nc\n", - " \n", - " \n" - ] - } - ], + "outputs": [], "source": [ "outf = pnc.pncopen(ocnoutpath, format='ioapi')\n", "print(outf.FILEDESC)\n", @@ -909,7 +934,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -917,27 +942,7 @@ "id": "dLE6gkWBgbVq", "outputId": "a6db4df5-835b-430d-e7ba-41889a6cdb1f" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/work/MOD3DATA/2016_12US1/surface/12US1_surf.ncf\n", - "output/12US1/OCEAN_08_L3m_MC_CHL_chlor_a_12US1.nc\n", - "output/12US1/OCEAN_09_L3m_MC_CHL_chlor_a_12US1.nc\n", - "output/12US1/OCEAN_10_L3m_MC_CHL_chlor_a_12US1.nc\n", - "output/12US1/OCEAN_11_L3m_MC_CHL_chlor_a_12US1.nc\n", - "output/12US1/OCEAN_12_L3m_MC_CHL_chlor_a_12US1.nc\n", - "output/12US1/OCEAN_01_L3m_MC_CHL_chlor_a_12US1.nc\n", - "output/12US1/OCEAN_02_L3m_MC_CHL_chlor_a_12US1.nc\n", - "output/12US1/OCEAN_03_L3m_MC_CHL_chlor_a_12US1.nc\n", - "output/12US1/OCEAN_04_L3m_MC_CHL_chlor_a_12US1.nc\n", - "output/12US1/OCEAN_05_L3m_MC_CHL_chlor_a_12US1.nc\n", - "output/12US1/OCEAN_06_L3m_MC_CHL_chlor_a_12US1.nc\n", - "output/12US1/OCEAN_07_L3m_MC_CHL_chlor_a_12US1.nc\n" - ] - } - ], + "outputs": [], "source": [ "with zipfile.ZipFile('downloaddmschlo.zip', 'w') as zf:\n", " print(ocnpath)\n", @@ -964,9 +969,9 @@ "provenance": [] }, "kernelspec": { - "display_name": "anaconda", + "display_name": "Python 3", "language": "python", - "name": "anaconda" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -978,7 +983,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.1" + "version": "3.6.3" } }, "nbformat": 4,