initial commit

code/authenticate.py

@@ -0,0 +1,29 @@
+from utils import *
+
+urs = 'urs.earthdata.nasa.gov'    # Earthdata URL endpoint for authentication
+prompts = ['Enter NASA Earthdata Login Username: ',
+           'Enter NASA Earthdata Login Password: ']
+
+# Determine the OS (Windows machines usually use an '_netrc' file)
+netrc_name = "_netrc" if system()=="Windows" else ".netrc"
+
+# Determine if netrc file exists, and if so, if it includes NASA Earthdata Login Credentials
+try:
+    netrcDir = os.path.expanduser(f"~/{netrc_name}")
+    netrc(netrcDir).authenticators(urs)[0]
+
+# Below, create a netrc file and prompt user for NASA Earthdata Login Username and Password
+except FileNotFoundError:
+    homeDir = os.path.expanduser("~")
+    Popen('touch {0}{2} | echo machine {1} >> {0}{2}'.format(homeDir + os.sep, urs, netrc_name), shell=True)
+    Popen('echo login {} >> {}{}'.format(getpass(prompt=prompts[0]), homeDir + os.sep, netrc_name), shell=True)
+    Popen('echo \'password {} \'>> {}{}'.format(getpass(prompt=prompts[1]), homeDir + os.sep, netrc_name), shell=True)
+    # Set restrictive permissions
+    Popen('chmod 0600 {0}{1}'.format(homeDir + os.sep, netrc_name), shell=True)
+
+    # Determine OS and edit netrc file if it exists but is not set up for NASA Earthdata Login
+except TypeError:
+    homeDir = os.path.expanduser("~")
+    Popen('echo machine {1} >> {0}{2}'.format(homeDir + os.sep, urs, netrc_name), shell=True)
+    Popen('echo login {} >> {}{}'.format(getpass(prompt=prompts[0]), homeDir + os.sep, netrc_name), shell=True)
+    Popen('echo \'password {} \'>> {}{}'.format(getpass(prompt=prompts[1]), homeDir + os.sep, netrc_name), shell=True)

code/end.py

@@ -0,0 +1 @@
+pass

code/harmony_connect_process.py

@@ -0,0 +1,47 @@
+import sys
+from utils import *
+import warnings
+import matplotlib.pyplot as plt
+
+
+short_name = 'MODIS_T-JPL-L2P-v2019.0'
+s3_url = 's3://podaac-ops-cumulus-protected/MODIS_T-JPL-L2P-v2019.0/20210820033500-JPL-L2P_GHRSST-SSTskin-MODIS_T-N-v02.0-fv01.0.nc'
+
+home_dir = os.path.expanduser("~")
+plot_file_name = os.path.join(home_dir, "harmony_plot_1.png")
+
+if os.path.isfile(plot_file_name):
+  os.remove(plot_file_name)
+
+try:
+  harmony_client = Client()
+  request = Request(
+    collection=Collection(id=short_name),
+    spatial=BBox(-97.77667, 21.20806, -83.05197, 30.16605),
+    temporal={
+      'start': dt.datetime(2021, 8, 20),
+      'stop': dt.datetime(2021, 8, 21),
+    },
+  )
+  job_id = harmony_client.submit(request)
+  harmony_client.wait_for_processing(job_id, show_progress=False)
+  data = harmony_client.result_json(job_id)
+  results = harmony_client.result_urls(job_id, link_type=LinkType.s3)
+  urls = list(results)
+  filename = '20210820033500-JPL-L2P_GHRSST-SSTskin-MODIS_T-N-v02.0-fv01.0_subsetted.nc4'
+
+  url = [url for url in urls if filename in url][0]
+  creds = harmony_client.aws_credentials()
+  s3_fs = s3fs.S3FileSystem(
+    key=creds['aws_access_key_id'],
+    secret=creds['aws_secret_access_key'],
+    token=creds['aws_session_token'],
+    client_kwargs={'region_name': 'us-west-2'},
+  )
+
+  f = s3_fs.open(url, mode='rb')
+  ds = xr.open_dataset(f)
+  ds.sea_surface_temperature.plot()
+  plt.savefig(plot_file_name)
+except Exception as e:
+  print(e)

code/podaac_s3.py

@@ -0,0 +1,62 @@
+import os
+import matplotlib.pyplot as plt
+from authenticate import *
+
+home_dir = os.path.expanduser("~")
+plot_file_name = os.path.join(home_dir, "podaac_s3_plot_1.png")
+
+if os.path.isfile(plot_file_name):
+  os.remove(plot_file_name)
+
+urs = 'urs.earthdata.nasa.gov'    # Earthdata URL endpoint for authentication
+prompts = ['Enter NASA Earthdata Login Username: ',
+           'Enter NASA Earthdata Login Password: ']
+
+# Determine the OS (Windows machines usually use an '_netrc' file)
+netrc_name = "_netrc" if system()=="Windows" else ".netrc"
+
+# Determine if netrc file exists, and if so, if it includes NASA Earthdata Login Credentials
+try:
+    netrcDir = os.path.expanduser(f"~/{netrc_name}")
+    netrc(netrcDir).authenticators(urs)[0]
+
+# Below, create a netrc file and prompt user for NASA Earthdata Login Username and Password
+except FileNotFoundError:
+    homeDir = os.path.expanduser("~")
+    Popen('touch {0}{2} | echo machine {1} >> {0}{2}'.format(homeDir + os.sep, urs, netrc_name), shell=True)
+    Popen('echo login {} >> {}{}'.format(getpass(prompt=prompts[0]), homeDir + os.sep, netrc_name), shell=True)
+    Popen('echo \'password {} \'>> {}{}'.format(getpass(prompt=prompts[1]), homeDir + os.sep, netrc_name), shell=True)
+    # Set restrictive permissions
+    Popen('chmod 0600 {0}{1}'.format(homeDir + os.sep, netrc_name), shell=True)
+
+    # Determine OS and edit netrc file if it exists but is not set up for NASA Earthdata Login
+except TypeError:
+    homeDir = os.path.expanduser("~")
+    Popen('echo machine {1} >> {0}{2}'.format(homeDir + os.sep, urs, netrc_name), shell=True)
+    Popen('echo login {} >> {}{}'.format(getpass(prompt=prompts[0]), homeDir + os.sep, netrc_name), shell=True)
+    Popen('echo \'password {} \'>> {}{}'.format(getpass(prompt=prompts[1]), homeDir + os.sep, netrc_name), shell=True)
+
+s3_cred_endpoint = {
+    'podaac':'https://archive.podaac.earthdata.nasa.gov/s3credentials',
+    'gesdisc': 'https://data.gesdisc.earthdata.nasa.gov/s3credentials',
+    'lpdaac':'https://data.lpdaac.earthdatacloud.nasa.gov/s3credentials',
+    'ornldaac': 'https://data.ornldaac.earthdata.nasa.gov/s3credentials',
+    'ghrcdaac': 'https://data.ghrc.earthdata.nasa.gov/s3credentials'
+}
+
+def get_temp_creds(provider):
+    return requests.get(s3_cred_endpoint[provider]).json()
+temp_creds_req = get_temp_creds('podaac')
+fs_s3 = s3fs.S3FileSystem(anon=False, 
+                          key=temp_creds_req['accessKeyId'], 
+                          secret=temp_creds_req['secretAccessKey'], 
+                          token=temp_creds_req['sessionToken'],
+                          client_kwargs={'region_name':'us-west-2'})
+short_name = 'MODIS_T-JPL-L2P-v2019.0'
+s3_url = 's3://podaac-ops-cumulus-protected/MODIS_T-JPL-L2P-v2019.0/20210820033500-JPL-L2P_GHRSST-SSTskin-MODIS_T-N-v02.0-fv01.0.nc'
+
+s3_file_obj = fs_s3.open(s3_url, mode='rb')
+ssh_ds = xr.open_dataset(s3_file_obj)
+ssh_ds.sea_surface_temperature.plot()
+plt.savefig(plot_file_name)
+s3_file_obj.close()

code/process.json

@@ -0,0 +1,41 @@
+[{
+  "id" : "iy50ea",
+  "name" : "utils",
+  "description" : "python",
+  "code" : "# Earthdata Login\nfrom netrc import netrc\nfrom subprocess import Popen\nfrom platform import system\nfrom getpass import getpass\nimport os\n\n# Direct access\nimport requests\nimport s3fs\nimport xarray as xr\nimport hvplot.xarray\n\n# Harmony\nfrom harmony import BBox, Client, Collection, Request, LinkType\nfrom harmony.config import Environment\nfrom pprint import pprint\nimport datetime as dt",
+  "lang" : "python",
+  "owner" : "111111",
+  "confidential" : "FALSE"
+},{
+  "id" : "y2amjw",
+  "name" : "authenticate",
+  "description" : null,
+  "code" : "from utils import *\n\nurs = 'urs.earthdata.nasa.gov'    # Earthdata URL endpoint for authentication\nprompts = ['Enter NASA Earthdata Login Username: ',\n           'Enter NASA Earthdata Login Password: ']\n\n# Determine the OS (Windows machines usually use an '_netrc' file)\nnetrc_name = \"_netrc\" if system()==\"Windows\" else \".netrc\"\n\n# Determine if netrc file exists, and if so, if it includes NASA Earthdata Login Credentials\ntry:\n    netrcDir = os.path.expanduser(f\"~/{netrc_name}\")\n    netrc(netrcDir).authenticators(urs)[0]\n\n# Below, create a netrc file and prompt user for NASA Earthdata Login Username and Password\nexcept FileNotFoundError:\n    homeDir = os.path.expanduser(\"~\")\n    Popen('touch {0}{2} | echo machine {1} >> {0}{2}'.format(homeDir + os.sep, urs, netrc_name), shell=True)\n    Popen('echo login {} >> {}{}'.format(getpass(prompt=prompts[0]), homeDir + os.sep, netrc_name), shell=True)\n    Popen('echo \\'password {} \\'>> {}{}'.format(getpass(prompt=prompts[1]), homeDir + os.sep, netrc_name), shell=True)\n    # Set restrictive permissions\n    Popen('chmod 0600 {0}{1}'.format(homeDir + os.sep, netrc_name), shell=True)\n\n    # Determine OS and edit netrc file if it exists but is not set up for NASA Earthdata Login\nexcept TypeError:\n    homeDir = os.path.expanduser(\"~\")\n    Popen('echo machine {1} >> {0}{2}'.format(homeDir + os.sep, urs, netrc_name), shell=True)\n    Popen('echo login {} >> {}{}'.format(getpass(prompt=prompts[0]), homeDir + os.sep, netrc_name), shell=True)\n    Popen('echo \\'password {} \\'>> {}{}'.format(getpass(prompt=prompts[1]), homeDir + os.sep, netrc_name), shell=True)",
+  "lang" : "python",
+  "owner" : "111111",
+  "confidential" : "FALSE"
+},{
+  "id" : "cqp5y5",
+  "name" : "podaac_s3",
+  "description" : null,
+  "code" : "import os\nimport matplotlib.pyplot as plt\nfrom authenticate import *\n\nhome_dir = os.path.expanduser(\"~\")\nplot_file_name = os.path.join(home_dir, \"podaac_s3_plot_1.png\")\n\nif os.path.isfile(plot_file_name):\n  os.remove(plot_file_name)\n\nurs = 'urs.earthdata.nasa.gov'    # Earthdata URL endpoint for authentication\nprompts = ['Enter NASA Earthdata Login Username: ',\n           'Enter NASA Earthdata Login Password: ']\n\n# Determine the OS (Windows machines usually use an '_netrc' file)\nnetrc_name = \"_netrc\" if system()==\"Windows\" else \".netrc\"\n\n# Determine if netrc file exists, and if so, if it includes NASA Earthdata Login Credentials\ntry:\n    netrcDir = os.path.expanduser(f\"~/{netrc_name}\")\n    netrc(netrcDir).authenticators(urs)[0]\n\n# Below, create a netrc file and prompt user for NASA Earthdata Login Username and Password\nexcept FileNotFoundError:\n    homeDir = os.path.expanduser(\"~\")\n    Popen('touch {0}{2} | echo machine {1} >> {0}{2}'.format(homeDir + os.sep, urs, netrc_name), shell=True)\n    Popen('echo login {} >> {}{}'.format(getpass(prompt=prompts[0]), homeDir + os.sep, netrc_name), shell=True)\n    Popen('echo \\'password {} \\'>> {}{}'.format(getpass(prompt=prompts[1]), homeDir + os.sep, netrc_name), shell=True)\n    # Set restrictive permissions\n    Popen('chmod 0600 {0}{1}'.format(homeDir + os.sep, netrc_name), shell=True)\n\n    # Determine OS and edit netrc file if it exists but is not set up for NASA Earthdata Login\nexcept TypeError:\n    homeDir = os.path.expanduser(\"~\")\n    Popen('echo machine {1} >> {0}{2}'.format(homeDir + os.sep, urs, netrc_name), shell=True)\n    Popen('echo login {} >> {}{}'.format(getpass(prompt=prompts[0]), homeDir + os.sep, netrc_name), shell=True)\n    Popen('echo \\'password {} \\'>> {}{}'.format(getpass(prompt=prompts[1]), homeDir + os.sep, netrc_name), shell=True)\n\ns3_cred_endpoint = {\n    'podaac':'https://archive.podaac.earthdata.nasa.gov/s3credentials',\n    'gesdisc': 'https://data.gesdisc.earthdata.nasa.gov/s3credentials',\n    'lpdaac':'https://data.lpdaac.earthdatacloud.nasa.gov/s3credentials',\n    'ornldaac': 'https://data.ornldaac.earthdata.nasa.gov/s3credentials',\n    'ghrcdaac': 'https://data.ghrc.earthdata.nasa.gov/s3credentials'\n}\n\ndef get_temp_creds(provider):\n    return requests.get(s3_cred_endpoint[provider]).json()\ntemp_creds_req = get_temp_creds('podaac')\nfs_s3 = s3fs.S3FileSystem(anon=False, \n                          key=temp_creds_req['accessKeyId'], \n                          secret=temp_creds_req['secretAccessKey'], \n                          token=temp_creds_req['sessionToken'],\n                          client_kwargs={'region_name':'us-west-2'})\nshort_name = 'MODIS_T-JPL-L2P-v2019.0'\ns3_url = 's3://podaac-ops-cumulus-protected/MODIS_T-JPL-L2P-v2019.0/20210820033500-JPL-L2P_GHRSST-SSTskin-MODIS_T-N-v02.0-fv01.0.nc'\n\ns3_file_obj = fs_s3.open(s3_url, mode='rb')\nssh_ds = xr.open_dataset(s3_file_obj)\nssh_ds.sea_surface_temperature.plot()\nplt.savefig(plot_file_name)\ns3_file_obj.close()",
+  "lang" : "python",
+  "owner" : "111111",
+  "confidential" : "FALSE"
+},{
+  "id" : "dh8stf",
+  "name" : "end",
+  "description" : "python",
+  "code" : "pass",
+  "lang" : "python",
+  "owner" : "111111",
+  "confidential" : "FALSE"
+},{
+  "id" : "u09gh6",
+  "name" : "harmony_connect_process",
+  "description" : null,
+  "code" : "import sys\nfrom utils import *\nimport warnings\nimport matplotlib.pyplot as plt\n\n\nshort_name = 'MODIS_T-JPL-L2P-v2019.0'\ns3_url = 's3://podaac-ops-cumulus-protected/MODIS_T-JPL-L2P-v2019.0/20210820033500-JPL-L2P_GHRSST-SSTskin-MODIS_T-N-v02.0-fv01.0.nc'\n\nhome_dir = os.path.expanduser(\"~\")\nplot_file_name = os.path.join(home_dir, \"harmony_plot_1.png\")\n\nif os.path.isfile(plot_file_name):\n  os.remove(plot_file_name)\n\ntry:\n  harmony_client = Client()\n  request = Request(\n    collection=Collection(id=short_name),\n    spatial=BBox(-97.77667, 21.20806, -83.05197, 30.16605),\n    temporal={\n      'start': dt.datetime(2021, 8, 20),\n      'stop': dt.datetime(2021, 8, 21),\n    },\n  )\n  job_id = harmony_client.submit(request)\n  harmony_client.wait_for_processing(job_id, show_progress=False)\n  data = harmony_client.result_json(job_id)\n  results = harmony_client.result_urls(job_id, link_type=LinkType.s3)\n  urls = list(results)\n  filename = '20210820033500-JPL-L2P_GHRSST-SSTskin-MODIS_T-N-v02.0-fv01.0_subsetted.nc4'\n\n  url = [url for url in urls if filename in url][0]\n  creds = harmony_client.aws_credentials()\n  s3_fs = s3fs.S3FileSystem(\n    key=creds['aws_access_key_id'],\n    secret=creds['aws_secret_access_key'],\n    token=creds['aws_session_token'],\n    client_kwargs={'region_name': 'us-west-2'},\n  )\n\n  f = s3_fs.open(url, mode='rb')\n  ds = xr.open_dataset(f)\n  ds.sea_surface_temperature.plot()\n  plt.savefig(plot_file_name)\nexcept Exception as e:\n  print(e)",
+  "lang" : "python",
+  "owner" : "111111",
+  "confidential" : "FALSE"
+}]

code/utils.py

@@ -0,0 +1,18 @@
+# Earthdata Login
+from netrc import netrc
+from subprocess import Popen
+from platform import system
+from getpass import getpass
+import os
+
+# Direct access
+import requests
+import s3fs
+import xarray as xr
+import hvplot.xarray
+
+# Harmony
+from harmony import BBox, Client, Collection, Request, LinkType
+from harmony.config import Environment
+from pprint import pprint
+import datetime as dt