Merge pull request #1 from rachellombardi/rachellombardi-patch-1

Rachellombardi patch 1

.github/workflows/validate-caches.yml

@@ -0,0 +1,20 @@
+name: Validate StashCache configs
+on:
+  schedule:
+    # Run daily
+    - cron: '0 0 * * *'
+
+jobs:
+  validate-caches:
+    name: Validate Topology caches
+    if: startsWith(github.repository, 'opensciencegrid/')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9.15
+      - name: Compare StashCache config lists
+        run: |
+          ./src/tests/diff_cache_configs.py

.github/workflows/validate-code.yml

@@ -10,7 +10,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6
+          python-version: 3.9.15
       - name: Install packages
         run: |
           sudo apt-get update
@@ -23,10 +23,12 @@ jobs:
       - name: Test API
         run: |
           export TOPOLOGY_CONFIG=$PWD/src/config-ci.py
+          export FLASK_DEBUG=1
           py.test ./src/tests/test_api.py
       - name: Test StashCache
         run: |
           export TOPOLOGY_CONFIG=$PWD/src/config-ci.py
+          export FLASK_DEBUG=1
           py.test ./src/tests/test_stashcache.py
       - name: Test cacher
         run: |

.github/workflows/validate-data.yml

@@ -11,7 +11,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6
+          python-version: 3.9.15
       - name: Install packages
         run: |
           sudo apt-get update

Dockerfile

@@ -1,4 +1,4 @@
-FROM opensciencegrid/software-base:3.6-el8-release
+FROM opensciencegrid/software-base:3.6-al8-release
 
 # Install dependencies (application, Apache)
 RUN \
@@ -7,8 +7,8 @@ RUN \
       gcc \
       git \
       libyaml-devel \
-      python3-devel \
-      python3-pip \
+      python39-devel \
+      python39-pip \
     && yum install -y \
       fetch-crl \
       httpd \
@@ -19,10 +19,12 @@ RUN \
       /usr/bin/pkill \
     && yum clean all && rm -rf /var/cache/yum/*
 
+RUN alternatives --set python3 /usr/bin/python3.9
+
 WORKDIR /app
 
 # Install application dependencies
-COPY requirements-apache.txt src/ ./
+COPY requirements-apache.txt requirements-rootless.txt ./
 RUN pip3 install --no-cache-dir -r requirements-apache.txt
 
 # Create data directory, and gather SSH keys for git
@@ -36,6 +38,8 @@ RUN echo "45 */6 * * * root /usr/sbin/fetch-crl -q -r 21600 -p 10" >  /etc/cron.
     echo "@reboot      root /usr/sbin/fetch-crl -q          -p 10" >> /etc/cron.d/fetch-crl && \
     echo "0 0 * * *    root /usr/bin/pkill -USR1 httpd"            >  /etc/cron.d/httpd
 
+# Install application
+COPY src/ ./
 
 # Set up Apache configuration
 # Remove default SSL config: default certs don't exist on EL8 so the

Dockerfile.client

@@ -1,6 +1,6 @@
 FROM quay.io/centos/centos:stream8
 
-LABEL maintainer="OSG Software <help@opensciencegrid.org>"
+LABEL maintainer="OSG Software <help@osg-htc.org>"
 LABEL name="OSG 3.5 OSG-Notify client"
 
 RUN yum -y install https://repo.opensciencegrid.org/osg/3.5/osg-3.5-el8-release-latest.rpm \

README.md

@@ -146,4 +146,4 @@ for the VO(s) that you support:
 | LHC VOs                     | [GGUS](https://ggus.eu)                          |
 | Anyone else                 | [Helpdesk](https://support.opensciencegrid.org) |
 
-Or email us at help@opensciencegrid.org.
+Or email us at help@osg-htc.org.

bin/next_downtime_id

@@ -4,6 +4,7 @@ set -o pipefail
 cd "$(dirname "$0")"/..
 
 unset GREP_OPTIONS
-egrep -h '^ *ID: *[0-9]{1,9}\>' topology/*/*/*_downtime.yaml |
+# After 18 digits we might start running into integer overflow issues...
+egrep -h '^ *ID: *[0-9]{1,18}\>' topology/*/*/*_downtime.yaml |
   awk '$2 > max {max = $2}; END {print max + 1}'
 

bin/osg-notify

@@ -5,6 +5,7 @@
 A simple tool for generating notification emails to the OSG
 """
 import os
+import re
 import sys
 import getpass
 import smtplib
@@ -157,7 +158,12 @@ def network_ok(bypass_dns_check):
 
 def replace_smart_quotes_and_dashes(contents):
     # Replace smart quotes and em/en dashes
-    replaced = contents.replace('“','"').replace('”','"').replace("–","-").replace("—","-")
+    regex_sub_map = [(r'[“”]', '"'),
+                     (r'[‘’]', "'"),
+                     (r'[—–]', '-')]
+    replaced = contents
+    for pattern, sub in regex_sub_map:
+        replaced = re.sub(pattern, sub, replaced)
     return replaced
 
 def has_non_printable_ascii_characters(contents):

docker/apache.conf

@@ -1,11 +1,11 @@
 Listen 8080
 Listen 8443
 
-LoadModule wsgi_module /usr/local/lib64/python3.6/site-packages/mod_wsgi/server/mod_wsgi-py36.cpython-36m-x86_64-linux-gnu.so
+LoadModule wsgi_module /usr/local/lib64/python3.9/site-packages/mod_wsgi/server/mod_wsgi-py39.cpython-39-x86_64-linux-gnu.so
 WSGIPythonPath /app
 
 # Run apps in separate processes to stop yaml.CSafeLoader import-time error
-WSGIDaemonProcess topology  home=/app
+WSGIDaemonProcess topology  home=/app processes=5
 WSGIDaemonProcess topomerge home=/app
 
 # vhost for topology, SSL terminated here (for gridsite auth)
@@ -35,6 +35,14 @@ WSGIDaemonProcess topomerge home=/app
 
   WSGIScriptAlias /        /app/topology.wsgi         process-group=topology application-group=topology
 
+  # Enable compression for text files
+  AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/javascript application/json
+
+  # Enable memory caching
+  CacheSocache shmcb
+  CacheSocacheMaxSize 102400
+  CacheEnable socache /
+
 </VirtualHost>
 
 # Separate vhost for map, no SSL required (terminated in traefik)

mappings/project_institution.yaml

@@ -10,6 +10,7 @@ BNL: "Brookhaven National Laboratory"
 BU: "Boston University"
 BYUI: "Brigham Young University Idaho"
 Caltech: "California Institute of Technology"
+CedarsSinai: "Cedars-Sinai Medical Center"
 Cincinnati: "University of Cincinnati"
 Clarkson: "Clarkson University"
 Coe: "Coe College"
@@ -39,6 +40,7 @@ Mines: "Colorado School of Mines"
 MIT: "Massachusetts Institute of Technology"
 Mizzou: "University of Missouri"
 MMC: "Meharry Medical College"
+MSKCC: "Memorial Sloan Kettering Cancer Center"
 MSU: "Montana State University"
 MTU: "Michigan Technological University"
 NCSU: "North Carolina State University"
@@ -48,21 +50,27 @@ NJIT: "New Jersey Institute of Technology"
 NMSU: "New Mexico State University"
 NOAA: "National Oceanic and Atmospheric Administration"
 Northeastern: "Northeastern University"
+NSHE: "Nevada System of Higher Education"
+OHSU: "Oregon Health & Science University"
 OSU: "The Ohio State University"
 Pitt: "University of Pittsburgh"
 PortlandState: "Portland State University"
+PSI: "Planetary Science Institute"
 PSU: "Pennsylvania State University"
 Repertoire: "Repertoire Immune Medicines"
 Rochester: "University of Rochester"
 Rowan: "Rowan University"
 RPI: "Rensselaer Polytechnic Institute"
 SalemState: "Salem State University"
+SBU: "State University of New York at Stony Brook"
 SIUE: "Southern Illinois University Edwardsville"
 SDSU: "San Diego State University"
+SUNYUpstateMed: "SUNY Upstate Medical University"
 SWOSU: "Southwest Oklahoma State University"
 Syracuse: "Syracuse University"
 TNTech: "Tennessee Tech University"
 TexasAM: "Texas A&M University"
+UALR: "University of Arkansas at Little Rock"
 UCBerkeley: "University of California, Berkeley"
 UCDenver: "University of Colorado Denver"
 UCHC: "University of Connecticut Health Center"
@@ -80,6 +88,7 @@ UMCES: "University of Maryland Center for Environmental Science"
 UMiss: "University of Mississippi"
 UNL: "University of Nebraska - Lincoln"
 USD: "University of South Dakota"
+USDA: "United States Department of Agriculture"
 USheffield: "University of Sheffield"
 UTEP: "University of Texas at El Paso"
 UTulsa: "University of Tulsa"

projects/Arizona_Chan_Steward.yaml

@@ -0,0 +1,6 @@
+Department: Department of Astronomy and Steward Observatory
+Description: Scalable astronomical data analytics for the Department of Astronomy
+  and Steward Observatory at the University of Arizona
+FieldOfScience: Physics
+Organization: University of Arizona
+PIName: Chi-kwan Chan

projects/Arkansas_UITSStaff.yaml

@@ -0,0 +1,9 @@
+Description: Research computing staff at University of Arkansas https://directory.uark.edu/departmental/uits/university-information-technology-serv
+Department: Information Technology
+FieldOfScience: Research Computing
+Organization: University of Arkansas
+PIName: Don DuRousseau
+
+Sponsor:
+  CampusGrid:
+    Name: OSG Connect

projects/BNL_Venugopalan.yaml

@@ -0,0 +1,7 @@
+Description: >
+  We investigate the dynamics of strongly interacting field theories through a variety of numerical methods, 
+  ranging from classical lattice simulations to tensor network projects.
+Department: Physics
+FieldOfScience: Physics
+Organization: Brookhaven National Laboratory
+PIName: Raju Venugopalan

projects/BrighamAndWomens_ Baratono.yaml

@@ -0,0 +1,5 @@
+Description: Studying how atrophy and connectivity to atrophy impacts cognitive and psychological outcomes in patients with a variety of neurodegenerative disorders
+Department: Neurology
+FieldOfScience: Biological and Biomedical Sciences
+Organization: Brigham and Women's Hospital
+PIName: Sheena R Baratono

projects/CMB_Petravick.yaml

@@ -0,0 +1 @@
+Illinois_Petravick.yaml

projects/CMU_Romagnoli.yaml

@@ -0,0 +1,9 @@
+Department: Electrical and computer engineering
+Description: Deep Reinforcement Learning (RL) for Secure Control of UAVs via Software Rejuvenation
+FieldOfScience: Electrical Engineering
+Organization: Carnegie-Mellon University
+PIName: Raffaele Romagnoli
+
+Sponsor:
+  CampusGrid:
+    Name: OSG Connect

projects/CSUN_Jiang.yaml

@@ -0,0 +1,9 @@
+Description: >
+  Wildfire Prediction for California using various remote sensing data from the past decades. 
+  This is a computation intensive research project that involves applying machine learning models for data analysis and prediction, 
+  and also computation-intensive work for data visualization. 
+  This project will involve collaborations from both internal and external students at California State University, Northridge.
+Department:  Computer Science
+FieldOfScience: Computer and Information Sciences
+Organization: California State University, Northridge
+PIName: Xunfei Jiang

projects/CSU_Buchanan.yaml

@@ -0,0 +1,5 @@
+Description: Modeling of magnetorheological elastomers
+Department: Physics
+FieldOfScience: Physics
+Organization: Colorado State University
+PIName: Kristen Buchanan

projects/Caltech_Vallisneri.yaml

@@ -0,0 +1,12 @@
+Description: >
+  The NANOGrav collaboration is a cross-university, cross-discipline collection of astrophysicists, 
+  data analysts, and engineers who are currently working to detect a gravitational wave background 
+  via Pulsar Timing Arrays (PTAs).  Our group's current projects are related to cross-validation and 
+  posterior predictive checking methods for parameter estimation and detection analyses for Bayesian 
+  PTA studies. Another project is related to increasing computational efficiency of PTA analyses by 
+  developing likelihood reweighting methods for PTAs. More information on NANOGrav can be 
+  found here:  http://nanograv.org/
+Department:  California Institute of Technology
+FieldOfScience: Astronomy and Astrophysics
+Organization: California Institute of Technology
+PIName: Michele Vallisneri

projects/CaseWestern_Zhang.yaml

@@ -0,0 +1,9 @@
+Description: A project involving neuroimaging and genetics data for neurodegenerative disease. 
+Department: Dept. of Population & Quantitative Health Sciences (PQHS)
+FieldOfScience: Biological and Biomedical Sciences
+Organization: Case Western Reserve University
+PIName: Lijun Zhang
+
+Sponsor:
+  CampusGrid:
+    Name: OSG Connect

projects/CedarsSinai_Meyer.yaml

@@ -0,0 +1,10 @@
+Description: >
+  The Platform for Single-Cell Science is a tool for improving both the reproducibility and accessibility of analytical 
+  pipelines developed for single-cell multiomics. Researchers will be able to upload their data, create an analysis pipeline
+  using our javascript designer, and link their results to a publication. The raw data, analysis, and results will be made 
+  available for interactive exploration or download when users are ready to publish. We are hoping to understand if there 
+  is a way we can use OSG by sending jobs that are prepared on our website hosted on AWS to the OSG for execution.
+Department: Computational Biomedicine
+FieldOfScience: Biological and Biomedical Sciences 
+Organization: Cedars-Sinai Medical Center
+PIName: Jesse Meyer

projects/Cornell_Sandoz.yaml

@@ -0,0 +1,5 @@
+Description: We study bacterial survival. Particularly, we are interested in cell wall modifications in response to changing environments. https://www.ksandozlab.com/new-page-2
+Department: Population Medicine and Diagnostic Sciences
+FieldOfScience: Biological and Biomedical Sciences
+Organization: Cornell University
+PIName: Kelsi Sandoz

projects/Dartmouth_Chaboyer.yaml

@@ -0,0 +1,9 @@
+Description: >
+  Computational stellar models are widely used in astrophysics and are often consulted to interpret observations of starlight.
+  My research group aims to improve stellar models by incorporating updated physics into the models, creating a database of 
+  stellar models for use by other researchers, and to use these improved stellar models to study a variety of issues related 
+  to galactic archelogy and the formation of galaxies. https://stellar.host.dartmouth.edu
+Department: Physics and Astronomy
+FieldOfScience: Astronomy and Astrophysics
+Organization: Dartmouth College
+PIName: Brian Chaboyer

projects/Duke_Charbonneau.yaml

@@ -0,0 +1,8 @@
+Description: >
+  Periodic microphases universally emerge in systems for which short-range inter-particle attraction is frustrated by long-range repulsion.
+  The morphological richness of these phases makes them desirable material targets, but our relatively coarse understanding of even simple models limits our grasp of their assembly. 
+  The OSG computing resources will enable us to explore more solutions of the equilibrium phase behavior of a family of similar microscopic microphase formers through specialized Monte Carlo simulations.
+Department: Chemistry
+FieldOfScience: Computational Condensed Matter Physics
+Organization: Duke University
+PIName: Patrick Charbonneau

projects/ETHZ_Zhang.yaml

@@ -0,0 +1,8 @@
+Department: Computer Science
+Description: To make machine learning techniques widely accessible. 
+FieldOfScience: Computer Science
+Organization: ETH Zurich
+PIName: Ce Zhang
+Sponsor:
+  CampusGrid:
+    Name: OSG Connect

projects/EWMS_Riedel_Startup.yaml

@@ -0,0 +1,8 @@
+Department: Physics
+Description: To develop an Observation Management System Framework which will help alleviating multiple types of astrophysical data and respective workflow management. The Event Workflow Management System (EWMS)is a workload manager designed for processing events (simulated readouts from a particle physics detector, recorded data points, images, etc.) with complex workflows.
+FieldOfScience: Astrophysics
+Organization: University of Wisconsin-Madison
+PIName: Benedikt Riedel
+Sponsor:
+  CampusGrid:
+    Name: PATh Facility

projects/Etown_Wittmeyer.yaml

@@ -0,0 +1,11 @@
+Description: I am interested in examining experience-dependent neuroplasticity and individual differences in humans as it pertains to learning and memory.  In particular, I analyze structural magnetic resonance imaging (sMRI) data from various neuroimaging data-sharing platforms to explore changes in gray matter across learning and/or correlate learning performance with various cognitive and demographic factors.
+Department: Psychology
+FieldOfScience: Psychology and Life Sciences
+Organization: Elizabethtown College
+PIName: Jennifer Legault Wittmeyer
+
+ID: '591'
+
+Sponsor:
+  CampusGrid:
+    Name: OSG Connect

projects/Eureka_Danehkar.yaml

@@ -0,0 +1,9 @@
+Description: Modeling Reflection around Black Holes
+Department: Eureka Scientific
+FieldOfScience: Astronomy and Astrophysics
+Organization: Eureka Scientific, Inc.
+PIName: Ashkbiz Danehkar
+
+Sponsor:
+  CampusGrid:
+    Name: OSG Connect

projects/FIU_Hamid.yaml

@@ -0,0 +1,9 @@
+Description:  https://fphlm.cs.fiu.edu/
+Department:  IHRC
+FieldOfScience: Ocean Sciences and Marine Sciences
+Organization: Florida International University
+PIName: Shahid Hamid
+
+Sponsor:
+  CampusGrid:
+    Name: OSG Connect

projects/FSU_RCC.yaml

@@ -0,0 +1,11 @@
+Description: The Research Computing Center at Florida State University
+  enables research and education by maintaining a diverse campus
+  cyberinfrastructure 
+Department: Research Computing Center
+FieldOfScience: Research Computing
+Organization: Florida State University
+PIName: Paul van der Mark
+
+Sponsor:
+  CampusGrid:
+    Name: OSG Connect