Default set in metadata and logic validation

aiidateam · Oct 2, 2023 · d5d36b7 · d5d36b7
1 parent b3fd7c8
commit d5d36b7
Show file tree

Hide file tree

Showing 11 changed files with 153 additions and 83 deletions.
diff --git a/daint.cscs.ch/gpu/computer-setup.yml b/daint.cscs.ch/gpu/computer-setup.yml
@@ -1,5 +1,5 @@
 ---
-label: "{{ label | default('daint-gpu') }}"
+label: '{{ label }}'
 hostname: daint.cscs.ch
 description: Piz Daint supercomputer at CSCS Lugano, Switzerland, using the GPU nodes. HyperThreading is off
 transport: core.ssh
@@ -12,27 +12,47 @@ work_dir: /scratch/snx3000/{username}/aiida/
 mpirun_command: srun -n {tot_num_mpiprocs}
 prepend_text: |-
     ### computer prepend_text start ###
-    #SBATCH --partition={{ slurm_partition | default('normal') }}
+    #SBATCH --partition={{ slurm_partition }}
     #SBATCH --account={{ slurm_account }}
     #SBATCH --constraint=gpu
-    #SBATCH --hint=nomultithread
+    #SBATCH --hint={{ multithreading }}
+
     export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
     export CRAY_CUDA_MPS=1
     ulimit -s unlimited
     ### computer prepend_text end ###
 metadata:
+    tooltip: |
+        <p>
+            <a href="https://user.cscs.ch/access/running/piz_daint/" target="_blank">Piz Daint</a> supercomputer at CSCS Lugano, Switzerland, hybrid partition.
+        </p>
     label:
+        default: daint-gpu
         description: A short name to identify the computer
         type: text
         key_display: Computer Label
     slurm_partition:
+        default: normal
         description: The slurm partition to submit jobs to
         type: list
         options:
             - normal
             - debug
+            - large
+            - long
+            - low
+            - prepost
         key_display: Slurm partition
     slurm_account:
         description: The slurm account to submit jobs to
         type: text
         key_display: Slurm account
+    multithreading:
+        default: nomultithread
+        description: The multithreading hint
+        type: list
+        options:
+            - nomultithread
+            - multithread
+        key_display: Multithreading hint
+
diff --git a/daint.cscs.ch/mc/computer-setup.yml b/daint.cscs.ch/mc/computer-setup.yml
@@ -1,5 +1,5 @@
 ---
-label: "{{ label | default('daint-mc') }}"
+label: '{{ label }}'
 hostname: daint.cscs.ch
 description: Piz Daint supercomputer at CSCS Lugano, Switzerland, multicore partition.
 transport: core.ssh
@@ -9,27 +9,46 @@ shebang: '#!/bin/bash'
 mpirun_command: srun -n {tot_num_mpiprocs}
 mpiprocs_per_machine: 36
 prepend_text: |-
-    #SBATCH --partition={{ slurm_partition | default('normal') }}
+    #SBATCH --partition={{ slurm_partition }}
     #SBATCH --account={{ slurm_account }}
     #SBATCH --constraint=mc
     #SBATCH --cpus-per-task=1
+    #SBATCH --hint={{ multithreading }}
 
     export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
     source $MODULESHOME/init/bash
     ulimit -s unlimited
 metadata:
+    tooltip: |
+        <p>
+            <a href="https://user.cscs.ch/access/running/piz_daint/" target="_blank">Piz Daint</a> supercomputer at CSCS Lugano, Switzerland, multicore partition.
+        </p>
     label:
+        default: daint-mc
         description: A short name to identify the computer
         type: text
         key_display: Computer Label
     slurm_partition:
+        default: normal
         description: The slurm partition to submit jobs to
         type: list
         options:
             - normal
             - debug
+            - large
+            - long
+            - low
+            - prepost
         key_display: Slurm partition
     slurm_account:
         description: The slurm account to submit jobs to
         type: text
         key_display: Slurm account
+    multithreading:
+        default: nomultithread
+        description: The multithreading hint
+        type: list
+        options:
+            - nomultithread
+            - multithread
+        key_display: Multithreading hint
diff --git a/gh_page/generate_json.py b/gh_page/generate_json.py
@@ -74,9 +74,12 @@ def main():
         link = domain_path / 'default'
         if link.exists() and link.is_symlink():
             domain_data['default'] = Path.readlink(link).name
+        else:
+            domain_data['default'] = list(domain_data.keys())[0]
 
         data[domain] = domain_data
 
+
     # Validate the data.
     schema = json.loads((root_path / 'gh_page' / 'resource.schema.json').read_text())
     for domain, domain_data in data.items():
@@ -85,6 +88,14 @@ def main():
         except Exception as e:
             raise Exception(f"Invalid data for domain: {domain}") from e
 
+        # I use for iteration to walk through all the keys in the domain_data, but more efficient way is to
+        # treat it as database.
+        for metadata in extract_nested_metadata(domain_data):
+            for value in metadata.values():
+                try:
+                    validata_metadata(value)
+                except Exception as e:
+                    raise Exception(f"Invalid metadata in {domain}: {value}") from e
 
     # Store the data in a JSON file.
     with open(root_path/ 'gh_page' / 'out' / 'database.json', 'w') as filep:
@@ -94,5 +105,29 @@ def main():
     with open(root_path / 'gh_page' / 'out' / 'resource.schema.json', 'w') as filep:
         json.dump(schema, filep, indent=4)
 
+def validata_metadata(data: dict):
+    """validata the metadata for deep logic.
+    For example, if the type is a list, then the default value should be in the list.
+    """
+    if isinstance(data, str):
+        return
+    if data.get("type") == "list":
+        if "options" not in data:
+            raise Exception(f"Invalid metadata: {data}, options not in list type matadata.")
+
+        if "default" in data and data["default"] not in data["options"]:
+            raise Exception(f"""Invalid metadata: {data}, default value "{data['default']}" not in options.""")
+
+def extract_nested_metadata(data: dict):
+    """Extract the nested metadata from the data."""
+    for key, value in data.items():
+        if key == "metadata":
+            yield value
+        elif isinstance(value, dict):
+            yield from extract_nested_metadata(value)
+        else:
+            continue
+
+
 if "__main__" == __name__:
     main()
diff --git a/gh_page/resource.schema.json b/gh_page/resource.schema.json
@@ -9,12 +9,18 @@
                 "type": "string"
             }
         },
+        "required": ["default"],
         "additionalProperties": {
             "$ref": "#/definitions/resource"
         }
     },
     "metadata": {
-        "type": "object",
+        "type": ["object"],
+        "properties": {
+            "tooltip": {
+                "type": "string"
+            }
+        },
         "additionalProperties": {
             "type": "object",
             "properties": {
@@ -34,7 +40,8 @@
                 "key_display": {
                     "type": "string"
                 }
-            }
+            },
+            "required": ["type"]
         }
     },
     "resource": {

diff --git a/merlin.psi.ch/cpu/codes/cp2k-9.1.yml b/merlin.psi.ch/cpu/codes/cp2k-9.1.yml
diff --git a/merlin.psi.ch/cpu/computer-setup.yml b/merlin.psi.ch/cpu/computer-setup.yml
@@ -1,37 +1,46 @@
 ---
-label: "{{ label | default('merlin-cpu') }}"
+label: '{{ label }}'
 hostname: merlin-l-01.psi.ch
-description: Merlin6 HPC at PSI.
+description: Merlin6 HPC at PSI (cpu section).
 transport: core.ssh
 scheduler: core.slurm
 work_dir: /shared-scratch/{username}/aiida_run/
 shebang: '#!/bin/bash'
 mpirun_command: srun -n {tot_num_mpiprocs}
 mpiprocs_per_machine: 44
 prepend_text: |-
-    #SBATCH --partition={{ slurm_partition | default('general') }}
-    #SBATCH --account={{ slurm_account }}
-    #SBATCH --constraint=mc
-    #SBATCH --cpus-per-task=1
-    #SBATCH --cluster=merlin6
+    #SBATCH --partition={{ slurm_partition }}
     #SBATCH --account=merlin
+    #SBATCH --cluster=merlin6
 
+    #SBATCH --cpus-per-task=1
+    #SBATCH --hint={{ multithreading }}
 
-    export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
     ulimit -s unlimited
 metadata:
+    tooltip: |
+        <p>
+            <a href="https://lsm-hpce.gitpages.psi.ch/merlin6/" target="_blank">Merlin</a> HPC at PSI.
+        </p>
     label:
+        default: merlin-cpu
         description: A short name to identify the computer
         type: text
         key_display: Computer Label
     slurm_partition:
+        default: general
         description: The slurm partition to submit jobs to
         type: list
         options:
             - general
-            - debug
+            - daily
+            - hourly
         key_display: Slurm partition
-    slurm_account:
-        description: The slurm account to submit jobs to
-        type: text
-        key_display: Slurm account
+    multithreading:
+        default: nomultithread
+        description: The multithreading hint
+        type: list
+        options:
+            - nomultithread
+            - multithread
+        key_display: Multithreading hint
diff --git a/merlin.psi.ch/default b/merlin.psi.ch/default
diff --git a/....psi.ch/cpu/codes/QE-7.2-exe-template.yml → ....psi.ch/gpu/codes/QE-7.0-exe-template.yml b/....psi.ch/cpu/codes/QE-7.2-exe-template.yml → ....psi.ch/gpu/codes/QE-7.0-exe-template.yml
@@ -1,11 +1,12 @@
 ---
-label: '{{ code_binary_name }}-7.2'
+label: '{{ code_binary_name }}-7.0'
 description: The code {{ code_binary_name }} of Quantum ESPRESSO compiled for daint-mc
 default_calc_job_plugin: quantumespresso.{{ code_binary_name }}
-filepath_executable: /apps/dom/UES/jenkins/7.0.UP03/21.09/dom-mc/software/QuantumESPRESSO/7.2-CrayIntel-21.09/bin/{{ code_binary_name }}.x
+filepath_executable: /opt/psi/MPI/qe/7.0/openmpi/4.1.3_slurm/gcc/11.2.0/bin/{{ code_binary_name }}.x
 prepend_text: |
-    module load daint-mc
-    module load QuantumESPRESSO
+    module use unstable;
+    module load gcc/11.2.0 openmpi/4.1.3_slurm
+    module load qe/7.0
 append_text: ''
 metadata:
     code_binary_name:

diff --git a/merlin.psi.ch/gpu/codes/QE-7.2-exe-template.yml b/merlin.psi.ch/gpu/codes/QE-7.2-exe-template.yml
diff --git a/merlin.psi.ch/gpu/codes/cp2k-9.1.yml b/merlin.psi.ch/gpu/codes/cp2k-9.1.yml
diff --git a/merlin.psi.ch/gpu/computer-setup.yml b/merlin.psi.ch/gpu/computer-setup.yml
@@ -1,38 +1,54 @@
 ---
-label: "{{ label | default('daint-gpu') }}"
-hostname: daint.cscs.ch
-description: Piz Daint supercomputer at CSCS Lugano, Switzerland, using the GPU nodes. HyperThreading is off
+label: '{{ label }}'
+hostname: merlin-l-01.psi.ch
+description: Merlin6 HPC at PSI (gpu section).
 transport: core.ssh
 scheduler: core.slurm
-shebang: '#!/bin/bash -l'
-mpiprocs_per_machine: 12
-num_cores_per_mpiproc: 1
-queue_name: normal
-work_dir: /scratch/snx3000/{username}/aiida/
+work_dir: /shared-scratch/{username}/aiida_run/
+shebang: '#!/bin/bash'
 mpirun_command: srun -n {tot_num_mpiprocs}
+mpiprocs_per_machine: 20
 prepend_text: |-
-    ### computer prepend_text start ###
-    #SBATCH --partition={{ slurm_partition | default('normal') }}
-    #SBATCH --account={{ slurm_account }}
-    #SBATCH --constraint=gpu
-    #SBATCH --hint=nomultithread
-    export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
-    export CRAY_CUDA_MPS=1
+    #SBATCH --partition={{ slurm_partition }}
+    #SBATCH --account=merlin
+    #SBATCH --cluster=gmerlin6
+    #SBATCH --constraint={{ slurm_constraint }}
+
+    #SBATCH --hint={{ multithreading }}
+
     ulimit -s unlimited
-    ### computer prepend_text end ###
 metadata:
+    tooltip: |
+        <p>
+            <a href="https://lsm-hpce.gitpages.psi.ch/merlin6/" target="_blank">Merlin</a> HPC at PSI (gpu).
+        </p>
     label:
+        default: merlin-gpu
         description: A short name to identify the computer
         type: text
         key_display: Computer Label
     slurm_partition:
+        default: gpu
         description: The slurm partition to submit jobs to
         type: list
         options:
-            - normal
-            - debug
+            - gpu
+            - gpu-short
         key_display: Slurm partition
-    slurm_account:
-        description: The slurm account to submit jobs to
-        type: text
-        key_display: Slurm account
+    slurm_constraint:
+        default: gpumem_8gb
+        description: Specify the GPU by the amount of memory available in the GPU card itself.
+        type: list
+        options:
+            - gpumem_8gb
+            - gpumem_11gb
+            - gpumem_40gb
+        key_display: Slurm constraint
+    multithreading:
+        default: nomultithread
+        description: The multithreading hint
+        type: list
+        options:
+            - nomultithread
+            - multithread
+        key_display: Multithreading hint