Changes for 2.11 release

Makefile

@@ -121,11 +121,11 @@ test:
 
 format_check:
 	@echo "Running gofmt -l to check for code formatting issues..."
-	@test -z $(shell gofmt -l -s internal/commandfile/ internal/core/ internal/cpu/ internal/progress/ internal/target/ cmd/orchestrator/ cmd/collector/ cmd/reporter/ cmd/pmu2metrics/ cmd/msrread/ cmd/msrwrite/) || { echo "[WARN] Formatting issues detected. Resolve with 'make format'"; exit 1; }
+	@test -z $(shell gofmt -l -s ./) || { echo "[WARN] Formatting issues detected. Resolve with 'make format'"; exit 1; }
 	@echo "gofmt detected no issues"
 
 check: format_check
 
 format:
-	gofmt -l -w -s internal/commandfile/ internal/core/ internal/cpu/ internal/progress/ internal/target/ orchestrator/ collector/ reporter/ pmu2metrics/ rdmsr/ wrmsr/
+	gofmt -l -w -s ./
 

README.md

@@ -16,7 +16,7 @@ Data can be collected from a single remote target by providing the login credent
 ./svr-info -ip 10.100.222.123 -user fred -key ~/.ssh/id_rsa
 ```
 ## Multiple Targets
-Data can be collected from multiple remote targets by placing login credentials of the targets in a 'targets' file and then referencing that targets file on the svr-info command line. See the included [targets.example](src/orchestrator/targets.example) file for the required file format.
+Data can be collected from multiple remote targets by placing login credentials of the targets in a 'targets' file and then referencing that targets file on the svr-info command line. See the included [targets.example](cmd/orchestrator/targets.example) file for the required file format.
 ```
 ./svr-info -targets <targets file>
 ```
@@ -50,19 +50,13 @@ For example, Intel® Memory Latency Checker can be downloaded from here: [MLC](h
 We welcome bug reports, questions and feature requests. Please submit via Github Issues.
 ## Building svr-info
 Due to the large number of build dependencies required, a Docker container-based build environment is provided. Assuming your system has Docker installed (instructions not provided here), the following steps are required to build svr-info:
-- `builder/build_docker_image` creates the docker image
-- `builder/build` runs `make dist` in the container
+- `builder/build` creates the necessary docker images and runs make in the container
 After a successful build, you will find the build output in the `dist` folder.
 
-Other builder commands available:
-- `builder/test` runs the automated tests in the container via `make test`
-- `builder/shell` starts the container and provides a bash prompt useful for debugging build problems
 ### Incremental Builds
-After a complete build using the build container, you can perform incremental builds directly on your host assuming dependencies are installed there. This can make the code/build/test cycle much quicker than rebuilding everything using the Docker container. You can look at the Dockerfile in the builder directory to get the build dependencies for everything or, more likely, you only need go(lang) so install the latest and get to work.
+After a complete build using the build container, you can perform incremental builds directly on your host assuming dependencies are installed there. This can make the code/build/test cycle much quicker than rebuilding everything using the Docker container.
 
-From the project's root directory, you can use the makefile. There are quite a few targets. Most useful may be `make apps`.  This will build all the go-based apps.
-
-If you are working on a single go-based app. You can run `go build` in the app's source directory to build it.
+If you are working on a single go-based app. You can run `go build` to build it.
 
 ### Including Additional Collection Tools In The Build
 Additional data collection tools can be built into the svr-info distribution by placing binaries in the bin directory before starting the build.

RELEASE_NOTES

@@ -3,9 +3,22 @@ Intel® System Health Inspector (AKA svr-info)
 
 Fully Supported Platforms
 - Xeon Micro-Architectures: SRF,EMR,SPR,CPX,ICX,CLX,SKX,BDX,HSX
-- Operating Systems: Ubuntu 16.04, 18.04, 20.04, 22.04, CentOS 7, Amazon Linux 2, Debian 11, RHEL 9, Rocky Linux 8
+- Operating Systems: Ubuntu 18.04, 20.04, 22.04, 24.04, CentOS 7, Amazon Linux 2, Debian 11, RHEL 9, Rocky Linux 8
   Note: svr-info may work on other micro-architectures and Linux distributions, but has not been thoroughly tested
 
+2.11.0
+Features Added
+- Report Efficiency Latency Control (ELC) configuration on SRF and GNR
+- Report PCIe address, NUMA node, and more for NVME drives
+- Add support for PMU profiling on AWS ICX and SPR VMs ... and SRF servers.
+- Report L3 size on GNR
+- Processor TDP added to Summary field of Excel 'brief' report
+
+Bugs Fixed
+- Fix turbo benchmark on systems with many cores, e.g., SRF
+- Fix memory performance benchmark when run on systems with large huge page sizes
+- Fix Excel report corruption when NaN values result from '-profile pmu'
+
 2.10.0
 Features Added
 - Support for Sierra Forest Xeon CPUs

cmd/orchestrator/resources/collector_reports.yaml.tmpl

@@ -60,8 +60,52 @@ commands:
   - label: maximum frequency
     command: cat /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq
     parallel: true
-  - label: lsblk -r -o
-    command: lsblk -r -o NAME,MODEL,SIZE,MOUNTPOINT,FSTYPE,RQ-SIZE,MIN-IO -e7 -e1
+  - label: disk info
+    command: |-
+        echo "NAME|MODEL|SIZE|MOUNTPOINT|FSTYPE|RQ-SIZE|MIN-IO|FIRMWARE|ADDR|NUMA|LINKSPEED|LINKWIDTH|MAXLINKSPEED|MAXLINKWIDTH"
+        lsblk -r -o NAME,MODEL,SIZE,MOUNTPOINT,FSTYPE,RQ-SIZE,MIN-IO -e7 -e1 \
+        | cut -d' ' -f1,2,3,4,5,6,7 --output-delimiter='|' \
+        | while IFS='|' read -r name model size mountpoint fstype rqsize minio ;
+        do
+            # skip the lsblk output header
+            if [ "$name" = "NAME" ] ; then
+                continue
+            fi
+            fw=""
+            addr=""
+            numa=""
+            curlinkspeed=""
+            curlinkwidth=""
+            maxlinkspeed=""
+            maxlinkwidth=""
+            # replace \x20 with space in model
+            model=${model//\\x20/ }
+            # if name refers to an NVMe device e.g, nvme0n1 - nvme99n99
+            if [[ $name =~ ^(nvme[0-9]+)n[0-9]+$ ]]; then
+                # get the name without the namespace
+                nvme=${BASH_REMATCH[1]}
+                if [ -f /sys/block/"$name"/device/firmware_rev ] ; then
+                    fw=$( cat /sys/block/"$name"/device/firmware_rev )
+                fi
+                if [ -f /sys/block/"$name"/device/address ] ; then
+                    addr=$( cat /sys/block/"$name"/device/address )
+                fi
+                if [ -d "/sys/block/$name/device/${nvme}" ]; then
+                    numa=$( cat /sys/block/"$name"/device/"${nvme}"/numa_node )
+                    curlinkspeed=$( cat /sys/block/"$name"/device/"${nvme}"/device/current_link_speed )
+                    curlinkwidth=$( cat /sys/block/"$name"/device/"${nvme}"/device/current_link_width )
+                    maxlinkspeed=$( cat /sys/block/"$name"/device/"${nvme}"/device/max_link_speed )
+                    maxlinkwidth=$( cat /sys/block/"$name"/device/"${nvme}"/device/max_link_width )
+                elif [ -d "/sys/block/$name/device/device" ]; then
+                    numa=$( cat /sys/block/"$name"/device/device/numa_node )
+                    curlinkspeed=$( cat /sys/block/"$name"/device/device/current_link_speed )
+                    curlinkwidth=$( cat /sys/block/"$name"/device/device/current_link_width )
+                    maxlinkspeed=$( cat /sys/block/"$name"/device/device/max_link_speed )
+                    maxlinkwidth=$( cat /sys/block/"$name"/device/device/max_link_width )
+                fi
+            fi
+            echo "$name|$model|$size|$mountpoint|$fstype|$rqsize|$minio|$fw|$addr|$numa|$curlinkspeed|$curlinkwidth|$maxlinkspeed|$maxlinkwidth"
+        done
     parallel: true
   - label: df -h
     command: df -h
@@ -199,19 +243,77 @@ commands:
     command: pcm-tpmi 2 0x18 -d -b 15:21
     superuser: true
     parallel: true
-  - label: active idle utilization point
+  - label: efficiency latency control
     command: |-
-        msrwrite 0xb0 0x80000694  # must write this value to this MSR before reading 0xb1
-        msrread -f 15:8 0xb1 # ACTIVE IDLE - UTILIZATION POINT
-    superuser: true
-    modprobe: msr
-    parallel: true
-  - label: active idle mesh frequency
-    command: |-
-        msrwrite 0xb0 0x80000694  # must write this value to this MSR before reading 0xb1
-        msrread -f 7:0 0xb1 # ACTIVE IDLE - MESH FREQUENCY
+      # Script derived from bhs-power-mode script in Intel PCM repository
+      # Run the pcm-tpmi command to determine I/O and compute dies
+      output=$(pcm-tpmi 2 0x10 -d -b 26:26)
+
+      # Parse the output to build lists of I/O and compute dies
+      io_dies=()
+      compute_dies=()
+      declare -A die_types
+      while read -r line; do
+          if [[ $line == *"instance 0"* ]]; then
+              die=$(echo "$line" | grep -oP 'entry \K[0-9]+')
+              if [[ $line == *"value 1"* ]]; then
+                  die_types[$die]="IO"
+            io_dies+=("$die")
+              elif [[ $line == *"value 0"* ]]; then
+                  die_types[$die]="Compute"
+            compute_dies+=("$die")
+              fi
+          fi
+      done <<< "$output"
+
+      # Function to extract and calculate metrics from the value
+      extract_and_print_metrics() {
+          local value=$1
+          local socket_id=$2
+          local die=$3
+          local die_type=${die_types[$die]}
+
+          # Extract bits and calculate metrics
+          local min_ratio=$(( (value >> 15) & 0x7F ))
+          local max_ratio=$(( (value >> 8) & 0x7F ))
+          local eff_latency_ctrl_ratio=$(( (value >> 22) & 0x7F ))
+          local eff_latency_ctrl_low_threshold=$(( (value >> 32) & 0x7F ))
+          local eff_latency_ctrl_high_threshold=$(( (value >> 40) & 0x7F ))
+          local eff_latency_ctrl_high_threshold_enable=$(( (value >> 39) & 0x1 ))
+
+          # Convert to MHz or percentage
+          min_ratio=$(( min_ratio * 100 ))
+          max_ratio=$(( max_ratio * 100 ))
+          eff_latency_ctrl_ratio=$(( eff_latency_ctrl_ratio * 100 ))
+          eff_latency_ctrl_low_threshold=$(( (eff_latency_ctrl_low_threshold * 100) / 127 ))
+          eff_latency_ctrl_high_threshold=$(( (eff_latency_ctrl_high_threshold * 100) / 127 ))
+
+          # Print metrics
+          echo -n "$socket_id,$die,$die_type,$min_ratio,$max_ratio,$eff_latency_ctrl_ratio,"
+          if [ $die_type == "IO" ] ; then
+              echo "$eff_latency_ctrl_low_threshold,$eff_latency_ctrl_high_threshold,$eff_latency_ctrl_high_threshold_enable"
+          else
+              echo ",,"
+          fi
+      }
+
+      # Print CSV header
+      echo "Socket,Die,Type,MIN_RATIO (MHz),MAX_RATIO (MHz),ELC_RATIO (MHz),ELC_LOW_THRESHOLD (%),ELC_HIGH_THRESHOLD (%),ELC_HIGH_THRESHOLD_ENABLE"
+
+      # Iterate over all dies and run pcm-tpmi for each to get the metrics
+      for die in "${!die_types[@]}"; do
+          output=$(pcm-tpmi 2 0x18 -d -e "$die")
+
+          # Parse the output and extract metrics for each socket
+          while read -r line; do
+              if [[ $line == *"Read value"* ]]; then
+                  value=$(echo "$line" | grep -oP 'value \K[0-9]+')
+                  socket_id=$(echo "$line" | grep -oP 'instance \K[0-9]+')
+                  extract_and_print_metrics "$value" "$socket_id" "$die"
+              fi
+          done <<< "$output"
+      done
     superuser: true
-    modprobe: msr
     parallel: true
   - label: ipmitool sel time get
     command: LC_ALL=C ipmitool sel time get
@@ -242,6 +344,10 @@ commands:
     superuser: true
     modprobe: msr
     parallel: true
+  - label: pmu driver version
+    command: dmesg | grep -A 1 "Intel PMU driver" | tail -1 | awk '{print $NF}'
+    superuser: true
+    parallel: true
   - label: lspci -vmm
     command: lspci -vmm
     parallel: true
@@ -415,21 +521,31 @@ commands:
   - label: Memory MLC Loaded Latency Test
     command: |-
         # measure memory loaded latency
+        #  need at least 2 GB (2,097,152 KB) of huge pages per NUMA node
+        min_kb=2097152
         numa_nodes=$( lscpu | grep "NUMA node(s):" | awk '{print $3}' )
+        size_huge_pages_kb=$( cat /proc/meminfo | grep Hugepagesize | awk '{print $2}' )
         orig_num_huge_pages=$( cat /proc/sys/vm/nr_hugepages )
-        new_num_huge_pages=$( echo "$numa_nodes * 1000" | bc )
-        echo $new_num_huge_pages > /proc/sys/vm/nr_hugepages
+        needed_num_huge_pages=$( echo "$numa_nodes * $min_kb / $size_huge_pages_kb" | bc )
+        if [ $needed_num_huge_pages -gt $orig_num_huge_pages ]; then
+          echo $needed_num_huge_pages > /proc/sys/vm/nr_hugepages
+        fi
         mlc --loaded_latency
         echo $orig_num_huge_pages > /proc/sys/vm/nr_hugepages
     modprobe: msr
     superuser: true
   - label: Memory MLC Bandwidth
     command: |-
         # measure memory bandwidth matrix
+        #  need at least 2 GB (2,097,152 KB) of huge pages per NUMA node
+        min_kb=2097152
         numa_nodes=$( lscpu | grep "NUMA node(s):" | awk '{print $3}' )
+        size_huge_pages_kb=$( cat /proc/meminfo | grep Hugepagesize | awk '{print $2}' )
         orig_num_huge_pages=$( cat /proc/sys/vm/nr_hugepages )
-        new_num_huge_pages=$( echo "$numa_nodes * 1000" | bc )
-        echo $new_num_huge_pages > /proc/sys/vm/nr_hugepages
+        needed_num_huge_pages=$( echo "$numa_nodes * $min_kb / $size_huge_pages_kb" | bc )
+        if [ $needed_num_huge_pages -gt $orig_num_huge_pages ]; then
+          echo $needed_num_huge_pages > /proc/sys/vm/nr_hugepages
+        fi
         mlc --bandwidth_matrix
         echo $orig_num_huge_pages > /proc/sys/vm/nr_hugepages
     modprobe: msr
@@ -451,7 +567,7 @@ commands:
   - label: CPU Turbo Test
     command: |-
         # measure tdp and all-core turbo frequency
-        ((turbostat -i 2 2>/dev/null &) ; stress-ng --cpu 1 -t 20s 2>&1 ; stress-ng --cpu 0 -t 60s 2>&1 ; pkill -9 -f turbostat) | awk '$0~"stress" {print $0} $1=="Package" || $1=="CPU" || $1=="Core" || $1=="Node" {if(f!=1) print $0;f=1} $1=="-" {print $0}'
+        ((turbostat --show 'Package','Core','Bzy_MHz','PkgWatt','PkgTmp' -i 2 &) ; stress-ng --cpu 1 -t 20s 2>&1 ; stress-ng --cpu 0 -t 60s 2>&1 ; pkill -9 -f turbostat) | awk '$0~"stress" {print $0} $1=="Package" || $1=="CPU" || $1=="Core" || $1=="Node" {if(f!=1) print $0;f=1} $1=="-" {print $0}'
     superuser: true
     modprobe: msr
   - label: CPU Idle