overhead benchmarks

.gitignore

@@ -160,3 +160,13 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+*.DS_Store*
+
+*outputs/
+*unsloth_compiled_cache/
+*.hatchet
+*.nsys-rep
+*trainer_output/
+*trace.json
+*unsloth_trace*.json

data_grapher.py

@@ -0,0 +1,55 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+
+# 1) Load your data
+df = pd.read_csv('all_results.csv')
+
+# 2) Compute mean of the four methods per script
+metrics = ['baseline', 'proton', 'torch', 'nsys']
+grouped = (
+    df
+    .groupby('script')[metrics]
+    .mean()
+    .reset_index()
+)
+print(grouped)
+
+# 3) Compute % difference from baseline
+#    (value - baseline) / baseline * 100
+for m in metrics:
+    # print(grouped[m])
+    grouped[m] = (grouped[m] - grouped['baseline']) / grouped['baseline'] * 100
+    print(m)
+
+# 4) Prepare for plotting
+x_labels  = grouped['script']
+data      = grouped[metrics]
+n_groups  = len(x_labels)
+n_metrics = len(metrics)
+bar_width = 0.8 / n_metrics
+x         = np.arange(n_groups)
+
+# 5) Create the grouped bar chart
+fig, ax = plt.subplots(figsize=(10, 5))
+
+for i, m in enumerate(metrics):
+    ax.bar(
+        x + i * bar_width,
+        data[m],
+        width=bar_width,
+        label=m
+    )
+
+# 6) Formatting
+ax.set_xlabel('Script')
+ax.set_ylabel('Percent Difference from Baseline (%)')
+ax.set_title('Average Performance Difference from Baseline by Script')
+ax.set_xticks(x + bar_width*(n_metrics-1)/2)
+ax.set_xticklabels(x_labels, rotation=30, ha='right')
+ax.axhline(0, color='black', linewidth=0.8, linestyle='--')  # reference line at 0%
+ax.legend(title='Implementation')
+
+plt.tight_layout()
+plt.savefig('overhead_bench.png', dpi=300)
+plt.show()

end2end/graph.py

@@ -0,0 +1,109 @@
+import re
+import csv
+import sys
+import argparse
+import matplotlib.pyplot as plt
+import numpy as np
+
+def parse_experiments(log_path):
+    # Patterns for experiment headers
+    first_hdr = re.compile(r"Timing\s+(?P<model>\S+)\s")
+    prof_hdr = re.compile(r"^(?P<profiler>NSYS|PROTON|TORCH)\s*$")
+    run_pattern = re.compile(r"Run\s*#?(?P<number>\d+)[: of]*.*?(?P<duration>\d+\.\d+)\s*seconds")
+
+    experiments = []
+    current = None
+    current_model = None
+
+    with open(log_path, 'r', encoding='utf-8') as f:
+        for raw_line in f:
+            line = raw_line.strip()
+
+            # Detect first header: Timing <model> NONE
+            m1 = first_hdr.search(line)
+            if m1:
+                current_model = m1.group('model')
+                current = {'model': current_model, 'profiler_type': 'NONE', 'runs': []}
+                experiments.append(current)
+                continue
+
+            # Detect subsequent headers: NSYS, PROTON, TORCH
+            m2 = prof_hdr.search(line)
+            if m2 and current_model:
+                profiler = m2.group('profiler')
+                current = {'model': current_model, 'profiler_type': profiler, 'runs': []}
+                experiments.append(current)
+                continue
+
+            # Parse run durations
+            if current:
+                run = run_pattern.search(line)
+                if run:
+                    current['runs'].append({'run_number': int(run.group('number')), 'duration_s': float(run.group('duration'))})
+    return experiments
+
+
+def write_csv(experiments, out_path):
+    with open(out_path, 'w', newline='', encoding='utf-8') as csvfile:
+        writer = csv.DictWriter(csvfile, fieldnames=['model','profiler_type','run_number','duration_s'])
+        writer.writeheader()
+        for exp in experiments:
+            for run in exp['runs']:
+                writer.writerow({'model': exp['model'],'profiler_type': exp['profiler_type'],'run_number': run['run_number'],'duration_s': run['duration_s']})
+
+
+def plot_experiments(experiments):
+    # Group by model and profiler
+    models = sorted({exp['model'] for exp in experiments})
+    # Calculate baseline (NONE) averages per model
+    base_avgs = {}
+    for model in models:
+        runs = next((exp['runs'] for exp in experiments if exp['model']==model and exp['profiler_type']=='NONE'), [])
+        base_avgs[model] = (sum(r['duration_s'] for r in runs)/len(runs)) if runs else 0.0
+
+    # Define desired profiler order, excluding NONE
+    all_profs = {exp['profiler_type'] for exp in experiments if exp['profiler_type']!='NONE'}
+    ordered = ['PROTON', 'NSYS', 'TORCH']
+    profilers = [p for p in ordered if p in all_profs]
+
+    # Calculate percentage difference from NONE
+    pct_diff = {prof: [] for prof in profilers}
+    for model in models:
+        base = base_avgs.get(model, 0.0)
+        for prof in profilers:
+            runs = next((exp['runs'] for exp in experiments if exp['model']==model and exp['profiler_type']==prof), [])
+            avg = (sum(r['duration_s'] for r in runs)/len(runs)) if runs else 0.0
+            percent = ((avg - base) / base) * 100 if base > 0 else 0.0
+            pct_diff[prof].append(percent)
+
+    # Plot grouped bar chart
+    x = np.arange(len(models))
+    width = 0.8 / len(profilers)
+    fig, ax = plt.subplots()
+    for i, prof in enumerate(profilers):
+        ax.bar(x + i * width, pct_diff[prof], width, label=prof)
+
+    ax.set_xticks(x + width * (len(profilers) - 1) / 2)
+    ax.set_xticklabels(models, rotation=45, ha='right')
+    ax.set_ylabel('Percentage Increase in Training Time (%)')
+    ax.set_title('Percentage Overhead of Profilers for Unsloth Model Training')
+    ax.legend()
+    plt.tight_layout()
+    plt.savefig('unsloth_profiling.png')
+    # plt.show()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Extract and optionally plot experiment timings.')
+    parser.add_argument('log_file', help='Path to the log file')
+    parser.add_argument('out_csv', help='Output CSV file path')
+    parser.add_argument('--plot', action='store_true', help='Show grouped bar chart with percent difference')
+    args = parser.parse_args()
+
+    experiments = parse_experiments(args.log_file)
+    write_csv(experiments, args.out_csv)
+    total_runs = sum(len(exp['runs']) for exp in experiments)
+    print(f"Extracted {total_runs} runs across {len(experiments)} experiments to {args.out_csv}")
+
+    if args.plot:
+        plot_experiments(experiments)

end2end/multimodal/all_timing.sh

@@ -0,0 +1,19 @@
+
+#!/usr/bin/env bash
+echo "NO PROFILER"
+./profile-wrapper.sh 2 python training.py
+
+echo "--------------------------------------------"
+
+echo "NSYS"
+./profile-wrapper.sh 2 nsys profile --trace=cuda --sample=none --cpuctxsw=none python training.py
+
+echo "--------------------------------------------"
+
+echo "PROTON"
+./profile-wrapper.sh 2 proton training.py
+
+echo "--------------------------------------------"
+
+echo "TORCH"
+./profile-wrapper.sh 2 python training.py --profile_torch

end2end/multimodal/profile-wrapper.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if (( $# < 2 )); then
+  echo "Usage: $0 <num_runs> <wrapped command...>"
+  echo "Example: $0 5 nsys python myscript.py arg1"
+  exit 1
+fi
+
+# Number of times to run
+num_runs=$1
+shift
+
+# Array to hold elapsed times
+declare -a times
+
+for ((i=1; i<=num_runs; i++)); do
+  echo
+  echo ">>> Run #$i of $num_runs"
+
+  # capture output to a temp log
+  logfile=$(mktemp /tmp/profile-log.XXXXXX)
+
+  # run the wrapped command
+  start_marker=""
+  # start time comes from script’s own START_PROFILE line; end time we record
+  "$@" 2>&1 | tee "$logfile"
+  exit_code=${PIPESTATUS[0]}
+
+  # get end timestamp
+  end_ts=$(date +%s.%N)
+
+  # extract the start timestamp from the first START_PROFILE line
+  start_line=$(grep -m1 '^START_PROFILE: ' "$logfile" || true)
+  if [[ -z "$start_line" ]]; then
+    echo "ERROR: no START_PROFILE found in run #$i" >&2
+    rm -f "$logfile"
+    exit 1
+  fi
+  start_ts=${start_line#START_PROFILE:\ }
+
+  # compute elapsed
+  elapsed=$(echo "$end_ts - $start_ts" | bc)
+
+  # store
+  times+=("$elapsed")
+
+  # report this run
+  printf "Run %2d: %s seconds\n" "$i" "$elapsed"
+
+  rm -f "$logfile"
+
+  # if the wrapped command failed, stop early
+  if [[ $exit_code -ne 0 ]]; then
+    echo "Wrapped command exited with code $exit_code. Aborting."
+    exit $exit_code
+  fi
+done
+
+# Summary: compute min, max, avg via bc
+min=${times[0]}
+max=${times[0]}
+sum=0
+for t in "${times[@]}"; do
+  # compare floats: use bc
+  sleep(5)
+  is_less=$(echo "$t < $min" | bc)
+  (( is_less )) && min=$t
+  is_greater=$(echo "$t > $max" | bc)
+  (( is_greater )) && max=$t
+  sum=$(echo "$sum + $t" | bc)
+done
+
+avg=$(echo "$sum / $num_runs" | bc -l)
+
+echo
+echo "=== SUMMARY over $num_runs runs ==="
+echo "  Min elapsed : $min seconds"
+echo "  Max elapsed : $max seconds"
+echo "  Avg elapsed : $avg seconds"
+echo "==================================="
+
+exit 0
+