diff --git a/css/components.css b/css/components.css index da54c52..3032abf 100644 --- a/css/components.css +++ b/css/components.css @@ -367,6 +367,40 @@ button:focus, white-space: nowrap; } +/* Selection checkbox column (fixed slim width) */ +.data-table.has-select-col th.select-col, +.data-table.has-select-col td.select-col { + width: 36px; + min-width: 36px; + max-width: 36px; + text-align: center; + white-space: nowrap; +} + +/* Prevent first (model) column from expanding when selection column present */ +.data-table.has-select-col th.sortable[data-sort="name"], +.data-table.has-select-col td:first-of-type + td { + width: auto; + white-space: normal; +} + +/* Ensure model column remains flexible when select column exists */ +.data-table.has-select-col th:nth-child(2), +.data-table.has-select-col td:nth-child(2) { + width: 40%; + min-width: 180px; + max-width: 350px; + white-space: normal; + word-wrap: break-word; + text-align: left; /* override default % resolved right alignment */ +} + +/* Keep % Resolved right-aligned (now 3rd column when select column exists) */ +.data-table.has-select-col th:nth-child(3), +.data-table.has-select-col td:nth-child(3) { + text-align: right; +} + /* Cards */ .card { background-color: var(--color-background); @@ -616,6 +650,19 @@ button:focus, } } +/* Modal basic styles */ +.modal { display: none; position: fixed; inset: 0; z-index: var(--z-modal); } +.modal.show { display: block; } +.modal-backdrop { position: absolute; inset: 0; background: rgba(0,0,0,0.45); } +.modal-dialog { position: relative; background: var(--color-background); color: var(--color-text); width: min(720px, calc(100vw - 2rem)); margin: 5vh auto; border-radius: var(--radius-lg); box-shadow: var(--shadow-xl); border: 1.5px solid var(--color-border); resize: both; overflow: auto; min-width: 400px; min-height: 300px; max-width: 90vw; max-height: 90vh; display: flex; flex-direction: column; } +.modal-dialog-small { width: min(480px, calc(100vw - 2rem)); min-width: 320px; min-height: auto; resize: none; } +.modal-dialog-large { width: min(1200px, 90vw); height: min(800px, 90vh); } +.modal-header { display: flex; align-items: center; justify-content: space-between; padding: 0.75rem 1rem; border-bottom: 1.5px solid var(--color-border); } +.modal-body { padding: 1rem; overflow: auto; flex: 1; display: flex; flex-direction: column; } +.modal-close { background: transparent; border: none; cursor: pointer; color: var(--color-text-secondary); } +.chart-container { flex: 1; display: flex; flex-direction: column; min-height: 0; position: relative; } +.chart-container canvas { flex: 1; min-height: 260px; } + @media (max-width: 992px) { /* On mobile and tablets */ .table-responsive { @@ -919,3 +966,77 @@ button:focus, text-decoration-thickness: 2px; text-decoration-color: var(--color-text-muted); } + +/* New Feature Badge */ +.new-badge { + position: absolute; + top: 100%; + left: 50%; + transform: translateX(-50%); + margin-top: 2px; + background: linear-gradient(135deg, var(--color-accent), var(--color-accent-dark)); + color: white; + padding: 0.25rem 0.5rem; + border-radius: var(--radius-full); + font-size: 0.7rem; + font-weight: var(--weight-medium); + white-space: nowrap; + box-shadow: 0 2px 8px rgba(59, 130, 246, 0.4); + z-index: 100; + animation: newBadgeAnimation 6s ease-in-out forwards; + pointer-events: none; +} + +.dark-mode .new-badge { + background: linear-gradient(135deg, var(--blue-400), var(--blue-600)); + box-shadow: 0 2px 8px rgba(59, 130, 246, 0.6); +} + +.new-badge-button { + top: -12px; + left: auto; + right: -45px; +} + +@keyframes newBadgeAnimation { + 0% { + opacity: 0; + transform: translateX(-50%) scale(0.8); + } + 10% { + opacity: 1; + transform: translateX(-50%) scale(1); + } + 15% { + transform: translateX(-50%) scale(1.1); + } + 20% { + transform: translateX(-50%) scale(1); + } + 30% { + transform: translateX(-50%) scale(1.05); + } + 35% { + transform: translateX(-50%) scale(1); + } + 45% { + transform: translateX(-50%) scale(1.05); + } + 50% { + transform: translateX(-50%) scale(1); + } + 60% { + transform: translateX(-50%) scale(1.05); + } + 65% { + transform: translateX(-50%) scale(1); + } + 85% { + opacity: 1; + transform: translateX(-50%) scale(1); + } + 100% { + opacity: 0; + transform: translateX(-50%) scale(0.8); + } +} diff --git a/js/analysis.js b/js/analysis.js new file mode 100644 index 0000000..bf4dfb5 --- /dev/null +++ b/js/analysis.js @@ -0,0 +1,523 @@ +// Analysis and comparison features for leaderboard +(function() { + let compareChart = null; + let resizeObserver = null; + let chartTheme = 'dark'; // 'light' or 'dark' + + function getLeaderboardData() { + const dataScript = document.getElementById('leaderboard-data'); + if (!dataScript) return null; + try { + return JSON.parse(dataScript.textContent); + } catch (e) { + console.error('Error parsing leaderboard data:', e); + return null; + } + } + + function getSelectedModels() { + const container = document.getElementById('leaderboard-container'); + const active = container ? container.querySelector('.tabcontent.active') : null; + if (!active) return []; + const checkboxes = active.querySelectorAll('input.row-select:checked'); + + // Get full leaderboard data (it's a direct array, not wrapped in a property) + const leaderboardData = getLeaderboardData(); + + const activeLeaderboard = leaderboardData?.find(lb => { + return active.id === `leaderboard-${lb.name}`; + }); + + return Array.from(checkboxes).map(cb => { + const row = cb.closest('tr'); + const costCell = row ? row.querySelector('td:nth-child(4) .number') : null; + const costText = costCell ? costCell.textContent.trim().replace('$', '') : ''; + const cost = costText ? parseFloat(costText) : null; + + const modelName = cb.getAttribute('data-model'); + + // Find full model data from leaderboard + const fullModelData = activeLeaderboard?.results?.find(r => r.name === modelName); + + return { + name: modelName, + resolved: parseFloat(cb.getAttribute('data-resolved')) || 0, + cost: cost, + per_instance_details: fullModelData?.per_instance_details || null + }; + }); + } + + function getThemeColors(theme) { + if (theme === 'light') { + return { + background: '#ffffff', + gridColor: 'rgba(0, 0, 0, 0.1)', + textColor: '#333333', + barBackground: 'rgba(37, 99, 235, 0.6)', + barBorder: 'rgba(37, 99, 235, 1)' + }; + } else { + return { + background: 'transparent', + gridColor: 'rgba(255, 255, 255, 0.1)', + textColor: '#ffffff', + barBackground: 'rgba(37, 99, 235, 0.6)', + barBorder: 'rgba(37, 99, 235, 1)' + }; + } + } + + function openModal() { + const selected = getSelectedModels(); + if (!selected.length) { + openNoSelectionModal(); + return; + } + const modal = document.getElementById('compare-modal'); + if (!modal) return; + modal.classList.add('show'); + modal.setAttribute('aria-hidden', 'false'); + renderChart(); + setupResizeObserver(); + } + + function openNoSelectionModal() { + const modal = document.getElementById('no-selection-modal'); + if (!modal) return; + modal.classList.add('show'); + modal.setAttribute('aria-hidden', 'false'); + } + + function closeNoSelectionModal() { + const modal = document.getElementById('no-selection-modal'); + if (!modal) return; + modal.classList.remove('show'); + modal.setAttribute('aria-hidden', 'true'); + } + + function selectAll() { + const container = document.getElementById('leaderboard-container'); + const active = container ? container.querySelector('.tabcontent.active') : null; + if (!active) return; + + // First uncheck all + const allCheckboxes = active.querySelectorAll('input.row-select'); + allCheckboxes.forEach(cb => cb.checked = false); + + // Get visible rows (not filtered out) + const visibleRows = Array.from(active.querySelectorAll('tbody tr:not(.no-results)')) + .filter(row => row.style.display !== 'none'); + + // Select all visible rows + visibleRows.forEach(row => { + const checkbox = row.querySelector('input.row-select'); + if (checkbox) { + checkbox.checked = true; + } + }); + + closeNoSelectionModal(); + openModal(); + } + + function selectTopN(n, openWeightsOnly = false) { + const container = document.getElementById('leaderboard-container'); + const active = container ? container.querySelector('.tabcontent.active') : null; + if (!active) return; + + // First uncheck all + const allCheckboxes = active.querySelectorAll('input.row-select'); + allCheckboxes.forEach(cb => cb.checked = false); + + // Get visible rows (not filtered out) + let visibleRows = Array.from(active.querySelectorAll('tbody tr:not(.no-results)')) + .filter(row => row.style.display !== 'none'); + + // Filter by open weights if requested + if (openWeightsOnly) { + visibleRows = visibleRows.filter(row => row.getAttribute('data-os_model') === 'true'); + } + + // Select top N visible rows (or all if n is null/undefined) + const rowsToSelect = n ? visibleRows.slice(0, n) : visibleRows; + rowsToSelect.forEach(row => { + const checkbox = row.querySelector('input.row-select'); + if (checkbox) { + checkbox.checked = true; + } + }); + + closeNoSelectionModal(); + openModal(); + } + + function closeModal() { + const modal = document.getElementById('compare-modal'); + if (!modal) return; + modal.classList.remove('show'); + modal.setAttribute('aria-hidden', 'true'); + teardownResizeObserver(); + } + + function setupResizeObserver() { + const modalDialog = document.querySelector('#compare-modal .modal-dialog'); + if (!modalDialog) return; + + if (resizeObserver) { + resizeObserver.disconnect(); + } + + resizeObserver = new ResizeObserver(() => { + if (compareChart) { + compareChart.resize(); + } + }); + + resizeObserver.observe(modalDialog); + } + + function teardownResizeObserver() { + if (resizeObserver) { + resizeObserver.disconnect(); + resizeObserver = null; + } + } + + function renderChart() { + const selected = getSelectedModels(); + const empty = document.getElementById('compare-empty'); + const canvas = document.getElementById('compare-chart'); + if (!canvas) return; + + // Hide matrix tooltip if it exists (safety check) + const matrixTooltip = document.getElementById('matrix-tooltip'); + if (matrixTooltip) { + matrixTooltip.style.display = 'none'; + } + + if (compareChart) { + compareChart.destroy(); + compareChart = null; + } + + if (!selected.length) { + if (empty) empty.style.display = ''; + return; + } + if (empty) empty.style.display = 'none'; + + const ctx = canvas.getContext('2d'); + const colors = getThemeColors(chartTheme); + + // Set canvas background via container + const chartContainer = canvas.closest('.chart-container'); + if (chartContainer) { + chartContainer.style.backgroundColor = colors.background; + } + + // Plugin to draw background on the chart + const backgroundPlugin = { + id: 'customCanvasBackgroundColor', + beforeDraw: (chart, args, options) => { + const {ctx, chartArea} = chart; + if (!chartArea) return; + ctx.save(); + ctx.fillStyle = colors.background; + ctx.fillRect(0, 0, chart.width, chart.height); + ctx.restore(); + } + }; + + const chartTypeSelect = document.getElementById('compare-chart-type'); + const chartType = chartTypeSelect ? chartTypeSelect.value : 'bar'; + + // Delegate to specific chart renderers + if (chartType === 'scatter') { + compareChart = renderScatterChart(ctx, selected, colors, backgroundPlugin); + if (!compareChart) { + if (empty) { + empty.textContent = 'No cost data available for selected models.'; + empty.style.display = ''; + } + } + } else if (chartType === 'cumulative-cost') { + compareChart = renderCumulativeChart(ctx, selected, colors, backgroundPlugin, 'cost', false); + if (!compareChart) { + if (empty) { + empty.textContent = 'No per-instance cost data available for selected models.'; + empty.style.display = ''; + } + } + } else if (chartType === 'cumulative-cost-resolved') { + compareChart = renderCumulativeChart(ctx, selected, colors, backgroundPlugin, 'cost', true); + if (!compareChart) { + if (empty) { + empty.textContent = 'No per-instance cost data available for resolved instances.'; + empty.style.display = ''; + } + } + } else if (chartType === 'cumulative-steps') { + compareChart = renderCumulativeChart(ctx, selected, colors, backgroundPlugin, 'api_calls', false); + if (!compareChart) { + if (empty) { + empty.textContent = 'No per-instance API call data available for selected models.'; + empty.style.display = ''; + } + } + } else if (chartType === 'cumulative-steps-resolved') { + compareChart = renderCumulativeChart(ctx, selected, colors, backgroundPlugin, 'api_calls', true); + if (!compareChart) { + if (empty) { + empty.textContent = 'No per-instance API call data available for resolved instances.'; + empty.style.display = ''; + } + } + } else if (chartType === 'grouped-bar') { + compareChart = renderGroupedBarChart(ctx, selected, colors, backgroundPlugin); + if (!compareChart) { + if (empty) { + empty.textContent = 'No per-instance data available for selected models.'; + empty.style.display = ''; + } + } + } else if (chartType === 'resolved-instances-matrix') { + const chunkSelector = document.getElementById('matrix-chunk-selector'); + const chunkStart = chunkSelector ? parseInt(chunkSelector.value) : 0; + compareChart = renderResolvedInstancesMatrix(ctx, selected, colors, backgroundPlugin, chunkStart, 100); + if (!compareChart) { + if (empty) { + empty.textContent = 'No per-instance data available for selected models.'; + empty.style.display = ''; + } + } + } else if (chartType === 'resolved-vs-avg-cost') { + compareChart = renderResolvedVsAvgCostChart(ctx, selected, colors, backgroundPlugin); + if (!compareChart) { + if (empty) { + empty.textContent = 'No per-instance data available for selected models.'; + empty.style.display = ''; + } + } + } else if (chartType === 'resolved-vs-cost-limit') { + compareChart = renderResolvedVsLimitChart(ctx, selected, colors, backgroundPlugin, 'cost'); + if (!compareChart) { + if (empty) { + empty.textContent = 'No per-instance cost data available for selected models.'; + empty.style.display = ''; + } + } + } else if (chartType === 'resolved-vs-step-limit') { + compareChart = renderResolvedVsLimitChart(ctx, selected, colors, backgroundPlugin, 'api_calls'); + if (!compareChart) { + if (empty) { + empty.textContent = 'No per-instance API call data available for selected models.'; + empty.style.display = ''; + } + } + } else { + // Default to bar chart + compareChart = renderBarChart(ctx, selected, colors, backgroundPlugin); + } + } + + function toggleChartTheme() { + chartTheme = chartTheme === 'light' ? 'dark' : 'light'; + updateThemeButton(); + renderChart(); + } + + function updateThemeButton() { + const btn = document.getElementById('chart-theme-toggle'); + if (!btn) return; + if (chartTheme === 'light') { + btn.innerHTML = ' Dark'; + btn.title = 'Switch to dark mode'; + } else { + btn.innerHTML = ' Light'; + btn.title = 'Switch to light mode'; + } + } + + function downloadJSON() { + const selected = getSelectedModels(); + if (!selected.length) return; + + const data = { + timestamp: new Date().toISOString(), + models: selected.map(s => { + const modelData = { + name: s.name, + resolved: s.resolved, + cost: s.cost + }; + + // Include per_instance_details if available + if (s.per_instance_details) { + modelData.per_instance_details = s.per_instance_details; + } + + return modelData; + }) + }; + + const jsonStr = JSON.stringify(data, null, 2); + const blob = new Blob([jsonStr], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `swe-bench-comparison-${new Date().toISOString().split('T')[0]}.json`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } + + function downloadPNG() { + if (!compareChart) return; + + const canvas = document.getElementById('compare-chart'); + if (!canvas) return; + + canvas.toBlob((blob) => { + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `swe-bench-chart-${new Date().toISOString().split('T')[0]}.png`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + }); + } + + function initEvents() { + // Open via delegated event to handle dynamic rendering + document.addEventListener('click', (e) => { + const trigger = e.target && typeof e.target.closest === 'function' ? e.target.closest('#compare-btn') : null; + if (trigger) { + e.preventDefault(); + e.stopPropagation(); + openModal(); + } + }); + + // Close via backdrop or close button/icon + const modal = document.getElementById('compare-modal'); + if (modal) { + modal.addEventListener('click', (e) => { + const closeEl = e.target && typeof e.target.closest === 'function' ? e.target.closest('[data-close="true"]') : null; + if (closeEl) { + e.preventDefault(); + closeModal(); + } + }); + } + + const chartType = document.getElementById('compare-chart-type'); + const chunkSelector = document.getElementById('matrix-chunk-selector'); + + if (chartType) { + chartType.addEventListener('change', () => { + // Show/hide chunk selector for matrix chart + if (chunkSelector) { + chunkSelector.style.display = chartType.value === 'resolved-instances-matrix' ? '' : 'none'; + } + renderChart(); + }); + } + + if (chunkSelector) { + chunkSelector.addEventListener('change', () => { + if (chartType && chartType.value === 'resolved-instances-matrix') { + renderChart(); + } + }); + } + + const themeToggle = document.getElementById('chart-theme-toggle'); + if (themeToggle) { + themeToggle.addEventListener('click', (e) => { + e.preventDefault(); + toggleChartTheme(); + }); + } + + // Quickselect button handler + const quickselectBtn = document.getElementById('quickselect-btn'); + if (quickselectBtn) { + quickselectBtn.addEventListener('click', (e) => { + e.preventDefault(); + closeModal(); + openNoSelectionModal(); + }); + } + + // Download JSON button handler + const downloadJsonBtn = document.getElementById('download-json-btn'); + if (downloadJsonBtn) { + downloadJsonBtn.addEventListener('click', (e) => { + e.preventDefault(); + downloadJSON(); + }); + } + + // Download PNG button handler + const downloadPngBtn = document.getElementById('download-png-btn'); + if (downloadPngBtn) { + downloadPngBtn.addEventListener('click', (e) => { + e.preventDefault(); + downloadPNG(); + }); + } + + // No selection modal close handlers + const noSelectionModal = document.getElementById('no-selection-modal'); + if (noSelectionModal) { + noSelectionModal.addEventListener('click', (e) => { + const closeEl = e.target && typeof e.target.closest === 'function' ? e.target.closest('[data-close="true"]') : null; + if (closeEl) { + e.preventDefault(); + closeNoSelectionModal(); + } + }); + } + + // Quick select buttons + const selectTop10 = document.getElementById('select-top-10'); + if (selectTop10) { + selectTop10.addEventListener('click', () => selectTopN(10, false)); + } + + const selectTop20 = document.getElementById('select-top-20'); + if (selectTop20) { + selectTop20.addEventListener('click', () => selectTopN(20, false)); + } + + const selectAllBtn = document.getElementById('select-all'); + if (selectAllBtn) { + selectAllBtn.addEventListener('click', () => selectAll()); + } + + const selectAllOW = document.getElementById('select-all-ow'); + if (selectAllOW) { + selectAllOW.addEventListener('click', () => selectTopN(null, true)); + } + + document.addEventListener('change', (e) => { + if (e.target && e.target.classList.contains('row-select')) { + if (document.getElementById('compare-modal')?.classList.contains('show')) { + renderChart(); + } + } + }); + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', initEvents); + } else { + initEvents(); + } +})(); + + diff --git a/js/charts/barChart.js b/js/charts/barChart.js new file mode 100644 index 0000000..bc35e3f --- /dev/null +++ b/js/charts/barChart.js @@ -0,0 +1,64 @@ +// Bar chart for performance comparison +function renderBarChart(ctx, selected, colors, backgroundPlugin) { + const labels = selected.map(s => s.name); + const values = selected.map(s => s.resolved); + + return new Chart(ctx, { + type: 'bar', + data: { + labels, + datasets: [{ + label: '% Resolved', + data: values, + backgroundColor: colors.barBackground, + borderColor: colors.barBorder, + borderWidth: 1, + }] + }, + options: { + responsive: true, + maintainAspectRatio: false, + animation: false, + scales: { + y: { + beginAtZero: true, + title: { + display: true, + text: '% Resolved', + color: colors.textColor + }, + ticks: { + callback: (v) => v + '%', + color: colors.textColor + }, + grid: { + color: colors.gridColor + } + }, + x: { + title: { + display: true, + text: 'Model', + color: colors.textColor + }, + ticks: { + color: colors.textColor + }, + grid: { + color: colors.gridColor + } + } + }, + plugins: { + legend: { display: false }, + tooltip: { + callbacks: { + label: (ctx) => `${ctx.parsed.y.toFixed(2)}%` + } + } + } + }, + plugins: [backgroundPlugin] + }); +} + diff --git a/js/charts/cumulativeCostChart.js b/js/charts/cumulativeCostChart.js new file mode 100644 index 0000000..b6f27d0 --- /dev/null +++ b/js/charts/cumulativeCostChart.js @@ -0,0 +1,164 @@ +// Cumulative distribution chart (generic for cost or api_calls) +// Parameters: +// metric: 'cost' or 'api_calls' +// resolvedOnly: if true, only include instances where resolved === true +function renderCumulativeChart(ctx, selected, colors, backgroundPlugin, metric = 'cost', resolvedOnly = false) { + // Filter models that have per_instance_details + const modelsWithDetails = selected.filter(s => s.per_instance_details !== null); + + if (modelsWithDetails.length === 0) { + return null; + } + + // Generate distinct colors for each model + const colorPalette = [ + 'rgb(37, 99, 235)', // blue + 'rgb(220, 38, 38)', // red + 'rgb(22, 163, 74)', // green + 'rgb(234, 88, 12)', // orange + 'rgb(168, 85, 247)', // purple + 'rgb(236, 72, 153)', // pink + 'rgb(14, 165, 233)', // cyan + 'rgb(234, 179, 8)', // yellow + 'rgb(156, 163, 175)', // gray + 'rgb(251, 146, 60)', // amber + ]; + + const datasets = modelsWithDetails.map((model, idx) => { + // Extract values from per_instance_details + const allDetails = Object.values(model.per_instance_details); + + let values = allDetails + .filter(d => !resolvedOnly || d.resolved === true) + .map(d => d[metric]); + + // If no values after filtering, skip this model + if (values.length === 0) { + return null; + } + + // Sort values + const sortedValues = [...values].sort((a, b) => a - b); + + // Create cumulative distribution + const cumulativeData = sortedValues.map((value, i) => ({ + x: value, + y: ((i + 1) / sortedValues.length) * 100 + })); + + const color = colorPalette[idx % colorPalette.length]; + + return { + label: model.name, + data: cumulativeData, + borderColor: color, + backgroundColor: 'transparent', + borderWidth: 2, + pointRadius: 0, + pointHoverRadius: 4, + tension: 0, + stepped: false + }; + }).filter(d => d !== null); + + // If no valid datasets after filtering, return null + if (datasets.length === 0) { + return null; + } + + // Calculate x-axis range + const allValues = modelsWithDetails.flatMap(m => + Object.values(m.per_instance_details) + .filter(d => !resolvedOnly || d.resolved === true) + .map(d => d[metric]) + ); + const maxValue = Math.max(...allValues); + + // Configure labels based on metric + const isApiCalls = metric === 'api_calls'; + const xAxisLabel = isApiCalls ? 'API Calls per Instance' : 'Cost per Instance ($)'; + const xTickCallback = isApiCalls ? (v) => v.toFixed(0) : (v) => '$' + v.toFixed(2); + const tooltipCallback = isApiCalls + ? (ctx) => `${ctx.dataset.label}: ${ctx.parsed.x.toFixed(0)} calls (${ctx.parsed.y.toFixed(1)}%)` + : (ctx) => `${ctx.dataset.label}: $${ctx.parsed.x.toFixed(2)} (${ctx.parsed.y.toFixed(1)}%)`; + + return new Chart(ctx, { + type: 'line', + data: { datasets }, + options: { + responsive: true, + maintainAspectRatio: false, + animation: false, + scales: { + x: { + type: 'linear', + beginAtZero: true, + max: maxValue * 1.05, + title: { + display: true, + text: xAxisLabel, + color: colors.textColor, + font: { size: 14 } + }, + ticks: { + callback: xTickCallback, + color: colors.textColor, + font: { size: 12 } + }, + grid: { + color: colors.gridColor + } + }, + y: { + beginAtZero: true, + max: 100, + title: { + display: true, + text: 'Cumulative % of Instances', + color: colors.textColor, + font: { size: 14 } + }, + ticks: { + callback: (v) => v + '%', + color: colors.textColor, + font: { size: 12 } + }, + grid: { + color: colors.gridColor + } + } + }, + plugins: { + legend: { + display: true, + position: 'top', + labels: { + color: colors.textColor, + font: { size: 12 }, + usePointStyle: true, + padding: 10 + } + }, + tooltip: { + mode: 'nearest', + intersect: false, + callbacks: { + label: tooltipCallback + } + } + }, + interaction: { + mode: 'nearest', + axis: 'x', + intersect: false + } + }, + plugins: [backgroundPlugin] + }); +} + +// Backward compatibility wrapper +function renderCumulativeCostChart(ctx, selected, colors, backgroundPlugin) { + return renderCumulativeChart(ctx, selected, colors, backgroundPlugin, 'cost', false); +} + diff --git a/js/charts/groupedBarChart.js b/js/charts/groupedBarChart.js new file mode 100644 index 0000000..5868a9d --- /dev/null +++ b/js/charts/groupedBarChart.js @@ -0,0 +1,200 @@ +// Grouped bar chart by repository +function renderGroupedBarChart(ctx, selected, colors, backgroundPlugin) { + // Filter models that have per_instance_details + const modelsWithDetails = selected.filter(s => s.per_instance_details !== null); + + if (modelsWithDetails.length === 0) { + return null; + } + + // Generate distinct colors for each model + const colorPalette = [ + 'rgb(37, 99, 235)', // blue + 'rgb(220, 38, 38)', // red + 'rgb(22, 163, 74)', // green + 'rgb(234, 88, 12)', // orange + 'rgb(168, 85, 247)', // purple + 'rgb(236, 72, 153)', // pink + 'rgb(14, 165, 233)', // cyan + 'rgb(234, 179, 8)', // yellow + 'rgb(156, 163, 175)', // gray + 'rgb(251, 146, 60)', // amber + ]; + + // Extract repository name from instance ID + function getRepoFromInstanceId(instanceId) { + // Format: org__repo-1234 -> repo (just the repo part) + // Split by '-' and find where the number starts + const parts = instanceId.split('-'); + let repoParts = []; + + for (let i = 0; i < parts.length; i++) { + // Check if this part starts with a digit + if (parts[i] && /^\d/.test(parts[i])) { + // This is the start of the issue number, take everything before + repoParts = parts.slice(0, i); + break; + } + } + + if (repoParts.length === 0) { + repoParts = parts; // No number found, use all parts + } + + // Join parts back and replace __ with / + const fullPath = repoParts.join('-').replace(/__/g, '/'); + + // Extract just the repo name (after the last /) + const pathParts = fullPath.split('/'); + return pathParts[pathParts.length - 1]; + } + + // Collect all repositories and count instances per repo + // Use a Set per repo to avoid counting duplicates across models + const repoInstanceSets = {}; + modelsWithDetails.forEach(model => { + Object.keys(model.per_instance_details).forEach(instanceId => { + const repo = getRepoFromInstanceId(instanceId); + if (!repoInstanceSets[repo]) { + repoInstanceSets[repo] = new Set(); + } + repoInstanceSets[repo].add(instanceId); + }); + }); + + // Convert sets to counts + const repoInstanceCounts = {}; + Object.keys(repoInstanceSets).forEach(repo => { + repoInstanceCounts[repo] = repoInstanceSets[repo].size; + }); + + // Sort repositories by name + const repos = Object.keys(repoInstanceCounts).sort(); + + // For each model, calculate resolved percentage per repository + const datasets = modelsWithDetails.map((model, idx) => { + const repoData = repos.map(repo => { + // Find all instances for this repo + const instances = Object.entries(model.per_instance_details) + .filter(([instanceId, _]) => getRepoFromInstanceId(instanceId) === repo); + + if (instances.length === 0) { + return 0; + } + + // Calculate resolved percentage + const resolvedCount = instances.filter(([_, data]) => data.resolved === true).length; + return (resolvedCount / instances.length) * 100; + }); + + const color = colorPalette[idx % colorPalette.length]; + + return { + label: model.name, + data: repoData, + backgroundColor: color, + borderColor: color, + borderWidth: 1 + }; + }); + + // Limit to top N repositories if there are too many + const maxRepos = 20; + let displayRepos = repos; + let displayDatasets = datasets; + + if (repos.length > maxRepos) { + // Calculate average resolved percentage across all models for each repo + const repoScores = repos.map((repo, idx) => { + const avgScore = datasets.reduce((sum, dataset) => sum + dataset.data[idx], 0) / datasets.length; + return { repo, idx, avgScore }; + }); + + // Sort by average score descending and take top N + repoScores.sort((a, b) => b.avgScore - a.avgScore); + const topIndices = repoScores.slice(0, maxRepos).map(r => r.idx); + + displayRepos = topIndices.map(idx => repos[idx]); + displayDatasets = datasets.map(dataset => ({ + ...dataset, + data: topIndices.map(idx => dataset.data[idx]) + })); + } + + // Create labels with instance counts + const displayLabels = displayRepos.map(repo => { + const instanceCount = repoInstanceCounts[repo]; + return `${repo} (${instanceCount})`; + }); + + return new Chart(ctx, { + type: 'bar', + data: { + labels: displayLabels, + datasets: displayDatasets + }, + options: { + responsive: true, + maintainAspectRatio: false, + animation: false, + scales: { + x: { + title: { + display: true, + text: repos.length > maxRepos ? `Repository (top ${maxRepos} by avg performance)` : 'Repository', + color: colors.textColor, + font: { size: 14 } + }, + ticks: { + color: colors.textColor, + font: { size: 10 }, + maxRotation: 45, + minRotation: 45 + }, + grid: { + color: colors.gridColor + } + }, + y: { + beginAtZero: true, + max: 100, + title: { + display: true, + text: 'Resolved (%)', + color: colors.textColor, + font: { size: 14 } + }, + ticks: { + callback: (v) => v + '%', + color: colors.textColor, + font: { size: 12 } + }, + grid: { + color: colors.gridColor + } + } + }, + plugins: { + legend: { + display: true, + position: 'top', + labels: { + color: colors.textColor, + font: { size: 12 }, + usePointStyle: true, + padding: 10 + } + }, + tooltip: { + callbacks: { + label: (ctx) => { + return `${ctx.dataset.label}: ${ctx.parsed.y.toFixed(1)}%`; + } + } + } + } + }, + plugins: [backgroundPlugin] + }); +} + diff --git a/js/charts/resolvedInstancesMatrix.js b/js/charts/resolvedInstancesMatrix.js new file mode 100644 index 0000000..c117117 --- /dev/null +++ b/js/charts/resolvedInstancesMatrix.js @@ -0,0 +1,185 @@ +// Resolved instances matrix chart (GitHub contribution style) +function renderResolvedInstancesMatrix(ctx, selected, colors, backgroundPlugin, chunkStart = 0, chunkSize = 100) { + // Filter models that have per_instance_details + const modelsWithDetails = selected.filter(s => s.per_instance_details !== null); + + if (modelsWithDetails.length === 0) { + return null; + } + + // Get all unique instance IDs across all models + const allInstanceIds = new Set(); + modelsWithDetails.forEach(model => { + Object.keys(model.per_instance_details).forEach(id => allInstanceIds.add(id)); + }); + const allSortedInstanceIds = Array.from(allInstanceIds).sort(); + + // Slice to get the current chunk + const sortedInstanceIds = allSortedInstanceIds.slice(chunkStart, chunkStart + chunkSize); + + const canvas = ctx.canvas; + + // Handle high DPI displays + const dpr = window.devicePixelRatio || 1; + const rect = canvas.getBoundingClientRect(); + canvas.width = rect.width * dpr; + canvas.height = rect.height * dpr; + ctx.scale(dpr, dpr); + + const width = rect.width; + const height = rect.height; + + // Calculate margins and dimensions + const leftMargin = 120; // Space for model names + const topMargin = 30; // Minimal top margin + const rightMargin = 10; + const bottomMargin = 30; // Space for hover text + + const availableWidth = width - leftMargin - rightMargin; + const availableHeight = height - topMargin - bottomMargin; + + const cellWidth = Math.max(2, availableWidth / sortedInstanceIds.length); + const cellHeight = Math.max(20, availableHeight / modelsWithDetails.length); + + // Clear canvas and draw background + ctx.fillStyle = colors.background; + ctx.fillRect(0, 0, width, height); + + // Draw title + ctx.fillStyle = colors.textColor; + ctx.textAlign = 'left'; + ctx.font = 'bold 12px sans-serif'; + ctx.fillText('Resolved Instances Matrix (hover to see instance ID)', 10, 15); + + // Draw model names (y-axis labels) with line breaks + ctx.fillStyle = colors.textColor; + ctx.font = '11px sans-serif'; + ctx.textAlign = 'right'; + ctx.textBaseline = 'middle'; + + modelsWithDetails.forEach((model, rowIdx) => { + const y = topMargin + rowIdx * cellHeight + cellHeight / 2; + + // Insert line breaks every 12 characters + const displayName = model.name; + const lines = []; + for (let i = 0; i < displayName.length; i += 12) { + lines.push(displayName.substring(i, i + 12)); + } + + // Draw each line + const lineHeight = 12; + const totalHeight = lines.length * lineHeight; + const startY = y - (totalHeight / 2) + (lineHeight / 2); + + lines.forEach((line, lineIdx) => { + ctx.fillText(line, leftMargin - 5, startY + lineIdx * lineHeight); + }); + }); + + // Draw matrix cells + modelsWithDetails.forEach((model, rowIdx) => { + sortedInstanceIds.forEach((instanceId, colIdx) => { + const instanceData = model.per_instance_details[instanceId]; + const x = leftMargin + colIdx * cellWidth; + const y = topMargin + rowIdx * cellHeight; + + // Draw cell + if (!instanceData) { + // Instance not in this model's data + ctx.fillStyle = 'rgba(128, 128, 128, 0.1)'; + } else if (instanceData.resolved) { + // Resolved - green for all models + ctx.fillStyle = 'rgba(34, 197, 94, 0.8)'; + } else { + // Not resolved - gray + ctx.fillStyle = 'rgba(156, 163, 175, 0.4)'; + } + + ctx.fillRect(x, y, Math.max(1, cellWidth - 0.5), cellHeight - 1); + }); + }); + + // Create tooltip element if it doesn't exist + let tooltip = document.getElementById('matrix-tooltip'); + if (!tooltip) { + tooltip = document.createElement('div'); + tooltip.id = 'matrix-tooltip'; + tooltip.style.position = 'absolute'; + tooltip.style.backgroundColor = 'rgba(0, 0, 0, 0.85)'; + tooltip.style.color = 'white'; + tooltip.style.padding = '8px 12px'; + tooltip.style.borderRadius = '4px'; + tooltip.style.fontSize = '12px'; + tooltip.style.fontFamily = 'sans-serif'; + tooltip.style.pointerEvents = 'none'; + tooltip.style.zIndex = '10000'; + tooltip.style.display = 'none'; + tooltip.style.whiteSpace = 'nowrap'; + document.body.appendChild(tooltip); + } + + // Store event handlers for cleanup + const mouseMoveHandler = (e) => { + const rect = canvas.getBoundingClientRect(); + const mouseX = e.clientX - rect.left; + const mouseY = e.clientY - rect.top; + + // Check if mouse is over the matrix area + if (mouseX >= leftMargin && mouseX < width - rightMargin && + mouseY >= topMargin && mouseY < height - bottomMargin) { + + // Calculate which column + const colIdx = Math.floor((mouseX - leftMargin) / cellWidth); + + if (colIdx >= 0 && colIdx < sortedInstanceIds.length) { + const instanceId = sortedInstanceIds[colIdx]; + + // Show tooltip + tooltip.textContent = instanceId; + tooltip.style.display = 'block'; + tooltip.style.left = (e.pageX + 10) + 'px'; + tooltip.style.top = (e.pageY - 30) + 'px'; + } else { + tooltip.style.display = 'none'; + } + } else { + tooltip.style.display = 'none'; + } + }; + + const mouseLeaveHandler = () => { + tooltip.style.display = 'none'; + }; + + // Add event listeners + canvas.addEventListener('mousemove', mouseMoveHandler); + canvas.addEventListener('mouseleave', mouseLeaveHandler); + + // Return a Chart-like object for compatibility + return { + destroy: () => { + // Remove event listeners + canvas.removeEventListener('mousemove', mouseMoveHandler); + canvas.removeEventListener('mouseleave', mouseLeaveHandler); + + // Hide tooltip when chart is destroyed + const tooltip = document.getElementById('matrix-tooltip'); + if (tooltip) { + tooltip.style.display = 'none'; + } + }, + resize: () => { + renderResolvedInstancesMatrix(ctx, selected, colors, backgroundPlugin, chunkStart, chunkSize); + }, + update: () => {}, + data: { datasets: [] }, + options: {}, + _matrixMetadata: { + totalInstances: allSortedInstanceIds.length, + chunkStart: chunkStart, + chunkSize: chunkSize + } + }; +} + diff --git a/js/charts/resolvedVsAvgCostChart.js b/js/charts/resolvedVsAvgCostChart.js new file mode 100644 index 0000000..b31333d --- /dev/null +++ b/js/charts/resolvedVsAvgCostChart.js @@ -0,0 +1,174 @@ +// Resolved vs Average Cost chart +function renderResolvedVsAvgCostChart(ctx, selected, colors, backgroundPlugin) { + // Filter models that have per_instance_details + const modelsWithDetails = selected.filter(s => s.per_instance_details !== null); + + if (modelsWithDetails.length === 0) { + return null; + } + + // Generate distinct colors for each model + const colorPalette = [ + 'rgb(37, 99, 235)', // blue + 'rgb(220, 38, 38)', // red + 'rgb(22, 163, 74)', // green + 'rgb(234, 88, 12)', // orange + 'rgb(168, 85, 247)', // purple + 'rgb(236, 72, 153)', // pink + 'rgb(14, 165, 233)', // cyan + 'rgb(234, 179, 8)', // yellow + 'rgb(156, 163, 175)', // gray + 'rgb(251, 146, 60)', // amber + ]; + + const datasets = modelsWithDetails.map((model, idx) => { + const allDetails = Object.values(model.per_instance_details); + const totalInstances = allDetails.length; + + // Get all instances with their data + const instances = allDetails.map(d => ({ + api_calls: d.api_calls, + cost: d.cost, + resolved: d.resolved === true + })); + + // Find max api_calls + const maxApiCalls = Math.max(...instances.map(i => i.api_calls)); + + // For each step limit, calculate resolved % and average cost + const chartData = []; + + // Scan through step limits + for (let stepLimit = 0; stepLimit <= maxApiCalls; stepLimit++) { + // Filter instances with api_calls <= stepLimit + const filteredInstances = instances.filter(i => i.api_calls <= stepLimit); + + if (filteredInstances.length === 0) { + continue; + } + + // Calculate resolved count + const resolvedCount = filteredInstances.filter(i => i.resolved).length; + const resolvedPercentage = (resolvedCount / totalInstances) * 100; + + // Calculate average cost of filtered instances + const avgCost = filteredInstances.reduce((sum, i) => sum + i.cost, 0) / filteredInstances.length; + + chartData.push({ + x: avgCost, + y: resolvedPercentage, + stepLimit: stepLimit, + numInstances: filteredInstances.length + }); + } + + const color = colorPalette[idx % colorPalette.length]; + + return { + label: `${model.name} (${model.resolved.toFixed(1)}%)`, + data: chartData, + borderColor: color, + backgroundColor: 'transparent', + borderWidth: 2, + pointRadius: 0, + pointHoverRadius: 5, + tension: 0.1 + }; + }); + + // Calculate max resolved percentage across all datasets + const maxResolved = Math.max(...datasets.flatMap(d => d.data.map(p => p.y))); + const tentativeMax = maxResolved * 1.1; + // Round up to nearest 5 or 10 for cleaner y-axis labels + let yAxisMax; + if (tentativeMax > 100) { + yAxisMax = 100; + } else if (tentativeMax > 50) { + yAxisMax = Math.ceil(tentativeMax / 10) * 10; // Round to nearest 10 + } else { + yAxisMax = Math.ceil(tentativeMax / 5) * 5; // Round to nearest 5 + } + + return new Chart(ctx, { + type: 'line', + data: { datasets }, + options: { + responsive: true, + maintainAspectRatio: false, + animation: false, + scales: { + x: { + type: 'linear', + beginAtZero: true, + title: { + display: true, + text: 'Average Cost per Instance ($)', + color: colors.textColor, + font: { size: 14 } + }, + ticks: { + callback: (v) => '$' + v.toFixed(2), + color: colors.textColor, + font: { size: 12 } + }, + grid: { + color: colors.gridColor + } + }, + y: { + beginAtZero: true, + max: yAxisMax, + title: { + display: true, + text: 'Resolved (%)', + color: colors.textColor, + font: { size: 14 } + }, + ticks: { + callback: (v) => v + '%', + color: colors.textColor, + font: { size: 12 } + }, + grid: { + color: colors.gridColor + } + } + }, + plugins: { + legend: { + display: true, + position: 'top', + labels: { + color: colors.textColor, + font: { size: 12 }, + usePointStyle: true, + padding: 10 + } + }, + tooltip: { + mode: 'nearest', + intersect: false, + callbacks: { + label: (ctx) => { + const dataPoint = ctx.dataset.data[ctx.dataIndex]; + return [ + `${ctx.dataset.label}`, + `Avg cost: $${ctx.parsed.x.toFixed(3)}`, + `Resolved: ${ctx.parsed.y.toFixed(1)}%`, + `Step limit: ${dataPoint.stepLimit}`, + `Instances: ${dataPoint.numInstances}` + ]; + } + } + } + }, + interaction: { + mode: 'nearest', + axis: 'x', + intersect: false + } + }, + plugins: [backgroundPlugin] + }); +} + diff --git a/js/charts/resolvedVsLimitChart.js b/js/charts/resolvedVsLimitChart.js new file mode 100644 index 0000000..4df96cd --- /dev/null +++ b/js/charts/resolvedVsLimitChart.js @@ -0,0 +1,161 @@ +// Resolved vs cost/step limit chart +function renderResolvedVsLimitChart(ctx, selected, colors, backgroundPlugin, metric = 'cost') { + // Filter models that have per_instance_details + const modelsWithDetails = selected.filter(s => s.per_instance_details !== null); + + if (modelsWithDetails.length === 0) { + return null; + } + + // Generate distinct colors for each model + const colorPalette = [ + 'rgb(37, 99, 235)', // blue + 'rgb(220, 38, 38)', // red + 'rgb(22, 163, 74)', // green + 'rgb(234, 88, 12)', // orange + 'rgb(168, 85, 247)', // purple + 'rgb(236, 72, 153)', // pink + 'rgb(14, 165, 233)', // cyan + 'rgb(234, 179, 8)', // yellow + 'rgb(156, 163, 175)', // gray + 'rgb(251, 146, 60)', // amber + ]; + + const datasets = modelsWithDetails.map((model, idx) => { + const allDetails = Object.values(model.per_instance_details); + const totalInstances = allDetails.length; + + // Get all values and resolved status + const instances = allDetails.map(d => ({ + value: d[metric], + resolved: d.resolved === true + })); + + // Sort by value + instances.sort((a, b) => a.value - b.value); + + // Find max value for this model + const maxValue = Math.max(...instances.map(i => i.value)); + + // Create data points - for each instance, calculate cumulative resolved % + const chartData = []; + let resolvedCount = 0; + + // Add point at 0 + chartData.push({ x: 0, y: 0 }); + + // For each instance (sorted by value), check if resolved and update count + instances.forEach((instance, i) => { + if (instance.resolved) { + resolvedCount++; + } + + // Add a point at this value with the current resolved percentage + const resolvedPercentage = (resolvedCount / totalInstances) * 100; + chartData.push({ x: instance.value, y: resolvedPercentage }); + }); + + const color = colorPalette[idx % colorPalette.length]; + + return { + label: model.name, + data: chartData, + borderColor: color, + backgroundColor: 'transparent', + borderWidth: 2, + pointRadius: 0, + pointHoverRadius: 4, + tension: 0, + stepped: 'before' // Step function to show threshold behavior + }; + }); + + // Calculate x-axis range across all models + const allValues = modelsWithDetails.flatMap(m => + Object.values(m.per_instance_details).map(d => d[metric]) + ); + const maxValue = Math.max(...allValues); + + // Configure labels based on metric + const isApiCalls = metric === 'api_calls'; + const xAxisLabel = isApiCalls ? 'API Call Limit' : 'Cost Limit ($)'; + const xTickCallback = isApiCalls ? (v) => v.toFixed(0) : (v) => '$' + v.toFixed(2); + const tooltipCallback = isApiCalls + ? (ctx) => `${ctx.dataset.label}: ${ctx.parsed.x.toFixed(0)} calls → ${ctx.parsed.y.toFixed(1)}% resolved` + : (ctx) => `${ctx.dataset.label}: $${ctx.parsed.x.toFixed(2)} → ${ctx.parsed.y.toFixed(1)}% resolved`; + + return new Chart(ctx, { + type: 'line', + data: { datasets }, + options: { + responsive: true, + maintainAspectRatio: false, + animation: false, + scales: { + x: { + type: 'linear', + beginAtZero: true, + max: maxValue * 1.05, + title: { + display: true, + text: xAxisLabel, + color: colors.textColor, + font: { size: 14 } + }, + ticks: { + callback: xTickCallback, + color: colors.textColor, + font: { size: 12 } + }, + grid: { + color: colors.gridColor + } + }, + y: { + beginAtZero: true, + max: 100, + title: { + display: true, + text: 'Resolved (%)', + color: colors.textColor, + font: { size: 14 } + }, + ticks: { + callback: (v) => v + '%', + color: colors.textColor, + font: { size: 12 } + }, + grid: { + color: colors.gridColor + } + } + }, + plugins: { + legend: { + display: true, + position: 'top', + labels: { + color: colors.textColor, + font: { size: 12 }, + usePointStyle: true, + padding: 10 + } + }, + tooltip: { + mode: 'nearest', + intersect: false, + callbacks: { + label: tooltipCallback + } + } + }, + interaction: { + mode: 'nearest', + axis: 'x', + intersect: false + } + }, + plugins: [backgroundPlugin] + }); +} + diff --git a/js/charts/scatterChart.js b/js/charts/scatterChart.js new file mode 100644 index 0000000..8715715 --- /dev/null +++ b/js/charts/scatterChart.js @@ -0,0 +1,133 @@ +// Scatter chart for cost vs performance analysis +function renderScatterChart(ctx, selected, colors, backgroundPlugin) { + // Filter out models without cost data + const modelsWithCost = selected.filter(s => s.cost !== null && s.cost !== undefined && s.cost !== 0 && !isNaN(s.cost)); + + if (modelsWithCost.length === 0) { + return null; + } + + // Plugin to draw labels on scatter plot + const labelPlugin = { + id: 'scatterLabels', + afterDatasetsDraw: (chart, args, options) => { + if (chart.config.type !== 'scatter') return; + const {ctx, chartArea} = chart; + if (!chartArea) return; + + ctx.save(); + ctx.font = '12px sans-serif'; + ctx.fillStyle = colors.textColor; + ctx.textAlign = 'left'; + ctx.textBaseline = 'middle'; + + chart.data.datasets[0].data.forEach((point, index) => { + const meta = chart.getDatasetMeta(0); + const element = meta.data[index]; + if (!element) return; + + const x = element.x; + const y = element.y; + const label = modelsWithCost[index].name; + + // Draw label with slight offset + ctx.fillText(label, x + 5, y); + }); + + ctx.restore(); + } + }; + + const scatterData = modelsWithCost.map(s => ({ + x: s.cost, + y: s.resolved + })); + + // Calculate y-axis range with nice round numbers + const resolvedValues = modelsWithCost.map(s => s.resolved); + const minResolved = Math.min(...resolvedValues); + const maxResolved = Math.max(...resolvedValues); + + // Round to nice values (multiples of 5) + const yMinRaw = Math.max(0, minResolved * 0.9); + const yMaxRaw = Math.min(100, maxResolved * 1.05); + const yMin = Math.floor(yMinRaw / 5) * 5; // Round down to nearest 5 + const yMax = Math.ceil(yMaxRaw / 5) * 5; // Round up to nearest 5 + + // Calculate x-axis range with extra padding for labels + const costValues = modelsWithCost.map(s => s.cost); + const maxCost = Math.max(...costValues); + const xMax = maxCost * 1.15; // Add 15% padding for labels + + return new Chart(ctx, { + type: 'scatter', + data: { + datasets: [{ + label: 'Models', + data: scatterData, + backgroundColor: colors.barBackground, + borderColor: colors.barBorder, + borderWidth: 2, + pointRadius: 6, + pointHoverRadius: 8 + }] + }, + options: { + responsive: true, + maintainAspectRatio: false, + animation: false, + scales: { + y: { + min: yMin, + max: yMax, + title: { + display: true, + text: '% Resolved', + color: colors.textColor, + font: { size: 14 } + }, + ticks: { + stepSize: 5, + callback: (v) => v + '%', + color: colors.textColor, + font: { size: 12 } + }, + grid: { + color: colors.gridColor + } + }, + x: { + beginAtZero: true, + max: xMax, + title: { + display: true, + text: 'Average Cost ($)', + color: colors.textColor, + font: { size: 14 } + }, + ticks: { + callback: (v) => '$' + v.toFixed(2), + color: colors.textColor, + font: { size: 12 } + }, + grid: { + color: colors.gridColor + } + } + }, + plugins: { + legend: { display: false }, + tooltip: { + callbacks: { + label: (ctx) => { + const model = modelsWithCost[ctx.dataIndex]; + return `${model.name}: $${ctx.parsed.x.toFixed(2)}, ${ctx.parsed.y.toFixed(2)}%`; + } + } + } + } + }, + plugins: [backgroundPlugin, labelPlugin] + }); +} + diff --git a/js/leaderboardFilters.js b/js/leaderboardFilters.js index 2758742..bc343ea 100644 --- a/js/leaderboardFilters.js +++ b/js/leaderboardFilters.js @@ -327,6 +327,11 @@ function updateTable() { } else { noResultsMessage.style.display = 'none'; } + + // Update the select-all checkbox state after filtering + if (typeof updateSelectAllCheckbox === 'function') { + updateSelectAllCheckbox(); + } } // Updated Filter Button Logic diff --git a/js/mainResults.js b/js/mainResults.js index 53c5583..49fd924 100644 --- a/js/mainResults.js +++ b/js/mainResults.js @@ -16,6 +16,9 @@ const statusToNaturalLanguage = { const loadedLeaderboards = new Set(); let leaderboardData = null; +// Track which badges have been shown to avoid re-animating +const badgesShown = new Set(); + const sortState = { field: 'resolved', direction: 'desc' }; function loadLeaderboardData() { @@ -95,12 +98,14 @@ function renderLeaderboardTable(leaderboard) { const tableHtml = `
- +
+ ${isBashOnly ? '' : ''} ${isBashOnly ? '' : ''} + ${isBashOnly ? '' : ''} ${!isBashOnly ? '' : ''} @@ -115,18 +120,22 @@ function renderLeaderboardTable(leaderboard) { data-checked="${item.checked ? 'true' : 'false'}" data-tags="${item.tags ? item.tags.join(',') : ''}" > + ${isBashOnly ? `` : ''} - ${isBashOnly ? `` : ''} + ${isBashOnly ? `` : ''} + ${isBashOnly ? `` : ''} `).join('')} - @@ -157,6 +166,29 @@ function renderLeaderboardTable(leaderboard) { updateSortIndicators(); attachSortHandlers(leaderboard.name); + + if (isBashOnly) { + attachSelectAllHandler(leaderboard.name); + updateSelectAllCheckbox(); + } + + // Handle new badges - only show animation once per page load + const isBashOnlyTab = leaderboard.name.toLowerCase() === 'bash-only'; + const badges = container.querySelectorAll('.new-badge'); + badges.forEach(badge => { + const badgeKey = 'trajs-badge-' + leaderboard.name; + const hasBeenShown = badgesShown.has(badgeKey); + + if (!isBashOnlyTab || hasBeenShown) { + badge.style.display = 'none'; + } else { + badge.style.display = ''; + // Mark as shown after animation completes + badge.addEventListener('animationend', () => { + badgesShown.add(badgeKey); + }, { once: true }); + } + }); } function attachSortHandlers(leaderboardName) { @@ -203,6 +235,82 @@ function updateSortIndicators() { }); } +function attachSelectAllHandler(leaderboardName) { + const selectAllCheckbox = document.getElementById('select-all-checkbox'); + if (!selectAllCheckbox) return; + + selectAllCheckbox.addEventListener('change', (e) => { + const container = document.getElementById('leaderboard-container'); + const active = container ? container.querySelector('.tabcontent.active') : null; + if (!active) return; + + const visibleCheckboxes = Array.from(active.querySelectorAll('tbody tr:not(.no-results)')) + .filter(row => row.style.display !== 'none') + .map(row => row.querySelector('input.row-select')) + .filter(cb => cb !== null); + + const isChecked = e.target.checked; + visibleCheckboxes.forEach(cb => { + cb.checked = isChecked; + }); + + // Trigger chart update if modal is open + if (document.getElementById('compare-modal')?.classList.contains('show')) { + // Dispatch change event to trigger chart update + const changeEvent = new Event('change', { bubbles: true }); + if (visibleCheckboxes.length > 0) { + visibleCheckboxes[0].dispatchEvent(changeEvent); + } + } + }); + + // Listen for changes to individual checkboxes to update select-all state + const container = document.getElementById('leaderboard-container'); + if (container) { + container.addEventListener('change', (e) => { + if (e.target && e.target.classList.contains('row-select')) { + updateSelectAllCheckbox(); + } + }); + } +} + +function updateSelectAllCheckbox() { + const selectAllCheckbox = document.getElementById('select-all-checkbox'); + if (!selectAllCheckbox) return; + + const container = document.getElementById('leaderboard-container'); + const active = container ? container.querySelector('.tabcontent.active') : null; + if (!active) return; + + const visibleCheckboxes = Array.from(active.querySelectorAll('tbody tr:not(.no-results)')) + .filter(row => row.style.display !== 'none') + .map(row => row.querySelector('input.row-select')) + .filter(cb => cb !== null); + + if (visibleCheckboxes.length === 0) { + selectAllCheckbox.checked = false; + selectAllCheckbox.indeterminate = false; + return; + } + + const checkedCount = visibleCheckboxes.filter(cb => cb.checked).length; + + if (checkedCount === 0) { + selectAllCheckbox.checked = false; + selectAllCheckbox.indeterminate = false; + } else if (checkedCount === visibleCheckboxes.length) { + selectAllCheckbox.checked = true; + selectAllCheckbox.indeterminate = false; + } else { + selectAllCheckbox.checked = false; + selectAllCheckbox.indeterminate = true; + } +} + +// Make the function globally accessible for filter updates +window.updateSelectAllCheckbox = updateSelectAllCheckbox; + function updateLogViewer(inst_id, split, model) { if (inst_id == 'No Instance Selected') { const logViewer = document.querySelector('#log-viewer'); @@ -362,6 +470,29 @@ function openLeaderboard(leaderboardName) { if (typeof updateTable === 'function') { setTimeout(updateTable, 0); } + + // Show/hide compare button and badge based on leaderboard type + const compareBtn = document.getElementById('compare-btn'); + const compareButtonBadge = document.querySelector('.new-badge-button'); + const isBashOnlyTab = leaderboardName.toLowerCase() === 'bash-only'; + + if (compareBtn) { + if (isBashOnlyTab) { + compareBtn.style.display = ''; + } else { + compareBtn.style.display = 'none'; + } + } + + // Hide/show compare button badge based on tab and if already shown + if (compareButtonBadge) { + const hasBeenShown = badgesShown.has('compare-button-badge'); + if (!isBashOnlyTab || hasBeenShown) { + compareButtonBadge.style.display = 'none'; + } else { + compareButtonBadge.style.display = ''; + } + } } document.addEventListener('DOMContentLoaded', function() { @@ -395,6 +526,19 @@ document.addEventListener('DOMContentLoaded', function() { }); }); + // Handle compare button badge - only show once per page load, only on bash-only + const compareButtonBadge = document.querySelector('.new-badge-button'); + if (compareButtonBadge) { + const hasBeenShown = badgesShown.has('compare-button-badge'); + if (hasBeenShown) { + compareButtonBadge.style.display = 'none'; + } else { + compareButtonBadge.addEventListener('animationend', () => { + badgesShown.add('compare-button-badge'); + }, { once: true }); + } + } + // Load initial tab based on hash or default to Verified (mini-SWE-agent) const hash = window.location.hash.slice(1).toLowerCase(); const validTabs = ['bash-only', 'verified', 'lite', 'test', 'multimodal']; diff --git a/templates/_leaderboard_table.html b/templates/_leaderboard_table.html index eaa1a1a..b5b64d0 100644 --- a/templates/_leaderboard_table.html +++ b/templates/_leaderboard_table.html @@ -7,6 +7,13 @@
+
+ + New! +
Filters:
@@ -55,6 +62,93 @@ {{ leaderboard_tags | tojson }} + + + + + + + + + + + + + + + + +

SWE-bench Bash Only uses the SWE-bench Verified dataset with the mini-SWE-agent environment for all models [Post].
diff --git a/templates/pages/bash-only.html b/templates/pages/bash-only.html index 8cae316..a74b5e8 100644 --- a/templates/pages/bash-only.html +++ b/templates/pages/bash-only.html @@ -95,5 +95,4 @@

Citation

{% endblock %} {% block scripts_extra %} - -{% endblock %} \ No newline at end of file +{% endblock %} \ No newline at end of file
Model % ResolvedAvg. $TrajsNew!Org DateSite
- ${item.date >= "2025-06-25" ? '🆕' : ''} + ${!isBashOnly && item.date >= "2025-06-25" ? '🆕' : ''} ${item.oss ? '🤠' : ''} - ${item.checked ? '✅' : ''} + ${!isBashOnly && item.checked ? '✅' : ''}
${item.name}
${parseFloat(item.resolved).toFixed(2)}$${item.instance_cost !== null && item.instance_cost !== undefined ? parseFloat(item.instance_cost).toFixed(2) : ''}${item.instance_cost !== null && item.instance_cost !== undefined && item.instance_cost !== 0 && !isNaN(item.instance_cost) ? '$' + parseFloat(item.instance_cost).toFixed(2) : ''} + ${item.trajs_docent && item.trajs_docent !== false ? `` : '-'} + ${item.logo && item.logo.length > 0 ? `
@@ -142,7 +151,7 @@ function renderLeaderboardTable(leaderboard) {