From e958b031ee419f2510e84c89499b43fde903dd7a Mon Sep 17 00:00:00 2001
From: "J.J. Allaire" <jj.allaire@gmail.com>
Date: Thu, 12 Sep 2024 20:17:59 -0400
Subject: [PATCH] Built site for gh-pages

---
 .nojekyll           |  2 +-
 eval-logs.html      |  2 +-
 examples/index.html | 74 ++++++++++++++++++++++++++++++---------------
 log-viewer.html     |  2 +-
 search.json         |  2 +-
 sitemap.xml         |  4 +--
 tutorial.html       | 26 ++++++++--------
 vscode.html         |  2 +-
 workflow.html       |  2 +-
 9 files changed, 71 insertions(+), 45 deletions(-)

diff --git a/.nojekyll b/.nojekyll
index ce6368550..8065efae9 100644
--- a/.nojekyll
+++ b/.nojekyll
@@ -1 +1 @@
-0ebe3633
\ No newline at end of file
+7f2a0d51
\ No newline at end of file
diff --git a/eval-logs.html b/eval-logs.html
index edac293d9..36158e964 100644
--- a/eval-logs.html
+++ b/eval-logs.html
@@ -1102,7 +1102,7 @@ <h3 class="anchored" data-anchor-id="reading-logs">Reading Logs</h3>
     </div>
   </div>
 </footer>
-<script>var lightboxQuarto = GLightbox({"openEffect":"zoom","descPosition":"bottom","selector":".lightbox","loop":false,"closeEffect":"zoom"});
+<script>var lightboxQuarto = GLightbox({"selector":".lightbox","descPosition":"bottom","closeEffect":"zoom","openEffect":"zoom","loop":false});
 (function() {
   let previousOnload = window.onload;
   window.onload = () => {
diff --git a/examples/index.html b/examples/index.html
index c2c1663f6..59a89179d 100644
--- a/examples/index.html
+++ b/examples/index.html
@@ -100,7 +100,8 @@
 
     const options = {
       valueNames: ['listing-title','listing-description','listing-arxiv','listing-cite','listing-group','listing-demonstrates','listing-contributors',{ data: ['index'] },{ data: ['categories'] }],
-      
+      page: 25,
+    pagination: { item: "<li class='page-item'><a class='page page-link' href='#'></a></li>" },
       searchColumns: ["listing-title","listing-description","listing-arxiv","listing-cite","listing-group","listing-demonstrates","listing-contributors"],
     };
 
@@ -438,12 +439,34 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
 </li>
 
   
+    
+  
+<li class="example" data-index="2">
+    <div class="example-card">
+      <div class="example-icon fs-5">
+        <i class="bi bi-code"></i>
+      </div>
+      <div class="example-info">
+        <div class="listing-title">
+          <a href="https://github.com/UKGovernmentBEIS/inspect_ai/tree/main/benchmarks/swe_bench">
+            SWE-Bench: Resolving Real-World GitHub Issues
+          </a>
+        </div>
+        <div class="listing-description text-secondary">Software engineering problems drawn from real GitHub issues and corresponding pull requests across 12 popular Python repositories.
+Demonstrates sandboxing untrusted model code.
+</div>
+      </div>
+    </div>
+  
+</li>
+
+  
   
     
 <li class="group"><h2 href="#cybersecurity" class="anchored">Cybersecurity</h2></li>
     
   
-<li class="example" data-index="2">
+<li class="example" data-index="3">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-shield-lock"></i>
@@ -464,7 +487,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="3">
+<li class="example" data-index="4">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-shield-lock"></i>
@@ -488,7 +511,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
 <li class="group"><h2 href="#mathematics" class="anchored">Mathematics</h2></li>
     
   
-<li class="example" data-index="4">
+<li class="example" data-index="5">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-calculator"></i>
@@ -509,7 +532,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="5">
+<li class="example" data-index="6">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-calculator"></i>
@@ -530,7 +553,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="6">
+<li class="example" data-index="7">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-calculator"></i>
@@ -554,7 +577,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
 <li class="group"><h2 href="#reasoning" class="anchored">Reasoning</h2></li>
     
   
-<li class="example" data-index="7">
+<li class="example" data-index="8">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-boxes"></i>
@@ -574,7 +597,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="8">
+<li class="example" data-index="9">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-boxes"></i>
@@ -595,7 +618,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="9">
+<li class="example" data-index="10">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-boxes"></i>
@@ -616,7 +639,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="10">
+<li class="example" data-index="11">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-boxes"></i>
@@ -637,7 +660,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="11">
+<li class="example" data-index="12">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-boxes"></i>
@@ -658,7 +681,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="12">
+<li class="example" data-index="13">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-boxes"></i>
@@ -679,7 +702,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="13">
+<li class="example" data-index="14">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-boxes"></i>
@@ -700,7 +723,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="14">
+<li class="example" data-index="15">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-boxes"></i>
@@ -721,7 +744,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="15">
+<li class="example" data-index="16">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-boxes"></i>
@@ -742,7 +765,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="16">
+<li class="example" data-index="17">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-boxes"></i>
@@ -763,7 +786,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="17">
+<li class="example" data-index="18">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-boxes"></i>
@@ -787,7 +810,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
 <li class="group"><h2 href="#knowledge" class="anchored">Knowledge</h2></li>
     
   
-<li class="example" data-index="18">
+<li class="example" data-index="19">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-book"></i>
@@ -808,7 +831,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="19">
+<li class="example" data-index="20">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-book"></i>
@@ -829,7 +852,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="20">
+<li class="example" data-index="21">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-book"></i>
@@ -850,7 +873,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="21">
+<li class="example" data-index="22">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-book"></i>
@@ -871,7 +894,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="22">
+<li class="example" data-index="23">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-book"></i>
@@ -892,7 +915,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="23">
+<li class="example" data-index="24">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-book"></i>
@@ -913,7 +936,7 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
   
     
   
-<li class="example" data-index="24">
+<li class="example" data-index="25">
     <div class="example-card">
       <div class="example-icon fs-5">
         <i class="bi bi-book"></i>
@@ -935,6 +958,9 @@ <h1 class="title"><span id="sec-examples" class="quarto-section-identifier"><spa
 <div class="listing-no-matching d-none">
 No matching items
 </div>
+<nav id="examples-pagination" class="listing-pagination" aria-label="Page Navigation">
+  <ul class="pagination"></ul>
+</nav>
 </div>
 <div id="refs" role="list" style="display: none">
 
diff --git a/log-viewer.html b/log-viewer.html
index b7e712230..764c61e0c 100644
--- a/log-viewer.html
+++ b/log-viewer.html
@@ -990,7 +990,7 @@ <h2 class="anchored" data-anchor-id="viewer-embedding">Viewer Embedding</h2>
     </div>
   </div>
 </footer>
-<script>var lightboxQuarto = GLightbox({"selector":".lightbox","openEffect":"zoom","closeEffect":"zoom","descPosition":"bottom","loop":false});
+<script>var lightboxQuarto = GLightbox({"closeEffect":"zoom","loop":false,"descPosition":"bottom","selector":".lightbox","openEffect":"zoom"});
 (function() {
   let previousOnload = window.onload;
   window.onload = () => {
diff --git a/search.json b/search.json
index 7f60bd47f..6725c6f81 100644
--- a/search.json
+++ b/search.json
@@ -187,7 +187,7 @@
     "href": "examples/index.html",
     "title": "Examples",
     "section": "",
-    "text": "The examples below demonstrate a variety of evaluation types and techniques. If you have just begun learning Inspect, you might benefit from reviewing the Tutorial examples before exploring these.\n\n\n\n\n \n\n  \n  \n    \nCoding\n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            HumanEval: Evaluating Large Language Models Trained on Code\n          \n        \n        Evaluating correctness for synthesizing Python programs from docstrings. Demonstrates custom scorers and sandboxing untrusted model code.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            MBPP: Mostly Basic Python Problems\n          \n        \n        Measuring the ability of these models to synthesize short Python programs from natural language descriptions. Demonstrates custom scorers and sandboxing untrusted model code.\n\n      \n    \n  \n\n\n  \n  \n    \nCybersecurity\n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            InterCode: Capture the Flag\n          \n        \n        Measure expertise in coding, cryptography (i.e. binary exploitation, forensics), reverse engineering, and recognizing security vulnerabilities. Demonstrates tool use and sandboxing untrusted model code.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            GDM Dangerous Capabilities: Capture the Flag\n          \n        \n        CTF challenges covering web app vulnerabilities, off-the-shelf exploits, databases, Linux privilege escalation, password cracking and spraying. Demonstrates tool use and sandboxing untrusted model code.\n\n      \n    \n  \n\n\n  \n  \n    \nMathematics\n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            MATH: Measuring Mathematical Problem Solving\n          \n        \n        Dataset of 12,500 challenging competition mathematics problems. Demonstrates fewshot prompting and custom scorers.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            GSM8K: Training Verifiers to Solve Math Word Problems\n          \n        \n        Dataset of 8.5K high quality linguistically diverse grade school math word problems. Demostrates fewshot prompting.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            MathVista: Evaluating Mathematical Reasoning in Visual Contexts\n          \n        \n        Diverse mathematical and visual tasks that require fine-grained, deep visual understanding and compositional reasoning. Demonstrates multimodal inputs and custom scorers.\n\n      \n    \n  \n\n\n  \n  \n    \nReasoning\n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            ARC: AI2 Reasoning Challenge\n          \n        \n        Dataset of natural, grade-school science multiple-choice questions (authored for human tests).\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            HellaSwag: Can a Machine Really Finish Your Sentence?\n          \n        \n        Evaluting commonsense natural language inference: given an event description such as \"A woman sits at a piano,\" a machine must select the most likely followup.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            PIQA: Reasoning about Physical Commonsense in Natural Language\n          \n        \n        Measure physical commonsense reasoning (e.g. \"To apply eyeshadow without a brush, should I use a cotton swab or a toothpick?\")\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            BoolQ: Exploring the Surprising Difficulty of Natural Yes/No Questions\n          \n        \n        Reading comprehension dataset that queries for complex, non-factoid information, and require difficult entailment-like inference to solve.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs\n          \n        \n        Evaluates reading comprehension where models must resolve references in a question, perhaps to multiple input positions, and perform discrete operations over them (such as addition, counting, or sorting).\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            WINOGRANDE: An Adversarial Winograd Schema Challenge at Scale\n          \n        \n        Set of 273 expert-crafted pronoun resolution problems originally designed to be unsolvable for statistical models that rely on selectional preferences or word associations.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            RACE-H: A benchmark for testing reading comprehension and reasoning abilities of neural models\n          \n        \n        Reading comprehension tasks collected from the English exams for middle and high school Chinese students in the age range between 12 to 18.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark\n          \n        \n        Multimodal questions from college exams, quizzes, and textbooks, covering six core disciplinestasks, demanding college-level subject knowledge and deliberate reasoning. Demonstrates multimodel inputs.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            SQuAD: A Reading Comprehension Benchmark requiring reasoning over Wikipedia articles\n          \n        \n        Set of 100,000+ questions posed by crowdworkers on a set of Wikipedia articles, where the answer to each question is a segment of text from the corresponding reading passage.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            IFEval: Instruction-Following Evaluation for Large Language Models\n          \n        \n        Evaluates the ability to follow a set of \"verifiable instructions\" such as \"write in more than 400 words\" and \"mention the keyword of AI at least 3 times. Demonstrates custom scoring.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models\n          \n        \n        Questions from human-centric standardized exams, such as college entrance exams, law school admission tests, math competitions, and lawyer qualification tests. Demonstrates custom scoring.\n\n      \n    \n  \n\n\n  \n  \n    \nKnowledge\n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            MMLU: Measuring Massive Multitask Language Understanding\n          \n        \n        Evaluate models on 57 tasks including elementary mathematics, US history, computer science, law, and more.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark\n          \n        \n        An enhanced dataset designed to extend the mostly knowledge-driven MMLU benchmark by integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            GPQA: A Graduate-Level Google-Proof Q&A Benchmark\n          \n        \n        Challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry (experts at PhD level in the corresponding domains reach 65% accuracy).\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge\n          \n        \n        Measure question answering with commonsense prior knowledge.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            TruthfulQA: Measuring How Models Mimic Human Falsehoods\n          \n        \n        Measure whether a language model is truthful in generating answers to questions using questions that some humans would answer falsely due to a false belief or misconception.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            XSTest: A benchmark for identifying exaggerated safety behaviours in LLM's\n          \n        \n        Dataset with 250 safe prompts across ten prompt types that well-calibrated models should not refuse, and 200 unsafe prompts as contrasts that models, for most applications, should refuse.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            PubMedQA: A Dataset for Biomedical Research Question Answering\n          \n        \n        Novel biomedical question answering (QA) dataset collected from PubMed abstracts.\n\n      \n    \n  \n\n  \n\n\nNo matching items",
+    "text": "The examples below demonstrate a variety of evaluation types and techniques. If you have just begun learning Inspect, you might benefit from reviewing the Tutorial examples before exploring these.\n\n\n\n\n \n\n  \n  \n    \nCoding\n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            HumanEval: Evaluating Large Language Models Trained on Code\n          \n        \n        Evaluating correctness for synthesizing Python programs from docstrings. Demonstrates custom scorers and sandboxing untrusted model code.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            MBPP: Mostly Basic Python Problems\n          \n        \n        Measuring the ability of these models to synthesize short Python programs from natural language descriptions. Demonstrates custom scorers and sandboxing untrusted model code.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            SWE-Bench: Resolving Real-World GitHub Issues\n          \n        \n        Software engineering problems drawn from real GitHub issues and corresponding pull requests across 12 popular Python repositories.\nDemonstrates sandboxing untrusted model code.\n\n      \n    \n  \n\n\n  \n  \n    \nCybersecurity\n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            InterCode: Capture the Flag\n          \n        \n        Measure expertise in coding, cryptography (i.e. binary exploitation, forensics), reverse engineering, and recognizing security vulnerabilities. Demonstrates tool use and sandboxing untrusted model code.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            GDM Dangerous Capabilities: Capture the Flag\n          \n        \n        CTF challenges covering web app vulnerabilities, off-the-shelf exploits, databases, Linux privilege escalation, password cracking and spraying. Demonstrates tool use and sandboxing untrusted model code.\n\n      \n    \n  \n\n\n  \n  \n    \nMathematics\n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            MATH: Measuring Mathematical Problem Solving\n          \n        \n        Dataset of 12,500 challenging competition mathematics problems. Demonstrates fewshot prompting and custom scorers.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            GSM8K: Training Verifiers to Solve Math Word Problems\n          \n        \n        Dataset of 8.5K high quality linguistically diverse grade school math word problems. Demostrates fewshot prompting.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            MathVista: Evaluating Mathematical Reasoning in Visual Contexts\n          \n        \n        Diverse mathematical and visual tasks that require fine-grained, deep visual understanding and compositional reasoning. Demonstrates multimodal inputs and custom scorers.\n\n      \n    \n  \n\n\n  \n  \n    \nReasoning\n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            ARC: AI2 Reasoning Challenge\n          \n        \n        Dataset of natural, grade-school science multiple-choice questions (authored for human tests).\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            HellaSwag: Can a Machine Really Finish Your Sentence?\n          \n        \n        Evaluting commonsense natural language inference: given an event description such as \"A woman sits at a piano,\" a machine must select the most likely followup.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            PIQA: Reasoning about Physical Commonsense in Natural Language\n          \n        \n        Measure physical commonsense reasoning (e.g. \"To apply eyeshadow without a brush, should I use a cotton swab or a toothpick?\")\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            BoolQ: Exploring the Surprising Difficulty of Natural Yes/No Questions\n          \n        \n        Reading comprehension dataset that queries for complex, non-factoid information, and require difficult entailment-like inference to solve.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs\n          \n        \n        Evaluates reading comprehension where models must resolve references in a question, perhaps to multiple input positions, and perform discrete operations over them (such as addition, counting, or sorting).\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            WINOGRANDE: An Adversarial Winograd Schema Challenge at Scale\n          \n        \n        Set of 273 expert-crafted pronoun resolution problems originally designed to be unsolvable for statistical models that rely on selectional preferences or word associations.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            RACE-H: A benchmark for testing reading comprehension and reasoning abilities of neural models\n          \n        \n        Reading comprehension tasks collected from the English exams for middle and high school Chinese students in the age range between 12 to 18.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark\n          \n        \n        Multimodal questions from college exams, quizzes, and textbooks, covering six core disciplinestasks, demanding college-level subject knowledge and deliberate reasoning. Demonstrates multimodel inputs.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            SQuAD: A Reading Comprehension Benchmark requiring reasoning over Wikipedia articles\n          \n        \n        Set of 100,000+ questions posed by crowdworkers on a set of Wikipedia articles, where the answer to each question is a segment of text from the corresponding reading passage.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            IFEval: Instruction-Following Evaluation for Large Language Models\n          \n        \n        Evaluates the ability to follow a set of \"verifiable instructions\" such as \"write in more than 400 words\" and \"mention the keyword of AI at least 3 times. Demonstrates custom scoring.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models\n          \n        \n        Questions from human-centric standardized exams, such as college entrance exams, law school admission tests, math competitions, and lawyer qualification tests. Demonstrates custom scoring.\n\n      \n    \n  \n\n\n  \n  \n    \nKnowledge\n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            MMLU: Measuring Massive Multitask Language Understanding\n          \n        \n        Evaluate models on 57 tasks including elementary mathematics, US history, computer science, law, and more.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark\n          \n        \n        An enhanced dataset designed to extend the mostly knowledge-driven MMLU benchmark by integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            GPQA: A Graduate-Level Google-Proof Q&A Benchmark\n          \n        \n        Challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry (experts at PhD level in the corresponding domains reach 65% accuracy).\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge\n          \n        \n        Measure question answering with commonsense prior knowledge.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            TruthfulQA: Measuring How Models Mimic Human Falsehoods\n          \n        \n        Measure whether a language model is truthful in generating answers to questions using questions that some humans would answer falsely due to a false belief or misconception.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            XSTest: A benchmark for identifying exaggerated safety behaviours in LLM's\n          \n        \n        Dataset with 250 safe prompts across ten prompt types that well-calibrated models should not refuse, and 200 unsafe prompts as contrasts that models, for most applications, should refuse.\n\n      \n    \n  \n\n\n  \n    \n  \n\n    \n      \n        \n      \n      \n        \n          \n            PubMedQA: A Dataset for Biomedical Research Question Answering\n          \n        \n        Novel biomedical question answering (QA) dataset collected from PubMed abstracts.\n\n      \n    \n  \n\n  \n\n\nNo matching items",
     "crumbs": [
       "Basics",
       "<span class='chapter-number'>4</span>  <span class='chapter-title'>Examples</span>"
diff --git a/sitemap.xml b/sitemap.xml
index db95206e4..5e5bc3b03 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -14,7 +14,7 @@
   </url>
   <url>
     <loc>https://inspect.ai-safety-institute.org.uk/examples/index.html</loc>
-    <lastmod>2024-09-12T21:25:51.769Z</lastmod>
+    <lastmod>2024-09-13T00:16:44.594Z</lastmod>
   </url>
   <url>
     <loc>https://inspect.ai-safety-institute.org.uk/log-viewer.html</loc>
@@ -70,7 +70,7 @@
   </url>
   <url>
     <loc>https://inspect.ai-safety-institute.org.uk/interactivity.html</loc>
-    <lastmod>2024-09-06T20:10:53.798Z</lastmod>
+    <lastmod>2024-09-13T00:12:39.701Z</lastmod>
   </url>
   <url>
     <loc>https://inspect.ai-safety-institute.org.uk/eval-logs.html</loc>
diff --git a/tutorial.html b/tutorial.html
index a43bda399..b2062f8fe 100644
--- a/tutorial.html
+++ b/tutorial.html
@@ -442,7 +442,7 @@ <h2 class="anchored" data-anchor-id="sec-security-guide">Security Guide</h2>
 <section id="setup" class="level3 unlisted">
 <h3 class="unlisted anchored" data-anchor-id="setup">Setup</h3>
 <p>We’ll start by importing the functions we need from Inspect and defining a system message that orients the model to its role as a computer security expert.</p>
-<div id="c37d5810" class="cell">
+<div id="066c7905" class="cell">
 <div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai <span class="im">import</span> Task, <span class="bu">eval</span>, task</span>
 <span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai.dataset <span class="im">import</span> csv_dataset</span>
 <span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai.scorer <span class="im">import</span> model_graded_fact</span>
@@ -459,7 +459,7 @@ <h3 class="unlisted anchored" data-anchor-id="setup">Setup</h3>
 <section id="eval" class="level3 unlisted">
 <h3 class="unlisted anchored" data-anchor-id="eval">Eval</h3>
 <p>Discerning whether the correct security guidance was provided by the model might provide difficult using only text matching algorithms. Here we use a model to read the response and assess the quality of the answer.</p>
-<div id="f919a6d3" class="cell">
+<div id="a06cd864" class="cell">
 <div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="at">@task</span></span>
 <span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> security_guide():</span>
 <span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a>    <span class="cf">return</span> Task(</span>
@@ -489,7 +489,7 @@ <h2 class="anchored" data-anchor-id="sec-hellaswag">HellaSwag</h2>
 <section id="setup-1" class="level3 unlisted">
 <h3 class="unlisted anchored" data-anchor-id="setup-1">Setup</h3>
 <p>We’ll start by importing the functions we need from Inspect, defining a system message, and writing a function to convert dataset records to samples (we need to do this to convert the index-based label in the dataset to a letter).</p>
-<div id="ebac4b26" class="cell">
+<div id="66d0f5c0" class="cell">
 <div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai <span class="im">import</span> Task, <span class="bu">eval</span>, task</span>
 <span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai.dataset <span class="im">import</span> Sample, hf_dataset</span>
 <span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai.scorer <span class="im">import</span> choice</span>
@@ -514,7 +514,7 @@ <h3 class="unlisted anchored" data-anchor-id="setup-1">Setup</h3>
 <section id="eval-1" class="level3 unlisted">
 <h3 class="unlisted anchored" data-anchor-id="eval-1">Eval</h3>
 <p>We’ll load the dataset from <a href="https://huggingface.co/datasets/Rowan/hellaswag">HuggingFace</a> using the <code>hf_dataset()</code> function. We’ll draw data from the validation split, and use the <code>record_to_sample()</code> function to parse the records (we’ll also pass <code>trust=True</code> to indicate that we are okay with Hugging Face executing the dataset loading code provided by hellaswag):</p>
-<div id="2df99f9c" class="cell">
+<div id="746eb36a" class="cell">
 <div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="at">@task</span></span>
 <span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> hellaswag():</span>
 <span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a>   </span>
@@ -574,7 +574,7 @@ <h3 class="unlisted anchored" data-anchor-id="setup-2">Setup</h3>
 <li><code>record_to_sample()</code> to convert raw records to samples. Note that we need a function rather than just mapping field names with a <code>FieldSpec</code> because the <strong>answer</strong> field in the dataset needs to be divided into reasoning and the actual answer (which appears at the very end after <code>####</code>).</li>
 <li><code>sample_to_fewshot()</code> to generate fewshot examples from samples.</li>
 </ol>
-<div id="c57dfc59" class="cell">
+<div id="b2cce099" class="cell">
 <div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai <span class="im">import</span> Task, task</span>
 <span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai.dataset <span class="im">import</span> Sample, hf_dataset</span>
 <span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai.scorer <span class="im">import</span> match</span>
@@ -621,7 +621,7 @@ <h3 class="unlisted anchored" data-anchor-id="setup-2">Setup</h3>
 <section id="eval-2" class="level3 unlisted">
 <h3 class="unlisted anchored" data-anchor-id="eval-2">Eval</h3>
 <p>We’ll load the dataset from <a href="https://huggingface.co/datasets/gsm8k">HuggingFace</a> using the <code>hf_dataset()</code> function. By default we use 10 fewshot examples, but the <code>fewshot</code> task arg can be used to turn this up, down, or off. The <code>fewshot_seed</code> is provided for stability of fewshot examples across runs.</p>
-<div id="df8cb6d7" class="cell">
+<div id="039cc439" class="cell">
 <div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="at">@task</span></span>
 <span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> gsm8k(fewshot<span class="op">=</span><span class="dv">10</span>, fewshot_seed<span class="op">=</span><span class="dv">42</span>):</span>
 <span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a>    <span class="co"># build plan dynamically (may or may not be doing fewshot)</span></span>
@@ -688,7 +688,7 @@ <h2 class="anchored" data-anchor-id="sec-mathematics">Mathematics</h2>
 <section id="setup-3" class="level3 unlisted">
 <h3 class="unlisted anchored" data-anchor-id="setup-3">Setup</h3>
 <p>We’ll start by importing the functions we need from Inspect and defining a prompt that asks the model to reason step by step and respond with its answer on a line at the end. It also nudges the model not to enclose its answer in <code>\boxed</code>, a LaTeX command for displaying equations that models often use in math output.</p>
-<div id="0142a773" class="cell">
+<div id="7be23953" class="cell">
 <div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> re</span>
 <span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a></span>
 <span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai <span class="im">import</span> Task, task</span>
@@ -726,7 +726,7 @@ <h3 class="unlisted anchored" data-anchor-id="setup-3">Setup</h3>
 <section id="eval-3" class="level3 unlisted">
 <h3 class="unlisted anchored" data-anchor-id="eval-3">Eval</h3>
 <p>Here is the basic setup for our eval. We <code>shuffle</code> the dataset so that when we use <code>--limit</code> to develop on smaller slices we get some variety of inputs and results:</p>
-<div id="e4485602" class="cell">
+<div id="e236215e" class="cell">
 <div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="at">@task</span></span>
 <span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> math(shuffle<span class="op">=</span><span class="va">True</span>):</span>
 <span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a>    <span class="cf">return</span> Task(</span>
@@ -749,7 +749,7 @@ <h3 class="unlisted anchored" data-anchor-id="eval-3">Eval</h3>
 <span id="cb12-20"><a href="#cb12-20" aria-hidden="true" tabindex="-1"></a>    )</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
 </div>
 <p>The heart of this eval isn’t in the task definition though, rather it’s in how we grade the output. Math expressions can be logically equivalent but not literally the same. Consequently, we’ll use a model to assess whether the output and the target are logically equivalent. the <code>expression_equivalence()</code> custom scorer implements this:</p>
-<div id="d226aeb8" class="cell">
+<div id="bd62ac79" class="cell">
 <div class="sourceCode cell-code" id="cb13"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb13-1"><a href="#cb13-1" aria-hidden="true" tabindex="-1"></a><span class="at">@scorer</span>(metrics<span class="op">=</span>[accuracy(), stderr()])</span>
 <span id="cb13-2"><a href="#cb13-2" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> expression_equivalence():</span>
 <span id="cb13-3"><a href="#cb13-3" aria-hidden="true" tabindex="-1"></a>    <span class="cf">async</span> <span class="kw">def</span> score(state: TaskState, target: Target):</span>
@@ -830,7 +830,7 @@ <h2 class="anchored" data-anchor-id="sec-tool-use">Tool Use</h2>
 <section id="addition" class="level3 unlisted">
 <h3 class="unlisted anchored" data-anchor-id="addition">Addition</h3>
 <p>We’ll demonstrate with a simple tool that adds two numbers, using the <code>@tool</code> decorator to register it with the system:</p>
-<div id="111d5dd7" class="cell">
+<div id="a2ffcc0d" class="cell">
 <div class="sourceCode cell-code" id="cb17"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb17-1"><a href="#cb17-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai <span class="im">import</span> Task, <span class="bu">eval</span>, task</span>
 <span id="cb17-2"><a href="#cb17-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai.dataset <span class="im">import</span> Sample</span>
 <span id="cb17-3"><a href="#cb17-3" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> inspect_ai.scorer <span class="im">import</span> includes, match</span>
@@ -865,7 +865,7 @@ <h3 class="unlisted anchored" data-anchor-id="addition">Addition</h3>
 <span id="cb19-3"><a href="#cb19-3" aria-hidden="true" tabindex="-1"></a>    y: Second number to add.</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
 <p>Type annotations and descriptions are <em>required</em> for tool declarations so that the model can be informed which types to pass back to the tool function and what the purpose of each parameter is.</p>
 <p>Now that we’ve defined the tool, we can use it in an evaluation by passing it to the <code>use_tools()</code> function.</p>
-<div id="e4b52b32" class="cell">
+<div id="1fc5d2f0" class="cell">
 <div class="sourceCode cell-code" id="cb20"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb20-1"><a href="#cb20-1" aria-hidden="true" tabindex="-1"></a><span class="at">@task</span></span>
 <span id="cb20-2"><a href="#cb20-2" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> addition_problem():</span>
 <span id="cb20-3"><a href="#cb20-3" aria-hidden="true" tabindex="-1"></a>    <span class="cf">return</span> Task(</span>
@@ -894,7 +894,7 @@ <h3 class="unlisted anchored" data-anchor-id="task">Task</h3>
 <ol start="2" type="1">
 <li><code>ctf_agent()</code>, which defines the agent’s plan. The plan consists principally of using <code>bash()</code> and <code>python()</code> tools in a loop until the flag is discovered. We’ll describe this function in more detail below.</li>
 </ol>
-<div id="f0e82eac" class="cell">
+<div id="fc756b14" class="cell">
 <div class="sourceCode cell-code" id="cb22"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb22-1"><a href="#cb22-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> textwrap <span class="im">import</span> dedent</span>
 <span id="cb22-2"><a href="#cb22-2" aria-hidden="true" tabindex="-1"></a></span>
 <span id="cb22-3"><a href="#cb22-3" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> dataset <span class="im">import</span> read_dataset</span>
@@ -920,7 +920,7 @@ <h3 class="unlisted anchored" data-anchor-id="task">Task</h3>
 </div>
 <p>Note that we specify <code>sandbox="docker"</code> to ensure that code generated from the model is run in a secure <a href="agents.html#sec-sandbox-environments">sandbox environment</a>.</p>
 <p>Here is the definition of the agent:</p>
-<div id="7cf65c81" class="cell">
+<div id="34dccceb" class="cell">
 <div class="sourceCode cell-code" id="cb23"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb23-1"><a href="#cb23-1" aria-hidden="true" tabindex="-1"></a><span class="at">@plan</span></span>
 <span id="cb23-2"><a href="#cb23-2" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> ctf_agent(max_attempts<span class="op">=</span><span class="dv">3</span>):</span>
 <span id="cb23-3"><a href="#cb23-3" aria-hidden="true" tabindex="-1"></a>    SYSTEM_MESSAGE <span class="op">=</span> dedent(<span class="st">"""</span></span>
diff --git a/vscode.html b/vscode.html
index a4db0b27a..f0af4d437 100644
--- a/vscode.html
+++ b/vscode.html
@@ -913,7 +913,7 @@ <h2 class="anchored" data-anchor-id="troubleshooting">Troubleshooting</h2>
     </div>
   </div>
 </footer>
-<script>var lightboxQuarto = GLightbox({"openEffect":"zoom","closeEffect":"zoom","descPosition":"bottom","selector":".lightbox","loop":false});
+<script>var lightboxQuarto = GLightbox({"openEffect":"zoom","selector":".lightbox","loop":false,"descPosition":"bottom","closeEffect":"zoom"});
 (function() {
   let previousOnload = window.onload;
   window.onload = () => {
diff --git a/workflow.html b/workflow.html
index f58802284..1c79798ca 100644
--- a/workflow.html
+++ b/workflow.html
@@ -1135,7 +1135,7 @@ <h2 class="anchored" data-anchor-id="eval-suites">Eval Suites</h2>
     </div>
   </div>
 </footer>
-<script>var lightboxQuarto = GLightbox({"closeEffect":"zoom","loop":false,"selector":".lightbox","descPosition":"bottom","openEffect":"zoom"});
+<script>var lightboxQuarto = GLightbox({"selector":".lightbox","closeEffect":"zoom","loop":false,"descPosition":"bottom","openEffect":"zoom"});
 (function() {
   let previousOnload = window.onload;
   window.onload = () => {