diff --git a/CHANGELOG.md b/CHANGELOG.md index 94034faf4..4adcfdd66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - Log viewer: Add timestamps to transcript events. - Log viewer: Metadata which contains images will now render the images. - Log viewer: Show custom tool call views in messages display. +- Log viewer: Allow filtering samples by compound expressions including multiple scorers. - Bugfix: Correctly read and forward image detail property. - Bugfix: Correct resolution of global eval override of task or sample sandboxes. - Bugfix: Don't do eval log listing on background threads (s3fs can deadlock when run from mutliple threads) diff --git a/src/inspect_ai/_view/www/App.css b/src/inspect_ai/_view/www/App.css index 9d200e024..1b7d02864 100644 --- a/src/inspect_ai/_view/www/App.css +++ b/src/inspect_ai/_view/www/App.css @@ -702,6 +702,10 @@ table.table.table-sm td { overflow: unset; } +.custom-dropdown-item:hover { + background-color: var(--bs-secondary-bg); +} + pre[class*="language-"].tool-output, .tool-output { background-color: #f8f8f8; diff --git a/src/inspect_ai/_view/www/dist/assets/index.css b/src/inspect_ai/_view/www/dist/assets/index.css index 26c7f5c71..8c6d5f811 100644 --- a/src/inspect_ai/_view/www/dist/assets/index.css +++ b/src/inspect_ai/_view/www/dist/assets/index.css @@ -14975,6 +14975,10 @@ table.table.table-sm td { overflow: unset; } +.custom-dropdown-item:hover { + background-color: var(--bs-secondary-bg); +} + pre[class*="language-"].tool-output, .tool-output { background-color: #f8f8f8; diff --git a/src/inspect_ai/_view/www/dist/assets/index.js b/src/inspect_ai/_view/www/dist/assets/index.js index 9c79e1f26..f4267ba76 100644 --- a/src/inspect_ai/_view/www/dist/assets/index.js +++ b/src/inspect_ai/_view/www/dist/assets/index.js @@ -1049,11 +1049,11 @@ var prism = { exports: {} }; if (rematch && pos2 >= rematch.reach) { break; } - var str = currentNode.value; + var str2 = currentNode.value; if (tokenList.length > text2.length) { return; } - if (str instanceof Token2) { + if (str2 instanceof Token2) { continue; } var removeCount = 1; @@ -1081,19 +1081,19 @@ var prism = { exports: {} }; p2 += k2.value.length; } removeCount--; - str = text2.slice(pos2, p2); + str2 = text2.slice(pos2, p2); match2.index -= pos2; } else { - match2 = matchPattern(pattern, 0, str, lookbehind); + match2 = matchPattern(pattern, 0, str2, lookbehind); if (!match2) { continue; } } var from = match2.index; var matchStr = match2[0]; - var before = str.slice(0, from); - var after = str.slice(from + matchStr.length); - var reach = pos2 + str.length; + var before = str2.slice(0, from); + var after = str2.slice(from + matchStr.length); + var reach = pos2 + str2.length; if (rematch && reach > rematch.reach) { rematch.reach = reach; } @@ -7624,40 +7624,40 @@ const formatTime = (seconds) => { )} min ${seconds % 60} sec`; } }; -function formatPrettyDecimal(num) { - const numDecimalPlaces = num.toString().includes(".") ? num.toString().split(".")[1].length : 0; +function formatPrettyDecimal(num2) { + const numDecimalPlaces = num2.toString().includes(".") ? num2.toString().split(".")[1].length : 0; if (numDecimalPlaces === 0) { - return num.toFixed(1); + return num2.toFixed(1); } else if (numDecimalPlaces > 3) { - return num.toFixed(3); + return num2.toFixed(3); } else { - return num.toString(); + return num2.toString(); } } -function formatDecimalNoTrailingZeroes(num) { - if (typeof num !== "number") { - return num; +function formatDecimalNoTrailingZeroes(num2) { + if (typeof num2 !== "number") { + return num2; } - if (num.toString().includes(".")) { - const decimal = num.toString().split(".")[1]; + if (num2.toString().includes(".")) { + const decimal = num2.toString().split(".")[1]; const trimmed = decimal.replace(/\.?0+$/, ""); - return num.toFixed(trimmed.length); + return num2.toFixed(trimmed.length); } else { - return num.toFixed(0); + return num2.toFixed(0); } } -function toTitleCase(str) { - return str.split(" ").map((w2) => w2[0].toUpperCase() + w2.substr(1).toLowerCase()).join(" "); +function toTitleCase(str2) { + return str2.split(" ").map((w2) => w2[0].toUpperCase() + w2.substr(1).toLowerCase()).join(" "); } -function formatNoDecimal(num) { - if (typeof num !== "number") { - return num; +function formatNoDecimal(num2) { + if (typeof num2 !== "number") { + return num2; } - const rounded = Math.round(num); + const rounded = Math.round(num2); return rounded.toFixed(0); } -function formatNumber(num) { - return num.toLocaleString(navigator.language, { +function formatNumber(num2) { + return num2.toLocaleString(navigator.language, { minimumFractionDigits: 0, maximumFractionDigits: 5 }); @@ -9978,28 +9978,28 @@ class EntityDecoder { * @param offset The offset at which the entity begins. Should be 0 if this is not the first call. * @returns The number of characters that were consumed, or -1 if the entity is incomplete. */ - write(str, offset2) { + write(str2, offset2) { switch (this.state) { case EntityDecoderState.EntityStart: { - if (str.charCodeAt(offset2) === CharCodes.NUM) { + if (str2.charCodeAt(offset2) === CharCodes.NUM) { this.state = EntityDecoderState.NumericStart; this.consumed += 1; - return this.stateNumericStart(str, offset2 + 1); + return this.stateNumericStart(str2, offset2 + 1); } this.state = EntityDecoderState.NamedEntity; - return this.stateNamedEntity(str, offset2); + return this.stateNamedEntity(str2, offset2); } case EntityDecoderState.NumericStart: { - return this.stateNumericStart(str, offset2); + return this.stateNumericStart(str2, offset2); } case EntityDecoderState.NumericDecimal: { - return this.stateNumericDecimal(str, offset2); + return this.stateNumericDecimal(str2, offset2); } case EntityDecoderState.NumericHex: { - return this.stateNumericHex(str, offset2); + return this.stateNumericHex(str2, offset2); } case EntityDecoderState.NamedEntity: { - return this.stateNamedEntity(str, offset2); + return this.stateNamedEntity(str2, offset2); } } } @@ -10012,22 +10012,22 @@ class EntityDecoder { * @param offset The current offset. * @returns The number of characters that were consumed, or -1 if the entity is incomplete. */ - stateNumericStart(str, offset2) { - if (offset2 >= str.length) { + stateNumericStart(str2, offset2) { + if (offset2 >= str2.length) { return -1; } - if ((str.charCodeAt(offset2) | TO_LOWER_BIT) === CharCodes.LOWER_X) { + if ((str2.charCodeAt(offset2) | TO_LOWER_BIT) === CharCodes.LOWER_X) { this.state = EntityDecoderState.NumericHex; this.consumed += 1; - return this.stateNumericHex(str, offset2 + 1); + return this.stateNumericHex(str2, offset2 + 1); } this.state = EntityDecoderState.NumericDecimal; - return this.stateNumericDecimal(str, offset2); + return this.stateNumericDecimal(str2, offset2); } - addToNumericResult(str, start2, end2, base2) { + addToNumericResult(str2, start2, end2, base2) { if (start2 !== end2) { const digitCount = end2 - start2; - this.result = this.result * Math.pow(base2, digitCount) + parseInt(str.substr(start2, digitCount), base2); + this.result = this.result * Math.pow(base2, digitCount) + parseInt(str2.substr(start2, digitCount), base2); this.consumed += digitCount; } } @@ -10040,18 +10040,18 @@ class EntityDecoder { * @param offset The current offset. * @returns The number of characters that were consumed, or -1 if the entity is incomplete. */ - stateNumericHex(str, offset2) { + stateNumericHex(str2, offset2) { const startIdx = offset2; - while (offset2 < str.length) { - const char = str.charCodeAt(offset2); + while (offset2 < str2.length) { + const char = str2.charCodeAt(offset2); if (isNumber(char) || isHexadecimalCharacter(char)) { offset2 += 1; } else { - this.addToNumericResult(str, startIdx, offset2, 16); + this.addToNumericResult(str2, startIdx, offset2, 16); return this.emitNumericEntity(char, 3); } } - this.addToNumericResult(str, startIdx, offset2, 16); + this.addToNumericResult(str2, startIdx, offset2, 16); return -1; } /** @@ -10063,18 +10063,18 @@ class EntityDecoder { * @param offset The current offset. * @returns The number of characters that were consumed, or -1 if the entity is incomplete. */ - stateNumericDecimal(str, offset2) { + stateNumericDecimal(str2, offset2) { const startIdx = offset2; - while (offset2 < str.length) { - const char = str.charCodeAt(offset2); + while (offset2 < str2.length) { + const char = str2.charCodeAt(offset2); if (isNumber(char)) { offset2 += 1; } else { - this.addToNumericResult(str, startIdx, offset2, 10); + this.addToNumericResult(str2, startIdx, offset2, 10); return this.emitNumericEntity(char, 2); } } - this.addToNumericResult(str, startIdx, offset2, 10); + this.addToNumericResult(str2, startIdx, offset2, 10); return -1; } /** @@ -10119,12 +10119,12 @@ class EntityDecoder { * @param offset The current offset. * @returns The number of characters that were consumed, or -1 if the entity is incomplete. */ - stateNamedEntity(str, offset2) { + stateNamedEntity(str2, offset2) { const { decodeTree } = this; let current = decodeTree[this.treeIndex]; let valueLength = (current & BinTrieFlags.VALUE_LENGTH) >> 14; - for (; offset2 < str.length; offset2++, this.excess++) { - const char = str.charCodeAt(offset2); + for (; offset2 < str2.length; offset2++, this.excess++) { + const char = str2.charCodeAt(offset2); this.treeIndex = determineBranch(decodeTree, current, this.treeIndex + Math.max(1, valueLength), char); if (this.treeIndex < 0) { return this.result === 0 || // If we are parsing an attribute @@ -10208,15 +10208,15 @@ class EntityDecoder { } function getDecoder(decodeTree) { let ret = ""; - const decoder = new EntityDecoder(decodeTree, (str) => ret += fromCodePoint$1(str)); - return function decodeWithTrie(str, decodeMode) { + const decoder = new EntityDecoder(decodeTree, (str2) => ret += fromCodePoint$1(str2)); + return function decodeWithTrie(str2, decodeMode) { let lastIndex = 0; let offset2 = 0; - while ((offset2 = str.indexOf("&", offset2)) >= 0) { - ret += str.slice(lastIndex, offset2); + while ((offset2 = str2.indexOf("&", offset2)) >= 0) { + ret += str2.slice(lastIndex, offset2); decoder.startEntity(decodeMode); const len = decoder.write( - str, + str2, // Skip the "&" offset2 + 1 ); @@ -10227,7 +10227,7 @@ function getDecoder(decodeTree) { lastIndex = offset2 + len; offset2 = len === 0 ? lastIndex + 1 : lastIndex; } - const result = ret + str.slice(lastIndex); + const result = ret + str2.slice(lastIndex); ret = ""; return result; }; @@ -10259,8 +10259,8 @@ function determineBranch(decodeTree, current, nodeIdx, char) { } const htmlDecoder = getDecoder(htmlDecodeTree); getDecoder(xmlDecodeTree); -function decodeHTML(str, mode = DecodingMode.Legacy) { - return htmlDecoder(str, mode); +function decodeHTML(str2, mode = DecodingMode.Legacy) { + return htmlDecoder(str2, mode); } function _class$1(obj) { return Object.prototype.toString.call(obj); @@ -10344,17 +10344,17 @@ function replaceEntityPattern(match2, name) { } return match2; } -function unescapeMd(str) { - if (str.indexOf("\\") < 0) { - return str; +function unescapeMd(str2) { + if (str2.indexOf("\\") < 0) { + return str2; } - return str.replace(UNESCAPE_MD_RE, "$1"); + return str2.replace(UNESCAPE_MD_RE, "$1"); } -function unescapeAll(str) { - if (str.indexOf("\\") < 0 && str.indexOf("&") < 0) { - return str; +function unescapeAll(str2) { + if (str2.indexOf("\\") < 0 && str2.indexOf("&") < 0) { + return str2; } - return str.replace(UNESCAPE_ALL_RE, function(match2, escaped, entity2) { + return str2.replace(UNESCAPE_ALL_RE, function(match2, escaped, entity2) { if (escaped) { return escaped; } @@ -10372,15 +10372,15 @@ const HTML_REPLACEMENTS = { function replaceUnsafeChar(ch3) { return HTML_REPLACEMENTS[ch3]; } -function escapeHtml(str) { - if (HTML_ESCAPE_TEST_RE.test(str)) { - return str.replace(HTML_ESCAPE_REPLACE_RE, replaceUnsafeChar); +function escapeHtml(str2) { + if (HTML_ESCAPE_TEST_RE.test(str2)) { + return str2.replace(HTML_ESCAPE_REPLACE_RE, replaceUnsafeChar); } - return str; + return str2; } const REGEXP_ESCAPE_RE = /[.?*+^$[\]\\(){}|-]/g; -function escapeRE$1(str) { - return str.replace(REGEXP_ESCAPE_RE, "\\$&"); +function escapeRE$1(str2) { + return str2.replace(REGEXP_ESCAPE_RE, "\\$&"); } function isSpace(code2) { switch (code2) { @@ -10452,12 +10452,12 @@ function isMdAsciiPunct(ch3) { return false; } } -function normalizeReference(str) { - str = str.trim().replace(/\s+/g, " "); +function normalizeReference(str2) { + str2 = str2.trim().replace(/\s+/g, " "); if ("ẞ".toLowerCase() === "Ṿ") { - str = str.replace(/ẞ/g, "ß"); + str2 = str2.replace(/ẞ/g, "ß"); } - return str.toLowerCase().toUpperCase(); + return str2.toLowerCase().toUpperCase(); } const lib$1 = { mdurl, ucmicro }; const utils = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Object.defineProperty({ @@ -10512,7 +10512,7 @@ function parseLinkLabel(state, start2, disableNested) { state.pos = oldPos; return labelEnd; } -function parseLinkDestination(str, start2, max2) { +function parseLinkDestination(str2, start2, max2) { let code2; let pos2 = start2; const result = { @@ -10520,10 +10520,10 @@ function parseLinkDestination(str, start2, max2) { pos: 0, str: "" }; - if (str.charCodeAt(pos2) === 60) { + if (str2.charCodeAt(pos2) === 60) { pos2++; while (pos2 < max2) { - code2 = str.charCodeAt(pos2); + code2 = str2.charCodeAt(pos2); if (code2 === 10) { return result; } @@ -10532,7 +10532,7 @@ function parseLinkDestination(str, start2, max2) { } if (code2 === 62) { result.pos = pos2 + 1; - result.str = unescapeAll(str.slice(start2 + 1, pos2)); + result.str = unescapeAll(str2.slice(start2 + 1, pos2)); result.ok = true; return result; } @@ -10546,7 +10546,7 @@ function parseLinkDestination(str, start2, max2) { } let level = 0; while (pos2 < max2) { - code2 = str.charCodeAt(pos2); + code2 = str2.charCodeAt(pos2); if (code2 === 32) { break; } @@ -10554,7 +10554,7 @@ function parseLinkDestination(str, start2, max2) { break; } if (code2 === 92 && pos2 + 1 < max2) { - if (str.charCodeAt(pos2 + 1) === 32) { + if (str2.charCodeAt(pos2 + 1) === 32) { break; } pos2 += 2; @@ -10580,12 +10580,12 @@ function parseLinkDestination(str, start2, max2) { if (level !== 0) { return result; } - result.str = unescapeAll(str.slice(start2, pos2)); + result.str = unescapeAll(str2.slice(start2, pos2)); result.pos = pos2; result.ok = true; return result; } -function parseLinkTitle(str, start2, max2, prev_state) { +function parseLinkTitle(str2, start2, max2, prev_state) { let code2; let pos2 = start2; const state = { @@ -10607,7 +10607,7 @@ function parseLinkTitle(str, start2, max2, prev_state) { if (pos2 >= max2) { return state; } - let marker = str.charCodeAt(pos2); + let marker = str2.charCodeAt(pos2); if (marker !== 34 && marker !== 39 && marker !== 40) { return state; } @@ -10619,10 +10619,10 @@ function parseLinkTitle(str, start2, max2, prev_state) { state.marker = marker; } while (pos2 < max2) { - code2 = str.charCodeAt(pos2); + code2 = str2.charCodeAt(pos2); if (code2 === state.marker) { state.pos = pos2 + 1; - state.str += unescapeAll(str.slice(start2, pos2)); + state.str += unescapeAll(str2.slice(start2, pos2)); state.ok = true; return state; } else if (code2 === 40 && state.marker === 41) { @@ -10633,7 +10633,7 @@ function parseLinkTitle(str, start2, max2, prev_state) { pos2++; } state.can_continue = true; - state.str += unescapeAll(str.slice(start2, pos2)); + state.str += unescapeAll(str2.slice(start2, pos2)); return state; } const helpers = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Object.defineProperty({ @@ -10657,9 +10657,9 @@ default_rules.fence = function(tokens, idx, options, env, slf) { let langName = ""; let langAttrs = ""; if (info) { - const arr = info.split(/(\s+)/g); - langName = arr[0]; - langAttrs = arr.slice(2).join(""); + const arr2 = info.split(/(\s+)/g); + langName = arr2[0]; + langAttrs = arr2.slice(2).join(""); } let highlighted; if (options.highlight) { @@ -11013,10 +11013,10 @@ StateCore.prototype.Token = Token; const NEWLINES_RE = /\r\n?|\n/g; const NULL_RE = /\0/g; function normalize(state) { - let str; - str = state.src.replace(NEWLINES_RE, "\n"); - str = str.replace(NULL_RE, "�"); - state.src = str; + let str2; + str2 = state.src.replace(NEWLINES_RE, "\n"); + str2 = str2.replace(NULL_RE, "�"); + state.src = str2; } function block(state) { let token2; @@ -11039,11 +11039,11 @@ function inline(state) { } } } -function isLinkOpen$1(str) { - return /^\s]/i.test(str); +function isLinkOpen$1(str2) { + return /^\s]/i.test(str2); } -function isLinkClose$1(str) { - return /^<\/a\s*>/i.test(str); +function isLinkClose$1(str2) { + return /^<\/a\s*>/i.test(str2); } function linkify$1(state) { const blockTokens = state.tokens; @@ -11197,8 +11197,8 @@ function replace(state) { const QUOTE_TEST_RE = /['"]/; const QUOTE_RE = /['"]/g; const APOSTROPHE = "’"; -function replaceAt(str, index, ch3) { - return str.slice(0, index) + ch3 + str.slice(index + 1); +function replaceAt(str2, index, ch3) { + return str2.slice(0, index) + ch3 + str2.slice(index + 1); } function process_inlines(tokens, state) { let j2; @@ -11554,30 +11554,30 @@ function getLine(state, line2) { const max2 = state.eMarks[line2]; return state.src.slice(pos2, max2); } -function escapedSplit(str) { +function escapedSplit(str2) { const result = []; - const max2 = str.length; + const max2 = str2.length; let pos2 = 0; - let ch3 = str.charCodeAt(pos2); + let ch3 = str2.charCodeAt(pos2); let isEscaped = false; let lastPos = 0; let current = ""; while (pos2 < max2) { if (ch3 === 124) { if (!isEscaped) { - result.push(current + str.substring(lastPos, pos2)); + result.push(current + str2.substring(lastPos, pos2)); current = ""; lastPos = pos2 + 1; } else { - current += str.substring(lastPos, pos2 - 1); + current += str2.substring(lastPos, pos2 - 1); lastPos = pos2; } } isEscaped = ch3 === 92; pos2++; - ch3 = str.charCodeAt(pos2); + ch3 = str2.charCodeAt(pos2); } - result.push(current + str.substring(lastPos)); + result.push(current + str2.substring(lastPos)); return result; } function table(state, startLine, endLine, silent) { @@ -12246,11 +12246,11 @@ function reference(state, startLine, _endLine, silent) { const max3 = state.eMarks[nextLine2]; return state.src.slice(pos3, max3 + 1); } - let str = state.src.slice(pos2, max2 + 1); - max2 = str.length; + let str2 = state.src.slice(pos2, max2 + 1); + max2 = str2.length; let labelEnd = -1; for (pos2 = 1; pos2 < max2; pos2++) { - const ch3 = str.charCodeAt(pos2); + const ch3 = str2.charCodeAt(pos2); if (ch3 === 91) { return false; } else if (ch3 === 93) { @@ -12259,32 +12259,32 @@ function reference(state, startLine, _endLine, silent) { } else if (ch3 === 10) { const lineContent = getNextLine(nextLine); if (lineContent !== null) { - str += lineContent; - max2 = str.length; + str2 += lineContent; + max2 = str2.length; nextLine++; } } else if (ch3 === 92) { pos2++; - if (pos2 < max2 && str.charCodeAt(pos2) === 10) { + if (pos2 < max2 && str2.charCodeAt(pos2) === 10) { const lineContent = getNextLine(nextLine); if (lineContent !== null) { - str += lineContent; - max2 = str.length; + str2 += lineContent; + max2 = str2.length; nextLine++; } } } } - if (labelEnd < 0 || str.charCodeAt(labelEnd + 1) !== 58) { + if (labelEnd < 0 || str2.charCodeAt(labelEnd + 1) !== 58) { return false; } for (pos2 = labelEnd + 2; pos2 < max2; pos2++) { - const ch3 = str.charCodeAt(pos2); + const ch3 = str2.charCodeAt(pos2); if (ch3 === 10) { const lineContent = getNextLine(nextLine); if (lineContent !== null) { - str += lineContent; - max2 = str.length; + str2 += lineContent; + max2 = str2.length; nextLine++; } } else if (isSpace(ch3)) ; @@ -12292,7 +12292,7 @@ function reference(state, startLine, _endLine, silent) { break; } } - const destRes = state.md.helpers.parseLinkDestination(str, pos2, max2); + const destRes = state.md.helpers.parseLinkDestination(str2, pos2, max2); if (!destRes.ok) { return false; } @@ -12305,12 +12305,12 @@ function reference(state, startLine, _endLine, silent) { const destEndLineNo = nextLine; const start2 = pos2; for (; pos2 < max2; pos2++) { - const ch3 = str.charCodeAt(pos2); + const ch3 = str2.charCodeAt(pos2); if (ch3 === 10) { const lineContent = getNextLine(nextLine); if (lineContent !== null) { - str += lineContent; - max2 = str.length; + str2 += lineContent; + max2 = str2.length; nextLine++; } } else if (isSpace(ch3)) ; @@ -12318,15 +12318,15 @@ function reference(state, startLine, _endLine, silent) { break; } } - let titleRes = state.md.helpers.parseLinkTitle(str, pos2, max2); + let titleRes = state.md.helpers.parseLinkTitle(str2, pos2, max2); while (titleRes.can_continue) { const lineContent = getNextLine(nextLine); if (lineContent === null) break; - str += lineContent; + str2 += lineContent; pos2 = max2; - max2 = str.length; + max2 = str2.length; nextLine++; - titleRes = state.md.helpers.parseLinkTitle(str, pos2, max2, titleRes); + titleRes = state.md.helpers.parseLinkTitle(str2, pos2, max2, titleRes); } let title; if (pos2 < max2 && start2 !== pos2 && titleRes.ok) { @@ -12338,19 +12338,19 @@ function reference(state, startLine, _endLine, silent) { nextLine = destEndLineNo; } while (pos2 < max2) { - const ch3 = str.charCodeAt(pos2); + const ch3 = str2.charCodeAt(pos2); if (!isSpace(ch3)) { break; } pos2++; } - if (pos2 < max2 && str.charCodeAt(pos2) !== 10) { + if (pos2 < max2 && str2.charCodeAt(pos2) !== 10) { if (title) { title = ""; pos2 = destEndPos; nextLine = destEndLineNo; while (pos2 < max2) { - const ch3 = str.charCodeAt(pos2); + const ch3 = str2.charCodeAt(pos2); if (!isSpace(ch3)) { break; } @@ -12358,10 +12358,10 @@ function reference(state, startLine, _endLine, silent) { } } } - if (pos2 < max2 && str.charCodeAt(pos2) !== 10) { + if (pos2 < max2 && str2.charCodeAt(pos2) !== 10) { return false; } - const label = normalizeReference(str.slice(1, labelEnd)); + const label = normalizeReference(str2.slice(1, labelEnd)); if (!label) { return false; } @@ -13438,11 +13438,11 @@ function autolink(state, silent) { } return false; } -function isLinkOpen(str) { - return /^\s]/i.test(str); +function isLinkOpen(str2) { + return /^\s]/i.test(str2); } -function isLinkClose(str) { - return /^<\/a\s*>/i.test(str); +function isLinkClose(str2) { + return /^<\/a\s*>/i.test(str2); } function isLetter(ch3) { const lc = ch3 | 32; @@ -13691,8 +13691,8 @@ ParserInline.prototype.tokenize = function(state) { state.pushPending(); } }; -ParserInline.prototype.parse = function(str, md, env, outTokens) { - const state = new this.State(str, md, env, outTokens); +ParserInline.prototype.parse = function(str2, md, env, outTokens) { + const state = new this.State(str2, md, env, outTokens); this.tokenize(state); const rules = this.ruler2.getRules(""); const len = rules.length; @@ -13768,8 +13768,8 @@ function isRegExp(obj) { function isFunction(obj) { return _class(obj) === "[object Function]"; } -function escapeRE(str) { - return str.replace(/[.?*+^$[\]\\(){}|-]/g, "\\$&"); +function escapeRE(str2) { + return str2.replace(/[.?*+^$[\]\\(){}|-]/g, "\\$&"); } const defaultOptions = { fuzzyLink: true, @@ -14075,8 +14075,8 @@ LinkifyIt.prototype.tlds = function tlds(list2, keepOld) { compile(this); return this; } - this.__tlds__ = this.__tlds__.concat(list2).sort().filter(function(el, idx, arr) { - return el !== arr[idx - 1]; + this.__tlds__ = this.__tlds__.concat(list2).sort().filter(function(el, idx, arr2) { + return el !== arr2[idx - 1]; }).reverse(); compile(this); return this; @@ -14498,8 +14498,8 @@ const config = { const BAD_PROTO_RE = /^(vbscript|javascript|file|data):/; const GOOD_DATA_RE = /^data:image\/(gif|png|jpeg|webp);/; function validateLink(url) { - const str = url.trim().toLowerCase(); - return BAD_PROTO_RE.test(str) ? GOOD_DATA_RE.test(str) : true; + const str2 = url.trim().toLowerCase(); + return BAD_PROTO_RE.test(str2) ? GOOD_DATA_RE.test(str2) : true; } const RECODE_HOSTNAME_FOR = ["http:", "https:", "mailto:"]; function normalizeLink(url) { @@ -14706,7 +14706,7 @@ const escape$1 = (content) => { } }); }; -function unescapeCodeHtmlEntities(str) { +function unescapeCodeHtmlEntities(str2) { const htmlEntities = { "<": "<", ">": ">", @@ -14714,7 +14714,7 @@ function unescapeCodeHtmlEntities(str) { "\": "\\", """: '"' }; - return str.replace( + return str2.replace( /(]*>)([\s\S]*?)(<\/code>)/gi, function(match2, starttag, content, endtag) { return starttag + content.replace( @@ -16567,7 +16567,7 @@ const isVscode = () => { }); }; const SampleScores = ({ sample, sampleDescriptor, scorer }) => { - const scores = scorer ? sampleDescriptor.scorer(sample, scorer).scores() : sampleDescriptor.selectedScorer(sample).scores(); + const scores = scorer ? sampleDescriptor.evalDescriptor.scorerDescriptor(sample, { scorer, name: scorer }).scores() : sampleDescriptor.selectedScorerDescriptor(sample).scores(); if (scores.length === 1) { return scores[0].rendered(); } else { @@ -16601,7 +16601,7 @@ const SampleScoreView = ({ }) => { var _a2, _b2, _c; if (!sampleDescriptor) { - return ""; + return m$1``; } const scoreInput = inputString(sample.input); if (sample.choices && sample.choices.length > 0) { @@ -16612,7 +16612,10 @@ const SampleScoreView = ({ }) ); } - const scorerDescriptor = sampleDescriptor.scorer(sample, scorer); + const scorerDescriptor = sampleDescriptor.evalDescriptor.scorerDescriptor( + sample, + { scorer, name: scorer } + ); const explanation = scorerDescriptor.explanation() || "(No Explanation)"; const answer = scorerDescriptor.answer(); return m$1` @@ -16724,37 +16727,40 @@ const SampleScoreView = ({ ` : ""} - ${((_a2 = sample == null ? void 0 : sample.score) == null ? void 0 : _a2.metadata) && Object.keys((_b2 = sample == null ? void 0 : sample.score) == null ? void 0 : _b2.metadata).length > 0 ? m$1` - - - + + + + + + + +
0 ? m$1` + + + - - - - - - - -
- Metadata -
- <${MetaDataView} - id="task-sample-score-metadata" - classes="tab-pane" - entries="${(_c = sample == null ? void 0 : sample.score) == null ? void 0 : _c.metadata}" - style=${{ marginTop: "1em" }} - /> -
` : ""} + > + Metadata +
+ <${MetaDataView} + id="task-sample-score-metadata" + classes="tab-pane" + entries="${// @ts-ignore + (_c = sample == null ? void 0 : sample.score) == null ? void 0 : _c.metadata}" + style=${{ marginTop: "1em" }} + /> +
` : ""} `; }; @@ -18269,11 +18275,11 @@ function diff(left2, right2) { } return defaultInstance$1.diff(left2, right2); } -const trimUnderscore = (str) => { - if (str.substring(0, 1) === "_") { - return str.slice(1); +const trimUnderscore = (str2) => { + if (str2.substring(0, 1) === "_") { + return str2.slice(1); } - return str; + return str2; }; const arrayKeyToSortNumber = (key2) => { if (key2 === "_t") { @@ -19898,6 +19904,7 @@ const kScoreTypeCategorical = "categorical"; const kScoreTypeNumeric = "numeric"; const kScoreTypeOther = "other"; const kScoreTypeObject = "object"; +const kScoreTypeBoolean = "boolean"; const kSampleAscVal = "sample-asc"; const kSampleDescVal = "sample-desc"; const kEpochAscVal = "epoch-asc"; @@ -20203,7 +20210,7 @@ const SampleSummary = ({ id, sample, style, sampleDescriptor }) => { clamp: true }); } - const fullAnswer = sample && sampleDescriptor ? sampleDescriptor.selectedScorer(sample).answer() : void 0; + const fullAnswer = sample && sampleDescriptor ? sampleDescriptor.selectedScorerDescriptor(sample).answer() : void 0; if (fullAnswer) { columns.push({ label: "Answer", @@ -20707,7 +20714,7 @@ const SampleRow = ({ > ${sample ? m$1` <${MarkdownDiv} - markdown=${sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScorer(sample).answer()} + markdown=${sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScorerDescriptor(sample).answer()} style=${{ paddingLeft: "0" }} class="no-last-para-padding" /> @@ -20962,7 +20969,7 @@ const groupBySample = (samples, sampleDescriptor, order2) => { } } }); - const groupCount = samples.length / sampleDescriptor.epochs; + const groupCount = samples.length / sampleDescriptor.evalDescriptor.epochs; const itemCount = samples.length / groupCount; const counter = getCounter(itemCount, groupCount, order2); return (sample, index, previousSample) => { @@ -20991,7 +20998,7 @@ const groupBySample = (samples, sampleDescriptor, order2) => { }; }; const groupByEpoch = (samples, sampleDescriptor, order2) => { - const groupCount = sampleDescriptor.epochs; + const groupCount = sampleDescriptor.evalDescriptor.epochs; const itemCount = samples.length / groupCount; const counter = getCounter(itemCount, groupCount, order2); return (sample, index, previousSample) => { @@ -23784,7 +23791,7 @@ const SortFilter = ({ sampleDescriptor, sort, setSort, epochs }) => { val: kEpochDescVal }); } - if ((_a2 = sampleDescriptor == null ? void 0 : sampleDescriptor.scoreDescriptor) == null ? void 0 : _a2.compare) { + if ((_a2 = sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScoreDescriptor) == null ? void 0 : _a2.compare) { options.push({ label: "score asc", val: kScoreAscVal @@ -23850,12 +23857,12 @@ const sortSamples = (sort, samples, samplesDescriptor) => { case kEpochDescVal: return b2.epoch - a2.epoch; case kScoreAscVal: - return samplesDescriptor.scoreDescriptor.compare( + return samplesDescriptor.selectedScoreDescriptor.compare( samplesDescriptor.selectedScore(a2).value, samplesDescriptor.selectedScore(b2).value ); case kScoreDescVal: - return samplesDescriptor.scoreDescriptor.compare( + return samplesDescriptor.selectedScoreDescriptor.compare( samplesDescriptor.selectedScore(b2).value, samplesDescriptor.selectedScore(a2).value ); @@ -23866,89 +23873,95 @@ const sortSamples = (sort, samples, samplesDescriptor) => { order: sort === kSampleAscVal || sort === kEpochAscVal || sort === kScoreAscVal ? "asc" : "desc" }; }; -const SampleFilter = ({ descriptor, filter, filterChanged }) => { - var _a2; - const updateCategoryValue = (e2) => { - const val = e2.currentTarget.value; - if (val === "all") { - filterChanged({}); - } else { - filterChanged({ - value: val, - type: kScoreTypeCategorical - }); - } - }; - switch ((_a2 = descriptor == null ? void 0 : descriptor.scoreDescriptor) == null ? void 0 : _a2.scoreType) { - case kScoreTypePassFail: { - const options = [{ text: "All", value: "all" }]; - options.push( - ...descriptor.scoreDescriptor.categories.map((cat) => { - return { text: cat.text, value: cat.val }; - }) - ); - return m$1`<${SelectFilter} - value=${filter.value || "all"} - options=${options} - onChange=${updateCategoryValue} - />`; - } - case kScoreTypeCategorical: { - const options = [{ text: "All", value: "all" }]; - options.push( - ...descriptor.scoreDescriptor.categories.map((cat) => { - return { text: cat, value: cat }; - }) - ); - return m$1`<${SelectFilter} - value=${filter.value || "all"} - options=${options} - onChange=${updateCategoryValue} - />`; - } - case kScoreTypeNumeric: { - return m$1` +const SampleFilter = ({ filter, filterError, filterChanged }) => { + const inputRef = A(null); + const tooltip = filterError ? `${filterError} + +${filterTooltip}` : filterTooltip; + return m$1` +
+ Filter: +
{ - filterChanged({ - value: e2.currentTarget.value, - type: kScoreTypeNumeric - }); - }} + filterChanged({ + value: e2.currentTarget.value + }); + }} + ref=${inputRef} /> - `; - } - case kScoreTypeObject: { - if (!descriptor.scoreDescriptor.categories) { - return ""; - } - const options = [{ text: "All", value: "all" }]; - options.push( - ...descriptor.scoreDescriptor.categories.map((cat) => { - return { text: cat.text, value: cat.value }; - }) - ); - return m$1`<${SelectFilter} - value=${filter.value || "all"} - options=${options} - onChange=${updateCategoryValue} - />`; - } - default: { - return void 0; - } - } + ${filter.value && m$1` + + `} +
+
+ `; }; -const SelectFilter = ({ value, options, onChange }) => { +const filterTooltip = ` +Filter samples by scores. Supported expressions: + • Arithmetic: +, -, *, /, mod, ^ + • Comparison: <, <=, >, >=, ==, !=, including chain comparisons, e.g. “10 <= x < 20” + • Boolean: and, or, not + • Regex matching: ~= (case-sensitive) + • Set operations: in, not in; e.g. “x in (1, 2, 3)” + • Functions: min, max, abs, round, floor, ceil, sqrt, log, log2, log10 +Click on the score name above to add it to the filter. +`.trim(); +const SelectScorer = ({ scores, score, setScore }) => { return m$1`
{ marginRight: "0.3em", marginLeft: "0.2em" }} - >Scores: - + <${ScoreSelector} + scores=${scores} + selectedIndex=${scoreIndex(score, scores)} + selectedIndexChanged=${(index) => { + setScore(scores[index]); + }} + />
`; }; -const SelectScorer = ({ scores, score, setScore }) => { - const scorers = scores.reduce((accum, scorer) => { - if (!accum.find((sc) => { - return scorer.scorer === sc.scorer; - })) { - accum.push(scorer); - } - return accum; - }, []); - if (scorers.length === 1) { - return m$1` -
- Score: - <${ScoreSelector} - scores=${scores} - selectedIndex=${scoreIndex(score, scores)} - selectedIndexChanged=${(index) => { - setScore(scores[index]); - }} - /> -
- `; - } else { - const scorerScores = scores.filter((sc) => { - return sc.scorer === score.scorer; - }); - const selectors = [ - m$1`<${ScorerSelector} - scorers=${scorers} - selectedIndex=${scorerIndex(score, scorers)} - selectedIndexChanged=${(index) => { - setScore(scorers[index]); - }} - />` - ]; - if (scorerScores.length > 1) { - selectors.push( - m$1`<${ScoreSelector} - style=${{ marginLeft: "1em" }} - scores=${scorerScores} - selectedIndex=${scoreIndex(score, scorerScores)} - selectedIndexChanged=${(index) => { - setScore(scorerScores[index]); - }} - />` - ); - } - return m$1` -
- Scorer: - ${selectors} -
- `; - } -}; const ScoreSelector = ({ scores, selectedIndex, @@ -24064,36 +23998,21 @@ const ScoreSelector = ({ }} > ${scores.map((score) => { - return m$1``; - })} - `; -}; -const ScorerSelector = ({ scorers, selectedIndex, selectedIndexChanged }) => { - return m$1``; }; const scoreIndex = (score, scores) => scores.findIndex((sc) => { return sc.name === score.name && sc.scorer === score.scorer; }); -const scorerIndex = (score, scores) => scores.findIndex((sc) => { - return sc.scorer === score.scorer; -}); const SampleTools = (props) => { const { epoch, setEpoch, filter, + filterError, filterChanged, sort, setSort, @@ -24123,13 +24042,6 @@ const SampleTools = (props) => { />` ); } - tools.push( - m$1`<${SampleFilter} - filter=${filter} - filterChanged=${filterChanged} - descriptor=${sampleDescriptor} - />` - ); tools.push( m$1`<${SortFilter} sampleDescriptor=${sampleDescriptor} @@ -24138,6 +24050,13 @@ const SampleTools = (props) => { epochs=${hasEpochs} />` ); + tools.push( + m$1`<${SampleFilter} + filter=${filter} + filterError=${filterError} + filterChanged=${filterChanged} + />` + ); return tools; }; const CopyButton = ({ value }) => { @@ -24198,119 +24117,2749 @@ const LabeledValue = ({
${children}
`; }; -const SecondaryBar = ({ - evalSpec, - evalPlan, - evalResults, - evalStats, - samples, - status, - style -}) => { - if (!evalSpec || status !== "success") { - return ""; - } - const staticColStyle = { - flexShrink: "0" - }; - const epochs = evalSpec.config.epochs || 1; - const hyperparameters = { - ...evalPlan == null ? void 0 : evalPlan.config, - ...evalSpec.task_args - }; - const hasConfig = Object.keys(hyperparameters).length > 0; - const values = []; - values.push({ - size: "minmax(12%, auto)", - value: m$1`<${LabeledValue} label="Dataset" style=${staticColStyle}> - <${DatasetSummary} - dataset=${evalSpec.dataset} - samples=${samples} - epochs=${epochs} /> - -` - }); - const label = (evalResults == null ? void 0 : evalResults.scores.length) > 1 ? "Scorers" : "Scorer"; - values.push({ - size: "minmax(12%, auto)", - value: m$1`<${LabeledValue} label="${label}" style=${staticColStyle} style=${{ justifySelf: hasConfig ? "left" : "center" }}> - <${ScorerSummary} - scorers=${evalResults == null ? void 0 : evalResults.scores} /> - ` - }); - if (hasConfig) { - values.push({ - size: "minmax(12%, auto)", - value: m$1`<${LabeledValue} label="Config" style=${{ justifySelf: "right" }}> - <${ParamSummary} params=${hyperparameters}/> - ` - }); - } - const totalDuration = formatDuration( - new Date(evalStats.started_at), - new Date(evalStats.completed_at) - ); - values.push({ - size: "minmax(12%, auto)", - value: m$1` - <${LabeledValue} label="Duration" style=${{ justifySelf: "right" }}> - ${totalDuration} - ` - }); - return m$1` - <${ExpandablePanel} style=${{ margin: "0", ...style }} collapse=${true} lines=${4}> -
{ - return val.size; - }).join(" ")}` - }} - > - ${values.map((val) => { - return val.value; - })} -
- - `; -}; -const DatasetSummary = ({ dataset, samples, epochs, style }) => { - if (!dataset) { - return ""; - } - return m$1` -
- ${dataset.name}${(samples == null ? void 0 : samples.length) ? m$1`${formatDataset(dataset.name, samples.length, epochs)}` : ""} -
- `; -}; -const ScorerSummary = ({ scorers }) => { - if (!scorers) { - return ""; - } - const uniqScorers = /* @__PURE__ */ new Set(); - scorers.forEach((scorer) => { - uniqScorers.add(scorer.name); - }); - return Array.from(uniqScorers).join(", "); -}; -const ParamSummary = ({ params }) => { - if (!params) { - return ""; - } - const paraValues = Object.keys(params).map((key2) => { - const val = params[key2]; - if (Array.isArray(val) || typeof val === "object") { - return `${key2}: ${JSON.stringify(val)}`; - } else { - return `${key2}: ${val}`; - } - }); - if (paraValues.length > 0) { +var _parser = function() { + var parser2 = { + trace: function trace() { + }, + yy: {}, + symbols_: { + error: 2, + expressions: 3, + e: 4, + EndOfExpression: 5, + "-": 6, + "+": 7, + "*": 8, + "/": 9, + "^": 10, + mod: 11, + and: 12, + or: 13, + not: 14, + if: 15, + then: 16, + else: 17, + in: 18, + notIn: 19, + "(": 20, + ")": 21, + Arguments: 22, + ",": 23, + Number: 24, + Symbol: 25, + String: 26, + of: 27, + Relation: 28, + "%": 29, + "?": 30, + ":": 31, + RelationalOperator: 32, + "==": 33, + "!=": 34, + "~=": 35, + "<": 36, + "<=": 37, + ">=": 38, + ">": 39, + $accept: 0, + $end: 1 + }, + terminals_: { + 2: "error", + 5: "EndOfExpression", + 6: "-", + 7: "+", + 8: "*", + 9: "/", + 10: "^", + 11: "mod", + 12: "and", + 13: "or", + 14: "not", + 15: "if", + 16: "then", + 17: "else", + 18: "in", + 19: "notIn", + 20: "(", + 21: ")", + 23: ",", + 24: "Number", + 25: "Symbol", + 26: "String", + 27: "of", + 29: "%", + 30: "?", + 31: ":", + 33: "==", + 34: "!=", + 35: "~=", + 36: "<", + 37: "<=", + 38: ">=", + 39: ">" + }, + productions_: [ + 0, + [3, 2], + [4, 2], + [4, 3], + [4, 3], + [4, 3], + [4, 3], + [4, 3], + [4, 3], + [4, 3], + [4, 3], + [4, 2], + [4, 6], + [4, 3], + [4, 3], + [4, 3], + [4, 5], + [4, 1], + [4, 1], + [4, 1], + [4, 3], + [4, 3], + [4, 4], + [4, 1], + [4, 3], + [4, 5], + [32, 1], + [32, 1], + [32, 1], + [32, 1], + [32, 1], + [32, 1], + [32, 1], + [28, 3], + [28, 3], + [22, 1], + [22, 3] + ], + performAction: function anonymous(yytext, yyleng, yylineno, yy, yystate, $$, _$) { + var $0 = $$.length - 1; + switch (yystate) { + case 1: + return $$[$0 - 1]; + case 2: + this.$ = ["(", "ops['-'](", $$[$0], ")", ")"]; + break; + case 3: + this.$ = [ + "(", + "ops['", + $$[$0 - 1], + "'](", + $$[$0 - 2], + ", ", + $$[$0], + ")", + ")" + ]; + break; + case 4: + this.$ = [ + "(", + "ops['", + $$[$0 - 1], + "'](", + $$[$0 - 2], + ", ", + $$[$0], + ")", + ")" + ]; + break; + case 5: + this.$ = [ + "(", + "ops['", + $$[$0 - 1], + "'](", + $$[$0 - 2], + ", ", + $$[$0], + ")", + ")" + ]; + break; + case 6: + this.$ = [ + "(", + "ops['", + $$[$0 - 1], + "'](", + $$[$0 - 2], + ", ", + $$[$0], + ")", + ")" + ]; + break; + case 7: + this.$ = [ + "(", + "ops['", + $$[$0 - 1], + "'](", + $$[$0 - 2], + ", ", + $$[$0], + ")", + ")" + ]; + break; + case 8: + this.$ = ["(", "ops.mod(", $$[$0 - 2], ", ", $$[$0], ")", ")"]; + break; + case 9: + this.$ = [ + "(", + "", + "std.coerceBoolean", + "(", + $$[$0 - 2], + ") && ", + "std.coerceBoolean", + "(", + $$[$0], + ")", + ")" + ]; + break; + case 10: + this.$ = [ + "(", + "", + "std.coerceBoolean", + "(", + $$[$0 - 2], + ") || ", + "std.coerceBoolean", + "(", + $$[$0], + ")", + ")" + ]; + break; + case 11: + this.$ = ["(", "! ", "std.coerceBoolean", "(", $$[$0], ")", ")"]; + break; + case 12: + this.$ = [ + "(", + "", + "std.coerceBoolean", + "(", + $$[$0 - 4], + ") ? ", + $$[$0 - 2], + " : ", + $$[$0], + "", + ")" + ]; + break; + case 13: + this.$ = ["(", "std.isSubset(", $$[$0 - 2], ", ", $$[$0], ")", ")"]; + break; + case 14: + this.$ = ["(", "!std.isSubset(", $$[$0 - 2], ", ", $$[$0], ")", ")"]; + break; + case 15: + this.$ = ["(", "", $$[$0 - 1], "", ")"]; + break; + case 16: + this.$ = ["(", "[ ", $$[$0 - 3], ", ", $$[$0 - 1], " ]", ")"]; + break; + case 17: + this.$ = ["", $$[$0], ""]; + break; + case 18: + this.$ = ["prop(", $$[$0], ", data)"]; + break; + case 19: + this.$ = ["", $$[$0], ""]; + break; + case 20: + this.$ = ["prop(", $$[$0 - 2], ", ", $$[$0], ")"]; + break; + case 21: + this.$ = ["call(", $$[$0 - 2], ")"]; + break; + case 22: + this.$ = ["call(", $$[$0 - 3], ", ", $$[$0 - 1], ")"]; + break; + case 23: + this.$ = yy.reduceRelation($$[$0]); + break; + case 24: + this.$ = [ + "std.warnDeprecated('modulo', ops['mod'](", + $$[$0 - 2], + ", ", + $$[$0], + "))" + ]; + break; + case 25: + this.$ = [ + "std.warnDeprecated('ternary', ", + "std.coerceBoolean", + "(", + $$[$0 - 4], + ") ? ", + $$[$0 - 2], + " : ", + $$[$0], + ")" + ]; + break; + case 26: + this.$ = ["=="]; + break; + case 27: + this.$ = ["!="]; + break; + case 28: + this.$ = ["~="]; + break; + case 29: + this.$ = ["<"]; + break; + case 30: + this.$ = ["<="]; + break; + case 31: + this.$ = [">="]; + break; + case 32: + this.$ = [">"]; + break; + case 33: + this.$ = [$$[$0 - 2], $$[$0 - 1], ...$$[$0]]; + break; + case 34: + this.$ = [$$[$0 - 2], $$[$0 - 1], $$[$0]]; + break; + case 35: + this.$ = ["", $$[$0], ""]; + break; + case 36: + this.$ = ["", $$[$0 - 2], ", ", $$[$0], ""]; + break; + } + }, + table: [ + { + 3: 1, + 4: 2, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 1: [3] + }, + { + 5: [1, 11], + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [1, 18], + 13: [1, 19], + 18: [1, 20], + 19: [1, 21], + 29: [1, 22], + 30: [1, 23], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 4: 32, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 33, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 34, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 35, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 22: 36, + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 5: [2, 17], + 6: [2, 17], + 7: [2, 17], + 8: [2, 17], + 9: [2, 17], + 10: [2, 17], + 11: [2, 17], + 12: [2, 17], + 13: [2, 17], + 16: [2, 17], + 17: [2, 17], + 18: [2, 17], + 19: [2, 17], + 21: [2, 17], + 23: [2, 17], + 29: [2, 17], + 30: [2, 17], + 31: [2, 17], + 33: [2, 17], + 34: [2, 17], + 35: [2, 17], + 36: [2, 17], + 37: [2, 17], + 38: [2, 17], + 39: [2, 17] + }, + { + 5: [2, 18], + 6: [2, 18], + 7: [2, 18], + 8: [2, 18], + 9: [2, 18], + 10: [2, 18], + 11: [2, 18], + 12: [2, 18], + 13: [2, 18], + 16: [2, 18], + 17: [2, 18], + 18: [2, 18], + 19: [2, 18], + 20: [1, 38], + 21: [2, 18], + 23: [2, 18], + 27: [1, 37], + 29: [2, 18], + 30: [2, 18], + 31: [2, 18], + 33: [2, 18], + 34: [2, 18], + 35: [2, 18], + 36: [2, 18], + 37: [2, 18], + 38: [2, 18], + 39: [2, 18] + }, + { + 5: [2, 19], + 6: [2, 19], + 7: [2, 19], + 8: [2, 19], + 9: [2, 19], + 10: [2, 19], + 11: [2, 19], + 12: [2, 19], + 13: [2, 19], + 16: [2, 19], + 17: [2, 19], + 18: [2, 19], + 19: [2, 19], + 21: [2, 19], + 23: [2, 19], + 29: [2, 19], + 30: [2, 19], + 31: [2, 19], + 33: [2, 19], + 34: [2, 19], + 35: [2, 19], + 36: [2, 19], + 37: [2, 19], + 38: [2, 19], + 39: [2, 19] + }, + { + 5: [2, 23], + 6: [2, 23], + 7: [2, 23], + 8: [2, 23], + 9: [2, 23], + 10: [2, 23], + 11: [2, 23], + 12: [2, 23], + 13: [2, 23], + 16: [2, 23], + 17: [2, 23], + 18: [2, 23], + 19: [2, 23], + 21: [2, 23], + 23: [2, 23], + 29: [2, 23], + 30: [2, 23], + 31: [2, 23], + 33: [2, 23], + 34: [2, 23], + 35: [2, 23], + 36: [2, 23], + 37: [2, 23], + 38: [2, 23], + 39: [2, 23] + }, + { + 1: [2, 1] + }, + { + 4: 39, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 40, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 41, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 42, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 43, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 44, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 45, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 46, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 47, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 48, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 49, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 50, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 52, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 51 + }, + { + 6: [2, 26], + 14: [2, 26], + 15: [2, 26], + 20: [2, 26], + 24: [2, 26], + 25: [2, 26], + 26: [2, 26] + }, + { + 6: [2, 27], + 14: [2, 27], + 15: [2, 27], + 20: [2, 27], + 24: [2, 27], + 25: [2, 27], + 26: [2, 27] + }, + { + 6: [2, 28], + 14: [2, 28], + 15: [2, 28], + 20: [2, 28], + 24: [2, 28], + 25: [2, 28], + 26: [2, 28] + }, + { + 6: [2, 29], + 14: [2, 29], + 15: [2, 29], + 20: [2, 29], + 24: [2, 29], + 25: [2, 29], + 26: [2, 29] + }, + { + 6: [2, 30], + 14: [2, 30], + 15: [2, 30], + 20: [2, 30], + 24: [2, 30], + 25: [2, 30], + 26: [2, 30] + }, + { + 6: [2, 31], + 14: [2, 31], + 15: [2, 31], + 20: [2, 31], + 24: [2, 31], + 25: [2, 31], + 26: [2, 31] + }, + { + 6: [2, 32], + 14: [2, 32], + 15: [2, 32], + 20: [2, 32], + 24: [2, 32], + 25: [2, 32], + 26: [2, 32] + }, + { + 5: [2, 2], + 6: [2, 2], + 7: [2, 2], + 8: [2, 2], + 9: [2, 2], + 10: [1, 16], + 11: [2, 2], + 12: [2, 2], + 13: [2, 2], + 16: [2, 2], + 17: [2, 2], + 18: [2, 2], + 19: [2, 2], + 21: [2, 2], + 23: [2, 2], + 29: [2, 2], + 30: [2, 2], + 31: [2, 2], + 32: 24, + 33: [2, 2], + 34: [2, 2], + 35: [2, 2], + 36: [2, 2], + 37: [2, 2], + 38: [2, 2], + 39: [2, 2] + }, + { + 5: [2, 11], + 6: [2, 11], + 7: [2, 11], + 8: [2, 11], + 9: [2, 11], + 10: [1, 16], + 11: [2, 11], + 12: [2, 11], + 13: [2, 11], + 16: [2, 11], + 17: [2, 11], + 18: [2, 11], + 19: [2, 11], + 21: [2, 11], + 23: [2, 11], + 29: [2, 11], + 30: [2, 11], + 31: [2, 11], + 32: 24, + 33: [2, 11], + 34: [2, 11], + 35: [2, 11], + 36: [2, 11], + 37: [2, 11], + 38: [2, 11], + 39: [2, 11] + }, + { + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [1, 18], + 13: [1, 19], + 16: [1, 53], + 18: [1, 20], + 19: [1, 21], + 29: [1, 22], + 30: [1, 23], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [1, 18], + 13: [1, 19], + 18: [1, 20], + 19: [1, 21], + 21: [1, 54], + 23: [2, 35], + 29: [1, 22], + 30: [1, 23], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 23: [1, 55] + }, + { + 4: 56, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 4: 59, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 21: [1, 57], + 22: 58, + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 5: [2, 3], + 6: [2, 3], + 7: [2, 3], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [2, 3], + 13: [2, 3], + 16: [2, 3], + 17: [2, 3], + 18: [2, 3], + 19: [2, 3], + 21: [2, 3], + 23: [2, 3], + 29: [1, 22], + 30: [2, 3], + 31: [2, 3], + 32: 24, + 33: [2, 3], + 34: [2, 3], + 35: [2, 3], + 36: [2, 3], + 37: [2, 3], + 38: [2, 3], + 39: [2, 3] + }, + { + 5: [2, 4], + 6: [2, 4], + 7: [2, 4], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [2, 4], + 13: [2, 4], + 16: [2, 4], + 17: [2, 4], + 18: [2, 4], + 19: [2, 4], + 21: [2, 4], + 23: [2, 4], + 29: [1, 22], + 30: [2, 4], + 31: [2, 4], + 32: 24, + 33: [2, 4], + 34: [2, 4], + 35: [2, 4], + 36: [2, 4], + 37: [2, 4], + 38: [2, 4], + 39: [2, 4] + }, + { + 5: [2, 5], + 6: [2, 5], + 7: [2, 5], + 8: [2, 5], + 9: [2, 5], + 10: [1, 16], + 11: [2, 5], + 12: [2, 5], + 13: [2, 5], + 16: [2, 5], + 17: [2, 5], + 18: [2, 5], + 19: [2, 5], + 21: [2, 5], + 23: [2, 5], + 29: [2, 5], + 30: [2, 5], + 31: [2, 5], + 32: 24, + 33: [2, 5], + 34: [2, 5], + 35: [2, 5], + 36: [2, 5], + 37: [2, 5], + 38: [2, 5], + 39: [2, 5] + }, + { + 5: [2, 6], + 6: [2, 6], + 7: [2, 6], + 8: [2, 6], + 9: [2, 6], + 10: [1, 16], + 11: [2, 6], + 12: [2, 6], + 13: [2, 6], + 16: [2, 6], + 17: [2, 6], + 18: [2, 6], + 19: [2, 6], + 21: [2, 6], + 23: [2, 6], + 29: [2, 6], + 30: [2, 6], + 31: [2, 6], + 32: 24, + 33: [2, 6], + 34: [2, 6], + 35: [2, 6], + 36: [2, 6], + 37: [2, 6], + 38: [2, 6], + 39: [2, 6] + }, + { + 5: [2, 7], + 6: [2, 7], + 7: [2, 7], + 8: [2, 7], + 9: [2, 7], + 10: [1, 16], + 11: [2, 7], + 12: [2, 7], + 13: [2, 7], + 16: [2, 7], + 17: [2, 7], + 18: [2, 7], + 19: [2, 7], + 21: [2, 7], + 23: [2, 7], + 29: [2, 7], + 30: [2, 7], + 31: [2, 7], + 32: 24, + 33: [2, 7], + 34: [2, 7], + 35: [2, 7], + 36: [2, 7], + 37: [2, 7], + 38: [2, 7], + 39: [2, 7] + }, + { + 5: [2, 8], + 6: [2, 8], + 7: [2, 8], + 8: [2, 8], + 9: [2, 8], + 10: [1, 16], + 11: [2, 8], + 12: [2, 8], + 13: [2, 8], + 16: [2, 8], + 17: [2, 8], + 18: [2, 8], + 19: [2, 8], + 21: [2, 8], + 23: [2, 8], + 29: [2, 8], + 30: [2, 8], + 31: [2, 8], + 32: 24, + 33: [2, 8], + 34: [2, 8], + 35: [2, 8], + 36: [2, 8], + 37: [2, 8], + 38: [2, 8], + 39: [2, 8] + }, + { + 5: [2, 9], + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [2, 9], + 13: [2, 9], + 16: [2, 9], + 17: [2, 9], + 18: [1, 20], + 19: [1, 21], + 21: [2, 9], + 23: [2, 9], + 29: [1, 22], + 30: [2, 9], + 31: [2, 9], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 5: [2, 10], + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [1, 18], + 13: [2, 10], + 16: [2, 10], + 17: [2, 10], + 18: [1, 20], + 19: [1, 21], + 21: [2, 10], + 23: [2, 10], + 29: [1, 22], + 30: [2, 10], + 31: [2, 10], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 5: [2, 13], + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [2, 13], + 13: [2, 13], + 16: [2, 13], + 17: [2, 13], + 18: [2, 13], + 19: [2, 13], + 21: [2, 13], + 23: [2, 13], + 29: [1, 22], + 30: [2, 13], + 31: [2, 13], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 5: [2, 14], + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [2, 14], + 13: [2, 14], + 16: [2, 14], + 17: [2, 14], + 18: [2, 14], + 19: [2, 14], + 21: [2, 14], + 23: [2, 14], + 29: [1, 22], + 30: [2, 14], + 31: [2, 14], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 5: [2, 24], + 6: [2, 24], + 7: [2, 24], + 8: [2, 24], + 9: [2, 24], + 10: [1, 16], + 11: [2, 24], + 12: [2, 24], + 13: [2, 24], + 16: [2, 24], + 17: [2, 24], + 18: [2, 24], + 19: [2, 24], + 21: [2, 24], + 23: [2, 24], + 29: [2, 24], + 30: [2, 24], + 31: [2, 24], + 32: 24, + 33: [2, 24], + 34: [2, 24], + 35: [2, 24], + 36: [2, 24], + 37: [2, 24], + 38: [2, 24], + 39: [2, 24] + }, + { + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [1, 18], + 13: [1, 19], + 18: [1, 20], + 19: [1, 21], + 29: [1, 22], + 30: [1, 23], + 31: [1, 60], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 5: [2, 33], + 6: [2, 33], + 7: [2, 33], + 8: [2, 33], + 9: [2, 33], + 10: [2, 33], + 11: [2, 33], + 12: [2, 33], + 13: [2, 33], + 16: [2, 33], + 17: [2, 33], + 18: [2, 33], + 19: [2, 33], + 21: [2, 33], + 23: [2, 33], + 29: [2, 33], + 30: [2, 33], + 31: [2, 33], + 33: [2, 33], + 34: [2, 33], + 35: [2, 33], + 36: [2, 33], + 37: [2, 33], + 38: [2, 33], + 39: [2, 33] + }, + { + 5: [2, 34], + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [2, 34], + 13: [2, 34], + 16: [2, 34], + 17: [2, 34], + 18: [2, 34], + 19: [2, 34], + 21: [2, 34], + 23: [2, 34], + 29: [1, 22], + 30: [2, 34], + 31: [2, 34], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 4: 61, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 5: [2, 15], + 6: [2, 15], + 7: [2, 15], + 8: [2, 15], + 9: [2, 15], + 10: [2, 15], + 11: [2, 15], + 12: [2, 15], + 13: [2, 15], + 16: [2, 15], + 17: [2, 15], + 18: [2, 15], + 19: [2, 15], + 21: [2, 15], + 23: [2, 15], + 29: [2, 15], + 30: [2, 15], + 31: [2, 15], + 33: [2, 15], + 34: [2, 15], + 35: [2, 15], + 36: [2, 15], + 37: [2, 15], + 38: [2, 15], + 39: [2, 15] + }, + { + 4: 62, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 5: [2, 20], + 6: [2, 20], + 7: [2, 20], + 8: [2, 20], + 9: [2, 20], + 10: [2, 20], + 11: [2, 20], + 12: [2, 20], + 13: [2, 20], + 16: [2, 20], + 17: [2, 20], + 18: [2, 20], + 19: [2, 20], + 21: [2, 20], + 23: [2, 20], + 29: [2, 20], + 30: [2, 20], + 31: [2, 20], + 32: 24, + 33: [2, 20], + 34: [2, 20], + 35: [2, 20], + 36: [2, 20], + 37: [2, 20], + 38: [2, 20], + 39: [2, 20] + }, + { + 5: [2, 21], + 6: [2, 21], + 7: [2, 21], + 8: [2, 21], + 9: [2, 21], + 10: [2, 21], + 11: [2, 21], + 12: [2, 21], + 13: [2, 21], + 16: [2, 21], + 17: [2, 21], + 18: [2, 21], + 19: [2, 21], + 21: [2, 21], + 23: [2, 21], + 29: [2, 21], + 30: [2, 21], + 31: [2, 21], + 33: [2, 21], + 34: [2, 21], + 35: [2, 21], + 36: [2, 21], + 37: [2, 21], + 38: [2, 21], + 39: [2, 21] + }, + { + 21: [1, 63], + 23: [1, 64] + }, + { + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [1, 18], + 13: [1, 19], + 18: [1, 20], + 19: [1, 21], + 21: [2, 35], + 23: [2, 35], + 29: [1, 22], + 30: [1, 23], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 4: 65, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [1, 18], + 13: [1, 19], + 17: [1, 66], + 18: [1, 20], + 19: [1, 21], + 29: [1, 22], + 30: [1, 23], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [1, 18], + 13: [1, 19], + 18: [1, 20], + 19: [1, 21], + 21: [1, 67], + 23: [2, 36], + 29: [1, 22], + 30: [1, 23], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 5: [2, 22], + 6: [2, 22], + 7: [2, 22], + 8: [2, 22], + 9: [2, 22], + 10: [2, 22], + 11: [2, 22], + 12: [2, 22], + 13: [2, 22], + 16: [2, 22], + 17: [2, 22], + 18: [2, 22], + 19: [2, 22], + 21: [2, 22], + 23: [2, 22], + 29: [2, 22], + 30: [2, 22], + 31: [2, 22], + 33: [2, 22], + 34: [2, 22], + 35: [2, 22], + 36: [2, 22], + 37: [2, 22], + 38: [2, 22], + 39: [2, 22] + }, + { + 4: 68, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 5: [2, 25], + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [1, 18], + 13: [1, 19], + 16: [2, 25], + 17: [2, 25], + 18: [1, 20], + 19: [1, 21], + 21: [2, 25], + 23: [2, 25], + 29: [1, 22], + 30: [1, 23], + 31: [2, 25], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 4: 69, + 6: [1, 3], + 14: [1, 4], + 15: [1, 5], + 20: [1, 6], + 24: [1, 7], + 25: [1, 8], + 26: [1, 9], + 28: 10 + }, + { + 5: [2, 16], + 6: [2, 16], + 7: [2, 16], + 8: [2, 16], + 9: [2, 16], + 10: [2, 16], + 11: [2, 16], + 12: [2, 16], + 13: [2, 16], + 16: [2, 16], + 17: [2, 16], + 18: [2, 16], + 19: [2, 16], + 21: [2, 16], + 23: [2, 16], + 29: [2, 16], + 30: [2, 16], + 31: [2, 16], + 33: [2, 16], + 34: [2, 16], + 35: [2, 16], + 36: [2, 16], + 37: [2, 16], + 38: [2, 16], + 39: [2, 16] + }, + { + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [1, 18], + 13: [1, 19], + 18: [1, 20], + 19: [1, 21], + 21: [2, 36], + 23: [2, 36], + 29: [1, 22], + 30: [1, 23], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + }, + { + 5: [2, 12], + 6: [1, 13], + 7: [1, 12], + 8: [1, 14], + 9: [1, 15], + 10: [1, 16], + 11: [1, 17], + 12: [1, 18], + 13: [1, 19], + 16: [2, 12], + 17: [2, 12], + 18: [1, 20], + 19: [1, 21], + 21: [2, 12], + 23: [2, 12], + 29: [1, 22], + 30: [1, 23], + 31: [2, 12], + 32: 24, + 33: [1, 25], + 34: [1, 26], + 35: [1, 27], + 36: [1, 28], + 37: [1, 29], + 38: [1, 30], + 39: [1, 31] + } + ], + defaultActions: { + 11: [2, 1] + }, + parseError: function parseError(str2, hash2) { + throw new Error(str2); + }, + parse: function parse3(input) { + var self2 = this, stack2 = [0], vstack = [null], lstack = [], table2 = this.table, yytext = "", yylineno = 0, yyleng = 0, recovering = 0, TERROR = 2, EOF = 1; + this.lexer.setInput(input); + this.lexer.yy = this.yy; + this.yy.lexer = this.lexer; + this.yy.parser = this; + if (typeof this.lexer.yylloc == "undefined") this.lexer.yylloc = {}; + var yyloc = this.lexer.yylloc; + lstack.push(yyloc); + var ranges = this.lexer.options && this.lexer.options.ranges; + if (typeof this.yy.parseError === "function") + this.parseError = this.yy.parseError; + function popStack(n2) { + stack2.length = stack2.length - 2 * n2; + vstack.length = vstack.length - n2; + lstack.length = lstack.length - n2; + } + function lex2() { + var token2; + token2 = self2.lexer.lex() || 1; + if (typeof token2 !== "number") { + token2 = self2.symbols_[token2] || token2; + } + return token2; + } + var symbol, preErrorSymbol, state, action, r2, yyval = {}, p2, len, newState, expected; + while (true) { + state = stack2[stack2.length - 1]; + if (this.defaultActions[state]) { + action = this.defaultActions[state]; + } else { + if (symbol === null || typeof symbol == "undefined") { + symbol = lex2(); + } + action = table2[state] && table2[state][symbol]; + } + if (typeof action === "undefined" || !action.length || !action[0]) { + var errStr = ""; + if (!recovering) { + expected = []; + for (p2 in table2[state]) + if (this.terminals_[p2] && p2 > 2) { + expected.push("'" + this.terminals_[p2] + "'"); + } + if (this.lexer.showPosition) { + errStr = "Parse error on line " + (yylineno + 1) + ":\n" + this.lexer.showPosition() + "\nExpecting " + expected.join(", ") + ", got '" + (this.terminals_[symbol] || symbol) + "'"; + } else { + errStr = "Parse error on line " + (yylineno + 1) + ": Unexpected " + (symbol == 1 ? "end of input" : "'" + (this.terminals_[symbol] || symbol) + "'"); + } + this.parseError(errStr, { + text: this.lexer.match, + token: this.terminals_[symbol] || symbol, + line: this.lexer.yylineno, + loc: yyloc, + expected + }); + } + if (recovering == 3) { + if (symbol == EOF) { + throw new Error(errStr || "Parsing halted."); + } + yyleng = this.lexer.yyleng; + yytext = this.lexer.yytext; + yylineno = this.lexer.yylineno; + yyloc = this.lexer.yylloc; + symbol = lex2(); + } + while (1) { + if (TERROR.toString() in table2[state]) { + break; + } + if (state === 0) { + throw new Error(errStr || "Parsing halted."); + } + popStack(1); + state = stack2[stack2.length - 1]; + } + preErrorSymbol = symbol == 2 ? null : symbol; + symbol = TERROR; + state = stack2[stack2.length - 1]; + action = table2[state] && table2[state][TERROR]; + recovering = 3; + } + if (action[0] instanceof Array && action.length > 1) { + throw new Error( + "Parse Error: multiple actions possible at state: " + state + ", token: " + symbol + ); + } + switch (action[0]) { + case 1: + stack2.push(symbol); + vstack.push(this.lexer.yytext); + lstack.push(this.lexer.yylloc); + stack2.push(action[1]); + symbol = null; + if (!preErrorSymbol) { + yyleng = this.lexer.yyleng; + yytext = this.lexer.yytext; + yylineno = this.lexer.yylineno; + yyloc = this.lexer.yylloc; + if (recovering > 0) recovering--; + } else { + symbol = preErrorSymbol; + preErrorSymbol = null; + } + break; + case 2: + len = this.productions_[action[1]][1]; + yyval.$ = vstack[vstack.length - len]; + yyval._$ = { + first_line: lstack[lstack.length - (len || 1)].first_line, + last_line: lstack[lstack.length - 1].last_line, + first_column: lstack[lstack.length - (len || 1)].first_column, + last_column: lstack[lstack.length - 1].last_column + }; + if (ranges) { + yyval._$.range = [ + lstack[lstack.length - (len || 1)].range[0], + lstack[lstack.length - 1].range[1] + ]; + } + r2 = this.performAction.call( + yyval, + yytext, + yyleng, + yylineno, + this.yy, + action[1], + vstack, + lstack + ); + if (typeof r2 !== "undefined") { + return r2; + } + if (len) { + stack2 = stack2.slice(0, -1 * len * 2); + vstack = vstack.slice(0, -1 * len); + lstack = lstack.slice(0, -1 * len); + } + stack2.push(this.productions_[action[1]][0]); + vstack.push(yyval.$); + lstack.push(yyval._$); + newState = table2[stack2[stack2.length - 2]][stack2[stack2.length - 1]]; + stack2.push(newState); + break; + case 3: + return true; + } + } + return true; + } + }; + var lexer = function() { + var lexer2 = { + EOF: 1, + parseError: function parseError(str2, hash2) { + if (this.yy.parser) { + this.yy.parser.parseError(str2, hash2); + } else { + throw new Error(str2); + } + }, + setInput: function(input) { + this._input = input; + this._more = this._less = this.done = false; + this.yylineno = this.yyleng = 0; + this.yytext = this.matched = this.match = ""; + this.conditionStack = ["INITIAL"]; + this.yylloc = { + first_line: 1, + first_column: 0, + last_line: 1, + last_column: 0 + }; + if (this.options.ranges) this.yylloc.range = [0, 0]; + this.offset = 0; + return this; + }, + input: function() { + var ch3 = this._input[0]; + this.yytext += ch3; + this.yyleng++; + this.offset++; + this.match += ch3; + this.matched += ch3; + var lines = ch3.match(/(?:\r\n?|\n).*/g); + if (lines) { + this.yylineno++; + this.yylloc.last_line++; + } else { + this.yylloc.last_column++; + } + if (this.options.ranges) this.yylloc.range[1]++; + this._input = this._input.slice(1); + return ch3; + }, + unput: function(ch3) { + var len = ch3.length; + var lines = ch3.split(/(?:\r\n?|\n)/g); + this._input = ch3 + this._input; + this.yytext = this.yytext.substr(0, this.yytext.length - len - 1); + this.offset -= len; + var oldLines = this.match.split(/(?:\r\n?|\n)/g); + this.match = this.match.substr(0, this.match.length - 1); + this.matched = this.matched.substr(0, this.matched.length - 1); + if (lines.length - 1) this.yylineno -= lines.length - 1; + var r2 = this.yylloc.range; + this.yylloc = { + first_line: this.yylloc.first_line, + last_line: this.yylineno + 1, + first_column: this.yylloc.first_column, + last_column: lines ? (lines.length === oldLines.length ? this.yylloc.first_column : 0) + oldLines[oldLines.length - lines.length].length - lines[0].length : this.yylloc.first_column - len + }; + if (this.options.ranges) { + this.yylloc.range = [r2[0], r2[0] + this.yyleng - len]; + } + return this; + }, + more: function() { + this._more = true; + return this; + }, + less: function(n2) { + this.unput(this.match.slice(n2)); + }, + pastInput: function() { + var past = this.matched.substr( + 0, + this.matched.length - this.match.length + ); + return (past.length > 20 ? "..." : "") + past.substr(-20).replace(/\n/g, ""); + }, + upcomingInput: function() { + var next = this.match; + if (next.length < 20) { + next += this._input.substr(0, 20 - next.length); + } + return (next.substr(0, 20) + (next.length > 20 ? "..." : "")).replace( + /\n/g, + "" + ); + }, + showPosition: function() { + var pre = this.pastInput(); + var c2 = new Array(pre.length + 1).join("-"); + return pre + this.upcomingInput() + "\n" + c2 + "^"; + }, + next: function() { + if (this.done) { + return this.EOF; + } + if (!this._input) this.done = true; + var token2, match2, tempMatch, index, lines; + if (!this._more) { + this.yytext = ""; + this.match = ""; + } + var rules = this._currentRules(); + for (var i = 0; i < rules.length; i++) { + tempMatch = this._input.match(this.rules[rules[i]]); + if (tempMatch && (!match2 || tempMatch[0].length > match2[0].length)) { + match2 = tempMatch; + index = i; + if (!this.options.flex) break; + } + } + if (match2) { + lines = match2[0].match(/(?:\r\n?|\n).*/g); + if (lines) this.yylineno += lines.length; + this.yylloc = { + first_line: this.yylloc.last_line, + last_line: this.yylineno + 1, + first_column: this.yylloc.last_column, + last_column: lines ? lines[lines.length - 1].length - lines[lines.length - 1].match(/\r?\n?/)[0].length : this.yylloc.last_column + match2[0].length + }; + this.yytext += match2[0]; + this.match += match2[0]; + this.matches = match2; + this.yyleng = this.yytext.length; + if (this.options.ranges) { + this.yylloc.range = [this.offset, this.offset += this.yyleng]; + } + this._more = false; + this._input = this._input.slice(match2[0].length); + this.matched += match2[0]; + token2 = this.performAction.call( + this, + this.yy, + this, + rules[index], + this.conditionStack[this.conditionStack.length - 1] + ); + if (this.done && this._input) this.done = false; + if (token2) return token2; + else return; + } + if (this._input === "") { + return this.EOF; + } else { + return this.parseError( + "Lexical error on line " + (this.yylineno + 1) + ". Unrecognized text.\n" + this.showPosition(), + { + text: "", + token: null, + line: this.yylineno + } + ); + } + }, + lex: function lex2() { + var r2 = this.next(); + if (typeof r2 !== "undefined") { + return r2; + } else { + return this.lex(); + } + }, + begin: function begin(condition) { + this.conditionStack.push(condition); + }, + popState: function popState() { + return this.conditionStack.pop(); + }, + _currentRules: function _currentRules() { + return this.conditions[this.conditionStack[this.conditionStack.length - 1]].rules; + }, + topState: function() { + return this.conditionStack[this.conditionStack.length - 2]; + }, + pushState: function begin(condition) { + this.begin(condition); + } + }; + lexer2.options = {}; + lexer2.performAction = function anonymous(yy, yy_, $avoiding_name_collisions, YY_START) { + switch ($avoiding_name_collisions) { + case 0: + return "*"; + case 1: + return "/"; + case 2: + return "-"; + case 3: + return "+"; + case 4: + return "^"; + case 5: + return "("; + case 6: + return ")"; + case 7: + return ","; + case 8: + return "=="; + case 9: + return "!="; + case 10: + return "~="; + case 11: + return ">="; + case 12: + return "<="; + case 13: + return "<"; + case 14: + return ">"; + case 15: + return "notIn"; + case 16: + return "and"; + case 17: + return "or"; + case 18: + return "not"; + case 19: + return "in"; + case 20: + return "of"; + case 21: + return "if"; + case 22: + return "then"; + case 23: + return "else"; + case 24: + return "mod"; + case 25: + break; + case 26: + return "Number"; + case 27: + yy_.yytext = JSON.stringify({ + name: yy_.yytext, + type: "unescaped" + }); + return "Symbol"; + case 28: + yy_.yytext = JSON.stringify({ + name: yy.buildString("'", yy_.yytext), + type: "single-quoted" + }); + return "Symbol"; + case 29: + yy_.yytext = JSON.stringify(yy.buildString('"', yy_.yytext)); + return "String"; + case 30: + return "%"; + case 31: + return "?"; + case 32: + return ":"; + case 33: + return "EndOfExpression"; + } + }; + lexer2.rules = [ + /^(?:\*)/, + /^(?:\/)/, + /^(?:-)/, + /^(?:\+)/, + /^(?:\^)/, + /^(?:\()/, + /^(?:\))/, + /^(?:\,)/, + /^(?:==)/, + /^(?:\!=)/, + /^(?:\~=)/, + /^(?:>=)/, + /^(?:<=)/, + /^(?:<)/, + /^(?:>)/, + /^(?:not\s+in[^\w])/, + /^(?:and[^\w])/, + /^(?:or[^\w])/, + /^(?:not[^\w])/, + /^(?:in[^\w])/, + /^(?:of[^\w])/, + /^(?:if[^\w])/, + /^(?:then[^\w])/, + /^(?:else[^\w])/, + /^(?:mod[^\w])/, + /^(?:\s+)/, + /^(?:[0-9]+(?:\.[0-9]+)?(?![0-9\.]))/, + /^(?:[a-zA-Z$_][\.a-zA-Z0-9$_]*)/, + /^(?:'(?:\\'|\\\\|[^'\\])*')/, + /^(?:"(?:\\"|\\\\|[^"\\])*")/, + /^(?:\%)/, + /^(?:\?)/, + /^(?::)/, + /^(?:$)/ + ]; + lexer2.conditions = { + INITIAL: { + rules: [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33 + ], + inclusive: true + } + }; + return lexer2; + }(); + parser2.lexer = lexer; + function Parser() { + this.yy = {}; + } + Parser.prototype = parser2; + parser2.Parser = Parser; + return new Parser(); +}(); +const parser = _parser; +_parser.Parser; +class UnknownFunctionError extends ReferenceError { + constructor(funcName) { + super(`Unknown function: ${funcName}()`); + __publicField(this, "I18N_STRING", "UNKNOWN_FUNCTION"); + this.functionName = funcName; + } +} +class UnknownPropertyError extends ReferenceError { + constructor(propName) { + super(`Property “${propName}” does not exist.`); + __publicField(this, "I18N_STRING", "UNKNOWN_PROPERTY"); + this.propertyName = propName; + } +} +class UnknownOptionError extends TypeError { + constructor(key2) { + super(`Unknown option: ${key2}`); + __publicField(this, "I18N_STRING", "UNKNOWN_OPTION"); + this.keyName = key2; + } +} +class UnexpectedTypeError extends TypeError { + constructor(expected, got) { + super(`Expected a ${expected}, but got a ${got} instead.`); + __publicField(this, "I18N_STRING", "UNEXPECTED_TYPE"); + this.expectedType = expected; + this.recievedType = got; + } +} +class InternalError extends Error { + constructor(message) { + super(message); + __publicField(this, "I18N_STRING", "INTERNAL"); + } +} +function hasOwnProperty(obj, prop) { + if (typeof obj === "object" || typeof obj === "function") { + return Object.prototype.hasOwnProperty.call(obj, prop); + } + return false; +} +function mod(a2, b2) { + return (a2 % b2 + b2) % b2; +} +function unbox(value) { + if (typeof value !== "object") return value; + if (value instanceof Number || value instanceof String || value instanceof Boolean) + return value.valueOf(); +} +function unwrap(value) { + if (Array.isArray(value) && value.length === 1) value = value[0]; + return unbox(value); +} +function prettyType(value) { + value = unwrap(value); + if (value === void 0) return "undefined"; + if (value === null) return "null"; + if (value === true) return "true"; + if (value === false) return "false"; + if (typeof value === "number") return "number"; + if (typeof value === "string") return "text"; + if (typeof value !== "object" && typeof value !== "function") + return "unknown type"; + if (Array.isArray(value)) return "list"; + return "object"; +} +function num(value) { + value = unwrap(value); + if (typeof value === "number") return value; + throw new UnexpectedTypeError("number", prettyType(value)); +} +function str(value) { + value = unwrap(value); + if (typeof value === "string") return value; + throw new UnexpectedTypeError("text", prettyType(value)); +} +function numstr(value) { + value = unwrap(value); + if (typeof value === "string" || typeof value === "number") return value; + throw new UnexpectedTypeError("text or number", prettyType(value)); +} +function bool(value) { + value = unwrap(value); + if (typeof value === "boolean") return value; + throw new UnexpectedTypeError( + "logical value (“true” or “false”)", + prettyType(value) + ); +} +function arr(value) { + if (value === void 0 || value === null) { + throw new UnexpectedTypeError("list", prettyType(value)); + } + if (Array.isArray(value)) { + return value; + } else { + return [value]; + } +} +function flatten(input) { + const stack2 = [...input]; + const res = []; + while (stack2.length) { + const next = stack2.pop(); + if (Array.isArray(next)) { + stack2.push(...next); + } else { + res.push(next); + } + } + return res.reverse(); +} +const std = { + isfn(fns, funcName) { + return hasOwnProperty(fns, funcName) && typeof fns[funcName] === "function"; + }, + unknown(funcName) { + throw new UnknownFunctionError(funcName); + }, + coerceArray: arr, + coerceNumber: num, + coerceNumberOrString: numstr, + coerceBoolean: bool, + isSubset(a2, b2) { + const A2 = arr(a2); + const B2 = arr(b2); + return A2.every((val) => B2.includes(val)); + }, + warnDeprecated: /* @__PURE__ */ function() { + const warnMax = 3; + let warnedTimes = { + ternary: 0, + modulo: 0 + }; + return (cause, value) => { + switch (cause) { + case "ternary": + if (warnedTimes.ternary++ >= warnMax) break; + console.warn( + "The use of ? and : as conditional operators has been deprecated in Filtrex v3 in favor of the if..then..else ternary operator. See issue #34 for more information." + ); + break; + case "modulo": + if (warnedTimes.modulo++ >= warnMax) break; + console.warn( + "The use of '%' as a modulo operator has been deprecated in Filtrex v3 in favor of the 'mod' operator. You can use it like this: '3 mod 2 == 1'. See issue #48 for more information." + ); + break; + } + return value; + }; + }(), + buildString(quote, literal2) { + quote = String(quote)[0]; + literal2 = String(literal2); + let built = ""; + if (literal2[0] !== quote || literal2[literal2.length - 1] !== quote) + throw new InternalError( + `Unexpected internal error: String literal doesn't begin/end with the right quotation mark.` + ); + for (let i = 1; i < literal2.length - 1; i++) { + if (literal2[i] === "\\") { + i++; + if (i >= literal2.length - 1) + throw new InternalError( + `Unexpected internal error: Unescaped backslash at the end of string literal.` + ); + if (literal2[i] === "\\") built += "\\"; + else if (literal2[i] === quote) built += quote; + else + throw new InternalError( + `Unexpected internal error: Invalid escaped character in string literal: ${literal2[i]}` + ); + } else if (literal2[i] === quote) { + throw new InternalError( + `Unexpected internal error: String literal contains unescaped quotation mark.` + ); + } else { + built += literal2[i]; + } + } + return built; + }, + reduceRelation(arr2) { + const declarations = []; + const comparisons = []; + let previousExpression = flatten([arr2[0]]).join(""); + let j2 = 0; + for (let i = 1; i < arr2.length - 1; i += 2) { + const expr = flatten([arr2[i + 1]]).join(""); + const tempVar = `tmp${j2++}`; + comparisons.push( + `ops["${arr2[i]}"](${previousExpression}, ${tempVar} = ${expr})` + ); + previousExpression = tempVar; + declarations.push(tempVar); + } + return `(function(){ var ${declarations.join(", ")}; return ${comparisons.join(" && ")};})()`; + } +}; +parser.yy = Object.create(std); +function compileExpression(expression, options) { + if (arguments.length > 2) throw new TypeError("Too many arguments."); + options = typeof options === "object" ? options : {}; + const knownOptions = [ + "extraFunctions", + "constants", + "customProp", + "operators" + ]; + let { extraFunctions, constants, customProp, operators } = options; + for (const key2 of Object.keys(options)) + if (!knownOptions.includes(key2)) throw new UnknownOptionError(key2); + let functions = { + abs: Math.abs, + ceil: Math.ceil, + floor: Math.floor, + log: Math.log, + log2: Math.log2, + log10: Math.log10, + max: Math.max, + min: Math.min, + round: Math.round, + sqrt: Math.sqrt, + exists: (v2) => v2 !== void 0 && v2 !== null, + empty: (v2) => v2 === void 0 || v2 === null || v2 === "" || Array.isArray(v2) && v2.length === 0 + }; + if (extraFunctions) { + for (const name of Object.keys(extraFunctions)) { + functions[name] = extraFunctions[name]; + } + } + let defaultOperators = { + "+": (a2, b2) => numstr(a2) + numstr(b2), + "-": (a2, b2) => b2 === void 0 ? -num(a2) : num(a2) - num(b2), + "*": (a2, b2) => num(a2) * num(b2), + "/": (a2, b2) => num(a2) / num(b2), + "^": (a2, b2) => Math.pow(num(a2), num(b2)), + mod: (a2, b2) => mod(num(a2), num(b2)), + "==": (a2, b2) => a2 === b2, + "!=": (a2, b2) => a2 !== b2, + "<": (a2, b2) => num(a2) < num(b2), + "<=": (a2, b2) => num(a2) <= num(b2), + ">=": (a2, b2) => num(a2) >= num(b2), + ">": (a2, b2) => num(a2) > num(b2), + "~=": (a2, b2) => RegExp(str(b2)).test(str(a2)) + }; + if (operators) { + for (const name of Object.keys(operators)) { + defaultOperators[name] = operators[name]; + } + } + operators = defaultOperators; + constants = constants ?? {}; + let js = flatten(parser.parse(expression)); + js.unshift("return "); + js.push(";"); + function nakedProp(name, obj, type) { + if (hasOwnProperty(obj ?? {}, name)) return obj[name]; + throw new UnknownPropertyError(name); + } + function safeGetter(obj) { + return function get2(name) { + if (hasOwnProperty(obj ?? {}, name)) return obj[name]; + throw new UnknownPropertyError(name); + }; + } + if (typeof customProp === "function") { + nakedProp = (name, obj, type) => customProp(name, safeGetter(obj), obj, type); + } + function createCall(fns) { + return function call(_ref) { + let { name } = _ref; + for (var _len = arguments.length, args = new Array(_len > 1 ? _len - 1 : 0), _key = 1; _key < _len; _key++) { + args[_key - 1] = arguments[_key]; + } + if (hasOwnProperty(fns, name) && typeof fns[name] === "function") + return fns[name](...args); + throw new UnknownFunctionError(name); + }; + } + function prop(_ref2, obj) { + let { name, type } = _ref2; + if (type === "unescaped" && hasOwnProperty(constants, name)) + return constants[name]; + return nakedProp(name, obj, type); + } + let func = new Function("call", "ops", "std", "prop", "data", js.join("")); + return function(data) { + try { + return func(createCall(functions), operators, std, prop, data); + } catch (e2) { + return e2; + } + }; +} +const coerceValue = (value, descriptor) => { + if (descriptor && descriptor.scoreType === kScoreTypeBoolean) { + return Boolean(value); + } else { + return value; + } +}; +const isFilteringSupportedForValue = (value) => ["string", "number", "boolean"].includes(typeof value); +const isFilteringSupportedForScore = (descriptor) => { + if (!descriptor) { + return false; + } + return [ + kScoreTypePassFail, + kScoreTypeCategorical, + kScoreTypeNumeric, + kScoreTypeBoolean + ].includes(descriptor.scoreType); +}; +const bannedShortScoreNames = (scores) => { + const used = /* @__PURE__ */ new Set(); + const banned = /* @__PURE__ */ new Set(); + for (const { scorer, name } of scores) { + banned.add(scorer); + if (used.has(name)) { + banned.add(name); + } else { + used.add(name); + } + } + return banned; +}; +const scoreVariables = (evalDescriptor, sampleScores) => { + const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores); + const variables = {}; + const addScore = (variableName, scoreLabel, value) => { + const coercedValue = coerceValue( + value, + evalDescriptor.scoreDescriptor(scoreLabel) + ); + if (isFilteringSupportedForValue(coercedValue)) { + variables[variableName] = coercedValue; + } + }; + for (const [scorer, score] of Object.entries(sampleScores)) { + addScore(scorer, { scorer, name: scorer }, score.value); + if (typeof score.value === "object") { + for (const [name, value] of Object.entries(score.value)) { + addScore(`${scorer}.${name}`, { scorer, name }, value); + if (!bannedShortNames.has(name)) { + addScore(name, { scorer, name }, value); + } + } + } + } + return variables; +}; +const scoreFilterItems = (evalDescriptor) => { + const items = []; + const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores); + const valueToString = (value) => typeof value === "string" ? `"${value}"` : String(value); + const addScore = (canonicalName, scoreLabel) => { + const descriptor = evalDescriptor.scoreDescriptor(scoreLabel); + if (!descriptor || !isFilteringSupportedForScore(descriptor)) { + items.push({ + canonicalName, + tooltip: void 0, + isFilterable: false, + suggestions: [] + }); + return; + } + var tooltip = `${canonicalName}: ${descriptor.scoreType}`; + var suggestions = []; + if (descriptor.min !== void 0 || descriptor.max !== void 0) { + const rounded = (num2) => { + return parseFloat(num2.toPrecision(3)).toString(); + }; + tooltip += ` +Range: ${rounded(descriptor.min)} to ${rounded(descriptor.max)}`; + } + if (descriptor.categories) { + tooltip += ` +Categories: ${descriptor.categories.map((cat) => cat.val).join(", ")}`; + suggestions = [ + canonicalName, + ...descriptor.categories.map( + (cat) => `${canonicalName} == ${valueToString(cat.val)}` + ) + ]; + } + items.push({ canonicalName, tooltip, isFilterable: true, suggestions }); + }; + for (const { name, scorer } of evalDescriptor.scores) { + const canonicalName = name !== scorer && bannedShortNames.has(name) ? `${scorer}.${name}` : name; + addScore(canonicalName, { name, scorer }); + } + return items; +}; +const addFragmentToFilter = (filter, fragment) => { + var value = filter.value || ""; + if (value.trim() && !value.endsWith(" ")) { + value = `${value} `; + } + if (value.trim() && !value.match(/ +(or|and) *$/)) { + value = `${value}and `; + } + value += fragment; + return { value }; +}; +const filterExpression = (evalDescriptor, sample, value) => { + try { + const expression = compileExpression(value); + const vars = scoreVariables(evalDescriptor, sample.scores); + const result = expression(vars); + if (typeof result === "boolean") { + return { matches: result, error: void 0 }; + } else if (result instanceof Error) { + throw result; + } else { + throw new TypeError( + `Filter expression returned a non-boolean value: ${result}` + ); + } + } catch (error2) { + return { matches: false, error: error2.message }; + } +}; +const SecondaryBar = ({ + evalSpec, + evalPlan, + evalResults, + evalStats, + samples, + evalDescriptor, + addToFilterExpression, + status, + style +}) => { + if (!evalSpec || status !== "success") { + return ""; + } + const staticColStyle = { + flexShrink: "0" + }; + const epochs = evalSpec.config.epochs || 1; + const hyperparameters = { + ...evalPlan == null ? void 0 : evalPlan.config, + ...evalSpec.task_args + }; + const hasConfig = Object.keys(hyperparameters).length > 0; + const values = []; + values.push({ + size: "minmax(12%, auto)", + value: m$1`<${LabeledValue} label="Dataset" style=${staticColStyle}> + <${DatasetSummary} + dataset=${evalSpec.dataset} + samples=${samples} + epochs=${epochs} /> + +` + }); + if (hasConfig) { + values.push({ + size: "minmax(12%, auto)", + value: m$1`<${LabeledValue} label="Config" style=${{ justifySelf: "center" }}> + <${ParamSummary} params=${hyperparameters}/> + ` + }); + } + const totalDuration = formatDuration( + new Date(evalStats.started_at), + new Date(evalStats.completed_at) + ); + values.push({ + size: "minmax(12%, auto)", + value: m$1` + <${LabeledValue} label="Duration" style=${{ justifySelf: "center" }}> + ${totalDuration} + ` + }); + const label = (evalResults == null ? void 0 : evalResults.scores.length) > 1 ? "Scorers" : "Scorer"; + values.push({ + size: "minmax(12%, auto)", + value: m$1`<${LabeledValue} label="${label}" style=${staticColStyle} style=${{ justifySelf: "right" }}> + <${ScorerSummary} + evalDescriptor=${evalDescriptor} + addToFilterExpression=${addToFilterExpression} /> + ` + }); + return m$1` + <${ExpandablePanel} style=${{ margin: "0", ...style }} collapse=${true} lines=${4}> +
{ + return val.size; + }).join(" ")}` + }} + > + ${values.map((val) => { + return val.value; + })} +
+ + `; +}; +const DatasetSummary = ({ dataset, samples, epochs, style }) => { + if (!dataset) { + return ""; + } + return m$1` +
+ ${dataset.name}${(samples == null ? void 0 : samples.length) ? m$1`${formatDataset(dataset.name, samples.length, epochs)}` : ""} +
+ `; +}; +const FilterableItem = ({ + item, + index, + openSuggestionIndex, + setOpenSuggestionIndex, + addToFilterExpression +}) => { + const handleClick = () => { + if (item.suggestions.length === 0) { + addToFilterExpression(item.canonicalName); + } else { + setOpenSuggestionIndex(openSuggestionIndex === index ? null : index); + } + }; + const handleSuggestionClick = (suggestion) => { + addToFilterExpression(suggestion); + setOpenSuggestionIndex(null); + }; + const popupRef = (el) => { + if (el && openSuggestionIndex === index) { + const rect = el.previousElementSibling.getBoundingClientRect(); + const viewportWidth = window.innerWidth; + const popupWidth = el.offsetWidth; + const finalLeft = rect.left + popupWidth > viewportWidth ? rect.right - popupWidth : rect.left; + el.style.setProperty("--popup-left", `${finalLeft}px`); + el.style.setProperty("--popup-top", `${rect.bottom + 4}px`); + } + }; + return m$1` +
+ + ${item.canonicalName} + + ${item.suggestions.length > 0 && // Use fixed position to avoid being clipped by `ExpandablePanel`. + m$1` +
+ ${item.suggestions.map( + (suggestion) => m$1` +
handleSuggestionClick(suggestion)} + > + ${suggestion} +
+ ` + )} +
+ `} +
+ `; +}; +const ScorerSummary = ({ evalDescriptor, addToFilterExpression }) => { + if (!evalDescriptor) { + return ""; + } + const items = scoreFilterItems(evalDescriptor); + const [openSuggestionIndex, setOpenSuggestionIndex] = h(null); + return m$1` + + ${Array.from(items).map( + (item, index) => m$1` + ${index > 0 ? ", " : ""} + ${item.isFilterable ? m$1`<${FilterableItem} + item=${item} + index=${index} + openSuggestionIndex=${openSuggestionIndex} + setOpenSuggestionIndex=${setOpenSuggestionIndex} + addToFilterExpression=${addToFilterExpression} + />` : m$1`${item.canonicalName}`} + ` + )} + + `; +}; +const ParamSummary = ({ params }) => { + if (!params) { + return ""; + } + const paraValues = Object.keys(params).map((key2) => { + const val = params[key2]; + if (Array.isArray(val) || typeof val === "object") { + return `${key2}: ${JSON.stringify(val)}`; + } else { + return `${key2}: ${val}`; + } + }); + if (paraValues.length > 0) { return m$1`${paraValues.join(", ")}`; @@ -24325,6 +26874,8 @@ const Navbar = ({ evalResults, evalStats, samples, + evalDescriptor, + addToFilterExpression, showToggle, offcanvas, status @@ -24463,6 +27014,8 @@ const Navbar = ({ evalResults=${evalResults} evalStats=${evalStats} samples=${samples} + evalDescriptor=${evalDescriptor} + addToFilterExpression=${addToFilterExpression} status=${status} style=${{ gridColumn: "1/-1" }} /> @@ -24698,7 +27251,9 @@ const WorkSpace = ({ epoch, setEpoch, filter, + filterError, setFilter, + addToFilterExpression, score, setScore, scores, @@ -24767,6 +27322,7 @@ const WorkSpace = ({ epochs=${epochs} setEpoch=${setEpoch} filter=${filter} + filterError=${filterError} filterChanged=${setFilter} sort=${sort} setSort=${setSort} @@ -24926,6 +27482,8 @@ const WorkSpace = ({ evalResults=${evalResults} evalStats=${evalStats} samples=${samples} + evalDescriptor=${samplesDescriptor.evalDescriptor} + addToFilterExpression=${addToFilterExpression} status=${evalStatus} tabs=${resolvedTabs} selectedTab=${selectedTab} @@ -24943,6 +27501,8 @@ const WorkspaceDisplay = ({ evalResults, evalStats, samples, + evalDescriptor, + addToFilterExpression, status, showToggle, selectedTab, @@ -25004,20 +27564,20 @@ const WorkspaceDisplay = ({ }); }, [tabs]); return m$1` - - <${Navbar} evalSpec=${evalSpec} evalPlan=${evalPlan} evalResults=${evalResults} evalStats=${evalStats} samples=${samples} + evalDescriptor=${evalDescriptor} + addToFilterExpression=${addToFilterExpression} status=${status} file=${logFileName} showToggle=${showToggle} - + offcanvas=${offcanvas} - /> + />
{
`; }; -const createsSamplesDescriptor = (scorers, samples, epochs, selectedScore) => { +const createEvalDescriptor = (scores, samples, epochs) => { if (!samples) { return void 0; } - const score = (sample, scorer = selectedScore == null ? void 0 : selectedScore.scorer) => { - if (sample.scores[scorer]) { - return sample.scores[scorer]; - } else { + const scoreValue = (sample, scoreLabel) => { + if (Object.keys(sample.scores).length === 0 || !scoreLabel) { return void 0; } - }; - const scoreValue = (sample) => { - if (Object.keys(sample.scores).length === 0 || !selectedScore) { - return void 0; - } - if (selectedScore.scorer !== selectedScore.name && sample.scores[selectedScore.scorer] && sample.scores[selectedScore.scorer].value) { - return sample.scores[selectedScore.scorer].value[selectedScore.name]; - } else if (sample.scores[selectedScore.name]) { - return sample.scores[selectedScore.name].value; + if (scoreLabel.scorer !== scoreLabel.name && sample.scores[scoreLabel.scorer] && sample.scores[scoreLabel.scorer].value) { + return sample.scores[scoreLabel.scorer].value[scoreLabel.name]; + } else if (sample.scores[scoreLabel.name]) { + return sample.scores[scoreLabel.name].value; } else { return void 0; } }; const scoreAnswer = (sample, scorer) => { if (sample) { - const sampleScore = score(sample, scorer); + const sampleScore = sample.scores[scorer]; if (sampleScore && sampleScore.answer) { return sampleScore.answer; } @@ -25232,48 +27785,149 @@ const createsSamplesDescriptor = (scorers, samples, epochs, selectedScore) => { }; const scoreExplanation = (sample, scorer) => { if (sample) { - const sampleScore = score(sample, scorer); + const sampleScore = sample.scores[scorer]; if (sampleScore && sampleScore.explanation) { return sampleScore.explanation; } } return void 0; }; - const uniqScoreValues = [ - ...new Set( - samples.filter((sample) => !!sample.scores).filter((sample) => { - if (!selectedScore) { - return true; + const scoreLabelKey = (scoreLabel) => { + return `${scoreLabel.scorer}.${scoreLabel.name}`; + }; + const scoreDescriptorMap = /* @__PURE__ */ new Map(); + for (const scoreLabel of scores) { + const uniqScoreValues = [ + ...new Set( + samples.filter((sample) => !!sample.scores).filter((sample) => { + if (!scoreLabel) { + return true; + } + if (scoreLabel.scorer !== scoreLabel.name) { + return Object.keys(sample.scores).includes(scoreLabel.scorer) && Object.keys(sample.scores[scoreLabel.scorer].value).includes( + scoreLabel.name + ); + } else { + return Object.keys(sample.scores).includes(scoreLabel.name); + } + }).map((sample) => { + return scoreValue(sample, scoreLabel); + }).filter((value) => { + return value !== null; + }) + ) + ]; + const uniqScoreTypes = [ + ...new Set(uniqScoreValues.map((scoreValue2) => typeof scoreValue2)) + ]; + for (const categorizer of scoreCategorizers) { + const scoreDescriptor2 = categorizer.describe( + uniqScoreValues, + uniqScoreTypes + ); + if (scoreDescriptor2) { + scoreDescriptorMap.set(scoreLabelKey(scoreLabel), scoreDescriptor2); + break; + } + } + } + const scoreDescriptor = (scoreLabel) => { + return scoreDescriptorMap.get(scoreLabelKey(scoreLabel)); + }; + const scoreRendered = (sample, scoreLabel) => { + const descriptor = scoreDescriptor(scoreLabel); + const score2 = scoreValue(sample, scoreLabel); + if (score2 === null || score2 === "undefined") { + return "null"; + } else if (descriptor.render) { + return descriptor.render(score2); + } else { + return score2; + } + }; + const scorerDescriptor = (sample, scoreLabel) => { + return { + explanation: () => { + return scoreExplanation(sample, scoreLabel.scorer); + }, + answer: () => { + return scoreAnswer(sample, scoreLabel.scorer); + }, + scores: () => { + if (!sample || !sample.scores) { + return []; } - if (selectedScore.scorer !== selectedScore.name) { - return Object.keys(sample.scores).includes(selectedScore.scorer) && Object.keys(sample.scores[selectedScore.scorer].value).includes( - selectedScore.name - ); + const myScoreDescriptor = scoreDescriptor(scoreLabel); + const scoreNames = scores.map((score2) => { + return score2.name; + }); + const sampleScorer = sample.scores[scoreLabel.scorer]; + const scoreVal = sampleScorer.value; + if (typeof scoreVal === "object") { + const names = Object.keys(scoreVal); + if (names.find((name) => { + return !scoreNames.includes(name); + })) { + return [ + { + name: scoreLabel.scorer, + rendered: () => { + return myScoreDescriptor.render(scoreVal); + } + } + ]; + } else { + const scores2 = names.map((name) => { + return { + name, + rendered: () => { + return myScoreDescriptor.render(scoreVal[name]); + } + }; + }); + return scores2; + } } else { - return Object.keys(sample.scores).includes(selectedScore.name); + return [ + { + name: scoreLabel.scorer, + rendered: () => { + return myScoreDescriptor.render(scoreVal); + } + } + ]; } - }).map((sample) => { - return scoreValue(sample); - }).filter((value) => { - return value !== null; - }) - ) - ]; - const uniqScoreTypes = [ - ...new Set(uniqScoreValues.map((scoreValue2) => typeof scoreValue2)) - ]; - let scoreDescriptor; - for (const categorizer of scoreCategorizers) { - scoreDescriptor = categorizer.describe(uniqScoreValues, uniqScoreTypes); - if (scoreDescriptor) { - break; - } + } + }; + }; + const score = (sample, scoreLabel) => { + return { + value: scoreValue(sample, scoreLabel), + render: () => { + return scoreRendered(sample, scoreLabel); + } + }; + }; + return { + epochs, + samples, + scores, + scorerDescriptor, + scoreDescriptor, + score, + scoreAnswer + }; +}; +const createSamplesDescriptor = (evalDescriptor, selectedScore) => { + if (!evalDescriptor) { + return void 0; } - const sizes = samples.reduce( + const sizes = evalDescriptor.samples.reduce( (previous, current) => { var _a2; const text2 = inputString(current.input).join(" "); - const scoreText = scoreValue(current) ? String(scoreValue(current)) : ""; + const scoreValue = evalDescriptor.score(current, selectedScore).value; + const scoreText = scoreValue ? String(scoreValue) : ""; previous[0] = Math.min(Math.max(previous[0], text2.length), 300); previous[1] = Math.min( Math.max(previous[1], arrayToString(current.target).length), @@ -25282,7 +27936,7 @@ const createsSamplesDescriptor = (scorers, samples, epochs, selectedScore) => { previous[2] = Math.min( Math.max( previous[2], - ((_a2 = scoreAnswer(current, selectedScore == null ? void 0 : selectedScore.name)) == null ? void 0 : _a2.length) || 0 + ((_a2 = evalDescriptor.scoreAnswer(current, selectedScore == null ? void 0 : selectedScore.name)) == null ? void 0 : _a2.length) || 0 ), 300 ); @@ -25326,88 +27980,12 @@ const createsSamplesDescriptor = (scorers, samples, epochs, selectedScore) => { score: maxSizes.score / base2 } }; - const scoreRendered = (sample) => { - const score2 = scoreValue(sample); - if (score2 === null || score2 === "undefined") { - return "null"; - } else if (scoreDescriptor.render) { - return scoreDescriptor.render(score2); - } else { - return score2; - } - }; - const scorerDescriptor = (sample, scorer) => { - return { - explanation: () => { - return scoreExplanation(sample, scorer); - }, - answer: () => { - return scoreAnswer(sample, scorer); - }, - scores: () => { - if (!sample || !sample.scores) { - return []; - } - const scoreNames = scorers.map((score2) => { - return score2.name; - }); - const sampleScorer = sample.scores[scorer]; - const scoreVal = sampleScorer.value; - if (typeof scoreVal === "object") { - const names = Object.keys(scoreVal); - if (names.find((name) => { - return !scoreNames.includes(name); - })) { - return [ - { - name: scorer, - rendered: () => { - return scoreDescriptor.render(scoreVal); - } - } - ]; - } else { - const scores = names.map((name) => { - return { - name, - rendered: () => { - return scoreDescriptor.render(scoreVal[name]); - } - }; - }); - return scores; - } - } else { - return [ - { - name: scorer, - rendered: () => { - return scoreDescriptor.render(scoreVal); - } - } - ]; - } - } - }; - }; return { - scoreDescriptor, - epochs, + evalDescriptor, messageShape, - selectedScore: (sample) => { - return { - value: scoreValue(sample), - render: () => { - return scoreRendered(sample); - } - }; - }, - scorer: (sample, scorer) => { - return scorerDescriptor(sample, scorer); - }, - selectedScorer: (sample) => { - return scorerDescriptor(sample, selectedScore == null ? void 0 : selectedScore.scorer); - } + selectedScoreDescriptor: evalDescriptor.scoreDescriptor(selectedScore), + selectedScore: (sample) => evalDescriptor.score(sample, selectedScore), + selectedScorerDescriptor: (sample) => evalDescriptor.scorerDescriptor(sample, selectedScore) }; }; const scoreCategorizers = [ @@ -25709,95 +28287,6 @@ const resolveAttachments = (value, attachments) => { } return value; }; -const filterFnForType = (filter) => { - if (filter.type) { - return filterFnsForType[filter.type]; - } else { - return void 0; - } -}; -const filterCategory = (descriptor, sample, value) => { - const score = descriptor.selectedScore(sample); - if (typeof score.value === "string") { - return score.value.toLowerCase() === (value == null ? void 0 : value.toLowerCase()); - } else if (typeof score.value === "object") { - return JSON.stringify(score.value) == value; - } else { - return String(score.value) === value; - } -}; -const filterText = (descriptor, sample, value) => { - const score = descriptor.selectedScore(sample); - if (!value) { - return true; - } else { - if (isNumeric(value)) { - if (typeof score.value === "number") { - return score.value === Number(value); - } else { - return Number(score.value) === Number(value); - } - } else { - const filters = [ - { - prefix: ">=", - fn: (score2, val) => { - return score2 >= val; - } - }, - { - prefix: "<=", - fn: (score2, val) => { - return score2 <= val; - } - }, - { - prefix: ">", - fn: (score2, val) => { - return score2 > val; - } - }, - { - prefix: "<", - fn: (score2, val) => { - return score2 < val; - } - }, - { - prefix: "=", - fn: (score2, val) => { - return score2 === val; - } - }, - { - prefix: "!=", - fn: (score2, val) => { - return score2 !== val; - } - } - ]; - for (const filter of filters) { - if (value == null ? void 0 : value.startsWith(filter.prefix)) { - const val = value.slice(filter.prefix.length).trim(); - if (!val) { - return true; - } - const num = Number(val); - return filter.fn(score.value, num); - } - } - if (typeof score.value === "string") { - return score.value.toLowerCase() === (value == null ? void 0 : value.toLowerCase()); - } else { - return String(score.value) === value; - } - } - } -}; -const filterFnsForType = { - [kScoreTypeCategorical]: filterCategory, - [kScoreTypeNumeric]: filterText -}; function App({ api: api2, initialState: initialState2 = void 0, @@ -25862,6 +28351,7 @@ function App({ const [sort, setSort] = h((initialState2 == null ? void 0 : initialState2.sort) || kDefaultSort); const [scores, setScores] = h((initialState2 == null ? void 0 : initialState2.scores) || []); const [score, setScore] = h(initialState2 == null ? void 0 : initialState2.score); + const [filterError, setFilterError] = h(initialState2 == null ? void 0 : initialState2.filterError); const [filteredSamples, setFilteredSamples] = h( (initialState2 == null ? void 0 : initialState2.filteredSamples) || [] ); @@ -25869,6 +28359,13 @@ function App({ const [groupByOrder, setGroupByOrder] = h( (initialState2 == null ? void 0 : initialState2.groupByOrder) || "asc" ); + const addToFilterExpression = (fragment) => { + setFilter(addFragmentToFilter(filter, fragment)); + const filterInput = document.getElementById("sample-filter-input"); + if (filterInput) { + filterInput.focus(); + } + }; const afterBodyElements = []; const saveState = q(() => { const state = { @@ -25995,42 +28492,51 @@ function App({ ] ); y(() => { - var _a3; + var _a3, _b3; const samples = ((_a3 = selectedLog == null ? void 0 : selectedLog.contents) == null ? void 0 : _a3.sampleSummaries) || []; + var newFilterError = void 0; const filtered = samples.filter((sample) => { if (epoch && epoch !== "all") { if (epoch !== sample.epoch + "") { return false; } } - const filterFn = filterFnForType(filter); - if (filterFn && filter.value) { - return filterFn(samplesDescriptor, sample, filter.value); + if (filter.value) { + const { matches, error: error2 } = filterExpression( + evalDescriptor, + sample, + filter.value + ); + newFilterError || (newFilterError = error2); + return matches; } else { return true; } }); const { sorted, order: order2 } = sortSamples(sort, filtered, samplesDescriptor); let grouping = "none"; - if ((samplesDescriptor == null ? void 0 : samplesDescriptor.epochs) > 1) { + if (((_b3 = samplesDescriptor == null ? void 0 : samplesDescriptor.evalDescriptor) == null ? void 0 : _b3.epochs) > 1) { if (byEpoch(sort) || epoch !== "all") { grouping = "epoch"; } else if (bySample(sort)) { grouping = "sample"; } } + setFilterError(newFilterError); setFilteredSamples(sorted); setGroupBy(grouping); setGroupByOrder(order2); }, [selectedLog, filter, sort, epoch]); - const samplesDescriptor = T(() => { + const evalDescriptor = T(() => { var _a3, _b3, _c2, _d2; - return createsSamplesDescriptor( + return createEvalDescriptor( scores, (_a3 = selectedLog.contents) == null ? void 0 : _a3.sampleSummaries, - ((_d2 = (_c2 = (_b3 = selectedLog.contents) == null ? void 0 : _b3.eval) == null ? void 0 : _c2.config) == null ? void 0 : _d2.epochs) || 1, - score + ((_d2 = (_c2 = (_b3 = selectedLog.contents) == null ? void 0 : _b3.eval) == null ? void 0 : _c2.config) == null ? void 0 : _d2.epochs) || 1 ); + }, [selectedLog, scores]); + const samplesDescriptor = T(() => { + return createSamplesDescriptor(evalDescriptor, score); }, [selectedLog, scores, score]); const refreshSampleTab = q( (sample) => { @@ -26453,7 +28959,9 @@ function App({ epoch=${epoch} setEpoch=${setEpoch} filter=${filter} + filterError=${filterError} setFilter=${setFilter} + addToFilterExpression=${addToFilterExpression} score=${score} setScore=${setScore} scores=${scores} diff --git a/src/inspect_ai/_view/www/package.json b/src/inspect_ai/_view/www/package.json index 13445b769..cf7beab84 100644 --- a/src/inspect_ai/_view/www/package.json +++ b/src/inspect_ai/_view/www/package.json @@ -31,6 +31,7 @@ "clipboard": "^2.0.11", "fast-json-patch": "^3.1.1", "fflate": "^0.8.2", + "filtrex": "^3.1.0", "htm": "^3.1.1", "json": "^11.0.0", "json5": "^2.2.3", diff --git a/src/inspect_ai/_view/www/src/App.mjs b/src/inspect_ai/_view/www/src/App.mjs index b2391190e..7908237e1 100644 --- a/src/inspect_ai/_view/www/src/App.mjs +++ b/src/inspect_ai/_view/www/src/App.mjs @@ -31,10 +31,16 @@ import { FindBand } from "./components/FindBand.mjs"; import { isVscode } from "./utils/Html.mjs"; import { getVscodeApi } from "./utils/vscode.mjs"; import { kDefaultSort } from "./constants.mjs"; -import { createsSamplesDescriptor } from "./samples/SamplesDescriptor.mjs"; +import { + createEvalDescriptor, + createSamplesDescriptor, +} from "./samples/SamplesDescriptor.mjs"; import { byEpoch, bySample, sortSamples } from "./samples/tools/SortFilter.mjs"; import { resolveAttachments } from "./utils/attachments.mjs"; -import { filterFnForType } from "./samples/tools/filters.mjs"; +import { + addFragmentToFilter, + filterExpression, +} from "./samples/tools/filters.mjs"; import { kEvalWorkspaceTabId, @@ -159,6 +165,7 @@ export function App({ const [score, setScore] = useState(initialState?.score); // Re-filter the samples + const [filterError, setFilterError] = useState(initialState?.filterError); const [filteredSamples, setFilteredSamples] = useState( initialState?.filteredSamples || [], ); @@ -167,6 +174,14 @@ export function App({ initialState?.groupByOrder || "asc", ); + const addToFilterExpression = (fragment) => { + setFilter(addFragmentToFilter(filter, fragment)); + const filterInput = document.getElementById("sample-filter-input"); + if (filterInput) { + filterInput.focus(); + } + }; + const afterBodyElements = []; const saveState = useCallback(() => { const state = { @@ -303,6 +318,7 @@ export function App({ useEffect(() => { const samples = selectedLog?.contents?.sampleSummaries || []; + var newFilterError = undefined; const filtered = samples.filter((sample) => { // Filter by epoch if specified if (epoch && epoch !== "all") { @@ -312,9 +328,14 @@ export function App({ } // Apply the filter - const filterFn = filterFnForType(filter); - if (filterFn && filter.value) { - return filterFn(samplesDescriptor, sample, filter.value); + if (filter.value) { + const { matches, error } = filterExpression( + evalDescriptor, + sample, + filter.value, + ); + newFilterError ||= error; + return matches; } else { return true; } @@ -325,7 +346,7 @@ export function App({ // Set the grouping let grouping = "none"; - if (samplesDescriptor?.epochs > 1) { + if (samplesDescriptor?.evalDescriptor?.epochs > 1) { if (byEpoch(sort) || epoch !== "all") { grouping = "epoch"; } else if (bySample(sort)) { @@ -333,18 +354,22 @@ export function App({ } } + setFilterError(newFilterError); setFilteredSamples(sorted); setGroupBy(grouping); setGroupByOrder(order); }, [selectedLog, filter, sort, epoch]); - const samplesDescriptor = useMemo(() => { - return createsSamplesDescriptor( + const evalDescriptor = useMemo(() => { + return createEvalDescriptor( scores, selectedLog.contents?.sampleSummaries, selectedLog.contents?.eval?.config?.epochs || 1, - score, ); + }, [selectedLog, scores]); + + const samplesDescriptor = useMemo(() => { + return createSamplesDescriptor(evalDescriptor, score); }, [selectedLog, scores, score]); const refreshSampleTab = useCallback( @@ -907,7 +932,9 @@ export function App({ epoch=${epoch} setEpoch=${setEpoch} filter=${filter} + filterError=${filterError} setFilter=${setFilter} + addToFilterExpression=${addToFilterExpression} score=${score} setScore=${setScore} scores=${scores} diff --git a/src/inspect_ai/_view/www/src/Types.mjs b/src/inspect_ai/_view/www/src/Types.mjs index 74278e548..3d08e76e4 100644 --- a/src/inspect_ai/_view/www/src/Types.mjs +++ b/src/inspect_ai/_view/www/src/Types.mjs @@ -26,7 +26,6 @@ /** * @typedef {Object} ScoreFilter * @property {string} [value] - * @property {string} [type] */ /** diff --git a/src/inspect_ai/_view/www/src/api/Types.mjs b/src/inspect_ai/_view/www/src/api/Types.mjs index 7b1bb9103..c5a0d5616 100644 --- a/src/inspect_ai/_view/www/src/api/Types.mjs +++ b/src/inspect_ai/_view/www/src/api/Types.mjs @@ -30,15 +30,26 @@ * @property { import("../types/log").Input } input * @property { import("../types/log").Target } target * @property { import("../types/log").Scores1 } scores + * @property { string } [error] * @property { import("../types/log").Type11 } [limit] */ /** -* @typedef {Object} Capabilities -* @property {boolean} downloadFiles - Indicates if file downloads are supported. -* @property {boolean} webWorkers - Indicates if web workers are supported. -* + * Fields shared by EvalSample and SampleSummary. + * Contains only fields that are copied verbatim in src/inspect_ai/log/_recorders/eval.py. + * + * @typedef {Object} SampleInterface + * @property { number | string } id + * @property { number } epoch + * @property { import("../types/log").Target } target + * @property { import("../types/log").Scores1 } scores + */ +/** + * @typedef {Object} Capabilities + * @property {boolean} downloadFiles - Indicates if file downloads are supported. + * @property {boolean} webWorkers - Indicates if web workers are supported. + */ /** * @typedef {Object} LogViewAPI diff --git a/src/inspect_ai/_view/www/src/navbar/Navbar.mjs b/src/inspect_ai/_view/www/src/navbar/Navbar.mjs index 0d1399cb7..d13d5c7ce 100644 --- a/src/inspect_ai/_view/www/src/navbar/Navbar.mjs +++ b/src/inspect_ai/_view/www/src/navbar/Navbar.mjs @@ -19,6 +19,8 @@ import { SecondaryBar } from "./SecondaryBar.mjs"; * @param {import("../types/log").EvalPlan} [props.evalPlan] - The EvalSpec * @param {import("../types/log").EvalStats} [props.evalStats] - The EvalStats * @param {import("../api/Types.mjs").SampleSummary[]} [props.samples] - the samples + * @param {import("../samples/SamplesDescriptor.mjs").EvalDescriptor} [props.evalDescriptor] - The EvalDescriptor + * @param {(fragment: string) => void} props.addToFilterExpression - add to the current filter expression * @param {string} [props.status] - the status * @param {boolean} props.offcanvas - Are we in offcanvas mode? * @param {boolean} props.showToggle - Should we show the toggle? @@ -32,6 +34,8 @@ export const Navbar = ({ evalResults, evalStats, samples, + evalDescriptor, + addToFilterExpression, showToggle, offcanvas, status, @@ -180,6 +184,8 @@ export const Navbar = ({ evalResults=${evalResults} evalStats=${evalStats} samples=${samples} + evalDescriptor=${evalDescriptor} + addToFilterExpression=${addToFilterExpression} status=${status} style=${{ gridColumn: "1/-1" }} /> diff --git a/src/inspect_ai/_view/www/src/navbar/SecondaryBar.mjs b/src/inspect_ai/_view/www/src/navbar/SecondaryBar.mjs index e101c1067..5e0232175 100644 --- a/src/inspect_ai/_view/www/src/navbar/SecondaryBar.mjs +++ b/src/inspect_ai/_view/www/src/navbar/SecondaryBar.mjs @@ -1,8 +1,10 @@ import { html } from "htm/preact"; +import { useState } from "preact/hooks"; import { LabeledValue } from "../components/LabeledValue.mjs"; import { formatDataset, formatDuration } from "../utils/Format.mjs"; import { ExpandablePanel } from "../components/ExpandablePanel.mjs"; +import { scoreFilterItems } from "../samples/tools/filters.mjs"; /** * Renders the Navbar @@ -13,6 +15,8 @@ import { ExpandablePanel } from "../components/ExpandablePanel.mjs"; * @param {import("../types/log").EvalResults} [props.evalResults] - The EvalResults * @param {import("../types/log").EvalStats} [props.evalStats] - The EvalStats * @param {import("../api/Types.mjs").SampleSummary[]} [props.samples] - the samples + * @param {import("../samples/SamplesDescriptor.mjs").EvalDescriptor} [props.evalDescriptor] - The EvalDescriptor + * @param {(fragment: string) => void} props.addToFilterExpression - add to the current filter expression * @param {string} [props.status] - the status * @param {Map} [props.style] - is this off canvas * @@ -24,6 +28,8 @@ export const SecondaryBar = ({ evalResults, evalStats, samples, + evalDescriptor, + addToFilterExpression, status, style, }) => { @@ -56,19 +62,10 @@ export const SecondaryBar = ({ `, }); - const label = evalResults?.scores.length > 1 ? "Scorers" : "Scorer"; - values.push({ - size: "minmax(12%, auto)", - value: html`<${LabeledValue} label="${label}" style=${staticColStyle} style=${{ justifySelf: hasConfig ? "left" : "center" }}> - <${ScorerSummary} - scorers=${evalResults?.scores} /> - `, - }); - if (hasConfig) { values.push({ size: "minmax(12%, auto)", - value: html`<${LabeledValue} label="Config" style=${{ justifySelf: "right" }}> + value: html`<${LabeledValue} label="Config" style=${{ justifySelf: "center" }}> <${ParamSummary} params=${hyperparameters}/> `, }); @@ -81,11 +78,21 @@ export const SecondaryBar = ({ values.push({ size: "minmax(12%, auto)", value: html` - <${LabeledValue} label="Duration" style=${{ justifySelf: "right" }}> + <${LabeledValue} label="Duration" style=${{ justifySelf: "center" }}> ${totalDuration} `, }); + const label = evalResults?.scores.length > 1 ? "Scorers" : "Scorer"; + values.push({ + size: "minmax(12%, auto)", + value: html`<${LabeledValue} label="${label}" style=${staticColStyle} style=${{ justifySelf: "right" }}> + <${ScorerSummary} + evalDescriptor=${evalDescriptor} + addToFilterExpression=${addToFilterExpression} /> + `, + }); + return html` <${ExpandablePanel} style=${{ margin: "0", ...style }} collapse=${true} lines=${4}>
{ `; }; -const ScorerSummary = ({ scorers }) => { - if (!scorers) { +const FilterableItem = ({ + item, + index, + openSuggestionIndex, + setOpenSuggestionIndex, + addToFilterExpression, +}) => { + const handleClick = () => { + if (item.suggestions.length === 0) { + addToFilterExpression(item.canonicalName); + } else { + setOpenSuggestionIndex(openSuggestionIndex === index ? null : index); + } + }; + + const handleSuggestionClick = (suggestion) => { + addToFilterExpression(suggestion); + setOpenSuggestionIndex(null); + }; + + /** @param {HTMLElement} el */ + const popupRef = (el) => { + if (el && openSuggestionIndex === index) { + const rect = el.previousElementSibling.getBoundingClientRect(); + const viewportWidth = window.innerWidth; + const popupWidth = el.offsetWidth; + const finalLeft = + rect.left + popupWidth > viewportWidth + ? rect.right - popupWidth + : rect.left; + el.style.setProperty("--popup-left", `${finalLeft}px`); + el.style.setProperty("--popup-top", `${rect.bottom + 4}px`); + } + }; + + return html` +
+ + ${item.canonicalName} + + ${item.suggestions.length > 0 && + // Use fixed position to avoid being clipped by `ExpandablePanel`. + html` +
+ ${item.suggestions.map( + (suggestion) => html` +
handleSuggestionClick(suggestion)} + > + ${suggestion} +
+ `, + )} +
+ `} +
+ `; +}; + +const ScorerSummary = ({ evalDescriptor, addToFilterExpression }) => { + if (!evalDescriptor) { return ""; } - const uniqScorers = new Set(); - scorers.forEach((scorer) => { - uniqScorers.add(scorer.name); - }); + const items = scoreFilterItems(evalDescriptor); + const [openSuggestionIndex, setOpenSuggestionIndex] = useState(null); - return Array.from(uniqScorers).join(", "); + return html` + + ${Array.from(items).map( + (item, index) => html` + ${index > 0 ? ", " : ""} + ${item.isFilterable + ? html`<${FilterableItem} + item=${item} + index=${index} + openSuggestionIndex=${openSuggestionIndex} + setOpenSuggestionIndex=${setOpenSuggestionIndex} + addToFilterExpression=${addToFilterExpression} + />` + : html`${item.canonicalName}`} + `, + )} + + `; }; /** diff --git a/src/inspect_ai/_view/www/src/samples/SampleDisplay.mjs b/src/inspect_ai/_view/www/src/samples/SampleDisplay.mjs index 014f8cf84..c0c898a51 100644 --- a/src/inspect_ai/_view/www/src/samples/SampleDisplay.mjs +++ b/src/inspect_ai/_view/www/src/samples/SampleDisplay.mjs @@ -352,6 +352,14 @@ const metadataViewsForSample = (id, sample) => { return sampleMetadatas; }; +/** + * @param {Object} props - The parameters for the component. + * @param {string} props.id - The unique identifier for the sample. + * @param {import("../types/log").EvalSample} props.sample - The sample. + * @param {Object} props.style - The style for the element + * @param {import("../samples/SamplesDescriptor.mjs").SamplesDescriptor} props.sampleDescriptor - The sample descriptor. + * @returns {import("preact").JSX.Element} The SampleSummary component. + */ const SampleSummary = ({ id, sample, style, sampleDescriptor }) => { const input = sampleDescriptor?.messageShape.normalized.input > 0 @@ -414,7 +422,7 @@ const SampleSummary = ({ id, sample, style, sampleDescriptor }) => { const fullAnswer = sample && sampleDescriptor - ? sampleDescriptor.selectedScorer(sample).answer() + ? sampleDescriptor.selectedScorerDescriptor(sample).answer() : undefined; if (fullAnswer) { columns.push({ diff --git a/src/inspect_ai/_view/www/src/samples/SampleList.mjs b/src/inspect_ai/_view/www/src/samples/SampleList.mjs index 3fbd749bf..1374edf8e 100644 --- a/src/inspect_ai/_view/www/src/samples/SampleList.mjs +++ b/src/inspect_ai/_view/www/src/samples/SampleList.mjs @@ -17,7 +17,22 @@ import { inputString } from "../utils/Format.mjs"; const kSampleHeight = 88; const kSeparatorHeight = 24; -// Convert samples to a datastructure which contemplates grouping, etc... +/** + * Convert samples to a datastructure which contemplates grouping, etc... + * + * @param {Object} props - The parameters for the component. + * @param {Object} props.listRef - The ref for the list. + * @param {import("./SamplesTab.mjs").ListItem[]} props.items - The samples. + * @param {import("../samples/SamplesDescriptor.mjs").SamplesDescriptor} props.sampleDescriptor - The sample descriptor. + * @param {Object} props.style - The style for the element + * @param {number} props.selectedIndex - The index of the selected sample. + * @param {(index: number) => void} props.setSelectedIndex - The function to set the selected sample index. + * @param {import("../Types.mjs").ScoreLabel} props.selectedScore - The function to get the selected score. + * @param {() => void} props.nextSample - The function to move to the next sample. + * @param {() => void} props.prevSample - The function to move to the previous sample. + * @param {(index: number) => void} props.showSample - The function to show the sample. + * @returns {import("preact").JSX.Element} The SampleList component. + */ export const SampleList = (props) => { const { listRef, @@ -93,6 +108,7 @@ export const SampleList = (props) => { } }, [selectedIndex, rowMap, listRef]); + /** @param {import("./SamplesTab.mjs").ListItem} item */ const renderRow = (item) => { if (item.type === "sample") { return html` @@ -192,6 +208,7 @@ export const SampleList = (props) => { // Count any sample errors and display a bad alerting the user // to any errors const errorCount = items?.reduce((previous, item) => { + // @ts-ignore if (item.data.error) { return previous + 1; } else { @@ -201,6 +218,7 @@ export const SampleList = (props) => { // Count limits const limitCount = items?.reduce((previous, item) => { + // @ts-ignore if (item.data.limit) { return previous + 1; } else { @@ -260,6 +278,17 @@ const SeparatorRow = ({ id, title, height }) => {
`; }; +/** + * @param {Object} props - The parameters for the component. + * @param {string} props.id - The unique identifier for the sample. + * @param {number} props.index - The index of the sample. + * @param {import("../api/Types.mjs").SampleSummary} props.sample - The sample. + * @param {import("../samples/SamplesDescriptor.mjs").SamplesDescriptor} props.sampleDescriptor - The sample descriptor. + * @param {number} props.height - The height of the sample row. + * @param {boolean} props.selected - Whether the sample is selected. + * @param {(index: number) => void} props.showSample - The function to show the sample. + * @returns {import("preact").JSX.Element} The SampleRow component. + */ const SampleRow = ({ id, index, @@ -339,7 +368,9 @@ const SampleRow = ({ ${sample ? html` <${MarkdownDiv} - markdown=${sampleDescriptor?.selectedScorer(sample).answer()} + markdown=${sampleDescriptor + ?.selectedScorerDescriptor(sample) + .answer()} style=${{ paddingLeft: "0" }} class="no-last-para-padding" /> diff --git a/src/inspect_ai/_view/www/src/samples/SampleScoreView.mjs b/src/inspect_ai/_view/www/src/samples/SampleScoreView.mjs index 22d201066..8e8339e04 100644 --- a/src/inspect_ai/_view/www/src/samples/SampleScoreView.mjs +++ b/src/inspect_ai/_view/www/src/samples/SampleScoreView.mjs @@ -13,6 +13,14 @@ const labelStyle = { ...TextStyle.secondary, }; +/** + * @param {Object} props - The component props. + * @param {import("../types/log").EvalSample} props.sample - The sample. + * @param {import("../samples/SamplesDescriptor.mjs").SamplesDescriptor} props.sampleDescriptor - The sample descriptor. + * @param {Object} props.style - The style for the element. + * @param {string} props.scorer - The scorer. + * @returns {import("preact").JSX.Element} The SampleScoreView component. + */ export const SampleScoreView = ({ sample, sampleDescriptor, @@ -20,7 +28,7 @@ export const SampleScoreView = ({ scorer, }) => { if (!sampleDescriptor) { - return ""; + return html``; } const scoreInput = inputString(sample.input); if (sample.choices && sample.choices.length > 0) { @@ -32,7 +40,10 @@ export const SampleScoreView = ({ ); } - const scorerDescriptor = sampleDescriptor.scorer(sample, scorer); + const scorerDescriptor = sampleDescriptor.evalDescriptor.scorerDescriptor( + sample, + { scorer, name: scorer }, + ); const explanation = scorerDescriptor.explanation() || "(No Explanation)"; const answer = scorerDescriptor.answer(); @@ -147,40 +158,47 @@ export const SampleScoreView = ({ ` : ""} - ${sample?.score?.metadata && - Object.keys(sample?.score?.metadata).length > 0 - ? html` - - - - - - - - - - -
- Metadata -
- <${MetaDataView} - id="task-sample-score-metadata" - classes="tab-pane" - entries="${sample?.score?.metadata}" - style=${{ marginTop: "1em" }} - /> -
` - : ""} + ${ + // @ts-ignore + sample?.score?.metadata && + // @ts-ignore + Object.keys(sample?.score?.metadata).length > 0 + ? html` + + + + + + + + + + +
+ Metadata +
+ <${MetaDataView} + id="task-sample-score-metadata" + classes="tab-pane" + entries="${ + // @ts-ignore + sample?.score?.metadata + }" + style=${{ marginTop: "1em" }} + /> +
` + : "" + } `; }; diff --git a/src/inspect_ai/_view/www/src/samples/SampleScores.mjs b/src/inspect_ai/_view/www/src/samples/SampleScores.mjs index 11e8d8725..4f257540f 100644 --- a/src/inspect_ai/_view/www/src/samples/SampleScores.mjs +++ b/src/inspect_ai/_view/www/src/samples/SampleScores.mjs @@ -1,9 +1,18 @@ import { html } from "htm/preact"; +/** + * @param {Object} props + * @param {import("../api/Types.mjs").SampleSummary} props.sample + * @param {import("../samples/SamplesDescriptor.mjs").SamplesDescriptor} props.sampleDescriptor + * @param {string} props.scorer + * @returns {import("preact").JSX.Element} + */ export const SampleScores = ({ sample, sampleDescriptor, scorer }) => { const scores = scorer - ? sampleDescriptor.scorer(sample, scorer).scores() - : sampleDescriptor.selectedScorer(sample).scores(); + ? sampleDescriptor.evalDescriptor + .scorerDescriptor(sample, { scorer, name: scorer }) + .scores() + : sampleDescriptor.selectedScorerDescriptor(sample).scores(); if (scores.length === 1) { return scores[0].rendered(); diff --git a/src/inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs b/src/inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs index 3c5282f5e..a0ef3a808 100644 --- a/src/inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs +++ b/src/inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs @@ -17,15 +17,26 @@ import { kScoreTypePassFail, } from "../constants.mjs"; +/** + * Represents a utility summary of the samples that doesn't change with the selected score. + * @typedef {Object} EvalDescriptor + * @property {number} epochs - The number of epochs. + * @property {import("../api/Types.mjs").SampleSummary[]} samples - The list of sample summaries. + * @property {import("../Types.mjs").ScoreLabel[]} scores - the list of available scores + * @property {(sample: import("../api/Types.mjs").SampleInterface, scoreLabel: import("../Types.mjs").ScoreLabel) => ScorerDescriptor} scorerDescriptor - Returns the scorer descriptor for a sample and a specified scorer. + * @property {(scoreLabel: import("../Types.mjs").ScoreLabel) => ScoreDescriptor} scoreDescriptor - Provides information about the score types and how to render them. + * @property {(sample: import("../api/Types.mjs").SampleInterface, scoreLabel: import("../Types.mjs").ScoreLabel) => SelectedScore} score - Returns information about a score for a sample. + * @property {(sample: import("../api/Types.mjs").SampleInterface, scorer: string) => string} scoreAnswer - Returns the answer for a sample and a specified scorer. + */ + /** * Represents a utility summary of the samples. * @typedef {Object} SamplesDescriptor - * @property {ScoreDescriptor} scoreDescriptor - Provides information about the score types and how to render them. - * @property {number} epochs - The number of epochs. + * @property {EvalDescriptor} evalDescriptor - The EvalDescriptor. * @property {MessageShape} messageShape - The normalized sizes of input, target, and answer messages. - * @property {(sample: import("../api/Types.mjs").SampleSummary) => SelectedScore} selectedScore - Returns the selected score for a sample. - * @property {(sample: import("../api/Types.mjs").SampleSummary, scorer: string) => ScorerDescriptor} scorer - Returns the scorer descriptor for a sample and a specified scorer. - * @property {(sample: import("../api/Types.mjs").SampleSummary) => ScorerDescriptor} selectedScorer - Returns the scorer descriptor for a sample using the selected scorer. + * @property {ScoreDescriptor} selectedScoreDescriptor - Provides information about the score types and how to render them. + * @property {(sample: import("../api/Types.mjs").SampleInterface) => SelectedScore} selectedScore - Returns the selected score for a sample. + * @property {(sample: import("../api/Types.mjs").SampleInterface) => ScorerDescriptor} selectedScorerDescriptor - Returns the scorer descriptor for a sample using the selected scorer. */ /** @@ -48,7 +59,7 @@ import { */ /** - * Represents the selected score for a sample, including its value and render function. + * Represents a score for a sample, including its value and render function. * @typedef {Object} SelectedScore * @property {import("../types/log").Value2} value - The value of the selected score. * @property {function(): any} render - Function to render the selected score. @@ -72,69 +83,48 @@ import { */ /** - * Provides a utility summary of the samples - * - * @param {import("../Types.mjs").ScoreLabel[]} scorers - the list of available scores + * @param {import("../Types.mjs").ScoreLabel[]} scores - the list of available scores * @param {import("../api/Types.mjs").SampleSummary[]} samples - the list of sample summaries * @param {number} epochs - The number of epochs - * @param {import("../Types.mjs").ScoreLabel} [selectedScore] - the currently selected score - * @returns {SamplesDescriptor} The SamplesDescriptor + * @returns {EvalDescriptor} The EvalDescriptor */ -export const createsSamplesDescriptor = ( - scorers, - samples, - epochs, - selectedScore, -) => { +export const createEvalDescriptor = (scores, samples, epochs) => { if (!samples) { return undefined; } /** - * @param {import("../api/Types.mjs").SampleSummary} sample - the currently selected score - * @param {string} scorer - the scorer name - * @returns {import("../types/log").Score} The Score - */ - const score = (sample, scorer = selectedScore?.scorer) => { - if (sample.scores[scorer]) { - return sample.scores[scorer]; - } else { - return undefined; - } - }; - - /** - * @param {import("../api/Types.mjs").SampleSummary} sample - the currently selected score + * @param {import("../api/Types.mjs").SampleInterface} sample - the currently selected score + * @param {import("../Types.mjs").ScoreLabel} scoreLabel - the score label * @returns {import("../types/log").Value2} The Score */ - const scoreValue = (sample) => { + const scoreValue = (sample, scoreLabel) => { // no scores, no value - if (Object.keys(sample.scores).length === 0 || !selectedScore) { + if (Object.keys(sample.scores).length === 0 || !scoreLabel) { return undefined; } if ( - selectedScore.scorer !== selectedScore.name && - sample.scores[selectedScore.scorer] && - sample.scores[selectedScore.scorer].value + scoreLabel.scorer !== scoreLabel.name && + sample.scores[scoreLabel.scorer] && + sample.scores[scoreLabel.scorer].value ) { - return sample.scores[selectedScore.scorer].value[selectedScore.name]; - } else if (sample.scores[selectedScore.name]) { - return sample.scores[selectedScore.name].value; + return sample.scores[scoreLabel.scorer].value[scoreLabel.name]; + } else if (sample.scores[scoreLabel.name]) { + return sample.scores[scoreLabel.name].value; } else { return undefined; } }; - // Retrieve the answer for a sample /** - * @param {import("../api/Types.mjs").SampleSummary} sample - the currently selected score + * @param {import("../api/Types.mjs").SampleInterface} sample - the currently selected score * @param {string} scorer - the scorer name * @returns {string} The answer */ const scoreAnswer = (sample, scorer) => { if (sample) { - const sampleScore = score(sample, scorer); + const sampleScore = sample.scores[scorer]; if (sampleScore && sampleScore.answer) { return sampleScore.answer; } @@ -143,162 +133,132 @@ export const createsSamplesDescriptor = ( } }; - // Retrieve the answer for a sample /** - * @param {import("../api/Types.mjs").SampleSummary} sample - the currently selected score + * @param {import("../api/Types.mjs").SampleInterface} sample - the currently selected score * @param {string} scorer - the scorer name * @returns {string} The explanation */ const scoreExplanation = (sample, scorer) => { if (sample) { - const sampleScore = score(sample, scorer); + const sampleScore = sample.scores[scorer]; if (sampleScore && sampleScore.explanation) { return sampleScore.explanation; } } return undefined; }; - const uniqScoreValues = [ - ...new Set( - samples - .filter((sample) => !!sample.scores) - .filter((sample) => { - // There is no selected scorer, so include this value - if (!selectedScore) { - return true; - } - if (selectedScore.scorer !== selectedScore.name) { - return ( - Object.keys(sample.scores).includes(selectedScore.scorer) && - Object.keys(sample.scores[selectedScore.scorer].value).includes( - selectedScore.name, - ) - ); - } else { - return Object.keys(sample.scores).includes(selectedScore.name); - } - }) - .map((sample) => { - return scoreValue(sample); - }) - .filter((value) => { - return value !== null; - }), - ), - ]; - const uniqScoreTypes = [ - ...new Set(uniqScoreValues.map((scoreValue) => typeof scoreValue)), - ]; - - /** @type {ScoreDescriptor} */ - let scoreDescriptor; - for (const categorizer of scoreCategorizers) { - scoreDescriptor = categorizer.describe(uniqScoreValues, uniqScoreTypes); - if (scoreDescriptor) { - break; - } - } + /** + * @param {import("../Types.mjs").ScoreLabel} scoreLabel + * @returns {string} + */ + const scoreLabelKey = (scoreLabel) => { + return `${scoreLabel.scorer}.${scoreLabel.name}`; + }; - // Find the total length of the value so we can compute an average - const sizes = samples.reduce( - (previous, current) => { - const text = inputString(current.input).join(" "); - const scoreText = scoreValue(current) ? String(scoreValue(current)) : ""; - previous[0] = Math.min(Math.max(previous[0], text.length), 300); - previous[1] = Math.min( - Math.max(previous[1], arrayToString(current.target).length), - 300, - ); - previous[2] = Math.min( - Math.max( - previous[2], - scoreAnswer(current, selectedScore?.name)?.length || 0, - ), - 300, - ); - previous[3] = Math.min( - Math.max(previous[3], current.limit ? current.limit.length : 0), - 50, - ); - previous[4] = Math.min( - Math.max(previous[4], String(current.id).length), - 10, - ); - previous[5] = Math.min(Math.max(previous[5], scoreText.length), 30); + /** + * The EvalDescriptor is memoized. Compute all descriptors now to avoid duplicate work. + * @type {Map} + */ + const scoreDescriptorMap = new Map(); + for (const scoreLabel of scores) { + const uniqScoreValues = [ + ...new Set( + samples + .filter((sample) => !!sample.scores) + .filter((sample) => { + // There is no selected scorer, so include this value + if (!scoreLabel) { + return true; + } - return previous; - }, - [0, 0, 0, 0, 0, 0], - ); + if (scoreLabel.scorer !== scoreLabel.name) { + return ( + Object.keys(sample.scores).includes(scoreLabel.scorer) && + Object.keys(sample.scores[scoreLabel.scorer].value).includes( + scoreLabel.name, + ) + ); + } else { + return Object.keys(sample.scores).includes(scoreLabel.name); + } + }) + .map((sample) => { + return scoreValue(sample, scoreLabel); + }) + .filter((value) => { + return value !== null; + }), + ), + ]; + const uniqScoreTypes = [ + ...new Set(uniqScoreValues.map((scoreValue) => typeof scoreValue)), + ]; - // normalize to base 1 - const maxSizes = { - input: Math.min(sizes[0], 300), - target: Math.min(sizes[1], 300), - answer: Math.min(sizes[2], 300), - limit: Math.min(sizes[3], 50), - id: Math.min(sizes[4], 10), - score: Math.min(sizes[4], 30), - }; - const base = - maxSizes.input + - maxSizes.target + - maxSizes.answer + - maxSizes.limit + - maxSizes.id + - maxSizes.score || 1; - const messageShape = { - raw: { - input: sizes[0], - target: sizes[1], - answer: sizes[2], - limit: sizes[3], - id: sizes[4], - score: sizes[5], - }, - normalized: { - input: maxSizes.input / base, - target: maxSizes.target / base, - answer: maxSizes.answer / base, - limit: maxSizes.limit / base, - id: maxSizes.id / base, - score: maxSizes.score / base, - }, + for (const categorizer of scoreCategorizers) { + const scoreDescriptor = categorizer.describe( + uniqScoreValues, + uniqScoreTypes, + ); + if (scoreDescriptor) { + scoreDescriptorMap.set(scoreLabelKey(scoreLabel), scoreDescriptor); + break; + } + } + } + + /** + * @param {import("../Types.mjs").ScoreLabel} scoreLabel + * @returns {ScoreDescriptor} + */ + const scoreDescriptor = (scoreLabel) => { + return scoreDescriptorMap.get(scoreLabelKey(scoreLabel)); }; - const scoreRendered = (sample) => { - const score = scoreValue(sample); + /** + * @param {import("../api/Types.mjs").SampleInterface} sample + * @param {import("../Types.mjs").ScoreLabel} scoreLabel + * @returns {any} + */ + const scoreRendered = (sample, scoreLabel) => { + const descriptor = scoreDescriptor(scoreLabel); + const score = scoreValue(sample, scoreLabel); if (score === null || score === "undefined") { return "null"; - } else if (scoreDescriptor.render) { - return scoreDescriptor.render(score); + } else if (descriptor.render) { + return descriptor.render(score); } else { return score; } }; - const scorerDescriptor = (sample, scorer) => { + /** + * @param {import("../api/Types.mjs").SampleInterface} sample + * @param {import("../Types.mjs").ScoreLabel} scoreLabel + * @returns {ScorerDescriptor} + */ + const scorerDescriptor = (sample, scoreLabel) => { return { explanation: () => { - return scoreExplanation(sample, scorer); + return scoreExplanation(sample, scoreLabel.scorer); }, answer: () => { - return scoreAnswer(sample, scorer); + return scoreAnswer(sample, scoreLabel.scorer); }, scores: () => { if (!sample || !sample.scores) { return []; } + const myScoreDescriptor = scoreDescriptor(scoreLabel); // Make a list of all the valid score names (this is // used to distinguish between dictionaries that contain // scores that should be treated as standlone scores and // dictionaries that just contain random values, which is allowed) - const scoreNames = scorers.map((score) => { + const scoreNames = scores.map((score) => { return score.name; }); - const sampleScorer = sample.scores[scorer]; + const sampleScorer = sample.scores[scoreLabel.scorer]; const scoreVal = sampleScorer.value; if (typeof scoreVal === "object") { const names = Object.keys(scoreVal); @@ -311,9 +271,9 @@ export const createsSamplesDescriptor = ( // we just treat it like an opaque dictionary return [ { - name: scorer, + name: scoreLabel.scorer, rendered: () => { - return scoreDescriptor.render(scoreVal); + return myScoreDescriptor.render(scoreVal); }, }, ]; @@ -324,7 +284,7 @@ export const createsSamplesDescriptor = ( return { name, rendered: () => { - return scoreDescriptor.render(scoreVal[name]); + return myScoreDescriptor.render(scoreVal[name]); }, }; }); @@ -333,9 +293,9 @@ export const createsSamplesDescriptor = ( } else { return [ { - name: scorer, + name: scoreLabel.scorer, rendered: () => { - return scoreDescriptor.render(scoreVal); + return myScoreDescriptor.render(scoreVal); }, }, ]; @@ -344,25 +304,119 @@ export const createsSamplesDescriptor = ( }; }; + /** + * @param {import("../api/Types.mjs").SampleInterface} sample + * @param {import("../Types.mjs").ScoreLabel} scoreLabel + * @returns {SelectedScore} + */ + const score = (sample, scoreLabel) => { + return { + value: scoreValue(sample, scoreLabel), + render: () => { + return scoreRendered(sample, scoreLabel); + }, + }; + }; + return { - scoreDescriptor, epochs, - messageShape, - selectedScore: (sample) => { - return { - value: scoreValue(sample), - render: () => { - return scoreRendered(sample); - }, - }; + samples, + scores, + scorerDescriptor, + scoreDescriptor, + score, + scoreAnswer, + }; +}; + +/** + * Provides a utility summary of the samples + * + * @param {EvalDescriptor} evalDescriptor - The EvalDescriptor. + * @param {import("../Types.mjs").ScoreLabel} selectedScore - Selected score. + * @returns {SamplesDescriptor} - The SamplesDescriptor. + */ +export const createSamplesDescriptor = (evalDescriptor, selectedScore) => { + if (!evalDescriptor) { + return undefined; + } + + // Find the total length of the value so we can compute an average + const sizes = evalDescriptor.samples.reduce( + (previous, current) => { + const text = inputString(current.input).join(" "); + const scoreValue = evalDescriptor.score(current, selectedScore).value; + const scoreText = scoreValue ? String(scoreValue) : ""; + previous[0] = Math.min(Math.max(previous[0], text.length), 300); + previous[1] = Math.min( + Math.max(previous[1], arrayToString(current.target).length), + 300, + ); + previous[2] = Math.min( + Math.max( + previous[2], + evalDescriptor.scoreAnswer(current, selectedScore?.name)?.length || 0, + ), + 300, + ); + previous[3] = Math.min( + Math.max(previous[3], current.limit ? current.limit.length : 0), + 50, + ); + previous[4] = Math.min( + Math.max(previous[4], String(current.id).length), + 10, + ); + previous[5] = Math.min(Math.max(previous[5], scoreText.length), 30); + + return previous; }, - scorer: (sample, scorer) => { - return scorerDescriptor(sample, scorer); + [0, 0, 0, 0, 0, 0], + ); + + // normalize to base 1 + const maxSizes = { + input: Math.min(sizes[0], 300), + target: Math.min(sizes[1], 300), + answer: Math.min(sizes[2], 300), + limit: Math.min(sizes[3], 50), + id: Math.min(sizes[4], 10), + score: Math.min(sizes[4], 30), + }; + const base = + maxSizes.input + + maxSizes.target + + maxSizes.answer + + maxSizes.limit + + maxSizes.id + + maxSizes.score || 1; + const messageShape = { + raw: { + input: sizes[0], + target: sizes[1], + answer: sizes[2], + limit: sizes[3], + id: sizes[4], + score: sizes[5], }, - selectedScorer: (sample) => { - return scorerDescriptor(sample, selectedScore?.scorer); + normalized: { + input: maxSizes.input / base, + target: maxSizes.target / base, + answer: maxSizes.answer / base, + limit: maxSizes.limit / base, + id: maxSizes.id / base, + score: maxSizes.score / base, }, }; + + return { + evalDescriptor, + messageShape, + selectedScoreDescriptor: evalDescriptor.scoreDescriptor(selectedScore), + selectedScore: (sample) => evalDescriptor.score(sample, selectedScore), + selectedScorerDescriptor: (sample) => + evalDescriptor.scorerDescriptor(sample, selectedScore), + }; }; /** diff --git a/src/inspect_ai/_view/www/src/samples/SamplesTab.mjs b/src/inspect_ai/_view/www/src/samples/SamplesTab.mjs index 8d9d17e98..39b8f6e9f 100644 --- a/src/inspect_ai/_view/www/src/samples/SamplesTab.mjs +++ b/src/inspect_ai/_view/www/src/samples/SamplesTab.mjs @@ -55,7 +55,9 @@ export const SamplesTab = ({ sampleScrollPositionRef, setSampleScrollPosition, }) => { + /** @type {[ListItem[], function(ListItem[]): void]} */ const [items, setItems] = useState([]); + /** @type {[ListItem[], function(ListItem[]): void]} */ const [sampleItems, setSampleItems] = useState([]); const sampleListRef = useRef(/** @type {HTMLElement|null} */ (null)); @@ -287,7 +289,7 @@ const groupBySample = (samples, sampleDescriptor, order) => { } } }); - const groupCount = samples.length / sampleDescriptor.epochs; + const groupCount = samples.length / sampleDescriptor.evalDescriptor.epochs; const itemCount = samples.length / groupCount; const counter = getCounter(itemCount, groupCount, order); return (sample, index, previousSample) => { @@ -328,7 +330,7 @@ const groupBySample = (samples, sampleDescriptor, order) => { * @returns {(sample: import("../api/Types.mjs").SampleSummary, index: number, previousSample: import("../api/Types.mjs").SampleSummary) => ListItem[]} The list */ const groupByEpoch = (samples, sampleDescriptor, order) => { - const groupCount = sampleDescriptor.epochs; + const groupCount = sampleDescriptor.evalDescriptor.epochs; const itemCount = samples.length / groupCount; const counter = getCounter(itemCount, groupCount, order); diff --git a/src/inspect_ai/_view/www/src/samples/SamplesTools.mjs b/src/inspect_ai/_view/www/src/samples/SamplesTools.mjs index 2a3eb5668..b92cb0cfe 100644 --- a/src/inspect_ai/_view/www/src/samples/SamplesTools.mjs +++ b/src/inspect_ai/_view/www/src/samples/SamplesTools.mjs @@ -10,6 +10,7 @@ export const SampleTools = (props) => { epoch, setEpoch, filter, + filterError, filterChanged, sort, setSort, @@ -43,14 +44,6 @@ export const SampleTools = (props) => { ); } - tools.push( - html`<${SampleFilter} - filter=${filter} - filterChanged=${filterChanged} - descriptor=${sampleDescriptor} - />`, - ); - tools.push( html`<${SortFilter} sampleDescriptor=${sampleDescriptor} @@ -60,5 +53,13 @@ export const SampleTools = (props) => { />`, ); + tools.push( + html`<${SampleFilter} + filter=${filter} + filterError=${filterError} + filterChanged=${filterChanged} + />`, + ); + return tools; }; diff --git a/src/inspect_ai/_view/www/src/samples/tools/SampleFilter.mjs b/src/inspect_ai/_view/www/src/samples/tools/SampleFilter.mjs index e720d4404..bbc5cbc5c 100644 --- a/src/inspect_ai/_view/www/src/samples/tools/SampleFilter.mjs +++ b/src/inspect_ai/_view/www/src/samples/tools/SampleFilter.mjs @@ -1,112 +1,26 @@ import { html } from "htm/preact"; import { FontSize, TextStyle } from "../../appearance/Fonts.mjs"; - -import { - kScoreTypeCategorical, - kScoreTypeNumeric, - kScoreTypeObject, - kScoreTypePassFail, -} from "../../constants.mjs"; +import { useRef } from "preact/hooks"; /** * Renders the Sample Filter Control * * @param {Object} props - The parameters for the component. - * @param {import("../SamplesDescriptor.mjs").SamplesDescriptor} props.descriptor - The sample descriptor * @param {(filter: import("../../Types.mjs").ScoreFilter) => void} props.filterChanged - Filter changed function * @param {import("../../Types.mjs").ScoreFilter} props.filter - Capabilities of the application host + * @param {string | undefined} props.filterError - The error in the filter expression, if any. * @returns {import("preact").JSX.Element | string} The TranscriptView component. */ -export const SampleFilter = ({ descriptor, filter, filterChanged }) => { - const updateCategoryValue = (e) => { - const val = e.currentTarget.value; - if (val === "all") { - filterChanged({}); - } else { - filterChanged({ - value: val, - type: kScoreTypeCategorical, - }); - } - }; - - switch (descriptor?.scoreDescriptor?.scoreType) { - case kScoreTypePassFail: { - const options = [{ text: "All", value: "all" }]; - options.push( - ...descriptor.scoreDescriptor.categories.map((cat) => { - return { text: cat.text, value: cat.val }; - }), - ); - return html`<${SelectFilter} - value=${filter.value || "all"} - options=${options} - onChange=${updateCategoryValue} - />`; - } - - case kScoreTypeCategorical: { - const options = [{ text: "All", value: "all" }]; - options.push( - ...descriptor.scoreDescriptor.categories.map((cat) => { - return { text: cat, value: cat }; - }), - ); - return html`<${SelectFilter} - value=${filter.value || "all"} - options=${options} - onChange=${updateCategoryValue} - />`; - } - - case kScoreTypeNumeric: { - // TODO: Create a real numeric slider control of some kind - return html` - { - filterChanged({ - value: e.currentTarget.value, - type: kScoreTypeNumeric, - }); - }} - /> - `; - } - - case kScoreTypeObject: { - if (!descriptor.scoreDescriptor.categories) { - return ""; - } - const options = [{ text: "All", value: "all" }]; - options.push( - ...descriptor.scoreDescriptor.categories.map((cat) => { - return { text: cat.text, value: cat.value }; - }), - ); - - return html`<${SelectFilter} - value=${filter.value || "all"} - options=${options} - onChange=${updateCategoryValue} - />`; - } +export const SampleFilter = ({ filter, filterError, filterChanged }) => { + const inputRef = useRef(null); + const tooltip = filterError + ? `${filterError}\n\n${filterTooltip}` + : filterTooltip; - default: { - return undefined; - } - } -}; - -const SelectFilter = ({ value, options, onChange }) => { return html`
{ marginRight: "0.3em", marginLeft: "0.2em", }} - >Scores:Filter: - +
+ { + filterChanged({ + value: e.currentTarget.value, + }); + }} + ref=${inputRef} + /> + ${filter.value && + html` + + `} +
`; }; + +const filterTooltip = ` +Filter samples by scores. Supported expressions: + • Arithmetic: +, -, *, /, mod, ^ + • Comparison: <, <=, >, >=, ==, !=, including chain comparisons, e.g. “10 <= x < 20” + • Boolean: and, or, not + • Regex matching: ~= (case-sensitive) + • Set operations: in, not in; e.g. “x in (1, 2, 3)” + • Functions: min, max, abs, round, floor, ceil, sqrt, log, log2, log10 +Click on the score name above to add it to the filter. +`.trim(); diff --git a/src/inspect_ai/_view/www/src/samples/tools/SelectScorer.mjs b/src/inspect_ai/_view/www/src/samples/tools/SelectScorer.mjs index b2ebd3acd..ddbc6872e 100644 --- a/src/inspect_ai/_view/www/src/samples/tools/SelectScorer.mjs +++ b/src/inspect_ai/_view/www/src/samples/tools/SelectScorer.mjs @@ -1,89 +1,37 @@ import { html } from "htm/preact"; import { FontSize, TextStyle } from "../../appearance/Fonts.mjs"; +/** + * @param {Object} props + * @param {import("../../Types.mjs").ScoreLabel[]} props.scores + * @param {import("../../Types.mjs").ScoreLabel} props.score + * @param {(score: import("../../Types.mjs").ScoreLabel) => void} props.setScore + * @returns {import("preact").JSX.Element} + */ export const SelectScorer = ({ scores, score, setScore }) => { - const scorers = scores.reduce((accum, scorer) => { - if ( - !accum.find((sc) => { - return scorer.scorer === sc.scorer; - }) - ) { - accum.push(scorer); - } - return accum; - }, []); - - if (scorers.length === 1) { - // There is only a single scorer in play, just show the list of available scores - return html` -
- Score: - <${ScoreSelector} - scores=${scores} - selectedIndex=${scoreIndex(score, scores)} - selectedIndexChanged=${(index) => { - setScore(scores[index]); - }} - /> -
- `; - } else { - // selected scorer - - const scorerScores = scores.filter((sc) => { - return sc.scorer === score.scorer; - }); - - const selectors = [ - html`<${ScorerSelector} - scorers=${scorers} - selectedIndex=${scorerIndex(score, scorers)} + return html` +
+ Scorer: + <${ScoreSelector} + scores=${scores} + selectedIndex=${scoreIndex(score, scores)} selectedIndexChanged=${(index) => { - setScore(scorers[index]); + setScore(scores[index]); }} - />`, - ]; - if (scorerScores.length > 1) { - selectors.push( - html`<${ScoreSelector} - style=${{ marginLeft: "1em" }} - scores=${scorerScores} - selectedIndex=${scoreIndex(score, scorerScores)} - selectedIndexChanged=${(index) => { - setScore(scorerScores[index]); - }} - />`, - ); - } - - // There are multiple scorers, so show a scorer selector and a r - return html` -
- Scorer: - ${selectors} -
- `; - } + /> +
+ `; }; const ScoreSelector = ({ @@ -102,23 +50,11 @@ const ScoreSelector = ({ }} > ${scores.map((score) => { - return html``; - })} - `; -}; - -const ScorerSelector = ({ scorers, selectedIndex, selectedIndexChanged }) => { - return html` is closed and only show it + // in the dropdown menu, but this requires a fully manual `; }; @@ -127,8 +63,3 @@ const scoreIndex = (score, scores) => scores.findIndex((sc) => { return sc.name === score.name && sc.scorer === score.scorer; }); - -const scorerIndex = (score, scores) => - scores.findIndex((sc) => { - return sc.scorer === score.scorer; - }); diff --git a/src/inspect_ai/_view/www/src/samples/tools/SortFilter.mjs b/src/inspect_ai/_view/www/src/samples/tools/SortFilter.mjs index 753abbf79..ed5a2bfe2 100644 --- a/src/inspect_ai/_view/www/src/samples/tools/SortFilter.mjs +++ b/src/inspect_ai/_view/www/src/samples/tools/SortFilter.mjs @@ -25,7 +25,7 @@ export const SortFilter = ({ sampleDescriptor, sort, setSort, epochs }) => { val: kEpochDescVal, }); } - if (sampleDescriptor?.scoreDescriptor?.compare) { + if (sampleDescriptor?.selectedScoreDescriptor?.compare) { options.push({ label: "score asc", val: kScoreAscVal, @@ -108,12 +108,12 @@ export const sortSamples = (sort, samples, samplesDescriptor) => { case kEpochDescVal: return b.epoch - a.epoch; case kScoreAscVal: - return samplesDescriptor.scoreDescriptor.compare( + return samplesDescriptor.selectedScoreDescriptor.compare( samplesDescriptor.selectedScore(a).value, samplesDescriptor.selectedScore(b).value, ); case kScoreDescVal: - return samplesDescriptor.scoreDescriptor.compare( + return samplesDescriptor.selectedScoreDescriptor.compare( samplesDescriptor.selectedScore(b).value, samplesDescriptor.selectedScore(a).value, ); diff --git a/src/inspect_ai/_view/www/src/samples/tools/filters.mjs b/src/inspect_ai/_view/www/src/samples/tools/filters.mjs index 1433ace06..9ee2bc3aa 100644 --- a/src/inspect_ai/_view/www/src/samples/tools/filters.mjs +++ b/src/inspect_ai/_view/www/src/samples/tools/filters.mjs @@ -1,114 +1,224 @@ -import { kScoreTypeCategorical, kScoreTypeNumeric } from "../../constants.mjs"; -import { isNumeric } from "../../utils/Type.mjs"; +import { compileExpression } from "filtrex"; +import { + kScoreTypeBoolean, + kScoreTypeCategorical, + kScoreTypeNumeric, + kScoreTypePassFail, +} from "../../constants.mjs"; /** - * Gets a filter function for the specified type + * Coerces a value to the type expected by the score. * - * @param {import("../../Types.mjs").ScoreFilter} filter - The parameters for the component. - * @returns {(descriptor: import("../SamplesDescriptor.mjs").SamplesDescriptor, sample: import("../../api/Types.mjs").SampleSummary, value: string) => boolean | undefined} the function + * @param {any} value + * @param {import("../../samples/SamplesDescriptor.mjs").ScoreDescriptor} descriptor + * @returns {any} */ -export const filterFnForType = (filter) => { - if (filter.type) { - return filterFnsForType[filter.type]; +const coerceValue = (value, descriptor) => { + if (descriptor && descriptor.scoreType === kScoreTypeBoolean) { + return Boolean(value); } else { - return undefined; + return value; } }; /** - * @type{(descriptor: import("../SamplesDescriptor.mjs").SamplesDescriptor, sample: import("../../api/Types.mjs").SampleSummary, value: string) => boolean} + * @param {any} value + * @returns {boolean} */ -const filterCategory = (descriptor, sample, value) => { - const score = descriptor.selectedScore(sample); - if (typeof score.value === "string") { - return score.value.toLowerCase() === value?.toLowerCase(); - } else if (typeof score.value === "object") { - return JSON.stringify(score.value) == value; - } else { - return String(score.value) === value; +const isFilteringSupportedForValue = (value) => + ["string", "number", "boolean"].includes(typeof value); + +/** + * @param {import("../../samples/SamplesDescriptor.mjs").ScoreDescriptor} descriptor + * @returns {boolean} + */ +const isFilteringSupportedForScore = (descriptor) => { + if (!descriptor) { + return false; } + return [ + kScoreTypePassFail, + kScoreTypeCategorical, + kScoreTypeNumeric, + kScoreTypeBoolean, + ].includes(descriptor.scoreType); }; /** - * @type{(descriptor: import("../SamplesDescriptor.mjs").SamplesDescriptor, sample: import("../../api/Types.mjs").SampleSummary, value: string) => boolean} + * Returns the names of scores that are not allowed to be used as short names in + * filter expressions because they are not unique. This should be applied only to + * the nested scores, not to the top-level scorer names. + * + * @param {import("../../Types.mjs").ScoreLabel[]} scores + * @returns {Set} */ -const filterText = (descriptor, sample, value) => { - const score = descriptor.selectedScore(sample); - if (!value) { - return true; - } else { - if (isNumeric(value)) { - if (typeof score.value === "number") { - return score.value === Number(value); - } else { - return Number(score.value) === Number(value); - } +const bannedShortScoreNames = (scores) => { + const used = new Set(); + const banned = new Set(); + for (const { scorer, name } of scores) { + banned.add(scorer); + if (used.has(name)) { + banned.add(name); } else { - const filters = [ - { - prefix: ">=", - fn: (score, val) => { - return score >= val; - }, - }, - { - prefix: "<=", - fn: (score, val) => { - return score <= val; - }, - }, - { - prefix: ">", - fn: (score, val) => { - return score > val; - }, - }, - { - prefix: "<", - fn: (score, val) => { - return score < val; - }, - }, - { - prefix: "=", - fn: (score, val) => { - return score === val; - }, - }, - { - prefix: "!=", - fn: (score, val) => { - return score !== val; - }, - }, - ]; + used.add(name); + } + } + return banned; +}; - for (const filter of filters) { - if (value?.startsWith(filter.prefix)) { - const val = value.slice(filter.prefix.length).trim(); - if (!val) { - return true; - } +/** + * Generates a dictionary of variables that can be used in the filter expression. + * High-level scorer metrics can be accessed by name directly. + * Child metrics are accessed using dot notation (e.g. `scorer_name.score_name`) or + * directly by name when it is unique. + * + * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor + * @param {import("../../types/log").Scores1} sampleScores + * @returns {Object} + */ +const scoreVariables = (evalDescriptor, sampleScores) => { + const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores); + const variables = {}; + + /** + * @param {import("../../Types.mjs").ScoreLabel} scoreLabel + * @param {any} value + */ + const addScore = (variableName, scoreLabel, value) => { + const coercedValue = coerceValue( + value, + evalDescriptor.scoreDescriptor(scoreLabel), + ); + if (isFilteringSupportedForValue(coercedValue)) { + variables[variableName] = coercedValue; + } + }; - const num = Number(val); - return filter.fn(score.value, num); + for (const [scorer, score] of Object.entries(sampleScores)) { + addScore(scorer, { scorer, name: scorer }, score.value); + if (typeof score.value === "object") { + for (const [name, value] of Object.entries(score.value)) { + addScore(`${scorer}.${name}`, { scorer, name }, value); + if (!bannedShortNames.has(name)) { + addScore(name, { scorer, name }, value); } } - if (typeof score.value === "string") { - return score.value.toLowerCase() === value?.toLowerCase(); - } else { - return String(score.value) === value; - } } } + return variables; }; /** - * A dictionary that maps filter types to their respective filter functions. + * @typedef {Object} ScoreFilterItem + * @property {string} canonicalName - The canonical name of the score. + * @property {string} tooltip - The informational tooltip for the score. + * @property {boolean} isFilterable - Whether the score can be used in a filter expression. + * @property {string[]} suggestions - Suggested expressions for the score. + */ + +/** + * Generates a dictionary of variables that can be used in the filter expression. + * High-level scorer metrics can be accessed by name directly. + * Child metrics are accessed using dot notation (e.g. `scorer_name.score_name`) or + * directly by name when it is unique. * - * @type {Record boolean>} + * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor + * @returns {ScoreFilterItem[]} */ -const filterFnsForType = { - [kScoreTypeCategorical]: filterCategory, - [kScoreTypeNumeric]: filterText, +export const scoreFilterItems = (evalDescriptor) => { + /** @type {ScoreFilterItem[]} */ + const items = []; + const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores); + const valueToString = (value) => + typeof value === "string" ? `"${value}"` : String(value); + + /** + * @param {string} canonicalName + * @param {import("../../Types.mjs").ScoreLabel} scoreLabel + */ + const addScore = (canonicalName, scoreLabel) => { + const descriptor = evalDescriptor.scoreDescriptor(scoreLabel); + if (!descriptor || !isFilteringSupportedForScore(descriptor)) { + items.push({ + canonicalName, + tooltip: undefined, + isFilterable: false, + suggestions: [], + }); + return; + } + var tooltip = `${canonicalName}: ${descriptor.scoreType}`; + var suggestions = []; + if (descriptor.min !== undefined || descriptor.max !== undefined) { + const rounded = (num) => { + // Additional round-trip to remove trailing zeros. + return parseFloat(num.toPrecision(3)).toString(); + }; + tooltip += `\nRange: ${rounded(descriptor.min)} to ${rounded(descriptor.max)}`; + } + if (descriptor.categories) { + tooltip += `\nCategories: ${descriptor.categories.map((cat) => cat.val).join(", ")}`; + suggestions = [ + canonicalName, + ...descriptor.categories.map( + (cat) => `${canonicalName} == ${valueToString(cat.val)}`, + ), + ]; + } + items.push({ canonicalName, tooltip, isFilterable: true, suggestions }); + }; + + for (const { name, scorer } of evalDescriptor.scores) { + const canonicalName = + name !== scorer && bannedShortNames.has(name) + ? `${scorer}.${name}` + : name; + addScore(canonicalName, { name, scorer }); + } + return items; +}; + +/** + * @param {import("../../Types.mjs").ScoreFilter} filter + * @param {string} fragment + * @returns {import("../../Types.mjs").ScoreFilter} + */ +export const addFragmentToFilter = (filter, fragment) => { + var value = filter.value || ""; + if (value.trim() && !value.endsWith(" ")) { + value = `${value} `; + } + if (value.trim() && !value.match(/ +(or|and) *$/)) { + value = `${value}and `; + } + value += fragment; + return { value }; +}; + +/** + * TODO: Add case-insensitive string comparison. + * TODO: Support filtering by things other than scores: metadata, transcript text, etc. + * + * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor + * @param {import("../../api/Types.mjs").SampleSummary} sample + * @param {string} value + * @returns {{matches: boolean, error: string | undefined}} + */ +export const filterExpression = (evalDescriptor, sample, value) => { + try { + const expression = compileExpression(value); + const vars = scoreVariables(evalDescriptor, sample.scores); + const result = expression(vars); + if (typeof result === "boolean") { + return { matches: result, error: undefined }; + } else if (result instanceof Error) { + throw result; + } else { + throw new TypeError( + `Filter expression returned a non-boolean value: ${result}`, + ); + } + } catch (error) { + return { matches: false, error: error.message }; + } }; diff --git a/src/inspect_ai/_view/www/src/workspace/WorkSpace.mjs b/src/inspect_ai/_view/www/src/workspace/WorkSpace.mjs index 7eccde803..dd1ae506b 100644 --- a/src/inspect_ai/_view/www/src/workspace/WorkSpace.mjs +++ b/src/inspect_ai/_view/www/src/workspace/WorkSpace.mjs @@ -66,7 +66,9 @@ import { debounce } from "../utils/sync.mjs"; * @param {(showing: boolean) => void} props.setShowingSampleDialog - Call to show the sample dialog * @param {(epoch: string) => void} props.setEpoch - set the current epoch * @param {import("../Types.mjs").ScoreFilter} props.filter - the current filter + * @param {string | undefined} props.filterError - whether there is an error in the filter expression * @param {(epoch: import("../Types.mjs").ScoreFilter) => void } props.setFilter - set the current filter + * @param {(fragment: string) => void } props.addToFilterExpression - add to the current filter expression * @param {import("../Types.mjs").ScoreLabel} props.score - The current selected scorer * @param {(score: import("../Types.mjs").ScoreLabel) => void} props.setScore - Set the current selected scorer * @param {import("../Types.mjs").ScoreLabel[]} props.scores - The current selected scorer @@ -113,7 +115,9 @@ export const WorkSpace = ({ epoch, setEpoch, filter, + filterError, setFilter, + addToFilterExpression, score, setScore, scores, @@ -191,6 +195,7 @@ export const WorkSpace = ({ epochs=${epochs} setEpoch=${setEpoch} filter=${filter} + filterError=${filterError} filterChanged=${setFilter} sort=${sort} setSort=${setSort} @@ -366,6 +371,8 @@ export const WorkSpace = ({ evalResults=${evalResults} evalStats=${evalStats} samples=${samples} + evalDescriptor=${samplesDescriptor.evalDescriptor} + addToFilterExpression=${addToFilterExpression} status=${evalStatus} tabs=${resolvedTabs} selectedTab=${selectedTab} @@ -384,6 +391,8 @@ const WorkspaceDisplay = ({ evalResults, evalStats, samples, + evalDescriptor, + addToFilterExpression, status, showToggle, selectedTab, @@ -454,20 +463,20 @@ const WorkspaceDisplay = ({ }, [tabs]); return html` - - <${Navbar} evalSpec=${evalSpec} evalPlan=${evalPlan} evalResults=${evalResults} evalStats=${evalStats} samples=${samples} + evalDescriptor=${evalDescriptor} + addToFilterExpression=${addToFilterExpression} status=${status} file=${logFileName} showToggle=${showToggle} - + offcanvas=${offcanvas} - /> + />