merge: dev into master

.gitignore

@@ -10,3 +10,4 @@ run_config.json
 .python-version
 *.pem
 config.yml
+.env

CHANGELOG.md

@@ -2,6 +2,51 @@
 
 ## [Unreleased]
 
+## [9.7.2] - 2024-12-09
+
+### Added
+
+- Support `.env`
+- Simplify time interval CQP covering whole days [#379](https://github.com/spraakbanken/korp-frontend/issues/379)
+- Track some events with Matomo: search, language switch
+- Select button in Corpus Updates section [#367](https://github.com/spraakbanken/korp-frontend/issues/367)
+- KWICs opened from the statistics should only query relevant corpora [#89](https://github.com/spraakbanken/korp-frontend/issues/89)
+- Alphabetic sorting of statistics columns [#37](https://github.com/spraakbanken/korp-frontend/issues/37)
+
+### Changed
+
+- Font is now a dependency, not checked-in files (and the font looks slightly different)
+- New loading spinners in result tabs
+- Undo override of Tailwind classname separator for Pug [#376](https://github.com/spraakbanken/korp-frontend/issues/376)
+- Extracted Karp backend usage into `app/scripts/karp.ts`
+- `stringifyFunc(key)` was renamed to `getStringifier(key)`
+- `stringify(key, x)` was removed, use `getStringifier(key)(x)` instead
+- `getStructValues()` of `structService` is refactored into two different functions, matching the two present use-cases:
+  - `getAttrValues()` for getting a flat list without counts
+  - `countAttrValues()` for getting a deep structure with counts
+- Search history is stored as parameters only, not full urls #118
+- Enabled the `noImplicitAny` TypeScript flag for added strictness, and fixed/refactored various parts as a consequence
+- The `hitCountHtml` util function now takes the numbers as a tuple
+- `reduceStringify()` now returns the stringifier, so it can be called only once per attribute
+
+### Fixed
+
+- In the corpus selector, an empty folder would add 1 to the parent folder's corpus count
+- Linking to corpus subfolder [#397](https://github.com/spraakbanken/korp-frontend/issues/397)
+- Searching by pressing Enter in Simple search is broken [#394](https://github.com/spraakbanken/korp-frontend/issues/394)
+- Barcode (aka hitsPicture) sometimes missing from KWIC tab [#395](https://github.com/spraakbanken/korp-frontend/issues/395)
+- Error when loading with restricted corpora selected [#398](https://github.com/spraakbanken/korp-frontend/issues/398)
+- Related words lookup must use OR [#401](https://github.com/spraakbanken/korp-frontend/issues/401)
+- Add support for annotations of the type 'set' in attribute filters [#116](https://github.com/spraakbanken/korp-frontend/issues/116)
+- Parallel mode is consistently checked against the `parallel` config setting, and not the mode name
+- Search history fails to select corpus [#405](https://github.com/spraakbanken/korp-frontend/issues/405)
+- Search history fails to distinguish options with same label [#406](https://github.com/spraakbanken/korp-frontend/issues/406)
+- The "X of Y corpora selected" phrase is not properly translated [#408](https://github.com/spraakbanken/korp-frontend/issues/408)
+- Empty localization strings sometimes render as localization key [#410](https://github.com/spraakbanken/korp-frontend/issues/410)
+- Wider filter lists [#412](https://github.com/spraakbanken/korp-frontend/issues/412)
+- Show intersection in attributes instead of union in comparison view [#56](https://github.com/spraakbanken/korp-frontend/issues/56)
+- Alphabetic sorting of statistics rows
+
 ## [9.7.1] - 2024-09-18
 
 ### Fixed
@@ -189,7 +234,7 @@
 - On repetition error (all tokens repeat from 0), restore red outline for input
 - Use `<match>` to constraint CQP subqueries (from statistics rows etc)
 
-## [9.5.0] - 2023-01-22
+## [9.5.0] - 2024-01-22
 
 ### Added
 
@@ -263,6 +308,7 @@
 - Lots of bug fixes for the sidebar
 
 [unreleased]: https://github.com/spraakbanken/korp-frontend/compare/master...dev
+[9.7.2]: https://github.com/spraakbanken/korp-frontend/releases/tag/v9.7.2
 [9.7.1]: https://github.com/spraakbanken/korp-frontend/releases/tag/v9.7.1
 [9.7.0]: https://github.com/spraakbanken/korp-frontend/releases/tag/v9.7.0
 [9.6.0]: https://github.com/spraakbanken/korp-frontend/releases/tag/v9.6.0

README.md

@@ -85,6 +85,8 @@ Host and port can be changed by the environment variables:
 - `KORP_HOST=<host>`
 - `KORP_PORT=<port>`
 
+Environment variables can be entered in the `.env` file, which is git-ignored.
+
 It is also possible to serve the frontend from HTTPS using the environment variables:
 - `KORP_HTTPS=true`
 - `KORP_KEY=<path_to_key>-key.pem`
@@ -100,8 +102,6 @@ mkcert -install
 Now use `korp.spraakbanken.gu.se` as the value for `KORP_HOST`. It must also be added
 to `/etc/hosts`.
 
-One way to set the environment variables automatically is to use [direnv](https://direnv.net/):
-
 # Branches, releases and versions
 
 Development is done on the `dev` branch. These changes are not necessarily yet stable and well-tested.

app/config/statistics_config.js → app/config/statistics_config.ts

@@ -3,23 +3,26 @@ import _ from "lodash"
 import settings from "@/settings"
 import { lemgramToHtml, regescape, saldoToHtml } from "@/util"
 import { locAttribute } from "@/i18n"
+import { Attribute } from "@/settings/config.types"
 
-let customFunctions = {}
+type Stringifier = (tokens: string[], ignoreCase?: boolean) => string
+
+let customFunctions: Record<string, Stringifier> = {}
 
 try {
     customFunctions = require("custom/statistics.js").default
 } catch (error) {
     console.log("No module for statistics functions available")
 }
 
-export function getCqp(hitValues, ignoreCase) {
+export function getCqp(hitValues: Record<string, string[]>[], ignoreCase: boolean): string {
     const positionalAttributes = ["word", ...Object.keys(settings.corpusListing.getCurrentAttributes())]
     let hasPositionalAttributes = false
 
-    var tokens = []
+    var tokens: string[] = []
     for (var i = 0; i < hitValues.length; i++) {
         var token = hitValues[i]
-        var andExpr = []
+        var andExpr: string[] = []
         for (var attribute in token) {
             if (token.hasOwnProperty(attribute)) {
                 var values = token[attribute]
@@ -39,11 +42,11 @@ export function getCqp(hitValues, ignoreCase) {
     return `<match> ${tokens.join(" ")} </match>`
 }
 
-function reduceCqp(type, tokens, ignoreCase) {
+function reduceCqp(type: string, tokens: string[], ignoreCase: boolean): string {
     let attrs = settings.corpusListing.getCurrentAttributes()
     if (attrs[type] && attrs[type].stats_cqp) {
         // A stats_cqp function should call regescape for the value as appropriate
-        return customFunctions[attrs[type].stats_cqp](tokens, ignoreCase)
+        return customFunctions[attrs[type].stats_cqp!](tokens, ignoreCase)
     }
     tokens = _.map(tokens, (val) => regescape(val))
     switch (type) {
@@ -55,11 +58,11 @@ function reduceCqp(type, tokens, ignoreCase) {
         case "sense":
         case "transformer-neighbour":
             if (tokens[0] === "") return "ambiguity(" + type + ") = 0"
-            else var res
+            let res: string
             if (tokens.length > 1) {
                 var key = tokens[0].split(":")[0]
 
-                var variants = []
+                const variants: string[][] = []
                 _.map(tokens, function (val) {
                     const parts = val.split(":")
                     if (variants.length == 0) {
@@ -68,15 +71,12 @@ function reduceCqp(type, tokens, ignoreCase) {
                     for (var idx = 1; idx < parts.length; idx++) variants[idx - 1].push(parts[idx])
                 })
 
-                variants = _.map(variants, function (variant) {
-                    return ":(" + variant.join("|") + ")"
-                })
-
-                res = key + variants.join("")
+                const variantsJoined = variants.map((variant) => ":(" + variant.join("|") + ")")
+                res = key + variantsJoined.join("")
             } else {
                 res = tokens[0]
             }
-            return type + " contains '" + res + "'"
+            return `${type} contains '${res}'`
         case "word":
             let s = 'word="' + tokens[0] + '"'
             if (ignoreCase) s = s + " %c"
@@ -101,74 +101,59 @@ function reduceCqp(type, tokens, ignoreCase) {
 }
 
 // Get the html (no linking) representation of the result for the statistics table
-export function reduceStringify(type, values, structAttributes) {
+export function reduceStringify(type: string, structAttr?: Attribute): (values: string[]) => string {
     let attrs = settings.corpusListing.getCurrentAttributes()
 
     if (attrs[type] && attrs[type].stats_stringify) {
-        return customFunctions[attrs[type].stats_stringify](values)
+        return customFunctions[attrs[type].stats_stringify!]
     }
 
     switch (type) {
         case "word":
         case "msd":
-            return values.join(" ")
+            return (values) => values.join(" ")
         case "pos":
-            var output = _.map(values, function (token) {
-                return locAttribute(attrs["pos"].translation, token)
-            }).join(" ")
-            return output
+            return (values) => values.map((token) => locAttribute(attrs["pos"].translation, token)).join(" ")
         case "saldo":
         case "prefix":
         case "suffix":
         case "lex":
         case "lemma":
         case "sense":
+            let stringify: (value: string, appendIndex?: boolean) => string
             if (type == "saldo" || type == "sense") {
-                var stringify = saldoToHtml
+                stringify = saldoToHtml
             } else if (type == "lemma") {
                 stringify = (lemma) => lemma.replace(/_/g, " ")
             } else {
                 stringify = lemgramToHtml
             }
 
-            var html = _.map(values, function (token) {
-                if (token === "") return "–"
-                return stringify(token.replace(/:.*/g, ""), true)
-            })
-
-            return html.join(" ")
+            return (values) =>
+                values.map((token) => (token === "" ? "–" : stringify(token.replace(/:.*/g, ""), true))).join(" ")
 
         case "transformer-neighbour":
-            return values.map((value) => value.replace(/:.*/g, "")).join(" ")
+            return (values) => values.map((value) => value.replace(/:.*/g, "")).join(" ")
 
         case "deprel":
-            var output = _.map(values, function (token) {
-                return locAttribute(attrs["deprel"].translation, token)
-            }).join(" ")
-            return output
+            return (values) => values.map((token) => locAttribute(attrs["deprel"].translation, token)).join(" ")
         case "msd_orig": // TODO: OMG this is corpus specific, move out to config ASAP (ASU corpus)
-            var output = _.map(values, function (token) {
-                return $("<span>").text(token).outerHTML()
-            }).join(" ")
-            return output
+            return (values) => values.map((token) => ($("<span>").text(token) as any).outerHTML()).join(" ")
         default:
             if (attrs[type]) {
                 // word attributes
-                return values.join(" ")
+                return (values) => values.join(" ")
             } else {
                 // structural attributes
-                var mapped = _.map(values, function (value) {
-                    if (structAttributes["set"] && value === "") {
-                        return "–"
-                    } else if (value === "") {
+                function stringify(value: string) {
+                    if (value === "") {
                         return "-"
-                    } else if (structAttributes.translation) {
-                        return locAttribute(structAttributes.translation, value)
-                    } else {
-                        return value
+                    } else if (structAttr?.translation) {
+                        return locAttribute(structAttr.translation, value)
                     }
-                })
-                return mapped.join(" ")
+                    return value
+                }
+                return (values) => values.map(stringify).join(" ")
             }
     }
 }

app/index.ts

@@ -3,6 +3,8 @@ import $ from "jquery"
 import currentMode from "@/mode"
 import { locationSearchGet } from "@/util"
 import { HashParams } from "@/urlparams"
+import "@fontsource/source-sans-pro/400.css"
+import "@fontsource/source-sans-pro/600.css"
 
 declare global {
     interface Window {
@@ -66,17 +68,6 @@ require("./lib/jquery.tooltip.pack.js")
 require("./scripts/main")
 require("./scripts/app")
 
-require("./scripts/controllers/comparison_controller")
-require("./scripts/controllers/kwic_controller")
-require("./scripts/controllers/example_controller")
-require("./scripts/controllers/statistics_controller")
-require("./scripts/controllers/trend_diagram_controller")
-require("./scripts/controllers/word_picture_controller")
-require("./scripts/controllers/map_controller")
-
-require("./scripts/text_reader_controller.js")
 require("./scripts/video_controllers.js")
-require("./scripts/extended.js")
-require("./scripts/struct_services.js")
-require("./scripts/filter_directives.js")
-require("./scripts/matomo.js")
+require("./scripts/backend/struct-service")
+require("./scripts/matomo")

app/lib/jquery.localize.js

@@ -49,11 +49,7 @@ const { locationSearchGet, getService } = require("@/util");
 		this.find("[rel^=localize]").each(function(i, elem) {
 			var elem = $(elem);
 			var key = elem.attr("rel").match(/localize\[(.*?)\]/)[1];
-			var value = valueForKey(key, data) || key;
-			var prefix = valueForKey($(this).data("locPrefix"), data) || "";
-			var suffix = valueForKey($(this).data("locSuffix"), data) || "";
-			if(prefix) prefix += ": "; 
-			value = prefix + value + suffix;
+			var value = valueForKey(key, data) ?? key;
 
 			if (elem.is('input')) {
 				elem.val(value);

app/markup/about.html

@@ -1,5 +1,5 @@
 <div class="modal-header">
-    <h2>Korp version 9.7.1</h2>
+    <h2>Korp version 9.7.2</h2>
     <span ng-click="clickX()" class="close-x">×</span>
 
 </div>

app/scripts/app.ts

@@ -29,6 +29,7 @@ import "@/components/korp-error"
 import { JQueryExtended } from "./jquery.types"
 import { LocationService } from "./urlparams"
 import { LocLangMap } from "@/i18n/types"
+import { getAllCorporaInFolders } from "./components/corpus-chooser/util"
 
 // load all custom components
 let customComponents: Record<string, IComponentOptions> = {}
@@ -126,7 +127,7 @@ korpApp.run([
         s.$on("$localeChangeSuccess", () => {
             // The fresh info in $locale only has the 2-letter code, not the 3-letter code that we use
             // Find the configured 3-letter UI language matching the new 2-letter locale
-            const lang = settings["languages"]
+            const lang = settings.languages
                 .map((language) => language.value)
                 .find((lang3) => tmhDynamicLocaleCache.get<ILocaleService>(lang3)?.id == $locale.id)
 
@@ -179,20 +180,7 @@ korpApp.run([
 
         async function initializeCorpusSelection(selectedIds: string[]): Promise<void> {
             // Resolve any folder ids to the contained corpus ids
-            const corpusIds: string[] = []
-            for (const id of selectedIds) {
-                // If it is a corpus, copy the id
-                if (settings.corpora[id]) {
-                    corpusIds.push(id)
-                }
-                // If it is not a corpus, but a folder
-                else if (settings.folders[id]) {
-                    // Resolve contained corpora
-                    corpusIds.push(...collectCorpusIdsInFolder(settings.folders[id]))
-                }
-            }
-            // Replace the possibly mixed list with the list of corpus-only ids
-            selectedIds = corpusIds
+            selectedIds = selectedIds.flatMap((id) => getAllCorporaInFolders(settings.folders, id))
 
             let loginNeededFor: CorpusTransformed[] = []
 
@@ -237,11 +225,9 @@ korpApp.run([
                                 <div>{{'access_partly_denied_continue' | loc:$root.lang}}</div>`,
                             onClose: () => {
                                 const neededIds = loginNeededFor.map((corpus) => corpus.id)
-                                let newIds = selectedIds.filter((corpusId) => !neededIds.includes(corpusId))
-                                if (newIds.length == 0) {
-                                    newIds = settings["preselected_corpora"]
-                                }
-                                initializeCorpusSelection(newIds)
+                                const filtered = selectedIds.filter((corpusId) => !neededIds.includes(corpusId))
+                                const selected = filtered.length ? filtered : settings["preselected_corpora"] || []
+                                initializeCorpusSelection(selected)
                             },
                         })
                     }
@@ -262,7 +248,7 @@ korpApp.run([
                     content: `{{'corpus_not_available' | loc:$root.lang}}`,
                     onClose: () => {
                         const validIds = selectedIds.filter((corpusId) => allCorpusIds.includes(corpusId))
-                        const newIds = validIds.length >= 0 ? validIds : settings["preselected_corpora"]
+                        const newIds = validIds.length ? validIds : settings["preselected_corpora"] || []
                         initializeCorpusSelection(newIds)
                     },
                 })
@@ -275,7 +261,7 @@ korpApp.run([
 
         // TODO the top bar could show even though the modal is open,
         // thus allowing switching modes or language when an error has occured.
-        s.openErrorModal = ({ content, resolvable = true, onClose = null, buttonText = null, translations = null }) => {
+        s.openErrorModal = ({ content, resolvable = true, onClose, buttonText, translations }) => {
             type ModalScope = IScope & {
                 translations?: LocLangMap
                 closeModal: () => void
@@ -315,7 +301,7 @@ korpApp.run([
         }
 
         function getCorporaFromHash(): string[] {
-            const corpus: string = $location.search().corpus
+            const corpus = $location.search().corpus
             return corpus ? corpus.split(",") : settings["preselected_corpora"] || []
         }