feat(lxl-querylang): grammar and highlighting (#1159)

* Add rudimentary grammar and tests
* Add highlighting etc
* Bundle language + extension in export

lxl-web/src/lib/components/Search.svelte

@@ -6,7 +6,8 @@
 	import addDefaultSearchParams from '$lib/utils/addDefaultSearchParams';
 	import getSortedSearchParams from '$lib/utils/getSortedSearchParams';
 	import BiSearch from '~icons/bi/search';
-	import { lxlQueryLanguage } from 'codemirror-lang-lxlquery';
+	import { lxlQuery } from 'codemirror-lang-lxlquery';
+	import '$lib/styles/lxlquery.css';
 
 	export let placeholder: string;
 	export let autofocus: boolean = false;
@@ -48,7 +49,7 @@
 		<SuperSearch
 			name="_q"
 			bind:value={q}
-			language={lxlQueryLanguage}
+			language={lxlQuery}
 			placeholder={$page.data.t('search.search')}
 		/>
 	{:else}

lxl-web/src/lib/styles/lxlquery.css

@@ -0,0 +1,28 @@
+.lxlq.qualifier {
+	color: green;
+}
+
+.lxlq.qualifier-key {
+	color: green;
+}
+
+.lxlq.qualifier-value {
+	color: green;
+}
+
+.lxlq.equal-operator,
+.lxlq.compare-operator {
+	color: darkolivegreen;
+}
+
+.lxlq.boolean-operator {
+	color: #935493;
+}
+
+.lxlq.boolean-query {
+	color: purple;
+}
+
+.lxlq.wildcard {
+	color: orange;
+}

lxl-web/src/routes/+layout.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
 	import '../app.css';
 	import NProgress from 'nprogress';
-	import '../nprogress.css';
+	import '$lib/styles/nprogress.css';
 	import { navigating } from '$app/stores';
 	import Matomo from '$lib/components/Matomo.svelte';
 	import CookieConsent from '$lib/components/CookieConsent.svelte';

packages/codemirror-lang-lxlquery/src/index.ts

@@ -1,22 +1,47 @@
 import { parser } from './syntax.grammar';
-import { LRLanguage, LanguageSupport } from '@codemirror/language';
-import { styleTags, tags as t } from '@lezer/highlight';
+import { LRLanguage, LanguageSupport, syntaxHighlighting } from '@codemirror/language';
+import { styleTags, Tag, tagHighlighter } from '@lezer/highlight';
+
+// custom tags attached to the language parser
+const customTags = {
+	Qualifier: Tag.define('Qualifier'),
+	QualifierKey: Tag.define('QualifierKey'),
+	QualifierValue: Tag.define('QualifierValue'),
+	EqualOperator: Tag.define('EqualOperator'),
+	CompareOperator: Tag.define('CompareOperator'),
+	BooleanQuery: Tag.define('BooleanQuery'),
+	BooleanOperator: Tag.define('BooleanOperator'),
+	Wildcard: Tag.define('Wildcard')
+};
 
 export const lxlQueryLanguage = LRLanguage.define({
 	name: 'Libris XL query',
 	parser: parser.configure({
-		props: [
-			styleTags({
-				Identifier: t.variableName,
-				String: t.string,
-				CompareOperator: t.compareOperator,
-				'( )': t.paren
-			})
-		]
+		props: [styleTags(customTags)]
 	}),
 	languageData: {}
 });
 
-export function lxlQuery() {
-	return new LanguageSupport(lxlQueryLanguage);
-}
+const highlighter = tagHighlighter(
+	[
+		{ tag: customTags.Qualifier, class: 'qualifier' },
+		{ tag: customTags.QualifierKey, class: 'qualifier-key' },
+		{ tag: customTags.QualifierValue, class: 'qualifier-value' },
+		{ tag: customTags.EqualOperator, class: 'equal-operator' },
+		{ tag: customTags.CompareOperator, class: 'compare-operator' },
+		{ tag: customTags.BooleanOperator, class: 'boolean-operator' },
+		{ tag: customTags.BooleanQuery, class: 'boolean-query' },
+		{ tag: customTags.Wildcard, class: 'wildcard' }
+	],
+	{
+		all: 'lxlq'
+	}
+);
+
+const highlighterExtension = syntaxHighlighting(highlighter);
+
+/**
+ * Libris XL query language together with a highlighter extension 
+ * that adds CSS classes for certain nodes
+ */
+export const lxlQuery = new LanguageSupport(lxlQueryLanguage, highlighterExtension)

packages/codemirror-lang-lxlquery/src/syntax.grammar

@@ -1,26 +1,57 @@
-@top Program { expression* }
+@top Query { term* }
 
-@skip { space | LineComment }
+@skip { space }
 
-expression {
-  Identifier |
-  String |
-  Boolean |
-  Application { "(" expression* ")" }
+term {
+  freetext |
+  Qualifier |
+  Group |
+  BooleanQuery
+}
+
+Group { "(" term* ")" } 
+
+BooleanQuery { 
+  (freetext | Qualifier) BooleanOperator (freetext | Qualifier) 
+  (BooleanOperator (freetext | Qualifier))+?
+}
+
+Qualifier {
+  (QualifierKey (EqualOperator | CompareOperator) QualifierValue) | reserved
+}
+
+QualifierKey {
+  identifier | string
+}
+
+QualifierValue {
+  freetext | Group
+}
+
+freetext {
+  (identifier | string | number) Wildcard?
 }
 
 @tokens {
-  Identifier { $[a-zA-Z_\-0-9]+ }
+  identifier { $[a-zA-ZåäöÅÄÖ]+ }
+
+  string { '"' (!["\\] | "\\" _)* '"' }
+
+  number { @digit+ }
+
+  EqualOperator { ":" | "=" }
+
+  CompareOperator { ">" | "<" | ">=" | "<=" }
 
-  String { '"' (!["\\] | "\\" _)* '"' }
+  BooleanOperator { "AND" | "OR" | "NOT" }
 
-  Boolean { "#t" | "#f" }
+  Wildcard { "*"+ }
 
-  LineComment { ";" ![\n]* }
+  reserved { "includeEplikt" | "includePreliminary" }
 
-  space { $[ \t\n\r]+ }
+  space { @whitespace+ }
 
-  "(" ")"
+  @precedence {BooleanOperator, reserved, identifier}
 }
 
 @detectDelim

packages/codemirror-lang-lxlquery/test/cases.txt

@@ -1,41 +1,171 @@
-# Booleans
+# Freetext words
 
-#t
-#f
+Astrid lindgren
 
 ==>
 
-Program(Boolean, Boolean)
+Query()
 
-# Identifiers
 
-one
-Two_Three
+# Freetext string
+
+"2001: A Space Odyssey"
+
+==>
+
+Query()
+
+
+# Freetext with Wildcard
+
+Astrid lind*
+
+==>
+
+Query(Wildcard)
+
+
+# Qualifier with EqualOperator
+
+ÅR:2024
+
+==>
+
+Query(
+  Qualifier(QualifierKey,EqualOperator,QualifierValue)
+)
+
+
+# Qualifier with CompareOperator
+
+ÅR>2024
+ÅR<=2024
 
 ==>
 
-Program(Identifier, Identifier)
+Query(
+  Qualifier(QualifierKey,CompareOperator,QualifierValue)
+  Qualifier(QualifierKey,CompareOperator,QualifierValue)
+)
+
+
+# Qualifier with string key
+
+"rdf:type":Text
+
+==>
+
+Query(
+  Qualifier(QualifierKey,EqualOperator,QualifierValue)
+)
+
+
+# Qualifier with string value
+
+contributor:"Astrid Lindgren"
+
+==>
+
+Query(
+  Qualifier(QualifierKey,EqualOperator,QualifierValue)
+)
+
+
+# Qualifier linked
+
+contributor:"libris:fcrtpljz1qp2bdv#it"
+
+==>
+
+Query(
+  Qualifier(QualifierKey,EqualOperator,QualifierValue)
+)
+
 
-# Strings
+# BooleanQuery
 
-"hello"
-"one\"\\two"
+sommar OR vinter NOT vår
 
 ==>
 
-Program(String, String)
+Query(
+  BooleanQuery(BooleanOperator,BooleanOperator)
+)
+
+
+# BooleanQuery - erroneous
+
+OR AND sommar
+
+==>
+
+Query(
+  ⚠(BooleanOperator),⚠(BooleanOperator)
+)
+
+
+# Combined: Qualifier and Freetext
+
+contributor:"Astrid Lindgren" Pippi Långstrump
+
+==>
+
+Query(
+  Qualifier(QualifierKey,EqualOperator,QualifierValue)
+)
+
+
+# Combined: Freetext BooleanQuery (Qualifier NOT Qualifier)
+
+träd* bibliografi:"sigel:DST" NOT typ:Text
+
+==>
+
+Query(
+  Wildcard,BooleanQuery(Qualifier(...),BooleanOperator,Qualifier(...))
+)
+
+
+# Combined: Freetext Qualifier (Group with BooleanQuery) Qualifier
+
+pippi långstrump språk:(engelska OR franska) ÅR>2002
+
+==>
+
+Query(
+  Qualifier(
+    QualifierKey,EqualOperator,QualifierValue(
+      Group(
+        BooleanQuery(BooleanOperator)
+      )
+    )
+  ),
+  Qualifier(...)
+)
+
+
+# Other filters: cover image
+
+image:*
+
+==>
+
+Query(Qualifier(...))
+
+
+# Other filters: include E-plikt
+
+Pippi includeEplikt
+
+==>
+
+Query(Qualifier)
+
 
-# Applications
+# Other filters: include preliminary
 
-(begin
-  (when #t
-    (print (concat "hello" " " "world")))
-  (print "DONE"))
+includePreliminary Pippi
 
 ==>
 
-Program(Application(
-  Identifier,
-  Application(Identifier, Boolean, Application(
-    Identifier, Application(Identifier, String, String, String)))
-  Application(Identifier, String)))
+Query(Qualifier)