Vml Library Support (#1602)

* grammar updates to allow date ranges and validate issue_dynamic

* checks that spawned block names exist

* wired up issue_dynamic form editor

* snippets for some block structures

* validate issue_dynamic arguments

* support SPACECRAFT_TIME() command and encoded_string arg

* tooltip for encoded string

* st warning

* extract functions for spawn and issue validation

* display byte array in selected command

* show decoded by array as read only in form view

* form view for library sequence

* tooltips for spawn and issue_dynamic args

* remove redundant display of command description

* check variable and global reference in issue*

* remove unused parameter

* svelte-check version doesn't support set union

* button to show error panel

* polish on completion suggestions

* add default arg for globals

* reduce nesting

* widen allowed input type on guard

* remove obosolete comment

* exempt variable declarations from checking if in scope

* spelling

* expand docs on issue vs issue_dynamic

* split up numerci cases

* readability of content assist

* stricter types

* readability of completion code

* add comment to make SeqN adaptation symmetric

* reduce conditional nesting

* fix svelte-check lint

* remove extraneous truthyness check

* reduce nesting

* reduce nesting

* Define constant for language name

* remove try/catch around svelte component creation

* enum for sequence types

* wrap form inputs in div for byte editor

* helper for computing library sequences for seqn

* missed file in prior commit

* smaller pieces for auto-complete

* better handling of typing new issue, issue_dynamic, etc

* minor cleanup

* Move SequenceTypes to enums file

* accept sonarqube false positive

src/components/sequencing/CommandPanel/SelectedCommand.svelte

@@ -16,7 +16,6 @@
   import type { ArgTextDef, TimeTagInfo } from '../../../types/sequencing';
   import type { CommandInfoMapper } from '../../../utilities/codemirror/commandInfoMapper';
   import { tooltip } from '../../../utilities/tooltip';
-  import Collapse from '../../Collapse.svelte';
   import AddMissingArgsButton from '../form/AddMissingArgsButton.svelte';
   import ArgEditor from '../form/ArgEditor.svelte';
   import StringEditor from '../form/StringEditor.svelte';
@@ -121,10 +120,6 @@
   {#if !!commandNode}
     {#if commandInfoMapper.nodeTypeHasArguments(commandNode)}
       {#if !!commandDef}
-        <fieldset>
-          <Collapse headerHeight={24} title={commandDef.stem} padContent={false}>{commandDef.description}</Collapse>
-        </fieldset>
-
         {#each editorArgInfoArray as argInfo}
           <ArgEditor
             {argInfo}

src/components/sequencing/SequenceEditor.svelte

@@ -3,7 +3,7 @@
 <script lang="ts">
   import { json } from '@codemirror/lang-json';
   import { indentService, syntaxTree } from '@codemirror/language';
-  import { lintGutter } from '@codemirror/lint';
+  import { lintGutter, openLintPanel } from '@codemirror/lint';
   import { Compartment, EditorState } from '@codemirror/state';
   import { type ViewUpdate } from '@codemirror/view';
   import type { SyntaxNode, Tree } from '@lezer/common';
@@ -20,7 +20,7 @@
   import ExpandIcon from 'bootstrap-icons/icons/arrow-bar-up.svg?component';
   import ClipboardIcon from 'bootstrap-icons/icons/clipboard.svg?component';
   import DownloadIcon from 'bootstrap-icons/icons/download.svg?component';
-  import { EditorView, basicSetup } from 'codemirror';
+  import { basicSetup, EditorView } from 'codemirror';
   import { debounce } from 'lodash-es';
   import { createEventDispatcher, onDestroy, onMount } from 'svelte';
   import { TOKEN_ERROR } from '../../constants/seq-n-grammar-constants';
@@ -43,27 +43,29 @@
     userSequences,
   } from '../../stores/sequencing';
   import type { User } from '../../types/app';
-  import type {
-    ArgTextDef,
-    IOutputFormat,
-    ISequenceAdaptation,
-    LibrarySequence,
-    Parcel,
-    TimeTagInfo,
+  import {
+    type ArgTextDef,
+    type IOutputFormat,
+    type ISequenceAdaptation,
+    type LibrarySequence,
+    type LibrarySequenceMap,
+    type Parcel,
+    type TimeTagInfo,
   } from '../../types/sequencing';
-  import { SeqLanguage, setupLanguageSupport } from '../../utilities/codemirror';
-  import { isFswCommandArgumentRepeat } from '../../utilities/codemirror/codemirror-utils';
+  import { setupLanguageSupport } from '../../utilities/codemirror';
+  import { isFswCommandArgumentRepeat, unquoteUnescape } from '../../utilities/codemirror/codemirror-utils';
   import type { CommandInfoMapper } from '../../utilities/codemirror/commandInfoMapper';
   import { seqNHighlightBlock, seqqNBlockHighlighter } from '../../utilities/codemirror/seq-n-highlighter';
-  import { SeqNCommandInfoMapper } from '../../utilities/codemirror/seq-n-tree-utils';
+  import { SeqNCommandInfoMapper, userSequenceToLibrarySequence } from '../../utilities/codemirror/seq-n-tree-utils';
   import { blockTheme } from '../../utilities/codemirror/themes/block';
   import {
     setupVmlLanguageSupport,
     vmlAdaptation,
     vmlBlockHighlighter,
     vmlHighlightBlock,
   } from '../../utilities/codemirror/vml/vml';
-  import { vmlAutoComplete } from '../../utilities/codemirror/vml/vmlAdaptation';
+  import { parseFunctionSignatures, vmlAutoComplete } from '../../utilities/codemirror/vml/vmlAdaptation';
+  import { librarySequenceToFswCommand } from '../../utilities/codemirror/vml/vmlBlockLibrary';
   import { vmlFormat } from '../../utilities/codemirror/vml/vmlFormatter';
   import { vmlLinter } from '../../utilities/codemirror/vml/vmlLinter';
   import { vmlTooltip } from '../../utilities/codemirror/vml/vmlTooltip';
@@ -73,7 +75,6 @@
   import { getCustomArgDef, inputLinter, outputLinter } from '../../utilities/sequence-editor/extension-points';
   import { seqNFormat } from '../../utilities/sequence-editor/sequence-autoindent';
   import { sequenceTooltip } from '../../utilities/sequence-editor/sequence-tooltip';
-  import { parseVariables } from '../../utilities/sequence-editor/to-seq-json';
   import { showFailureToast, showSuccessToast } from '../../utilities/toast';
   import { tooltip } from '../../utilities/tooltip';
   import Menu from '../menus/Menu.svelte';
@@ -101,8 +102,8 @@
   const debouncedSeqNHighlightBlock = debounce(seqNHighlightBlock, 250);
   const debouncedVmlHighlightBlock = debounce(vmlHighlightBlock, 250);
 
-  let clientHeightGridRightBottom: number;
-  let clientHeightGridRightTop: number;
+  let clientHeightGridRightBottom: number = 0;
+  let clientHeightGridRightTop: number = 0;
   let compartmentSeqJsonLinter: Compartment;
   let compartmentSeqLanguage: Compartment;
   let compartmentSeqLinter: Compartment;
@@ -113,6 +114,7 @@
   let commandDictionary: CommandDictionary | null;
   let disableCopyAndExport: boolean = true;
   let parameterDictionaries: ParameterDictionary[] = [];
+  let librarySequenceMap: LibrarySequenceMap = {};
   let librarySequences: LibrarySequence[] = [];
   let commandFormBuilderGrid: string;
   let editorOutputDiv: HTMLDivElement;
@@ -192,30 +194,36 @@
       }
     });
 
-    librarySequences = $userSequences
-      .filter(sequence => sequence.workspace_id === workspaceId && sequence.name !== sequenceName)
-      .map(sequence => {
-        const tree = SeqLanguage.parser.parse(sequence.definition);
-        return {
-          name: sequence.name,
-          parameters: parseVariables(tree.topNode, sequence.definition, 'ParameterDeclaration') ?? [],
-          tree,
-          workspace_id: sequence.workspace_id,
-        };
-      });
+    if (isInVmlMode) {
+      librarySequences = $userSequences
+        .filter(sequence => sequence.workspace_id === workspaceId)
+        .flatMap(sequence => parseFunctionSignatures(sequence.definition, sequence.workspace_id));
+    } else {
+      librarySequences = $userSequences
+        .filter(sequence => sequence.workspace_id === workspaceId && sequence.name !== sequenceName)
+        .map(userSequenceToLibrarySequence);
+    }
+
+    librarySequenceMap = Object.fromEntries(librarySequences.map(seq => [seq.name, seq]));
 
     if (unparsedCommandDictionary) {
       if (sequenceName && isInVmlMode) {
         getParsedCommandDictionary(unparsedCommandDictionary, user).then(parsedCommandDictionary => {
           commandDictionary = parsedCommandDictionary;
           editorSequenceView.dispatch({
-            effects: compartmentSeqLanguage.reconfigure(setupVmlLanguageSupport(vmlAutoComplete(commandDictionary))),
+            effects: compartmentSeqLanguage.reconfigure(
+              setupVmlLanguageSupport(
+                vmlAutoComplete(commandDictionary, $sequenceAdaptation.globals ?? [], librarySequenceMap),
+              ),
+            ),
           });
           editorSequenceView.dispatch({
-            effects: compartmentSeqLinter.reconfigure(vmlLinter(commandDictionary)),
+            effects: compartmentSeqLinter.reconfigure(
+              vmlLinter(commandDictionary, librarySequenceMap, $sequenceAdaptation.globals ?? []),
+            ),
           });
           editorSequenceView.dispatch({
-            effects: compartmentSeqTooltip.reconfigure(vmlTooltip(commandDictionary)),
+            effects: compartmentSeqTooltip.reconfigure(vmlTooltip(commandDictionary, librarySequenceMap)),
           });
         });
       } else {
@@ -237,6 +245,7 @@
           // Reconfigure sequence editor.
           editorSequenceView.dispatch({
             effects: [
+              // TODO: use librarySequenceMap here, requires a change to adaptations so defer until changing adaptation API
               compartmentSeqLanguage.reconfigure(
                 setupLanguageSupport(
                   $sequenceAdaptation.autoComplete(
@@ -288,8 +297,9 @@
 
   $: commandNode = commandInfoMapper.getContainingCommand(selectedNode);
   $: commandNameNode = commandInfoMapper.getNameNode(commandNode);
-  $: commandName = commandNameNode && editorSequenceView.state.sliceDoc(commandNameNode.from, commandNameNode.to);
-  $: commandDef = getCommandDef(commandDictionary, commandName ?? '');
+  $: commandName =
+    commandNameNode && unquoteUnescape(editorSequenceView.state.sliceDoc(commandNameNode.from, commandNameNode.to));
+  $: commandDef = getCommandDef(commandDictionary, librarySequenceMap, commandName ?? '');
   $: timeTagNode = getTimeTagInfo(editorSequenceView, commandNode);
   $: argInfoArray = getArgumentInfo(
     commandInfoMapper,
@@ -473,6 +483,10 @@
     toggleSeqJsonPreview = !toggleSeqJsonPreview;
   }
 
+  function showErrorPanel() {
+    openLintPanel(editorSequenceView);
+  }
+
   function formatDocument() {
     if (isInVmlMode) {
       vmlFormat(editorSequenceView);
@@ -496,8 +510,21 @@
     );
   }
 
-  function getCommandDef(commandDictionary: CommandDictionary | null, stemName: string): FswCommand | null {
-    return commandDictionary?.fswCommandMap[stemName] ?? null;
+  function getCommandDef(
+    commandDictionary: CommandDictionary | null,
+    librarySequenceMap: LibrarySequenceMap,
+    stemName: string,
+  ): FswCommand | null {
+    const commandDefFromCommandDictionary = commandDictionary?.fswCommandMap[stemName];
+    if (commandDefFromCommandDictionary) {
+      return commandDefFromCommandDictionary;
+    }
+
+    const librarySeqDef = librarySequenceMap[stemName];
+    if (librarySeqDef) {
+      return librarySequenceToFswCommand(librarySeqDef);
+    }
+    return null;
   }
 
   function getVariablesInScope(
@@ -600,6 +627,14 @@
         <SectionTitle>{title}</SectionTitle>
 
         <div class="right">
+          <button
+            use:tooltip={{ content: 'Show Error Panel', placement: 'top' }}
+            class="st-button icon-button secondary ellipsis"
+            on:click={showErrorPanel}
+          >
+            Error Panel
+          </button>
+
           <button
             use:tooltip={{ content: 'Format sequence whitespace', placement: 'top' }}
             class="st-button icon-button secondary ellipsis"

src/components/sequencing/StringTooltip.svelte

@@ -0,0 +1,19 @@
+<svelte:options immutable={true} />
+
+<script lang="ts">
+  export let message: string;
+</script>
+
+<div class="sequence-tooltip">
+  <div class="container">
+    {message}
+  </div>
+</div>
+
+<style>
+  .container {
+    align-items: center;
+    display: flex;
+    padding: 5px;
+  }
+</style>

src/components/sequencing/form/ArgEditor.svelte

@@ -18,6 +18,7 @@
   import AddMissingArgsButton from './AddMissingArgsButton.svelte';
   import ArgTitle from './ArgTitle.svelte';
   import BooleanEditor from './BooleanEditor.svelte';
+  import ByteArrayEditor from './ByteArrayEditor.svelte';
   import EnumEditor from './EnumEditor.svelte';
   import ExtraArgumentEditor from './ExtraArgumentEditor.svelte';
   import NumEditor from './NumEditor.svelte';
@@ -92,7 +93,9 @@
         }}
       />
     {/if}
-    {#if isSymbol && isFswCommandArgumentEnum(argDef)}
+    {#if argInfo.node && commandInfoMapper.isByteArrayArg(argInfo.node)}
+      <ByteArrayEditor value={argInfo.text ?? ''} argNode={argInfo.node} {commandInfoMapper} />
+    {:else if isSymbol && isFswCommandArgumentEnum(argDef)}
       <div class="st-typography-small-caps">Reference</div>
       <EnumEditor
         {argDef}

src/components/sequencing/form/ByteArrayEditor.svelte

@@ -0,0 +1,25 @@
+<svelte:options immutable={true} />
+
+<script lang="ts">
+  import type { SyntaxNode } from '@lezer/common';
+  import { decodeInt32Array } from '../../../utilities/codemirror/codemirror-utils';
+  import type { CommandInfoMapper } from '../../../utilities/codemirror/commandInfoMapper';
+
+  export let argNode: SyntaxNode;
+  export let commandInfoMapper: CommandInfoMapper;
+  export let value: string;
+
+  let decodedValue: string;
+
+  $: {
+    const arrayNodes = commandInfoMapper.getByteArrayElements && commandInfoMapper.getByteArrayElements(argNode, value);
+    if (arrayNodes) {
+      decodedValue = decodeInt32Array(arrayNodes);
+    }
+  }
+</script>
+
+<div>
+  <input class="st-input w-100" spellcheck="false" bind:value title="encoded string" disabled={true} />
+  <input class="st-input w-100" spellcheck="false" bind:value={decodedValue} title="decoded string" disabled={true} />
+</div>

src/enums/sequencing.ts

@@ -0,0 +1,4 @@
+export enum SequenceTypes {
+  LIBRARY = 'library',
+  USER = 'user',
+}

src/types/sequencing.ts

@@ -18,6 +18,7 @@ import type {
 import type { VariableDeclaration } from '@nasa-jpl/seq-json-schema/types';
 import type { EditorView } from 'codemirror';
 import type { DictionaryTypes } from '../enums/dictionaryTypes';
+import type { SequenceTypes } from '../enums/sequencing';
 import type { ArgDelegator } from '../utilities/sequence-editor/extension-points';
 import type { UserId } from './app';
 import type { GlobalType } from './global-type';
@@ -155,9 +156,12 @@ export type LibrarySequence = {
   name: string;
   parameters: VariableDeclaration[];
   tree: Tree;
+  type: SequenceTypes.LIBRARY;
   workspace_id: number;
 };
 
+export type LibrarySequenceMap = { [sequenceName: string]: LibrarySequence };
+
 export type UserSequenceInsertInput = Omit<UserSequence, 'created_at' | 'id' | 'owner' | 'updated_at'>;
 
 export type Workspace = {

src/utilities/codemirror/codemirror-utils.test.ts

@@ -146,7 +146,7 @@ describe('isHexValue', () => {
   });
 });
 
-describe('Command and argument typeguards', () => {
+describe('Command and argument type guards', () => {
   test('isFswCommand', () => {
     expect(
       isFswCommand({

src/utilities/codemirror/codemirror-utils.ts

@@ -23,6 +23,7 @@ import type { CommandInfoMapper } from './commandInfoMapper';
 export function isFswCommand(command: FswCommand | HwCommand): command is FswCommand {
   return (command as FswCommand).type === 'fsw_command';
 }
+
 export function isHwCommand(command: FswCommand | HwCommand): command is HwCommand {
   return (command as HwCommand).type === 'hw_command';
 }
@@ -181,3 +182,20 @@ export function parseNumericArg(argText: string, dictArgType: 'float' | 'integer
 export function isHexValue(argText: string) {
   return /^0x[\da-f]+$/i.test(argText);
 }
+
+export function decodeInt32Array(encoded: string[]) {
+  return encoded
+    .map(charAsHex => {
+      const n = Number(charAsHex);
+      return String.fromCodePoint((n >> 24) & 0xff, (n >> 16) & 0xff, (n >> 8) & 0xff, n & 0xff);
+    })
+    .join('');
+}
+
+export function encodeInt32Array(s: string) {
+  const encoded: string[] = [];
+  for (let i = 0; i < s.length; i += 4) {
+    encoded.push(s.codePointAt(i)?.toString(16) ?? '00');
+  }
+  return encoded;
+}

src/utilities/codemirror/commandInfoMapper.ts

@@ -14,6 +14,8 @@ export interface CommandInfoMapper {
   /** collects argument nodes from sub-tree of this command argument container */
   getArgumentsFromContainer(containerNode: SyntaxNode): SyntaxNode[];
 
+  getByteArrayElements?(node: SyntaxNode | null, arrayText: string): string[] | null;
+
   /** ascends parse tree to find scope to display in form editor */
   getContainingCommand(node: SyntaxNode | null): SyntaxNode | null;
 
@@ -34,6 +36,8 @@ export interface CommandInfoMapper {
   /** is argument node a variable, false implies literal */
   isArgumentNodeOfVariableType(argNode: SyntaxNode | null): boolean;
 
+  isByteArrayArg(argNode: SyntaxNode | null): boolean;
+
   /** checks if select list should be used */
   nodeTypeEnumCompatible(node: SyntaxNode | null): boolean;