Import from anywhere

.vscode/launch.json

@@ -40,6 +40,12 @@
       "request": "launch",
       "type": "node-terminal"
     },
+    {
+      "name": "Run Electric Vehicles Example",
+      "command": "npm run example:vehicles",
+      "request": "launch",
+      "type": "node-terminal"
+    },
     {
       "name": "Run Gas Reserve Example",
       "command": "npm run example:gas",

apps/docs/docs/dev/04-guides/05-standard-library.md

@@ -42,7 +42,7 @@ We use code generation to transform these `.jv` files into TypeScript files that
 
 ### 2. Builtin libraries
 
-The solution we chose to implement the standard library mechanism is close to the [built-in library tutorial](https://langium.org/guides/builtin-library/) by Langium. The following components are of interest:
+The solution we chose to implement the standard library mechanism is close to the [built-in library tutorial](https://langium.org/guides/workspace/) by Langium. The following components are of interest:
 
-- [JayveeWorkspaceManager](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/lib/builtin-library/jayvee-workspace-manager.ts) in the `language-server` that registers all libraries with the langium framework.
+- [JayveeWorkspaceManager](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/lib/workspace/jayvee-workspace-manager.ts) in the `language-server` that registers all libraries with the langium framework.
 - [StandardLibraryFileSystemProvider](https://github.com/jvalue/jayvee/tree/main/apps/vs-code-extension/src/standard-library-file-system-provider.ts) in the `vs-code-extension` that registers all libraries with the vscode plugin framework.

apps/vs-code-extension/src/language-server.ts

@@ -16,5 +16,4 @@ const { shared } = createJayveeServices({
   ...NodeFileSystem,
 });
 
-// Start the language server with the shared services
 startLanguageServer(shared);

libs/interpreter-lib/src/parsing-util.ts

@@ -6,7 +6,10 @@ import * as fs from 'node:fs';
 import path from 'node:path';
 
 import { type Logger } from '@jvalue/jayvee-execution';
-import { initializeWorkspace } from '@jvalue/jayvee-language-server';
+import {
+  type JayveeServices,
+  initializeWorkspace,
+} from '@jvalue/jayvee-language-server';
 import { type AstNode, type LangiumDocument } from 'langium';
 import { type LangiumServices } from 'langium/lsp';
 import { DiagnosticSeverity } from 'vscode-languageserver-protocol';
@@ -58,7 +61,7 @@ export async function extractDocumentFromFile(
  */
 export async function extractDocumentFromString(
   modelString: string,
-  services: LangiumServices,
+  services: JayveeServices,
   logger: Logger,
 ): Promise<LangiumDocument> {
   const document = services.shared.workspace.LangiumDocumentFactory.fromString(
@@ -104,7 +107,7 @@ export async function validateDocument(
 
 export async function extractAstNodeFromFile<T extends AstNode>(
   filePath: string,
-  services: LangiumServices,
+  services: JayveeServices,
   logger: Logger,
 ): Promise<T> {
   return (await extractDocumentFromFile(filePath, services, logger)).parseResult
@@ -113,7 +116,7 @@ export async function extractAstNodeFromFile<T extends AstNode>(
 
 export async function extractAstNodeFromString<T extends AstNode>(
   modelString: string,
-  services: LangiumServices,
+  services: JayveeServices,
   logger: Logger,
 ): Promise<T> {
   return (await extractDocumentFromString(modelString, services, logger))

libs/language-server/.gitignore

@@ -4,4 +4,4 @@
 
 # files generated by Langium
 src/lib/ast/generated
-src/lib/builtin-library/generated
+src/lib/workspace/generated

libs/language-server/src/lib/index.ts

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 export * from './ast';
-export * from './builtin-library';
+export * from './workspace';
 export * from './docs';
 export * from './services';
 export * from './util';

libs/language-server/src/lib/jayvee-module.ts

@@ -24,7 +24,6 @@ import {
 } from './ast/generated/module';
 import { ValueTypeProvider } from './ast/wrappers/value-type/primitive/primitive-value-type-provider';
 import { WrapperFactoryProvider } from './ast/wrappers/wrapper-factory-provider';
-import { JayveeWorkspaceManager } from './builtin-library/jayvee-workspace-manager';
 import { JayveeValueConverter } from './jayvee-value-converter';
 import {
   JayveeCodeActionProvider,
@@ -37,6 +36,8 @@ import {
 import { JayveeImportResolver } from './services/import-resolver';
 import { RuntimeParameterProvider } from './services/runtime-parameter-provider';
 import { JayveeValidationRegistry } from './validation/validation-registry';
+import { addDynamicFileImport } from './workspace';
+import { JayveeWorkspaceManager } from './workspace/jayvee-workspace-manager';
 
 /**
  * Declaration of custom services for the Jayvee language.
@@ -150,5 +151,7 @@ export function createJayveeServices(context: DefaultSharedModuleContext): {
   );
   shared.ServiceRegistry.register(Jayvee);
 
+  addDynamicFileImport(Jayvee);
+
   return { shared, Jayvee };
 }

libs/language-server/src/lib/lsp/import-dynamic-reference.spec.ts

@@ -0,0 +1,90 @@
+// SPDX-FileCopyrightText: 2024 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import path from 'node:path';
+
+import { NodeFileSystem } from 'langium/node';
+
+import {
+  type JayveeServices,
+  createJayveeServices,
+  isJayveeModel,
+} from '../../lib';
+import {
+  expectNoParserAndLexerErrors,
+  parseTestFileInWorkingDir,
+} from '../../test';
+import { type JayveeModel } from '../ast';
+
+describe('References to imported elements outside of the working directory', () => {
+  const WORKING_DIR = path.resolve(
+    __dirname,
+    '../../test/assets/import-dynamic-reference/models', // use the "deep" directory as working dir to avoid loading the "higher" dir
+  );
+  let services: JayveeServices;
+
+  async function parseModel(
+    relativeTestFilePath: string,
+  ): Promise<JayveeModel> {
+    const document = await parseTestFileInWorkingDir(
+      WORKING_DIR,
+      relativeTestFilePath,
+      services,
+    );
+    expectNoParserAndLexerErrors(document);
+
+    const parsedModel = document.parseResult.value;
+    assert(isJayveeModel(parsedModel), 'Test file is not valid Jayvee model');
+    return parsedModel;
+  }
+
+  beforeEach(() => {
+    // Create language services
+    services = createJayveeServices(NodeFileSystem).Jayvee;
+  });
+
+  const validCases: [string, string][] = [
+    // [test name, test file path]
+    [
+      'should resolve reference to imported element',
+      'valid-import-from-outside-workdir.jv',
+    ],
+  ];
+  test.each(validCases)('%s', async (_, relativeTestFilePath) => {
+    const model = await parseModel(relativeTestFilePath);
+
+    expect(model.pipelines.length).toEqual(1); // file contains one pipeline
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const blocks = model.pipelines[0]!.blocks;
+    expect(blocks.length).toEqual(1); // pipeline contains one block
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const reference = blocks[0]!.type; // of an imported block type
+
+    expect(reference.ref).toBeDefined();
+  });
+
+  const invalidCases: [string, string][] = [
+    // [test name, test file path]
+    [
+      'should not resolve reference to file with no jv extension',
+      'invalid-import-wrong-file-type.jv',
+    ],
+    [
+      'should not resolve reference to file that does not exist',
+      'invalid-import-not-existing-file.jv',
+    ],
+  ];
+  test.each(invalidCases)('%s', async (_, relativeTestFilePath) => {
+    const model = await parseModel(relativeTestFilePath);
+
+    expect(model.pipelines.length).toEqual(1); // file contains one pipeline
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const blocks = model.pipelines[0]!.blocks;
+    expect(blocks.length).toEqual(1); // pipeline contains one block
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const reference = blocks[0]!.type; // of an imported block type
+
+    expect(reference.ref).toBeUndefined();
+  });
+});

libs/language-server/src/lib/services/import-resolver.ts

@@ -20,8 +20,8 @@ import {
   isExportableElement,
   isJayveeModel,
 } from '../ast/generated/ast';
-import { getStdLib } from '../builtin-library/stdlib';
 import { type JayveeServices } from '../jayvee-module';
+import { getStdLib } from '../workspace/stdlib';
 
 export interface ImportDetails {
   element: ExportableElement;
@@ -52,6 +52,25 @@ export class JayveeImportResolver {
     this.availableElementsPerDocumentCache = new DocumentCache(services.shared);
   }
 
+  /**
+   * Finds all import URIs that could not be resolved.
+   */
+  findUnresolvedImportURIs(model: JayveeModel): URI[] {
+    const unresolvedURIs: URI[] = [];
+    for (const importDefinition of model.imports) {
+      const uri = this.resolveImportUri(importDefinition);
+      if (uri === undefined) {
+        continue;
+      }
+
+      const isDocumentResolved = this.documents.getDocument(uri) !== undefined;
+      if (!isDocumentResolved) {
+        unresolvedURIs.push(uri);
+      }
+    }
+    return unresolvedURIs;
+  }
+
   resolveImport(importDefinition: ImportDefinition): JayveeModel | undefined {
     const resolvedDocument = this.resolveImportedDocument(importDefinition);
     if (resolvedDocument === undefined) {

libs/language-server/src/lib/validation/checks/import-definition.spec.ts

@@ -100,13 +100,13 @@ describe('Validation of ImportDefinition', () => {
       expect(validationAcceptorMock).toHaveBeenNthCalledWith(
         1,
         'error',
-        'Import from "not-existing-imported-file-deeper.jv" could be resolved. Check if the file exists in the given location.',
+        'Import from "not-existing-imported-file-deeper.jv" could not be resolved. Check if the file exists in the given location.',
         expect.any(Object),
       );
       expect(validationAcceptorMock).toHaveBeenNthCalledWith(
         2,
         'error',
-        'Import from "./not-existing-imported-file-deeper.jv" could be resolved. Check if the file exists in the given location.',
+        'Import from "./not-existing-imported-file-deeper.jv" could not be resolved. Check if the file exists in the given location.',
         expect.any(Object),
       );
     });
@@ -121,13 +121,13 @@ describe('Validation of ImportDefinition', () => {
       expect(validationAcceptorMock).toHaveBeenNthCalledWith(
         1,
         'error',
-        'Import from "existing-imported-file.njv" could be resolved. Check if the file exists in the given location.',
+        'Import from "existing-imported-file.njv" could not be resolved. Check if the file exists in the given location.',
         expect.any(Object),
       );
       expect(validationAcceptorMock).toHaveBeenNthCalledWith(
         2,
         'error',
-        'Import from "./existing-imported-file.njv" could be resolved. Check if the file exists in the given location.',
+        'Import from "./existing-imported-file.njv" could not be resolved. Check if the file exists in the given location.',
         expect.any(Object),
       );
     });

libs/language-server/src/lib/validation/checks/import-definition.ts

@@ -98,7 +98,7 @@ function checkPathExists(
   if (resolvedImport === undefined) {
     props.validationContext.accept(
       'error',
-      `Import from "${importDefinition.path}" could be resolved. Check if the file exists in the given location.`,
+      `Import from "${importDefinition.path}" could not be resolved. Check if the file exists in the given location.`,
       {
         node: importDefinition,
         property: 'path',

libs/language-server/src/lib/validation/docs-constraint-examples.spec.ts

@@ -6,8 +6,8 @@ import { NodeFileSystem } from 'langium/node';
 
 import { validationHelper } from '../../test/langium-utils';
 import { getAllBuiltinConstraintTypes } from '../ast';
-import { initializeWorkspace } from '../builtin-library';
 import { createJayveeServices } from '../jayvee-module';
+import { initializeWorkspace } from '../workspace';
 
 describe('Validation of docs examples of ConstraintTypes', () => {
   it('should have no validation errors', async () => {

libs/language-server/src/lib/workspace/dynamic-file-import.ts

@@ -0,0 +1,74 @@
+// SPDX-FileCopyrightText: 2024 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { DocumentState } from 'langium';
+import { type URI } from 'vscode-uri';
+
+import { type JayveeModel, isJayveeModel } from '../ast';
+import { type JayveeServices } from '../jayvee-module';
+
+export function addDynamicFileImport(services: JayveeServices): void {
+  const documentBuilder = services.shared.workspace.DocumentBuilder;
+  const importResolver = services.ImportResolver;
+
+  documentBuilder.onBuildPhase(DocumentState.IndexedContent, async (docs) => {
+    for (const doc of docs) {
+      const model = doc.parseResult.value;
+      if (!isJayveeModel(model)) {
+        return;
+      }
+      const importURIs = importResolver.findUnresolvedImportURIs(model);
+
+      for (const importURI of importURIs) {
+        await loadDocumentFromFs(importURI, services);
+      }
+    }
+  });
+}
+
+async function loadDocumentFromFs(
+  importURI: URI,
+  services: JayveeServices,
+): Promise<void> {
+  const allowedFileExtensions = services.shared.ServiceRegistry.all.flatMap(
+    (e) => e.LanguageMetaData.fileExtensions,
+  );
+  const hasAllowedFileExtension = allowedFileExtensions.some((ext) =>
+    importURI.fsPath.endsWith(ext),
+  );
+  if (!hasAllowedFileExtension) {
+    return;
+  }
+
+  const langiumDocuments = services.shared.workspace.LangiumDocuments;
+  const documentBuilder = services.shared.workspace.DocumentBuilder;
+  const documentFactory = services.shared.workspace.LangiumDocumentFactory;
+
+  const file = await loadFileFromUri(importURI, services);
+  if (file === undefined) {
+    return;
+  }
+
+  const document = documentFactory.fromString<JayveeModel>(file, importURI);
+  await documentBuilder.build([document], { validation: true });
+
+  await services.shared.workspace.WorkspaceLock.write(() => {
+    if (!langiumDocuments.hasDocument(document.uri)) {
+      langiumDocuments.addDocument(document);
+    }
+  });
+}
+
+async function loadFileFromUri(
+  uri: URI,
+  services: JayveeServices,
+): Promise<string | undefined> {
+  const fileSystemProvider = services.shared.workspace.FileSystemProvider;
+
+  try {
+    return await fileSystemProvider.readFile(uri);
+  } catch (e) {
+    return undefined;
+  }
+}

libs/language-server/src/lib/builtin-library/index.ts → libs/language-server/src/lib/workspace/index.ts

@@ -3,4 +3,5 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 export { getStdLib } from './stdlib';
+export { addDynamicFileImport } from './dynamic-file-import';
 export * from './jayvee-workspace-manager';