feat!: Include referenced files relative to conf dir

This makes the file:// references used in the YAML relative to the
directory that YAML file is in instead of the working directory.

BREAKING CHANGE: please update your pipeline config file:// references.

src/lib/LDWorkbenchConfiguration.d.ts

@@ -18,6 +18,10 @@ export interface LDWorkbenchConfiguration {
    * An optional description for your pipeline.
    */
   description?: string;
+  /**
+   * The base directory for files referenced by file://... paths. Defaults to the parent directory of the YAML config file.
+   */
+  baseDir?: string;
   /**
    * The file where the final result of your pipeline is saved.
    */

src/utils/loadConfiguration.ts

@@ -1,6 +1,8 @@
 import type {LDWorkbenchConfiguration} from '../lib/LDWorkbenchConfiguration.js';
 import parseYamlFile from './parseYamlFile.js';
 import validate from './validate.js';
+import path from 'node:path';
+import {dirname} from 'path';
 
 /**
  * This is a wrapper for the YAML Parser and Schema Validator
@@ -9,12 +11,40 @@ import validate from './validate.js';
 export default function loadConfiguration(
   filePath: string
 ): LDWorkbenchConfiguration {
-  const configuration = parseYamlFile(filePath);
+  const configuration = parseYamlFile(filePath) as LDWorkbenchConfiguration;
   const errors = validate(configuration);
   if (errors !== null) {
     throw new Error(
       `The YAML file \`${filePath}\` is not a valid LD Workbench configuration file.`
     );
   }
-  return configuration as LDWorkbenchConfiguration;
+
+  return mapObject(
+    configuration,
+    inlineQueryFromFile(configuration.baseDir ?? dirname(filePath))
+  ) as LDWorkbenchConfiguration;
 }
+
+const mapObject = (obj: object, replacer: (from: string) => string): object => {
+  return Object.fromEntries(
+    Object.entries(obj).map(([key, value]) => {
+      if (typeof value === 'string' && value.startsWith('file://')) {
+        return [key, replacer(value)];
+      } else if (Array.isArray(value)) {
+        return [key, value.map(v => mapObject(v, replacer))];
+      } else if (typeof value === 'object') {
+        return [key, mapObject(value, replacer)];
+      } else {
+        return [key, value];
+      }
+    })
+  );
+};
+
+const inlineQueryFromFile =
+  (directory: string) =>
+  (prefixedFilePath: string): string => {
+    const filePath = prefixedFilePath.replace('file://', '');
+    const lookupDir = path.isAbsolute(filePath) ? '' : directory;
+    return `file://${path.resolve(lookupDir, filePath)}`;
+  };

src/utils/tests/static/correct/conf1.yml

@@ -6,13 +6,13 @@ description: >
 
 
 # This is optional, by default it will be stored in the data directory of the pipeline using filename 'statements.nt'
-destination: file://pipelines/data/example-pipeline.nt
+destination: file:///pipelines/data/example-pipeline.nt
 
 # The individual stages for your pipeline
 stages:
   - name: "Stage 1A"
     iterator:
-      query: file://static/example/iterator-stage-1.rq
-      endpoint: file://static/tests/iris.nt
-    generator: 
-      - query: file://static/example/generator-stage-1-1.rq
+      query: file:///static/example/iterator-stage-1.rq
+      endpoint: file:///static/tests/iris.nt
+    generator:
+      - query: file:///static/example/generator-stage-1-1.rq

src/utils/tests/static/correct/conf2.yml

@@ -6,13 +6,13 @@ description: >
 
 
 # This is optional, by default it will be stored in the data directory of the pipeline using filename 'statements.nt'
-destination: file://pipelines/data/example-pipeline.nt
+destination: file:///pipelines/data/example-pipeline.nt
 
 # The individual stages for your pipeline
 stages:
   - name: "Stage 1B"
     iterator:
-      query: file://static/example/iterator-stage-1.rq
-      endpoint: file://static/tests/iris.nt
-    generator: 
-      - query: file://static/example/generator-stage-1-1.rq
+      query: file:///static/example/iterator-stage-1.rq
+      endpoint: file:///static/tests/iris.nt
+    generator:
+      - query: file:///static/example/generator-stage-1-1.rq

src/utils/tests/static/single/conf.yml

@@ -6,19 +6,19 @@ description: >
 
 
 # This is optional, by default it will be stored in the data directory of the pipeline using filename 'statements.nt'
-destination: file://pipelines/data/example-pipeline.nt
+destination: file:///pipelines/data/example-pipeline.nt
 
 # The individual stages for your pipeline
 stages:
   - name: "Stage 1"
     iterator:
-      query: file://static/example/iterator-stage-1.rq
-      endpoint: file://static/tests/iris.nt
-    generator: 
-      - query: file://static/example/generator-stage-1-1.rq
+      query: file:///static/example/iterator-stage-1.rq
+      endpoint: file:///static/tests/iris.nt
+    generator:
+      - query: file:///static/example/generator-stage-1-1.rq
   - name: "Stage 2"
-    iterator: 
-      query: file://static/example/iterator-stage-2.rq
-    generator: 
-      - query: file://static/example/generator-stage-2.rq
-        endpoint: file://static/tests/wikidata.nt
+    iterator:
+      query: file:///static/example/iterator-stage-2.rq
+    generator:
+      - query: file:///static/example/generator-stage-2.rq
+        endpoint: file:///static/tests/wikidata.nt

src/utils/tests/utilities.test.ts

@@ -217,16 +217,16 @@ describe('Utilities', () => {
             name: 'Example Pipeline A',
             description:
               'This is an example pipeline. It uses files that are available in this repository  and SPARQL endpoints that should work.\n',
-            destination: 'file://pipelines/data/example-pipeline.nt',
+            destination: 'file:///pipelines/data/example-pipeline.nt',
             stages: [
               {
                 name: 'Stage 1A',
                 iterator: {
-                  query: 'file://static/example/iterator-stage-1.rq',
-                  endpoint: 'file://static/tests/iris.nt',
+                  query: 'file:///static/example/iterator-stage-1.rq',
+                  endpoint: 'file:///static/tests/iris.nt',
                 },
                 generator: [
-                  {query: 'file://static/example/generator-stage-1-1.rq'},
+                  {query: 'file:///static/example/generator-stage-1-1.rq'},
                 ],
               },
             ],
@@ -238,16 +238,16 @@ describe('Utilities', () => {
             name: 'Example Pipeline B',
             description:
               'This is an example pipeline. It uses files that are available in this repository  and SPARQL endpoints that should work.\n',
-            destination: 'file://pipelines/data/example-pipeline.nt',
+            destination: 'file:///pipelines/data/example-pipeline.nt',
             stages: [
               {
                 name: 'Stage 1B',
                 iterator: {
-                  query: 'file://static/example/iterator-stage-1.rq',
-                  endpoint: 'file://static/tests/iris.nt',
+                  query: 'file:///static/example/iterator-stage-1.rq',
+                  endpoint: 'file:///static/tests/iris.nt',
                 },
                 generator: [
-                  {query: 'file://static/example/generator-stage-1-1.rq'},
+                  {query: 'file:///static/example/generator-stage-1-1.rq'},
                 ],
               },
             ],
@@ -268,27 +268,27 @@ describe('Utilities', () => {
             name: 'Example Pipeline',
             description:
               'This is an example pipeline. It uses files that are available in this repository  and SPARQL endpoints that should work.\n',
-            destination: 'file://pipelines/data/example-pipeline.nt',
+            destination: 'file:///pipelines/data/example-pipeline.nt',
             stages: [
               {
                 name: 'Stage 1',
                 iterator: {
-                  query: 'file://static/example/iterator-stage-1.rq',
-                  endpoint: 'file://static/tests/iris.nt',
+                  query: 'file:///static/example/iterator-stage-1.rq',
+                  endpoint: 'file:///static/tests/iris.nt',
                 },
                 generator: [
-                  {query: 'file://static/example/generator-stage-1-1.rq'},
+                  {query: 'file:///static/example/generator-stage-1-1.rq'},
                 ],
               },
               {
                 name: 'Stage 2',
                 iterator: {
-                  query: 'file://static/example/iterator-stage-2.rq',
+                  query: 'file:///static/example/iterator-stage-2.rq',
                 },
                 generator: [
                   {
-                    query: 'file://static/example/generator-stage-2.rq',
-                    endpoint: 'file://static/tests/wikidata.nt',
+                    query: 'file:///static/example/generator-stage-2.rq',
+                    endpoint: 'file:///static/tests/wikidata.nt',
                   },
                 ],
               },

static/example/config.yml

@@ -6,22 +6,22 @@ description: >
 
 
 # This is optional, by default it will be stored in the data directory of the pipeline using filename 'statements.nt'
-destination: file://pipelines/data/example-pipeline.nt
+destination: file://../../pipelines/data/example-pipeline.nt
 
 # The individual stages for your pipeline
 stages:
   - name: "Stage 1"
     iterator:
-      query: file://static/example/iterator-stage-1.rq
+      query: file://iterator-stage-1.rq
       endpoint: https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql
     generator:
       # First generator
-      -  query: file://static/example/generator-stage-1-1.rq
+      - query: file://generator-stage-1-1.rq
       # Second generator
-      -  query: file://static/example/generator-stage-1-2.rq
+      - query: file://generator-stage-1-2.rq
   - name: "Stage 2"
     iterator:
-      query: file://static/example/iterator-stage-2.rq
+      query: file://iterator-stage-2.rq
     generator:
-      - query: file://static/example/generator-stage-2.rq
+      - query: file://generator-stage-2.rq
         endpoint: https://query.wikidata.org/sparql

static/ld-workbench.schema.json

@@ -14,6 +14,10 @@
       "type": "string",
       "description": "An optional description for your pipeline."
     },
+    "baseDir": {
+      "type": "string",
+      "description": "The base directory for files referenced by file://... paths. Defaults to the parent directory of the YAML config file."
+    },
     "destination": {
       "type": "string",
       "description": "The file where the final result of your pipeline is saved."
@@ -88,4 +92,4 @@
     }
   },
   "required": ["name", "stages"]
-}
+}