From 353f63f26e6a81f4aef1fc564df49d442384d27a Mon Sep 17 00:00:00 2001 From: David de Boer Date: Fri, 14 Jun 2024 13:54:42 +0200 Subject: [PATCH] chore: Standardize files --- CONTRIBUTING.md | 2 +- src/{cliArgs.ts => cli.ts} | 6 +- src/configuration.d.ts | 158 ++++++++++++++ src/{lib/File.class.ts => file.ts} | 4 +- src/{lib/Generator.class.ts => generator.ts} | 12 +- src/{lib/Iterator.class.ts => iterator.ts} | 12 +- src/lib/LDWorkbenchConfiguration.d.ts | 180 ---------------- src/lib/PreviousStage.class.ts | 31 --- src/main.ts | 20 +- src/{lib/Pipeline.class.ts => pipeline.ts} | 18 +- src/{lib/Stage.class.ts => stage.ts} | 42 +++- src/{lib/TriplyDB.class.ts => triply-db.ts} | 2 +- src/{lib => }/types.ts | 4 +- src/utils/getEndpoint.ts | 11 +- src/utils/getEngine.ts | 2 +- src/utils/getEngineSource.ts | 2 +- src/utils/guards.ts | 12 +- src/utils/loadConfiguration.ts | 10 +- src/utils/loadPipelines.ts | 6 +- test/PreviousStage.class.test.ts | 192 ----------------- test/{File.class.test.ts => file.test.ts} | 2 +- ...erator.class.test.ts => generator.test.ts} | 16 +- ...terator.class.test.ts => iterator.test.ts} | 12 +- ...ipeline.class.test.ts => pipeline.test.ts} | 24 +-- test/{Stage.class.test.ts => stage.test.ts} | 204 +++++++++++++++++- test/utils/utilities.test.ts | 25 ++- 26 files changed, 487 insertions(+), 522 deletions(-) rename src/{cliArgs.ts => cli.ts} (92%) create mode 100644 src/configuration.d.ts rename src/{lib/File.class.ts => file.ts} (94%) rename src/{lib/Generator.class.ts => generator.ts} (91%) rename src/{lib/Iterator.class.ts => iterator.ts} (90%) delete mode 100644 src/lib/LDWorkbenchConfiguration.d.ts delete mode 100644 src/lib/PreviousStage.class.ts rename src/{lib/Pipeline.class.ts => pipeline.ts} (94%) rename src/{lib/Stage.class.ts => stage.ts} (75%) rename src/{lib/TriplyDB.class.ts => triply-db.ts} (97%) rename src/{lib => }/types.ts (76%) delete mode 100644 test/PreviousStage.class.test.ts rename test/{File.class.test.ts => file.test.ts} (98%) rename test/{Generator.class.test.ts => generator.test.ts} (95%) rename test/{Iterator.class.test.ts => iterator.test.ts} (93%) rename test/{Pipeline.class.test.ts => pipeline.test.ts} (94%) rename test/{Stage.class.test.ts => stage.test.ts} (53%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d5b667b..492ef5b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,7 +14,7 @@ npm run compile npm run ld-workbench -- --config static/example ``` -The configuration of this project is validated and defined by [JSON Schema](https://json-schema.org). The schema is located in `./static/ld-workbench-schema.json`. To create the types from this schema, run `npm run util:json-schema-to-typescript`. This will regenerate `./src/types/LDWorkbenchConfiguration.d.ts`, do not modify this file by hand. +The configuration of this project is validated and defined by [JSON Schema](https://json-schema.org). The schema is located in `./static/ld-workbench-schema.json`. To create the types from this schema, run `npm run util:json-schema-to-typescript`. This will regenerate `./src/types/configuration.d.ts`, do not modify this file by hand. ## Committing changes diff --git a/src/cliArgs.ts b/src/cli.ts similarity index 92% rename from src/cliArgs.ts rename to src/cli.ts index 5d802a7..110c196 100644 --- a/src/cliArgs.ts +++ b/src/cli.ts @@ -27,7 +27,7 @@ program ) .version(version()); program.parse(); -export const cliArgs: { +export const cli: { config: string; pipeline?: string; stage?: string; @@ -35,8 +35,8 @@ export const cliArgs: { init?: boolean; } = program.opts(); -if (cliArgs.init !== undefined) { - if (Object.values(cliArgs).length !== 2) { +if (cli.init !== undefined) { + if (Object.values(cli).length !== 2) { error( 'The --init flag can not be used in conjunction with other CLI arguments.' ); diff --git a/src/configuration.d.ts b/src/configuration.d.ts new file mode 100644 index 0000000..5499923 --- /dev/null +++ b/src/configuration.d.ts @@ -0,0 +1,158 @@ +/* eslint-disable */ +/** + * This file was automatically generated by json-schema-to-typescript. + * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file, + * and run json-schema-to-typescript to regenerate this file. + */ + +/** + * JSON Schema for LDWorkbench. + * It helps with the writing of the configuration needed to run LDWorkbench pipelines. + */ +export interface Configuration { + /** + * The name of your pipeline. It must be unique over all your configurations. + */ + name: string; + /** + * An optional description for your pipeline. + */ + description?: string; + /** + * An optional base directory for files referenced by `file://...` paths. This defaults to the directory that contains the YAML configuration file. + */ + baseDir?: string; + /** + * The file where the final result of your pipeline is saved. + */ + destination?: string; + /** + * A pipeline stage consists of an iterator and one or more generators. + * + * @minItems 1 + */ + stages: [ + { + /** + * The name of the stage. It must be unique within the pipeline. + */ + name: string; + iterator: { + /** + * SPARQL SELECT query that returns a `$this` binding for each URI that will be passed to the generator(s). Either an inline string (`SELECT $this WHERE {...}`) or a reference to a file (`file://...`) that contains the query. + */ + query: string; + /** + * SPARQL endpoint for the iterator. If it starts with `file://`, a local RDF file is queried. If omitted the result of the previous stage is used. + */ + endpoint?: string; + /** + * Number of `$this` bindings retrieved per query. Defaults to the LIMIT value of your iterator query or 10 if no LIMIT is present. + */ + batchSize?: number; + /** + * Human-readable time delay for requests to the the iterator’s SPARQL endpoint (e.g. `5ms`, `100 milliseconds`, `1s`). + */ + delay?: string; + }; + /** + * @minItems 1 + */ + generator: [ + { + /** + * SPARQL CONSTRUCT query that takes a `$this` binding from the iterator and generates triples for it. Either an inline string (`CONSTRUCT $this schema:name ?name WHERE {$this ...}`) or a reference to a file (`file://...`) that contains the query. + */ + query: string; + /** + * The SPARQL endpoint for the generator. If it starts with `file://`, a local RDF file is queried. If omitted, the endpoint of the iterator is used. + */ + endpoint?: string; + /** + * Overrule the generator's behaviour of fetching results for 10 bindings of `$this` per request. + */ + batchSize?: number; + }, + ...{ + /** + * SPARQL CONSTRUCT query that takes a `$this` binding from the iterator and generates triples for it. Either an inline string (`CONSTRUCT $this schema:name ?name WHERE {$this ...}`) or a reference to a file (`file://...`) that contains the query. + */ + query: string; + /** + * The SPARQL endpoint for the generator. If it starts with `file://`, a local RDF file is queried. If omitted, the endpoint of the iterator is used. + */ + endpoint?: string; + /** + * Overrule the generator's behaviour of fetching results for 10 bindings of `$this` per request. + */ + batchSize?: number; + }[] + ]; + /** + * The optional path where the results are saved. If omitted, a temporary file will be created. + */ + destination?: string; + }, + ...{ + /** + * The name of the stage. It must be unique within the pipeline. + */ + name: string; + iterator: { + /** + * SPARQL SELECT query that returns a `$this` binding for each URI that will be passed to the generator(s). Either an inline string (`SELECT $this WHERE {...}`) or a reference to a file (`file://...`) that contains the query. + */ + query: string; + /** + * SPARQL endpoint for the iterator. If it starts with `file://`, a local RDF file is queried. If omitted the result of the previous stage is used. + */ + endpoint?: string; + /** + * Number of `$this` bindings retrieved per query. Defaults to the LIMIT value of your iterator query or 10 if no LIMIT is present. + */ + batchSize?: number; + /** + * Human-readable time delay for requests to the the iterator’s SPARQL endpoint (e.g. `5ms`, `100 milliseconds`, `1s`). + */ + delay?: string; + }; + /** + * @minItems 1 + */ + generator: [ + { + /** + * SPARQL CONSTRUCT query that takes a `$this` binding from the iterator and generates triples for it. Either an inline string (`CONSTRUCT $this schema:name ?name WHERE {$this ...}`) or a reference to a file (`file://...`) that contains the query. + */ + query: string; + /** + * The SPARQL endpoint for the generator. If it starts with `file://`, a local RDF file is queried. If omitted, the endpoint of the iterator is used. + */ + endpoint?: string; + /** + * Overrule the generator's behaviour of fetching results for 10 bindings of `$this` per request. + */ + batchSize?: number; + }, + ...{ + /** + * SPARQL CONSTRUCT query that takes a `$this` binding from the iterator and generates triples for it. Either an inline string (`CONSTRUCT $this schema:name ?name WHERE {$this ...}`) or a reference to a file (`file://...`) that contains the query. + */ + query: string; + /** + * The SPARQL endpoint for the generator. If it starts with `file://`, a local RDF file is queried. If omitted, the endpoint of the iterator is used. + */ + endpoint?: string; + /** + * Overrule the generator's behaviour of fetching results for 10 bindings of `$this` per request. + */ + batchSize?: number; + }[] + ]; + /** + * The optional path where the results are saved. If omitted, a temporary file will be created. + */ + destination?: string; + }[] + ]; +} diff --git a/src/lib/File.class.ts b/src/file.ts similarity index 94% rename from src/lib/File.class.ts rename to src/file.ts index e2dd736..d65d360 100644 --- a/src/lib/File.class.ts +++ b/src/file.ts @@ -6,11 +6,11 @@ import { mkdirSync, createReadStream, } from 'fs'; -import {isFile, isFilePathString} from '../utils/guards.js'; +import {isFile, isFilePathString} from './utils/guards.js'; import {dirname} from 'path'; import chalk from 'chalk'; import {type Ora} from 'ora'; -import type Pipeline from './Pipeline.class.js'; +import type Pipeline from './pipeline.js'; import {pipeline as streamPipeline} from 'stream/promises'; export default class File { diff --git a/src/lib/Generator.class.ts b/src/generator.ts similarity index 91% rename from src/lib/Generator.class.ts rename to src/generator.ts index fddb12e..eeb0918 100644 --- a/src/lib/Generator.class.ts +++ b/src/generator.ts @@ -1,12 +1,12 @@ import type {ConstructQuery} from 'sparqljs'; -import type Stage from './Stage.class.js'; -import getSPARQLQuery from '../utils/getSPARQLQuery.js'; +import type Stage from './stage.js'; +import getSPARQLQuery from './utils/getSPARQLQuery.js'; import type {Quad, NamedNode} from '@rdfjs/types'; -import getSPARQLQueryString from '../utils/getSPARQLQueryString.js'; -import getEndpoint from '../utils/getEndpoint.js'; +import getSPARQLQueryString from './utils/getSPARQLQueryString.js'; +import getEndpoint from './utils/getEndpoint.js'; import type {Endpoint, QueryEngine, QuerySource} from './types.js'; -import getEngine from '../utils/getEngine.js'; -import getEngineSource from '../utils/getEngineSource.js'; +import getEngine from './utils/getEngine.js'; +import getEngineSource from './utils/getEngineSource.js'; import EventEmitter from 'node:events'; const DEFAULT_BATCH_SIZE = 10; diff --git a/src/lib/Iterator.class.ts b/src/iterator.ts similarity index 90% rename from src/lib/Iterator.class.ts rename to src/iterator.ts index 2b2a5da..ad0cde9 100644 --- a/src/lib/Iterator.class.ts +++ b/src/iterator.ts @@ -1,14 +1,14 @@ import EventEmitter from 'node:events'; import type {SelectQuery} from 'sparqljs'; -import type Stage from './Stage.class.js'; +import type Stage from './stage.js'; import type {NamedNode} from '@rdfjs/types'; -import getSPARQLQuery from '../utils/getSPARQLQuery.js'; +import getSPARQLQuery from './utils/getSPARQLQuery.js'; import {type Bindings} from '@comunica/types'; -import getSPARQLQueryString from '../utils/getSPARQLQueryString.js'; -import getEndpoint from '../utils/getEndpoint.js'; +import getSPARQLQueryString from './utils/getSPARQLQueryString.js'; +import getEndpoint from './utils/getEndpoint.js'; import type {Endpoint, QueryEngine, QuerySource} from './types.js'; -import getEngine from '../utils/getEngine.js'; -import getEngineSource from '../utils/getEngineSource.js'; +import getEngine from './utils/getEngine.js'; +import getEngineSource from './utils/getEngineSource.js'; import parse from 'parse-duration'; const DEFAULT_LIMIT = 10; diff --git a/src/lib/LDWorkbenchConfiguration.d.ts b/src/lib/LDWorkbenchConfiguration.d.ts deleted file mode 100644 index 3b06366..0000000 --- a/src/lib/LDWorkbenchConfiguration.d.ts +++ /dev/null @@ -1,180 +0,0 @@ -/* eslint-disable */ -/** - * This file was automatically generated by json-schema-to-typescript. - * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file, - * and run json-schema-to-typescript to regenerate this file. - */ - -/** - * JSON Schema for LDWorkbench. - * It helps with the writing of the configuration needed to run LDWorkbench pipelines. - */ -export interface LDWorkbenchConfiguration { - /** - * The name of your pipeline, it must be unique over all your configurations. - */ - name: string; - /** - * An optional description for your pipeline. - */ - description?: string; - /** - * The base directory for files referenced by file://... paths. Defaults to the parent directory of the YAML config file. - */ - baseDir?: string; - /** - * The file where the final result of your pipeline is saved. - */ - destination?: string; - /** - * This is where you define the individual iterator/generator for each step. - * - * @minItems 1 - */ - stages: [ - { - /** - * The name of your pipeline step, it must be unique within one configuration. - */ - name: string; - iterator: { - /** - * Path (prefixed with "file://") or SPARQL Query - * that makes the iterator using SPARQL select. - */ - query: string; - /** - * The SPARQL endpoint for the iterator. - * If it starts with "file://", a local RDF file is queried. - * If ommmitted the result of the previous file is used. - */ - endpoint?: string; - /** - * Overrule the iterator's behaviour of fetching 10 results per request, regardless of any limit's in your query. - */ - batchSize?: number; - /** - * Human readable time delay for the iterator's SPARQL endpoint requests (e.g. '5ms', '100 milliseconds', '1s'). - */ - delay?: string; - }; - /** - * @minItems 1 - */ - generator: [ - { - /** - * Path (prefixed with "file://") or SPARQL Query - * that makes the generator using SPARQL construct. - */ - query: string; - /** - * The SPARQL endpoint for the generator. - * If it starts with "file://", a local RDF file is queried. - * If ommmitted the endpoint of the Iterator is used. - */ - endpoint?: string; - /** - * Overrule the generator's behaviour of fetching results for 10 bindings of $this per request. - */ - batchSize?: number; - }, - ...{ - /** - * Path (prefixed with "file://") or SPARQL Query - * that makes the generator using SPARQL construct. - */ - query: string; - /** - * The SPARQL endpoint for the generator. - * If it starts with "file://", a local RDF file is queried. - * If ommmitted the endpoint of the Iterator is used. - */ - endpoint?: string; - /** - * Overrule the generator's behaviour of fetching results for 10 bindings of $this per request. - */ - batchSize?: number; - }[] - ]; - /** - * The file where the results are saved. - * This is not a required property, - * if ommitted a temporary file will be created automatically. - */ - destination?: string; - }, - ...{ - /** - * The name of your pipeline step, it must be unique within one configuration. - */ - name: string; - iterator: { - /** - * Path (prefixed with "file://") or SPARQL Query - * that makes the iterator using SPARQL select. - */ - query: string; - /** - * The SPARQL endpoint for the iterator. - * If it starts with "file://", a local RDF file is queried. - * If ommmitted the result of the previous file is used. - */ - endpoint?: string; - /** - * Overrule the iterator's behaviour of fetching 10 results per request, regardless of any limit's in your query. - */ - batchSize?: number; - /** - * Human readable time delay for the iterator's SPARQL endpoint requests (e.g. '5ms', '100 milliseconds', '1s'). - */ - delay?: string; - }; - /** - * @minItems 1 - */ - generator: [ - { - /** - * Path (prefixed with "file://") or SPARQL Query - * that makes the generator using SPARQL construct. - */ - query: string; - /** - * The SPARQL endpoint for the generator. - * If it starts with "file://", a local RDF file is queried. - * If ommmitted the endpoint of the Iterator is used. - */ - endpoint?: string; - /** - * Overrule the generator's behaviour of fetching results for 10 bindings of $this per request. - */ - batchSize?: number; - }, - ...{ - /** - * Path (prefixed with "file://") or SPARQL Query - * that makes the generator using SPARQL construct. - */ - query: string; - /** - * The SPARQL endpoint for the generator. - * If it starts with "file://", a local RDF file is queried. - * If ommmitted the endpoint of the Iterator is used. - */ - endpoint?: string; - /** - * Overrule the generator's behaviour of fetching results for 10 bindings of $this per request. - */ - batchSize?: number; - }[] - ]; - /** - * The file where the results are saved. - * This is not a required property, - * if ommitted a temporary file will be created automatically. - */ - destination?: string; - }[] - ]; -} diff --git a/src/lib/PreviousStage.class.ts b/src/lib/PreviousStage.class.ts deleted file mode 100644 index 059c86f..0000000 --- a/src/lib/PreviousStage.class.ts +++ /dev/null @@ -1,31 +0,0 @@ -import {isPreviousStage} from '../utils/guards.js'; -import type Stage from './Stage.class.js'; - -export default class PreviousStage { - public readonly $id = 'PreviousStage'; - public constructor( - public readonly nextStage: Stage, - public readonly name: string - ) {} - - public load(): Stage { - if (!this.nextStage.pipeline.stages.has(this.name)) { - throw new Error( - `This is unexpected: missing stage "${this.name}" in stages.` - ); - } - const previousStage = this.nextStage.pipeline.getPreviousStage( - this.nextStage - ); - if (previousStage === undefined) { - throw new Error( - 'no endpoint was defined, but there is also no previous stage to use' - ); - } - return previousStage; - } - - public static is(value: unknown): value is PreviousStage { - return isPreviousStage(value); - } -} diff --git a/src/main.ts b/src/main.ts index ac8cfa5..2b68990 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,25 +3,25 @@ import inquirer from 'inquirer'; import chalk from 'chalk'; import {error} from './utils/error.js'; import version from './utils/version.js'; -import type {LDWorkbenchConfiguration} from './lib/LDWorkbenchConfiguration.js'; +import type {Configuration} from './configuration.js'; import loadPipelines from './utils/loadPipelines.js'; -import Pipeline from './lib/Pipeline.class.js'; -import {cliArgs} from './cliArgs.js'; +import Pipeline from './pipeline.js'; +import {cli} from './cli.js'; console.info( chalk.bold(`Welcome to LD Workbench version ${chalk.cyan(version())}`) ); async function main(): Promise { - const pipelines = loadPipelines(cliArgs.config ?? './pipelines/'); + const pipelines = loadPipelines(cli.config ?? './pipelines/'); const names = [...pipelines.keys()]; - let configuration: LDWorkbenchConfiguration | undefined; + let configuration: Configuration | undefined; - if (cliArgs.pipeline !== undefined) { - configuration = pipelines.get(cliArgs.pipeline); + if (cli.pipeline !== undefined) { + configuration = pipelines.get(cli.pipeline); if (configuration === undefined) { error( - `No pipeline named “${cliArgs.pipeline}” was found.`, + `No pipeline named “${cli.pipeline}” was found.`, 2, `Valid pipeline names are: ${names.map(name => `"${name}"`).join(', ')}` ); @@ -57,8 +57,8 @@ async function main(): Promise { try { const pipeline = new Pipeline(configuration, { - startFromStageName: cliArgs.stage, - silent: cliArgs.silent, + startFromStageName: cli.stage, + silent: cli.silent, }); await pipeline.run(); } catch (e) { diff --git a/src/lib/Pipeline.class.ts b/src/pipeline.ts similarity index 94% rename from src/lib/Pipeline.class.ts rename to src/pipeline.ts index cf93d91..1a15a4f 100644 --- a/src/lib/Pipeline.class.ts +++ b/src/pipeline.ts @@ -1,17 +1,17 @@ import ora, {Ora} from 'ora'; import kebabcase from 'lodash.kebabcase'; -import type {LDWorkbenchConfiguration} from './LDWorkbenchConfiguration.js'; +import type {Configuration} from './configuration.js'; import chalk from 'chalk'; -import Stage from './Stage.class.js'; -import formatDuration from '../utils/formatDuration.js'; +import Stage from './stage.js'; +import formatDuration from './utils/formatDuration.js'; import {millify} from 'millify'; -import File from './File.class.js'; +import File from './file.js'; import path from 'node:path'; import * as fs from 'node:fs'; -import {isFilePathString, isTriplyDBPathString} from '../utils/guards.js'; -import TriplyDB from './TriplyDB.class.js'; +import {isFilePathString, isTriplyDBPathString} from './utils/guards.js'; +import TriplyDB from './triply-db.js'; import prettyMilliseconds from 'pretty-ms'; -import {memoryConsumption} from '../utils/memory.js'; +import {memoryConsumption} from './utils/memory.js'; interface PipelineOptions { startFromStageName?: string; silent?: boolean; @@ -28,7 +28,7 @@ class Pipeline { private readonly opts?: PipelineOptions; public constructor( - private readonly $configuration: LDWorkbenchConfiguration, + private readonly $configuration: Configuration, pipelineOptions?: PipelineOptions ) { // create data folder: @@ -111,7 +111,7 @@ class Pipeline { } } - public get configuration(): LDWorkbenchConfiguration { + public get configuration(): Configuration { return this.$configuration; } diff --git a/src/lib/Stage.class.ts b/src/stage.ts similarity index 75% rename from src/lib/Stage.class.ts rename to src/stage.ts index 7f1ed30..bccb399 100644 --- a/src/lib/Stage.class.ts +++ b/src/stage.ts @@ -1,14 +1,15 @@ import EventEmitter from 'node:events'; -import File from './File.class.js'; -import {type LDWorkbenchConfiguration} from './LDWorkbenchConfiguration.js'; -import Iterator from './Iterator.class.js'; -import Generator from './Generator.class.js'; +import File from './file.js'; +import {Configuration} from './configuration.js'; +import Iterator from './iterator.js'; +import Generator from './generator.js'; import kebabcase from 'lodash.kebabcase'; -import type Pipeline from './Pipeline.class.js'; +import type Pipeline from './pipeline.js'; import path from 'node:path'; import {Writer} from 'n3'; import type {NamedNode} from '@rdfjs/types'; import type {WriteStream} from 'node:fs'; +import {isPreviousStage} from './utils/guards.js'; interface Events { generatorResult: [count: number]; @@ -25,7 +26,7 @@ export default class Stage extends EventEmitter { public constructor( public readonly pipeline: Pipeline, - public readonly configuration: LDWorkbenchConfiguration['stages'][0] + public readonly configuration: Configuration['stages'][0] ) { super(); try { @@ -123,3 +124,32 @@ export default class Stage extends EventEmitter { this.iterator.run(); } } + +export class PreviousStage { + public readonly $id = 'PreviousStage'; + public constructor( + public readonly nextStage: Stage, + public readonly name: string + ) {} + + public load(): Stage { + if (!this.nextStage.pipeline.stages.has(this.name)) { + throw new Error( + `This is unexpected: missing stage "${this.name}" in stages.` + ); + } + const previousStage = this.nextStage.pipeline.getPreviousStage( + this.nextStage + ); + if (previousStage === undefined) { + throw new Error( + 'no endpoint was defined, but there is also no previous stage to use' + ); + } + return previousStage; + } + + public static is(value: unknown): value is PreviousStage { + return isPreviousStage(value); + } +} diff --git a/src/lib/TriplyDB.class.ts b/src/triply-db.ts similarity index 97% rename from src/lib/TriplyDB.class.ts rename to src/triply-db.ts index 6741b03..795fe6f 100644 --- a/src/lib/TriplyDB.class.ts +++ b/src/triply-db.ts @@ -1,5 +1,5 @@ import {type Ora} from 'ora'; -import type Pipeline from './Pipeline.class.js'; +import type Pipeline from './pipeline.js'; import App from '@triply/triplydb'; const pattern = /^triplydb:\/\/([a-z0-9-]+)\/([a-z0-9-]+)$/; diff --git a/src/lib/types.ts b/src/types.ts similarity index 76% rename from src/lib/types.ts rename to src/types.ts index efc22c7..41b2b61 100644 --- a/src/lib/types.ts +++ b/src/types.ts @@ -1,7 +1,7 @@ -import type File from './File.class.js'; -import type PreviousStage from './PreviousStage.class.js'; +import type File from './file.js'; import {type QueryEngine as QueryEngineSparql} from '@comunica/query-sparql'; import {type QueryEngine as QueryEngineFile} from '@comunica/query-sparql-file'; +import {PreviousStage} from './stage.js'; export type Endpoint = File | URL | PreviousStage; export type QueryEngine = QueryEngineSparql | QueryEngineFile; diff --git a/src/utils/getEndpoint.ts b/src/utils/getEndpoint.ts index 1dc5167..71fc296 100644 --- a/src/utils/getEndpoint.ts +++ b/src/utils/getEndpoint.ts @@ -1,8 +1,7 @@ -import File from '../lib/File.class.js'; -import type {LDWorkbenchConfiguration} from '../lib/LDWorkbenchConfiguration.js'; -import PreviousStage from '../lib/PreviousStage.class.js'; -import type Stage from '../lib/Stage.class.js'; -import type {Endpoint} from '../lib/types.js'; +import File from '../file.js'; +import {Configuration} from '../configuration.js'; +import Stage, {PreviousStage} from '../stage.js'; +import type {Endpoint} from '../types.js'; import {isFilePathString} from './guards.js'; export default function getEndpoint(stage: Stage, type?: 'iterator'): Endpoint; @@ -16,7 +15,7 @@ export default function getEndpoint( type: 'iterator' | 'generator' = 'iterator', index?: number ): Endpoint { - const t: keyof LDWorkbenchConfiguration['stages'][number] = type; + const t: keyof Configuration['stages'][number] = type; const endpoint = t === 'generator' ? stage.configuration[t]?.[index!]?.endpoint diff --git a/src/utils/getEngine.ts b/src/utils/getEngine.ts index 3c711ee..54cb41a 100644 --- a/src/utils/getEngine.ts +++ b/src/utils/getEngine.ts @@ -1,6 +1,6 @@ import {QueryEngine as QueryEngineSparql} from '@comunica/query-sparql'; import {QueryEngine as QueryEngineFile} from '@comunica/query-sparql-file'; -import type {Endpoint} from '../lib/types.js'; +import type {Endpoint} from '../types.js'; export default function getEngine( endpoint: Endpoint diff --git a/src/utils/getEngineSource.ts b/src/utils/getEngineSource.ts index b6f19de..f913377 100644 --- a/src/utils/getEngineSource.ts +++ b/src/utils/getEngineSource.ts @@ -1,7 +1,7 @@ import {isPreviousStage} from './guards.js'; import {existsSync} from 'fs'; import path from 'path'; -import type {Endpoint, QuerySource} from '../lib/types.js'; +import type {Endpoint, QuerySource} from '../types.js'; export default function getEngineSource(endpoint: Endpoint): QuerySource { if (isPreviousStage(endpoint)) { diff --git a/src/utils/guards.ts b/src/utils/guards.ts index d632dd0..34fa1ae 100644 --- a/src/utils/guards.ts +++ b/src/utils/guards.ts @@ -1,12 +1,10 @@ -import type {LDWorkbenchConfiguration} from '../lib/LDWorkbenchConfiguration.js'; -import PreviousStage from '../lib/PreviousStage.class.js'; +import {Configuration} from '../configuration.js'; import validate from './validate.js'; -import TriplyDB from '../lib/TriplyDB.class.js'; -import File from '../lib/File.class.js'; +import TriplyDB from '../triply-db.js'; +import File from '../file.js'; +import {PreviousStage} from '../stage.js'; -export const isConfiguration = ( - value: unknown -): value is LDWorkbenchConfiguration => +export const isConfiguration = (value: unknown): value is Configuration => value !== null && typeof value === 'object' && validate(value) === null; type FilePathString = string; diff --git a/src/utils/loadConfiguration.ts b/src/utils/loadConfiguration.ts index d813802..134de35 100644 --- a/src/utils/loadConfiguration.ts +++ b/src/utils/loadConfiguration.ts @@ -1,4 +1,4 @@ -import type {LDWorkbenchConfiguration} from '../lib/LDWorkbenchConfiguration.js'; +import {Configuration} from '../configuration.js'; import parseYamlFile from './parseYamlFile.js'; import validate from './validate.js'; import path from 'node:path'; @@ -8,10 +8,8 @@ import {dirname} from 'path'; * This is a wrapper for the YAML Parser and Schema Validator * to provide a 1 step loader. */ -export default function loadConfiguration( - filePath: string -): LDWorkbenchConfiguration { - const configuration = parseYamlFile(filePath) as LDWorkbenchConfiguration; +export default function loadConfiguration(filePath: string): Configuration { + const configuration = parseYamlFile(filePath) as Configuration; const errors = validate(configuration); if (errors !== null) { throw new Error( @@ -22,7 +20,7 @@ export default function loadConfiguration( return mapObject( configuration, inlineQueryFromFile(configuration.baseDir ?? dirname(filePath)) - ) as LDWorkbenchConfiguration; + ) as Configuration; } const mapObject = (obj: object, replacer: (from: string) => string): object => { diff --git a/src/utils/loadPipelines.ts b/src/utils/loadPipelines.ts index 2529f24..ee6b159 100644 --- a/src/utils/loadPipelines.ts +++ b/src/utils/loadPipelines.ts @@ -1,13 +1,13 @@ import fs from 'fs'; -import type {LDWorkbenchConfiguration} from '../lib/LDWorkbenchConfiguration.js'; +import {Configuration} from '../configuration.js'; import chalk from 'chalk'; import glob from 'glob'; import loadConfiguration from './loadConfiguration.js'; export default function loadPipelines( configDirOrFile: string -): Map { - const pipelines = new Map(); +): Map { + const pipelines = new Map(); if (!fs.existsSync(configDirOrFile)) throw new Error( diff --git a/test/PreviousStage.class.test.ts b/test/PreviousStage.class.test.ts deleted file mode 100644 index e61c4d0..0000000 --- a/test/PreviousStage.class.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -import PreviousStage from '../src/lib/PreviousStage.class.js'; -import * as chai from 'chai'; -import chaiAsPromised from 'chai-as-promised'; -import type {LDWorkbenchConfiguration} from '../src/lib/LDWorkbenchConfiguration.js'; -import Pipeline from '../src/lib/Pipeline.class.js'; -import Stage from '../src/lib/Stage.class.js'; -chai.use(chaiAsPromised); -const expect = chai.expect; - -describe('PreviousStage Class', () => { - describe('constructor', () => { - it('should set properties correctly', () => { - const config: LDWorkbenchConfiguration = { - name: 'Example Pipeline', - description: - 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', - destination: 'file://pipelines/data/example-pipeline.nt', - stages: [ - { - name: 'Stage 1', - iterator: { - query: 'file://static/example/iterator-stage-1.rq', - endpoint: 'file://static/tests/iris.nt', - }, - generator: [ - { - query: 'file://static/example/generator-stage-1-1.rq', - }, - ], - }, - { - name: 'Stage 2', - iterator: { - query: 'file://static/example/iterator-stage-2.rq', - }, - generator: [ - { - query: 'file://static/example/generator-stage-2.rq', - endpoint: 'file://static/tests/wikidata.nt', - }, - ], - }, - ], - }; - const pipeline = new Pipeline(config, {silent: true}); - const stage: Stage = new Stage(pipeline, config.stages[1]); - const stagesSoFar = Array.from(stage.pipeline.stages.keys()); - const previousStage = new PreviousStage(stage, stagesSoFar.pop()!); - expect(previousStage).to.be.an.instanceOf(PreviousStage); - expect(previousStage).to.have.property('nextStage'); - expect(previousStage).to.have.property('name'); - expect(previousStage.$id).to.equal('PreviousStage'); - }); - }); - describe('load', () => { - it('should throw an error if the previous stage is not found', () => { - const config: LDWorkbenchConfiguration = { - name: 'Example Pipeline', - description: - 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', - destination: 'file://pipelines/data/example-pipeline.nt', - stages: [ - { - name: 'Stage 1', - iterator: { - query: 'file://static/example/iterator-stage-1.rq', - endpoint: 'file://static/tests/iris.nt', - }, - generator: [ - { - query: 'file://static/example/generator-stage-1-1.rq', - }, - ], - }, - { - name: 'Stage 2', - iterator: { - query: 'file://static/example/iterator-stage-2.rq', - }, - generator: [ - { - query: 'file://static/example/generator-stage-2.rq', - endpoint: 'file://static/tests/wikidata.nt', - }, - ], - }, - ], - }; - const pipeline = new Pipeline(config, {silent: true}); - const stage: Stage = new Stage(pipeline, config.stages[0]); - const stagesSoFar = Array.from(stage.pipeline.stages.keys()); - const previousStage = new PreviousStage(stage, stagesSoFar.pop()!); - expect(() => previousStage.load()).to.throw( - 'no endpoint was defined, but there is also no previous stage to use' - ); - }); - - it('should return the previous stage correctly', () => { - const config: LDWorkbenchConfiguration = { - name: 'Example Pipeline', - description: - 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', - destination: 'file://pipelines/data/example-pipeline.nt', - stages: [ - { - name: 'Stage 1', - iterator: { - query: 'file://static/example/iterator-stage-1.rq', - endpoint: 'file://static/tests/iris.nt', - }, - generator: [ - { - query: 'file://static/example/generator-stage-1-1.rq', - }, - ], - }, - { - name: 'Stage 2', - iterator: { - query: 'file://static/example/iterator-stage-2.rq', - endpoint: 'file://static/tests/iris.nt', - }, - generator: [ - { - query: 'file://static/example/generator-stage-2.rq', - endpoint: 'file://static/tests/wikidata.nt', - }, - ], - }, - ], - }; - const pipeline = new Pipeline(config, {silent: true}); - const stageTwo: Stage = new Stage(pipeline, config.stages[1]); - const stagesSoFar = Array.from(stageTwo.pipeline.stages.keys()); - const previousStage = new PreviousStage(stageTwo, stagesSoFar.pop()!); // should be stage one - const stage2 = pipeline.stages.get('Stage 2')!; - const testPreviousStage = pipeline.getPreviousStage(stage2); - expect(previousStage.load()).to.equal(testPreviousStage); - }); - }); - - describe('is', () => { - it('should return true for a valid PreviousStage instance', () => { - const config: LDWorkbenchConfiguration = { - name: 'Example Pipeline', - description: - 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', - destination: 'file://pipelines/data/example-pipeline.nt', - stages: [ - { - name: 'Stage 1', - iterator: { - query: 'file://static/example/iterator-stage-1.rq', - endpoint: 'file://static/tests/iris.nt', - }, - generator: [ - { - query: 'file://static/example/generator-stage-1-1.rq', - }, - ], - }, - { - name: 'Stage 2', - iterator: { - query: 'file://static/example/iterator-stage-2.rq', - endpoint: 'file://static/tests/iris.nt', - }, - generator: [ - { - query: 'file://static/example/generator-stage-2.rq', - endpoint: 'file://static/tests/wikidata.nt', - }, - ], - }, - ], - }; - const pipeline = new Pipeline(config, {silent: true}); - const stage: Stage = new Stage(pipeline, config.stages[1]); - const stagesSoFar = Array.from(stage.pipeline.stages.keys()); - const previousStage = new PreviousStage(stage, stagesSoFar.pop()!); - previousStage.load(); - const result = PreviousStage.is(previousStage); - expect(result).to.equal(true); - }); - - it('should return false for an invalid instance', () => { - const invalidInstance = {$id: 'invalid'}; - const result = PreviousStage.is(invalidInstance); - expect(result).to.equal(false); - }); - }); -}); diff --git a/test/File.class.test.ts b/test/file.test.ts similarity index 98% rename from test/File.class.test.ts rename to test/file.test.ts index 6d46107..8e94033 100644 --- a/test/File.class.test.ts +++ b/test/file.test.ts @@ -1,6 +1,6 @@ import * as fs from 'fs'; import * as path from 'path'; -import File from '../src/lib/File.class.js'; +import File from '../src/file.js'; import * as chai from 'chai'; import chaiAsPromised from 'chai-as-promised'; import {WriteStream} from 'node:fs'; diff --git a/test/Generator.class.test.ts b/test/generator.test.ts similarity index 95% rename from test/Generator.class.test.ts rename to test/generator.test.ts index 46de674..15f000b 100644 --- a/test/Generator.class.test.ts +++ b/test/generator.test.ts @@ -1,13 +1,13 @@ -import Generator from '../src/lib/Generator.class.js'; +import Generator from '../src/generator.js'; import {EventEmitter} from 'events'; -import Stage from '../src/lib/Stage.class.js'; -import Pipeline from '../src/lib/Pipeline.class.js'; +import Stage from '../src/stage.js'; +import Pipeline from '../src/pipeline.js'; import * as chai from 'chai'; import * as path from 'path'; import chaiAsPromised from 'chai-as-promised'; import {NamedNode} from 'n3'; import * as fs from 'fs'; -import type {LDWorkbenchConfiguration} from '../src/lib/LDWorkbenchConfiguration.js'; +import type {Configuration} from '../src/configuration.js'; import {fileURLToPath} from 'url'; import removeDirectory from '../src/utils/removeDir.js'; import {Quad} from '@rdfjs/types'; @@ -26,7 +26,7 @@ describe('Generator Class', () => { describe('constructor', () => { it('should set query, engine, endpoint, and source properties correctly', () => { - const configuration: LDWorkbenchConfiguration = { + const configuration: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -73,7 +73,7 @@ describe('Generator Class', () => { it('Should work with multiple generators in parallel for one pipeline', async () => { const filePath = 'pipelines/data/example-pipelineParallel.nt'; - const config: LDWorkbenchConfiguration = { + const config: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -121,7 +121,7 @@ describe('Generator Class', () => { it("Should work in batchSize for pipeline's generator", async () => { const filePath = 'pipelines/data/example-pipelineBatch.nt'; - const batchConfiguration: LDWorkbenchConfiguration = { + const batchConfiguration: Configuration = { name: 'Example Pipeline Batch', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -164,7 +164,7 @@ describe('Generator Class', () => { }); }); it.skip('should emit "data" and "end" events with the correct number of statements', async () => { - const configuration: LDWorkbenchConfiguration = { + const configuration: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', diff --git a/test/Iterator.class.test.ts b/test/iterator.test.ts similarity index 93% rename from test/Iterator.class.test.ts rename to test/iterator.test.ts index bef2762..0c195ec 100644 --- a/test/Iterator.class.test.ts +++ b/test/iterator.test.ts @@ -1,11 +1,11 @@ -import Iterator from '../src/lib/Iterator.class.js'; +import Iterator from '../src/iterator.js'; import {EventEmitter} from 'events'; -import Stage from '../src/lib/Stage.class.js'; -import Pipeline from '../src/lib/Pipeline.class.js'; +import Stage from '../src/stage.js'; +import Pipeline from '../src/pipeline.js'; import * as chai from 'chai'; import * as path from 'path'; import chaiAsPromised from 'chai-as-promised'; -import type {LDWorkbenchConfiguration} from '../src/lib/LDWorkbenchConfiguration.js'; +import {Configuration} from '../src/configuration.js'; import {fileURLToPath} from 'url'; import removeDirectory from '../src/utils/removeDir.js'; import {NamedNode} from '@rdfjs/types'; @@ -23,7 +23,7 @@ describe('Iterator Class', () => { describe('constructor', () => { it('should set query, endpoint, engine, $offset, and totalResults properties correctly', () => { - const configuration: LDWorkbenchConfiguration = { + const configuration: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -70,7 +70,7 @@ describe('Iterator Class', () => { }); describe.skip('run', () => { it('should emit "data" and "end" events with the correct $this and numResults', async () => { - const configuration: LDWorkbenchConfiguration = { + const configuration: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', diff --git a/test/Pipeline.class.test.ts b/test/pipeline.test.ts similarity index 94% rename from test/Pipeline.class.test.ts rename to test/pipeline.test.ts index 9ec7376..59987e5 100644 --- a/test/Pipeline.class.test.ts +++ b/test/pipeline.test.ts @@ -1,11 +1,11 @@ -import type {LDWorkbenchConfiguration} from '../src/lib/LDWorkbenchConfiguration.js'; -import Pipeline from '../src/lib/Pipeline.class.js'; +import type {Configuration} from '../src/configuration.js'; +import Pipeline from '../src/pipeline.js'; import * as chai from 'chai'; import * as path from 'path'; import chaiAsPromised from 'chai-as-promised'; -import Stage from '../src/lib/Stage.class.js'; +import Stage from '../src/stage.js'; import removeDirectory from '../src/utils/removeDir.js'; -import File from '../src/lib/File.class.js'; +import File from '../src/file.js'; chai.use(chaiAsPromised); const expect = chai.expect; @@ -19,7 +19,7 @@ describe('Pipeline Class', () => { describe('constructor', () => { it('should set properties correctly', () => { - const configuration: LDWorkbenchConfiguration = { + const configuration: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -65,7 +65,7 @@ describe('Pipeline Class', () => { describe('getPreviousStage', () => { it('should return the previous stage correctly', () => { - const configuration: LDWorkbenchConfiguration = { + const configuration: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -138,7 +138,7 @@ describe('Pipeline Class', () => { ], }, ], - } as unknown as LDWorkbenchConfiguration; + } as unknown as Configuration; const pipeline = new Pipeline(configuration, {silent: true}); const stage2: Stage = new Stage(pipeline, configuration.stages[1]); pipeline.getPreviousStage(stage2); @@ -152,7 +152,7 @@ describe('Pipeline Class', () => { 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', destination: 'file://pipelines/data/example-pipeline.nt', stages: [], - } as unknown as LDWorkbenchConfiguration; + } as unknown as Configuration; let failed = false; try { new Pipeline(invalidConfiguration, {silent: true}); @@ -200,7 +200,7 @@ describe('Pipeline Class', () => { ], }, ], - } as unknown as LDWorkbenchConfiguration; + } as unknown as Configuration; let failed = false; try { new Pipeline(invalidConfiguration, {silent: true}); @@ -223,7 +223,7 @@ describe('Pipeline Class', () => { expect(failed).to.equal(true); }); it("should throw an error if the pipeline's configuration has duplicate stage name", () => { - const configDuplicateStageName: LDWorkbenchConfiguration = { + const configDuplicateStageName: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -278,7 +278,7 @@ describe('Pipeline Class', () => { }); it('should succeed if pipeline is valid', () => { - const configDuplicateStageName: LDWorkbenchConfiguration = { + const configDuplicateStageName: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -330,7 +330,7 @@ describe('Pipeline Class', () => { describe('run', () => { it('should run the pipeline correctly', async () => { try { - const configuration: LDWorkbenchConfiguration = { + const configuration: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', diff --git a/test/Stage.class.test.ts b/test/stage.test.ts similarity index 53% rename from test/Stage.class.test.ts rename to test/stage.test.ts index 0d92dba..843c14f 100644 --- a/test/Stage.class.test.ts +++ b/test/stage.test.ts @@ -1,12 +1,12 @@ -import Stage from '../src/lib/Stage.class.js'; -import Pipeline from '../src/lib/Pipeline.class.js'; +import Stage, {PreviousStage} from '../src/stage.js'; +import Pipeline from '../src/pipeline.js'; import kebabcase from 'lodash.kebabcase'; -import Iterator from '../src/lib/Iterator.class.js'; -import Generator from '../src/lib/Generator.class.js'; +import Iterator from '../src/iterator.js'; +import Generator from '../src/generator.js'; import * as chai from 'chai'; import * as path from 'path'; import chaiAsPromised from 'chai-as-promised'; -import type {LDWorkbenchConfiguration} from '../src/lib/LDWorkbenchConfiguration.js'; +import type {Configuration} from '../src/configuration.js'; import {fileURLToPath} from 'url'; import removeDirectory from '../src/utils/removeDir.js'; import {NamedNode} from '@rdfjs/types'; @@ -25,7 +25,7 @@ describe('Stage Class', () => { describe('constructor', () => { it('should set properties correctly', () => { - const configuration: LDWorkbenchConfiguration = { + const configuration: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -74,7 +74,7 @@ describe('Stage Class', () => { describe('destinationPath', () => { it('should return the correct destination path', () => { - const configuration: LDWorkbenchConfiguration = { + const configuration: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -121,7 +121,7 @@ describe('Stage Class', () => { describe('name', () => { it('should return the correct stage name', () => { - const configuration: LDWorkbenchConfiguration = { + const configuration: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -162,7 +162,7 @@ describe('Stage Class', () => { describe.skip('run', () => { it('should run the stage correctly', async () => { - const configuration: LDWorkbenchConfiguration = { + const configuration: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -209,6 +209,7 @@ describe('Stage Class', () => { iteratorCount: number; statements: number; }> = []; + async function runStageWithPromise(): Promise { return new Promise((resolve, reject) => { stage.addListener('generatorResult', count => { @@ -227,6 +228,7 @@ describe('Stage Class', () => { stage.run(); }); } + await runStageWithPromise(); expect(iteratorEvents[0].event).to.equal('iteratorResult'); expect(iteratorEvents[0].namedNode.termType).to.equal('NamedNode'); @@ -244,3 +246,187 @@ describe('Stage Class', () => { }); }); }); + +describe('PreviousStage Class', () => { + describe('constructor', () => { + it('should set properties correctly', () => { + const config: Configuration = { + name: 'Example Pipeline', + description: + 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', + destination: 'file://pipelines/data/example-pipeline.nt', + stages: [ + { + name: 'Stage 1', + iterator: { + query: 'file://static/example/iterator-stage-1.rq', + endpoint: 'file://static/tests/iris.nt', + }, + generator: [ + { + query: 'file://static/example/generator-stage-1-1.rq', + }, + ], + }, + { + name: 'Stage 2', + iterator: { + query: 'file://static/example/iterator-stage-2.rq', + }, + generator: [ + { + query: 'file://static/example/generator-stage-2.rq', + endpoint: 'file://static/tests/wikidata.nt', + }, + ], + }, + ], + }; + const pipeline = new Pipeline(config, {silent: true}); + const stage: Stage = new Stage(pipeline, config.stages[1]); + const stagesSoFar = Array.from(stage.pipeline.stages.keys()); + const previousStage = new PreviousStage(stage, stagesSoFar.pop()!); + expect(previousStage).to.be.an.instanceOf(PreviousStage); + expect(previousStage).to.have.property('nextStage'); + expect(previousStage).to.have.property('name'); + expect(previousStage.$id).to.equal('PreviousStage'); + }); + }); + describe('load', () => { + it('should throw an error if the previous stage is not found', () => { + const config: Configuration = { + name: 'Example Pipeline', + description: + 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', + destination: 'file://pipelines/data/example-pipeline.nt', + stages: [ + { + name: 'Stage 1', + iterator: { + query: 'file://static/example/iterator-stage-1.rq', + endpoint: 'file://static/tests/iris.nt', + }, + generator: [ + { + query: 'file://static/example/generator-stage-1-1.rq', + }, + ], + }, + { + name: 'Stage 2', + iterator: { + query: 'file://static/example/iterator-stage-2.rq', + }, + generator: [ + { + query: 'file://static/example/generator-stage-2.rq', + endpoint: 'file://static/tests/wikidata.nt', + }, + ], + }, + ], + }; + const pipeline = new Pipeline(config, {silent: true}); + const stage: Stage = new Stage(pipeline, config.stages[0]); + const stagesSoFar = Array.from(stage.pipeline.stages.keys()); + const previousStage = new PreviousStage(stage, stagesSoFar.pop()!); + expect(() => previousStage.load()).to.throw( + 'no endpoint was defined, but there is also no previous stage to use' + ); + }); + + it('should return the previous stage correctly', () => { + const config: Configuration = { + name: 'Example Pipeline', + description: + 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', + destination: 'file://pipelines/data/example-pipeline.nt', + stages: [ + { + name: 'Stage 1', + iterator: { + query: 'file://static/example/iterator-stage-1.rq', + endpoint: 'file://static/tests/iris.nt', + }, + generator: [ + { + query: 'file://static/example/generator-stage-1-1.rq', + }, + ], + }, + { + name: 'Stage 2', + iterator: { + query: 'file://static/example/iterator-stage-2.rq', + endpoint: 'file://static/tests/iris.nt', + }, + generator: [ + { + query: 'file://static/example/generator-stage-2.rq', + endpoint: 'file://static/tests/wikidata.nt', + }, + ], + }, + ], + }; + const pipeline = new Pipeline(config, {silent: true}); + const stageTwo: Stage = new Stage(pipeline, config.stages[1]); + const stagesSoFar = Array.from(stageTwo.pipeline.stages.keys()); + const previousStage = new PreviousStage(stageTwo, stagesSoFar.pop()!); // should be stage one + const stage2 = pipeline.stages.get('Stage 2')!; + const testPreviousStage = pipeline.getPreviousStage(stage2); + expect(previousStage.load()).to.equal(testPreviousStage); + }); + }); + + describe('is', () => { + it('should return true for a valid PreviousStage instance', () => { + const config: Configuration = { + name: 'Example Pipeline', + description: + 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', + destination: 'file://pipelines/data/example-pipeline.nt', + stages: [ + { + name: 'Stage 1', + iterator: { + query: 'file://static/example/iterator-stage-1.rq', + endpoint: 'file://static/tests/iris.nt', + }, + generator: [ + { + query: 'file://static/example/generator-stage-1-1.rq', + }, + ], + }, + { + name: 'Stage 2', + iterator: { + query: 'file://static/example/iterator-stage-2.rq', + endpoint: 'file://static/tests/iris.nt', + }, + generator: [ + { + query: 'file://static/example/generator-stage-2.rq', + endpoint: 'file://static/tests/wikidata.nt', + }, + ], + }, + ], + }; + const pipeline = new Pipeline(config, {silent: true}); + const stage: Stage = new Stage(pipeline, config.stages[1]); + const stagesSoFar = Array.from(stage.pipeline.stages.keys()); + const previousStage = new PreviousStage(stage, stagesSoFar.pop()!); + previousStage.load(); + const result = PreviousStage.is(previousStage); + expect(result).to.equal(true); + }); + + it('should return false for an invalid instance', () => { + const invalidInstance = {$id: 'invalid'}; + const result = PreviousStage.is(invalidInstance); + expect(result).to.equal(false); + }); + }); +}); diff --git a/test/utils/utilities.test.ts b/test/utils/utilities.test.ts index c6a6bf1..da1ebe4 100644 --- a/test/utils/utilities.test.ts +++ b/test/utils/utilities.test.ts @@ -7,16 +7,15 @@ import { isFilePathString, isPreviousStage, } from '../../src/utils/guards.js'; -import Pipeline from '../../src/lib/Pipeline.class.js'; -import Stage from '../../src/lib/Stage.class.js'; -import PreviousStage from '../../src/lib/PreviousStage.class.js'; -import File from '../../src/lib/File.class.js'; +import Pipeline from '../../src/pipeline.js'; +import Stage, {PreviousStage} from '../../src/stage.js'; +import File from '../../src/file.js'; import path from 'path'; import loadPipelines from '../../src/utils/loadPipelines.js'; import chalk from 'chalk'; import assert from 'assert'; import getEndpoint from '../../src/utils/getEndpoint.js'; -import type {LDWorkbenchConfiguration} from '../../src/lib/LDWorkbenchConfiguration.js'; +import {Configuration} from '../../src/configuration.js'; import getEngine from '../../src/utils/getEngine.js'; import {QueryEngine as QueryEngineSparql} from '@comunica/query-sparql'; import {QueryEngine as QueryEngineFile} from '@comunica/query-sparql-file'; @@ -86,7 +85,7 @@ describe('Utilities', () => { }); describe('Type guards', () => { it('should assert a configuration', () => { - const configuration: LDWorkbenchConfiguration = { + const configuration: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -308,7 +307,7 @@ describe('Utilities', () => { describe('getEndpoint', () => { it('should return File when filePath is provided in Stage', () => { const filePath = 'file://path/to/file.txt'; - const config: LDWorkbenchConfiguration = { + const config: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -339,7 +338,7 @@ describe('Utilities', () => { }); it('should return URL when URL is provided in Stage', () => { const url = new URL('https://example.com').toString(); - const config: LDWorkbenchConfiguration = { + const config: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -370,7 +369,7 @@ describe('Utilities', () => { }); it('should throw error if invalid URL is provided', () => { const url = 'invalidExample'; // will be accepted - const config: LDWorkbenchConfiguration = { + const config: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -396,7 +395,7 @@ describe('Utilities', () => { }); it('should throw if stage has undefined endpoint and is first stage', () => { const endpoint = undefined; - const config: LDWorkbenchConfiguration = { + const config: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -422,7 +421,7 @@ describe('Utilities', () => { }); it('should return PreviousStage if stage has undefined endpoint', () => { const url = new URL('https://example.com').toString(); - const config: LDWorkbenchConfiguration = { + const config: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -472,7 +471,7 @@ describe('Utilities', () => { expect(result instanceof QueryEngineFile).to.equal(true); }); it('should return QueryEngineFile when input is PreviousStage', () => { - const config: LDWorkbenchConfiguration = { + const config: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', @@ -525,7 +524,7 @@ describe('Utilities', () => { }); }); it('should return engine source string when input is PreviousStage with destinationPath', async () => { - const config: LDWorkbenchConfiguration = { + const config: Configuration = { name: 'Example Pipeline', description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n',