feat: Option to request and include metadata from ldes

README.md

@@ -22,6 +22,7 @@ ldes-client <url> [-f] [--save <path>] [--pollInterval <number>] [--shape <shape
 - `--shape`: shape file to which LDES members should conform (overwrite LDES configured shape)
 - `-t` `--default-timezone`: default timezone to use for dates in tree:InBetweenRelation. `AoE|Z|±HH:mm` Default: `AoE`.
 - `--condition <condition_file>`: filter the LDES stream to only emit members that adhere to this condition
+- `-m`, `--metadata`: include metadata in the emitted members. Notifies the ldes server that it is interested in metadata.
 - Others soon coming
 
 

bin/cli.ts

@@ -25,6 +25,7 @@ let save: string | undefined;
 let onlyDefaultGraph: boolean = false;
 let loose: boolean = false;
 let defaultTimezone: string | undefined;
+let includeMetadata: boolean = false;
 
 const fetch_config: FetchConfig = {
     retry: {},
@@ -94,6 +95,7 @@ program
     )
     .option("--http-codes [codes...]", "What HTTP codes to retry")
     .option("-t --default-timezone <timezone>", "Default timezone for dates in tree:InBetweenRelation", "AoE")
+    .option("-m, --metadata", "include metadata in the output members")
     .action((url: string, program) => {
         urlIsView = program.urlIsView;
         noShape = !program.shape;
@@ -110,6 +112,7 @@ program
         materialize = program.materializeVersion;
         lastVersionOnly = program.lastVersionOnly;
         defaultTimezone = program.defaultTimezone;
+        includeMetadata = program.metadata;
 
         fetch_config.concurrent = parseInt(program.concurrent);
         if (program.basicAuth) {
@@ -165,6 +168,7 @@ async function main() {
             defaultTimezone,
             materialize,
             lastVersionOnly,
+            includeMetadata,
             fetch: enhanced_fetch(fetch_config),
         }),
         ordered,

lib/client.ts

@@ -380,6 +380,7 @@ export class Client {
                       this.ordered,
                       this.config.polling,
                       this.config.pollInterval,
+                      this.config.includeMetadata,
                   )
                 : new UnorderedStrategy(
                       this.memberManager,
@@ -388,6 +389,7 @@ export class Client {
                       factory,
                       this.config.polling,
                       this.config.pollInterval,
+                      this.config.includeMetadata,
                   );
 
         this.logger.debug(

lib/config.ts

@@ -23,6 +23,7 @@ export interface Config {
     onlyDefaultGraph?: boolean;
     materialize?: boolean;
     lastVersionOnly?: boolean;
+    includeMetadata?: boolean;
     fetch?: typeof fetch;
 }
 
@@ -41,6 +42,7 @@ const defaultConfig: Config = {
     defaultTimezone: "AoE",
     materialize: false,
     lastVersionOnly: false,
+    includeMetadata: false,
 };
 
 const defaultTarget: WithTarget = {

lib/pageFetcher.ts

@@ -1,4 +1,4 @@
-import { RdfDereferencer } from "rdf-dereference";
+import { IDereferenceOptions, RdfDereferencer } from "rdf-dereference";
 import { Notifier } from "./utils";
 import { extractRelations, Relation, Relations } from "./page";
 import { SimpleRelation } from "./relation";
@@ -93,12 +93,18 @@ export class Fetcher {
         this.closed = true;
     }
 
-    async fetch<S>(node: Node, state: S, notifier: Notifier<FetchEvent, S>) {
+    async fetch<S>(node: Node, state: S, notifier: Notifier<FetchEvent, S>, includeMetadata: boolean) {
         try {
-            const resp = await this.dereferencer.dereference(node.target, {
+            let options: IDereferenceOptions = {
                 localFiles: true,
                 fetch: this.fetch_f,
-            });
+            };
+            if (includeMetadata) {
+                options.headers = {
+                    Accept: "application/metadata+trig",
+                };
+            }
+            const resp = await this.dereferencer.dereference(node.target, options);
 
             node.target = resp.url;
 

lib/strategy/ordered.ts

@@ -51,6 +51,7 @@ export class OrderedStrategy {
     private polling: boolean;
     private toPoll: Heap<{ chain: RelationChain; expected: string[] }>;
     private pollInterval?: number;
+    private includeMetadata: boolean;
 
     private cancled = false;
 
@@ -64,13 +65,15 @@ export class OrderedStrategy {
         ordered: Ordered,
         polling: boolean,
         pollInterval?: number,
+        includeMetadata: boolean = false,
     ) {
         this.ordered = ordered;
         this.manager = memberManager;
         this.fetcher = fetcher;
         this.notifier = notifier;
         this.polling = polling;
         this.pollInterval = pollInterval;
+        this.includeMetadata = includeMetadata;
 
         this.toPoll = new Heap((a, b) => a.chain.ordering(b.chain));
         this.launchedRelations = new Heap((a, b) => a.ordering(b));
@@ -142,6 +145,7 @@ export class OrderedStrategy {
                         { target: chain.target, expected },
                         { chain, index },
                         this.fetchNotifier,
+                        this.includeMetadata,
                     );
                 },
             },

lib/strategy/unordered.ts

@@ -21,6 +21,7 @@ export class UnorderedStrategy {
     private cacheList: Node[] = [];
     private polling: boolean;
     private pollInterval?: number;
+    private includeMetadata: boolean;
 
     private canceled = false;
 
@@ -33,12 +34,14 @@ export class UnorderedStrategy {
         modulatorFactory: ModulatorFactory,
         polling: boolean,
         pollInterval?: number,
+        includeMetadata: boolean = false,
     ) {
         this.pollInterval = pollInterval;
         this.notifier = notifier;
         this.manager = memberManager;
         this.fetcher = fetcher;
         this.polling = polling;
+        this.includeMetadata = includeMetadata;
 
         // Callbacks for the fetcher
         // - seen: the strategy wanted to fetch an uri, but it was already seen
@@ -87,7 +90,7 @@ export class UnorderedStrategy {
 
         this.modulator = modulatorFactory.create<Node>("fetcher", [], {
             ready: ({ item, index }) =>
-                this.fetcher.fetch(item, { index }, this.fetchNotifier),
+                this.fetcher.fetch(item, { index }, this.fetchNotifier, this.includeMetadata),
         });
     }