diff --git a/docs/source_files/dynamic.md b/docs/source_files/dynamic.md index bdae460..55c0612 100644 --- a/docs/source_files/dynamic.md +++ b/docs/source_files/dynamic.md @@ -22,15 +22,14 @@ return a unique string that will help Saffron to identify of the implementation. } ``` Following, we will implement the `request` method, which is responsible to do -all the network requests. The response should include one or multiple objects of -type `AxiosResponse`. +all the network requests. In cases where a login is required to the remote website, it can be done from here. ```ts class Custom extends DynamicSourceFile { // ... - request(utils: Utils): Promise { + request(utils: Utils): Promise { // Request using utils.get to assign the axios config // passed in the global and/or source configurations return utils.get(utils.url); @@ -39,7 +38,7 @@ class Custom extends DynamicSourceFile { } ``` Lastly, we are going to implement the `parse` method, which is responsible to do -all the parsing. It will receive the requests responses from the `request` method +all the parsing. It will receive the payload returned from the `request` method and must return an array of Articles. ```ts @@ -57,6 +56,8 @@ class Custom extends DynamicSourceFile { ## Scrape ### `implementation` +Default value: `` + The name of the implementation we have configured. ## Utils diff --git a/package.json b/package.json index eb0822c..8376ab0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@unistudents/saffron", - "version": "6.0.0", + "version": "6.1.0", "description": "A fairly intuitive & powerful framework that enables you to collect & save articles and news from all over the web. ", "license": "MIT", "homepage": "https://github.com/unistudents/saffron#readme", diff --git a/src/components/parser.type.ts b/src/components/parser.type.ts index edd0901..44c9bef 100644 --- a/src/components/parser.type.ts +++ b/src/components/parser.type.ts @@ -15,7 +15,7 @@ type JsonSkipOptions = { }; export type ScrapeDynamic = { - implementation: string; + implementation?: string; }; export type ScrapeHTML = { diff --git a/src/components/types.ts b/src/components/types.ts index 3582b37..0e0c8e0 100644 --- a/src/components/types.ts +++ b/src/components/types.ts @@ -62,7 +62,7 @@ export type InstructionUrl = { export type ParserResult = InstructionUrl & { articles: Article[]; }; -export type RequestsResult = AxiosResponse | AxiosResponse[]; +export type RequestsResult = any; export type CallbackVoid = (...args: any[]) => void; @@ -103,7 +103,7 @@ export type SourceFile = { extra?: any; } & ({ type: 'dynamic' - scrape: ScrapeDynamic; + scrape?: ScrapeDynamic; } | { type: 'html' scrape: ScrapeHTML; diff --git a/src/modules/worker.ts b/src/modules/worker.ts index ea989ed..21429dc 100644 --- a/src/modules/worker.ts +++ b/src/modules/worker.ts @@ -37,7 +37,7 @@ export class Worker { utils.source = job.source; if (instructions.parserType === ParserType.DYNAMIC) { - const impName = instructions.dynamic.implementation; + const impName = instructions.dynamic.implementation ?? job.source.name; const dsf = dynamicSourceFiles.find(dsf => dsf.name() === impName); if(!dsf) { throw new Error(`could not find any implementation with name ${impName}`); diff --git a/src/parsers/dynamic.parser.ts b/src/parsers/dynamic.parser.ts index e7248ff..136a20f 100644 --- a/src/parsers/dynamic.parser.ts +++ b/src/parsers/dynamic.parser.ts @@ -10,10 +10,10 @@ export class DynamicParser extends Parser { // This exists only for typescript, it is not valid and will not run at runtime. const scrape = s as ScrapeDynamic; - if (typeof scrape !== 'object' || Array.isArray(scrape)) + if (typeof scrape !== 'undefined' && (typeof scrape !== 'object' || Array.isArray(scrape))) throw new Error("must be a JSON object"); - if (typeof scrape.implementation !== 'string' || scrape.implementation.length === 0) + if (typeof scrape.implementation !== 'undefined' && (typeof scrape.implementation !== 'string' || scrape.implementation.length === 0)) throw new Error(`implementation must be non empty string`); } diff --git a/tsconfig.json b/tsconfig.json index 0d743da..c04dbd7 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "ignoreDeprecations": "5.0", - + "target": "ES6", "module": "commonjs",