More flexible dynamic parsing

docs/source_files/dynamic.md

@@ -22,15 +22,14 @@ return a unique string that will help Saffron to identify of the implementation.
 }
 ```
 Following, we will implement the `request` method, which is responsible to do
-all the network requests. The response should include one or multiple objects of
-type `AxiosResponse`.
+all the network requests.
 
 In cases where a login is required to the remote website, it can be done from here.
 
 ```ts
 class Custom extends DynamicSourceFile {
     // ...
-    request(utils: Utils): Promise<RequestsResult> {
+    request(utils: Utils): Promise<any> {
         // Request using utils.get to assign the axios config
         // passed in the global and/or source configurations
         return utils.get(utils.url);
@@ -39,7 +38,7 @@ class Custom extends DynamicSourceFile {
 }
 ```
 Lastly, we are going to implement the `parse` method, which is responsible to do
-all the parsing. It will receive the requests responses from the `request` method
+all the parsing. It will receive the payload returned from the `request` method
 and must return an array of Articles.
 
 ```ts
@@ -57,6 +56,8 @@ class Custom extends DynamicSourceFile {
 ## Scrape
 
 ### `implementation`
+Default value: `<source name>`
+
 The name of the implementation we have configured.
 
 ## Utils

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@unistudents/saffron",
-    "version": "6.0.0",
+    "version": "6.1.0",
     "description": "A fairly intuitive & powerful framework that enables you to collect & save articles and news from all over the web. ",
     "license": "MIT",
     "homepage": "https://github.com/unistudents/saffron#readme",

src/components/parser.type.ts

@@ -15,7 +15,7 @@ type JsonSkipOptions = {
 };
 
 export type ScrapeDynamic = {
-    implementation: string;
+    implementation?: string;
 };
 
 export type ScrapeHTML = {

src/components/types.ts

@@ -62,7 +62,7 @@ export type InstructionUrl = {
 
 export type ParserResult = InstructionUrl & { articles: Article[]; };
 
-export type RequestsResult = AxiosResponse | AxiosResponse[];
+export type RequestsResult = any;
 
 export type CallbackVoid = (...args: any[]) => void;
 
@@ -103,7 +103,7 @@ export type SourceFile = {
     extra?: any;
 } & ({
     type: 'dynamic'
-    scrape: ScrapeDynamic;
+    scrape?: ScrapeDynamic;
 } | {
     type: 'html'
     scrape: ScrapeHTML;

src/modules/worker.ts

@@ -37,7 +37,7 @@ export class Worker {
             utils.source = job.source;
 
             if (instructions.parserType === ParserType.DYNAMIC) {
-                const impName = instructions.dynamic.implementation;
+                const impName = instructions.dynamic.implementation ?? job.source.name;
                 const dsf = dynamicSourceFiles.find(dsf => dsf.name() === impName);
                 if(!dsf) {
                     throw new Error(`could not find any implementation with name ${impName}`);

src/parsers/dynamic.parser.ts

@@ -10,10 +10,10 @@ export class DynamicParser extends Parser {
         // This exists only for typescript, it is not valid and will not run at runtime.
         const scrape = s as ScrapeDynamic;
 
-        if (typeof scrape !== 'object' || Array.isArray(scrape))
+        if (typeof scrape !== 'undefined' && (typeof scrape !== 'object' || Array.isArray(scrape)))
             throw new Error("must be a JSON object");
 
-        if (typeof scrape.implementation !== 'string' || scrape.implementation.length === 0)
+        if (typeof scrape.implementation !== 'undefined' && (typeof scrape.implementation !== 'string' || scrape.implementation.length === 0))
             throw new Error(`implementation must be non empty string`);
     }
 

tsconfig.json

@@ -1,7 +1,7 @@
 {
     "compilerOptions": {
         "ignoreDeprecations": "5.0",
-        
+
         "target": "ES6",
         "module": "commonjs",