diff --git a/packages/http/package.json b/packages/http/package.json index 01b4eb097..02ee1705b 100644 --- a/packages/http/package.json +++ b/packages/http/package.json @@ -41,7 +41,8 @@ "form-data": "^3.0.0", "lodash": "^4.17.19", "request": "^2.88.2", - "tough-cookie": "^4.0.0" + "tough-cookie": "^4.0.0", + "xml2js": "0.5.0" }, "devDependencies": { "@openfn/buildtools": "workspace:^1.0.2", diff --git a/packages/http/src/Adaptor.js b/packages/http/src/Adaptor.js index f85539d62..3ec8b3dcb 100644 --- a/packages/http/src/Adaptor.js +++ b/packages/http/src/Adaptor.js @@ -17,6 +17,7 @@ import nodeRequest from 'request'; import cheerio from 'cheerio'; import cheerioTableparser from 'cheerio-tableparser'; import tough from 'tough-cookie'; +import { parseStringPromise } from 'xml2js'; const { axios } = http; @@ -310,7 +311,8 @@ export function del(path, params, callback) { */ export function parseXML(body, script) { return state => { - const $ = cheerio.load(body); + const resolvedBody = expandReferences(body)(state); + const $ = cheerio.load(resolvedBody); cheerioTableparser($); if (script) { @@ -322,11 +324,36 @@ export function parseXML(body, script) { return composeNextState(state, { body: result }); } } else { - return composeNextState(state, { body: body }); + return composeNextState(state, { body: resolvedBody }); } }; } +/** + * The xml2json function takes an XML string and converts it to a JSON object. + * + * @param {string} xml - Pass in the xml string that we want to parse + * @param {object} options - Specify the options for the parseStringPromise function + * @param {function} callback - (Optional) Callback function + * @returns {Operation} + */ + +export function xml2json(xml, options, callback) { + return state => { + const resolvedXml = expandReferences(xml)(state); + const resolvedOptions = expandReferences(options)(state); + + return parseStringPromise(resolvedXml, resolvedOptions).then(result => { + console.log('Done converting xml to json ✅'); + + const nextState = { ...composeNextState(state, result), result }; + + if (callback) return callback(nextState); + return nextState; + }); + }; +} + /** * Make a request using the 'request' node module. This module is deprecated. * @example diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 943e98042..64e540b2c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -649,6 +649,9 @@ importers: tough-cookie: specifier: ^4.0.0 version: 4.1.3 + xml2js: + specifier: 0.5.0 + version: 0.5.0 devDependencies: '@openfn/buildtools': specifier: workspace:^1.0.2 @@ -2311,6 +2314,13 @@ packages: '@babel/highlight': 7.22.20 chalk: 2.4.2 + /@babel/code-frame@7.23.5: + resolution: {integrity: sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/highlight': 7.23.4 + chalk: 2.4.2 + /@babel/compat-data@7.22.20: resolution: {integrity: sha512-BQYjKbpXjoXwFW5jGqiizJQQT/aC7pFm9Ok1OWssonuguICi264lbgMzRp2ZMmRSlfkX6DsWDDcsrctK8Rwfiw==} engines: {node: '>=6.9.0'} @@ -2336,6 +2346,29 @@ packages: semver: 6.3.1 transitivePeerDependencies: - supports-color + dev: true + + /@babel/core@7.23.5: + resolution: {integrity: sha512-Cwc2XjUrG4ilcfOw4wBAK+enbdgwAcAJCfGUItPBKR7Mjw4aEfAFYrLxeRp4jWgtNIKn3n2AlBOfwwafl+42/g==} + engines: {node: '>=6.9.0'} + dependencies: + '@ampproject/remapping': 2.2.1 + '@babel/code-frame': 7.23.5 + '@babel/generator': 7.23.5 + '@babel/helper-compilation-targets': 7.22.15 + '@babel/helper-module-transforms': 7.23.3(@babel/core@7.23.5) + '@babel/helpers': 7.23.5 + '@babel/parser': 7.23.5 + '@babel/template': 7.22.15 + '@babel/traverse': 7.23.5 + '@babel/types': 7.23.5 + convert-source-map: 2.0.0 + debug: 4.3.4(supports-color@8.1.1) + gensync: 1.0.0-beta.2 + json5: 2.2.3 + semver: 6.3.1 + transitivePeerDependencies: + - supports-color /@babel/generator@7.22.15: resolution: {integrity: sha512-Zu9oWARBqeVOW0dZOjXc3JObrzuqothQ3y/n1kUtrjCoCPLkXUwMvOo/F/TCfoHMbWIFlWwpZtkZVb9ga4U2pA==} @@ -2345,6 +2378,16 @@ packages: '@jridgewell/gen-mapping': 0.3.3 '@jridgewell/trace-mapping': 0.3.19 jsesc: 2.5.2 + dev: true + + /@babel/generator@7.23.5: + resolution: {integrity: sha512-BPssCHrBD+0YrxviOa3QzpqwhNIXKEtOa2jQrm4FlmkC2apYgRnQcmPWiGZDlGxiNtltnUFolMe8497Esry+jA==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.5 + '@jridgewell/gen-mapping': 0.3.3 + '@jridgewell/trace-mapping': 0.3.20 + jsesc: 2.5.2 /@babel/helper-compilation-targets@7.22.15: resolution: {integrity: sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw==} @@ -2366,6 +2409,14 @@ packages: dependencies: '@babel/template': 7.22.15 '@babel/types': 7.22.19 + dev: true + + /@babel/helper-function-name@7.23.0: + resolution: {integrity: sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/template': 7.22.15 + '@babel/types': 7.23.5 /@babel/helper-hoist-variables@7.22.5: resolution: {integrity: sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==} @@ -2391,6 +2442,20 @@ packages: '@babel/helper-simple-access': 7.22.5 '@babel/helper-split-export-declaration': 7.22.6 '@babel/helper-validator-identifier': 7.22.20 + dev: true + + /@babel/helper-module-transforms@7.23.3(@babel/core@7.23.5): + resolution: {integrity: sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.5 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-module-imports': 7.22.15 + '@babel/helper-simple-access': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/helper-validator-identifier': 7.22.20 /@babel/helper-simple-access@7.22.5: resolution: {integrity: sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==} @@ -2408,6 +2473,10 @@ packages: resolution: {integrity: sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==} engines: {node: '>=6.9.0'} + /@babel/helper-string-parser@7.23.4: + resolution: {integrity: sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==} + engines: {node: '>=6.9.0'} + /@babel/helper-validator-identifier@7.22.20: resolution: {integrity: sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==} engines: {node: '>=6.9.0'} @@ -2425,6 +2494,17 @@ packages: '@babel/types': 7.22.19 transitivePeerDependencies: - supports-color + dev: true + + /@babel/helpers@7.23.5: + resolution: {integrity: sha512-oO7us8FzTEsG3U6ag9MfdF1iA/7Z6dz+MtFhifZk8C8o453rGJFFWUP1t+ULM9TUIAzC9uxXEiXjOiVMyd7QPg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/template': 7.22.15 + '@babel/traverse': 7.23.5 + '@babel/types': 7.23.5 + transitivePeerDependencies: + - supports-color /@babel/highlight@7.22.20: resolution: {integrity: sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==} @@ -2434,6 +2514,14 @@ packages: chalk: 2.4.2 js-tokens: 4.0.0 + /@babel/highlight@7.23.4: + resolution: {integrity: sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-validator-identifier': 7.22.20 + chalk: 2.4.2 + js-tokens: 4.0.0 + /@babel/parser@7.22.16: resolution: {integrity: sha512-+gPfKv8UWeKKeJTUxe59+OobVcrYHETCsORl61EmSkmgymguYk/X5bp7GuUIXaFsc6y++v8ZxPsLSSuujqDphA==} engines: {node: '>=6.0.0'} @@ -2441,6 +2529,13 @@ packages: dependencies: '@babel/types': 7.22.19 + /@babel/parser@7.23.5: + resolution: {integrity: sha512-hOOqoiNXrmGdFbhgCzu6GiURxUgM27Xwd/aPuu8RfHEZPBzL1Z54okAHAQjXfcQNwvrlkAmAp4SlRTZ45vlthQ==} + engines: {node: '>=6.0.0'} + hasBin: true + dependencies: + '@babel/types': 7.23.5 + /@babel/runtime@7.22.15: resolution: {integrity: sha512-T0O+aa+4w0u06iNmapipJXMV4HoUir03hpx3/YqXXhu9xim3w+dVphjFWl1OH8NbZHw5Lbm9k45drDkgq2VNNA==} engines: {node: '>=6.9.0'} @@ -2471,6 +2566,24 @@ packages: globals: 11.12.0 transitivePeerDependencies: - supports-color + dev: true + + /@babel/traverse@7.23.5: + resolution: {integrity: sha512-czx7Xy5a6sapWWRx61m1Ke1Ra4vczu1mCTtJam5zRTBOonfdJ+S/B6HYmGYu3fJtr8GGET3si6IhgWVBhJ/m8w==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/code-frame': 7.23.5 + '@babel/generator': 7.23.5 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-function-name': 7.23.0 + '@babel/helper-hoist-variables': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/parser': 7.23.5 + '@babel/types': 7.23.5 + debug: 4.3.4(supports-color@8.1.1) + globals: 11.12.0 + transitivePeerDependencies: + - supports-color /@babel/types@7.22.19: resolution: {integrity: sha512-P7LAw/LbojPzkgp5oznjE6tQEIWbp4PkkfrZDINTro9zgBRtI324/EYsiSI7lhPbpIQ+DCeR2NNmMWANGGfZsg==} @@ -2480,6 +2593,14 @@ packages: '@babel/helper-validator-identifier': 7.22.20 to-fast-properties: 2.0.0 + /@babel/types@7.23.5: + resolution: {integrity: sha512-ON5kSOJwVO6xXVRTvOI0eOnWe7VdUcIpsovGo9U/Br4Ie4UVFQTboO2cYnDhAGU6Fp+UxSiT+pMft0SMHfuq6w==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-string-parser': 7.23.4 + '@babel/helper-validator-identifier': 7.22.20 + to-fast-properties: 2.0.0 + /@changesets/apply-release-plan@6.1.4: resolution: {integrity: sha512-FMpKF1fRlJyCZVYHr3CbinpZZ+6MwvOtWUuO8uo+svcATEoc1zRDcj23pAurJ2TZ/uVz1wFHH6K3NlACy0PLew==} dependencies: @@ -3013,6 +3134,12 @@ packages: '@jridgewell/resolve-uri': 3.1.1 '@jridgewell/sourcemap-codec': 1.4.15 + /@jridgewell/trace-mapping@0.3.20: + resolution: {integrity: sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==} + dependencies: + '@jridgewell/resolve-uri': 3.1.1 + '@jridgewell/sourcemap-codec': 1.4.15 + /@jridgewell/trace-mapping@0.3.9: resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} dependencies: @@ -3095,7 +3222,7 @@ packages: resolution: {integrity: sha512-O0YUYgNkQpTadkUzviJShvw0JNz1rZex6aySq2rBF3I63qBo+OTi4TrQfr8lHqJXhrVMsBuW9tYOYwUGZ+aAqA==} hasBin: true dependencies: - '@babel/core': 7.22.20 + '@babel/core': 7.23.5 doctrine: 2.1.0 lodash.isequal: 4.5.0 yargs: 17.7.2 @@ -4867,10 +4994,10 @@ packages: engines: {node: '>=4'} dependencies: assertion-error: 1.1.0 - check-error: 1.0.2 + check-error: 1.0.3 deep-eql: 3.0.1 - get-func-name: 2.0.0 - loupe: 2.3.6 + get-func-name: 2.0.2 + loupe: 2.3.7 pathval: 1.1.1 type-detect: 4.0.8 dev: true @@ -4937,6 +5064,12 @@ packages: resolution: {integrity: sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA==} dev: true + /check-error@1.0.3: + resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==} + dependencies: + get-func-name: 2.0.2 + dev: true + /cheerio-select@2.1.0: resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} dependencies: @@ -5281,6 +5414,10 @@ packages: /convert-source-map@1.9.0: resolution: {integrity: sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==} + dev: true + + /convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} /convert-to-spaces@2.0.1: resolution: {integrity: sha512-rcQ1bsQO9799wq24uE5AM2tAILy4gXGIK/njFWcVQkGNZ96edlpY+A7bjwvzjYvLDyzmG1MmMLZhpcsb+klNMQ==} @@ -7030,6 +7167,10 @@ packages: resolution: {integrity: sha512-Hm0ixYtaSZ/V7C8FJrtZIuBBI+iSgL+1Aq82zSu8VQNB4S3Gk8e7Qs3VwBDJAhmRZcFqkl3tQu36g/Foh5I5ig==} dev: true + /get-func-name@2.0.2: + resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} + dev: true + /get-intrinsic@1.2.1: resolution: {integrity: sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==} dependencies: @@ -7466,7 +7607,7 @@ packages: dependencies: assert-plus: 1.0.0 jsprim: 1.4.2 - sshpk: 1.17.0 + sshpk: 1.18.0 dev: false /https-proxy-agent@5.0.1: @@ -8666,6 +8807,12 @@ packages: get-func-name: 2.0.0 dev: true + /loupe@2.3.7: + resolution: {integrity: sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==} + dependencies: + get-func-name: 2.0.2 + dev: true + /lru-cache@4.1.5: resolution: {integrity: sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==} dependencies: @@ -10192,6 +10339,11 @@ packages: resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} engines: {node: '>=6'} + /punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + dev: false + /q@1.5.1: resolution: {integrity: sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==} engines: {node: '>=0.6.0', teleport: '>=0.2.0'} @@ -10737,6 +10889,10 @@ packages: resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==} dev: false + /sax@1.3.0: + resolution: {integrity: sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==} + dev: false + /scmp@2.1.0: resolution: {integrity: sha512-o/mRQGk9Rcer/jEEw/yw4mwo3EU/NvYvp577/Btqrym9Qy5/MdWGBqipbALgd2lrdWTJ5/gqDusxfnQBxOxT2Q==} dev: false @@ -11103,8 +11259,8 @@ packages: nan: 2.18.0 dev: false - /sshpk@1.17.0: - resolution: {integrity: sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==} + /sshpk@1.18.0: + resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==} engines: {node: '>=0.10.0'} hasBin: true dependencies: @@ -11717,7 +11873,7 @@ packages: engines: {node: '>=6'} dependencies: psl: 1.9.0 - punycode: 2.3.0 + punycode: 2.3.1 universalify: 0.2.0 url-parse: 1.5.10 dev: false @@ -12428,7 +12584,7 @@ packages: resolution: {integrity: sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==} engines: {node: '>=4.0.0'} dependencies: - sax: 1.2.4 + sax: 1.3.0 xmlbuilder: 11.0.1 dev: false