From a0c59e403011af303c949679df98eaa8b5b1bacc Mon Sep 17 00:00:00 2001 From: sairaj mote Date: Fri, 12 Jan 2024 15:30:24 +0530 Subject: [PATCH] merging hashing and price history APIs --- index.html | 181 ++++++++++++++++++++++++++ index.js | 152 ++++------------------ index.min.html | 1 + index.min.js | 2 +- models/price-history.js | 22 ++++ models/price-history.min.js | 1 + package-lock.json | 249 ++++++++++++++++++++++++++++++++++++ package.json | 2 + routes/hash.js | 118 +++++++++++++++++ routes/hash.min.js | 1 + routes/price-history.js | 122 ++++++++++++++++++ routes/price-history.min.js | 1 + 12 files changed, 726 insertions(+), 126 deletions(-) create mode 100644 index.html create mode 100644 index.min.html create mode 100644 models/price-history.js create mode 100644 models/price-history.min.js create mode 100644 routes/hash.js create mode 100644 routes/hash.min.js create mode 100644 routes/price-history.js create mode 100644 routes/price-history.min.js diff --git a/index.html b/index.html new file mode 100644 index 0000000..3881725 --- /dev/null +++ b/index.html @@ -0,0 +1,181 @@ + + + + + + + RanchiMall Utility APIs + + + + +
+

+ Welcome to the RanchiMall Utility APIs! +

+

+ Endpoints: +

+
    +
  1. +

    + /price-history +

    +

    + Query parameters: +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ParameterRequiredDefaultformat | values
    fromNoNoneYYYY-MM-DD
    toNoNoneYYYY-MM-DD
    onNoNoneYYYY-MM-DD
    limitNo100all | <number>
    assetNobtcbtc
    currencyNoAllusd | inr
    +

    + Example: +

    + + /price-history?from=2020-01-01&to=2020-01-31 + +
    2. +
  2. +

    + /hash +

    + + + + + + + + + + + + + + + +
    TypePOST
    BodyJSON
    Body parameterurls [Array]
    +

    + Example: +

    + + fetch('https://utility-api.ranchimall.net/hash',{
    +   method: 'POST',
    +   headers: {
    +     'Content-Type': 'application/json'
    +   },
    +   body: JSON.stringify({ urls: [url] })
    + }).then(res => res.json()).then(console.log) +
    +
    + Output:
    + [{
    +   "url": url,
    +   "hash": hash
    + }] +     +
    3. +
+
+ + + \ No newline at end of file diff --git a/index.js b/index.js index 107f427..ab8269c 100644 --- a/index.js +++ b/index.js @@ -1,155 +1,57 @@ require('dotenv').config(); const express = require('express'); +const mongoose = require('mongoose'); const cors = require('cors'); -const axios = require('axios'); -const { createHash } = require('crypto'); const rateLimit = require('express-rate-limit'); -const { parse: parseUrl, URL } = require('url'); -const { parse: parseHtml } = require('node-html-parser'); +const path = require('path'); // Set up the allowed domains (replace with your specific domains) -const allowedDomains = process.env.ALLOWED_DOMAINS.split(','); +// const allowedDomains = process.env.ALLOWED_DOMAINS.split(','); const app = express(); +const PORT = process.env.PORT || 3000; +const HOST = process.env.HOST || '127.0.0.1'; + +// Middleware to parse JSON requests +app.use(express.json()); +// Middleware to enable CORS // pass the cors options to the cors middleware to enable CORS for the allowed domains // const corsOptions = { // origin: allowedDomains, // optionsSuccessStatus: 200, // Some legacy browsers (IE11, various SmartTVs) choke on 204 // } app.use(cors()); -const port = process.env.PORT || 3000; -const host = process.env.HOST || '0.0.0.0'; - -// Middleware to parse JSON requests -app.use(express.json()); -// Middleware to enable CORS - app.use( rateLimit({ windowMs: 1 * 60 * 1000, // 1 minute - max: 20, // limit each IP request per windowMs + max: 30, // limit each IP request per windowMs }) ); -app.get('/', (req, res) => { - res.send('Hello There!'); -}) -function addProtocolToUrl(url) { - if (!url.startsWith('http://') && !url.startsWith('https://')) { - url = 'https://' + url; - } - return url; -} - -function parseUrlWithoutHashAndQuery(fullUrl) { - fullUrl = addProtocolToUrl(fullUrl); - const parsedUrl = new URL(fullUrl); - - // Set the hash and search/query to empty strings - parsedUrl.hash = ''; - parsedUrl.search = ''; - - // Reconstruct the URL without hash and query - const urlWithoutHashAndQuery = parsedUrl.toString(); - - return urlWithoutHashAndQuery; -} -// hashContent function to hash the content of a file -async function hashContent(content) { - const hash = createHash('sha256'); - hash.update(content); - return hash.digest('hex'); -} - -// Recursive function to fetch and hash content, including linked resources -async function fetchAndHashContent(url, visitedUrls = new Set()) { - if (visitedUrls.has(url)) { - return ''; // Avoid fetching the same URL multiple times to prevent infinite loops - } - - visitedUrls.add(url); - const response = await axios.get(url, { responseType: 'arraybuffer' }); - const content = response.data.toString('utf-8'); - // Parse HTML content to identify linked resources - const root = parseHtml(content); - const linkedResources = root.querySelectorAll('link[rel="stylesheet"], script[src]'); - // Fetch and hash linked resources - const linkedResource = await Promise.all(linkedResources.map(async (resource) => { - const resourceUrl = parseUrl(resource.getAttribute('href') || resource.getAttribute('src'), true); - let absoluteResourceUrl = resourceUrl.href; - if (!resourceUrl.hostname) { - if (!resourceUrl.path.startsWith('/') && !url.endsWith('/')) - url += '/'; - absoluteResourceUrl = `${url}${resourceUrl.path}`; - } - const resourceContent = await fetchAndHashContent(absoluteResourceUrl, visitedUrls); - return `${resourceUrl.path}_${resourceContent}`; - })); - - // Combine the content and hashes of linked resources - return `${content}_${linkedResource.join('_')}`; -} - -const hashCache = new Map(); -// API endpoint to start the recursive download and hashing -app.post('/hash', async (req, res) => { - try { - let { urls } = req.body; - if (!urls) { - return res.status(400).json({ error: 'Missing in the request parameters' }); - } - if (!Array.isArray(urls)) - urls = [urls]; - const promises = urls.map(async (url) => { - const urlWithoutHashAndQuery = parseUrlWithoutHashAndQuery(url); - let hash; - // regex to identify owner and repo name from https://owner.github.io/repo-name - const githubRepoRegex = /https?:\/\/([\w-]+)\.github\.io\/([\w-]+)/; - if (githubRepoRegex.test(urlWithoutHashAndQuery) && urlWithoutHashAndQuery.match(githubRepoRegex)[1] === 'ranchimall') { - if (!hashCache.has(urlWithoutHashAndQuery)) { - await fetchAndSaveAppHash(urlWithoutHashAndQuery) - } - hash = hashCache.get(urlWithoutHashAndQuery).hash; - } else { - const hashedContent = await fetchAndHashContent(urlWithoutHashAndQuery); - hash = await hashContent(Buffer.from(hashedContent, 'utf-8')); - } - return { url, hash }; - }); - - const results = await Promise.all(promises); - res.json(results); - } catch (error) { - res.status(500).json({ error: error.message }); - } +// connect to MongoDB +mongoose.connect(`mongodb://${HOST}/price-history`); +const db = mongoose.connection; +db.on('error', console.error.bind(console, 'connection error:')); +db.once('open', () => { + console.log('Connected to MongoDB'); }); -async function fetchAndSaveAppHash(url, lastUpdated = Date.now()) { - const hashedContent = await fetchAndHashContent(url); - const hash = await hashContent(Buffer.from(hashedContent, 'utf-8')); - hashCache.set(url, { hash, lastUpdated }); -} -app.post('/gitwh', async (req, res) => { - try { - // ignore if request is not from github - if (!req.headers['user-agent'].startsWith('GitHub-Hookshot/')) - return; - const { repository: { pushed_at, organization, name, has_pages } } = req.body; - if (!has_pages) - return; - const url = `https://${organization}.github.io/${name}` - await fetchAndSaveAppHash(url, pushed_at) - res.json({ message: 'success' }); - } catch (err) { - res.status(500).json({ error: err.message }); - } +app.get("/", (req, res) => { + res.sendFile(path.join(__dirname, './index.min.html')); }) +const hash = require('./routes/hash') +app.use("/hash", hash); +const priceHistory = require('./routes/price-history') +app.use("/price-history", priceHistory); // Start the server -app.listen(port, host, () => { - console.log(`Server is running at http://${host}:${port}`); +app.listen(PORT, HOST, () => { + console.log(`Server is running at http://${HOST}:${PORT}`); }); +// TODO +//https://utility-api.ranchimall.net/hash/gitwh + // Export the Express API module.exports = app; diff --git a/index.min.html b/index.min.html new file mode 100644 index 0000000..f20c4a3 --- /dev/null +++ b/index.min.html @@ -0,0 +1 @@ + RanchiMall Utility APIs

Welcome to the RanchiMall Utility APIs!

Endpoints:

  1. /price-history

    Query parameters:

    Parameter Required Default format | values
    from No None YYYY-MM-DD
    to No None YYYY-MM-DD
    on No None YYYY-MM-DD
    limit No 100 all | <number>
    asset No btc btc
    currency No All usd | inr

    Example:

    /price-history?from=2020-01-01&to=2020-01-31

  2. /hash

    Type POST
    Body JSON
    Body parameter urls [Array]

    Example:

    fetch('https://utility-api.ranchimall.net/hash',{
      method: 'POST',
      headers: {
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({ urls: [url] })
    }).then(res => res.json()).then(console.log)

    Output:
    [{
      "url": url,
      "hash": hash
    }]
\ No newline at end of file diff --git a/index.min.js b/index.min.js index 4902f71..d816959 100644 --- a/index.min.js +++ b/index.min.js @@ -1 +1 @@ -require("dotenv").config();const express=require("express"),cors=require("cors"),axios=require("axios"),{createHash:createHash}=require("crypto"),rateLimit=require("express-rate-limit"),{parse:parseUrl,URL:URL}=require("url"),{parse:parseHtml}=require("node-html-parser"),allowedDomains=process.env.ALLOWED_DOMAINS.split(","),app=express();app.use(cors());const port=process.env.PORT||3e3,host=process.env.HOST||"0.0.0.0";function addProtocolToUrl(url){return url.startsWith("http://")||url.startsWith("https://")||(url="https://"+url),url}function parseUrlWithoutHashAndQuery(fullUrl){fullUrl=addProtocolToUrl(fullUrl);const parsedUrl=new URL(fullUrl);parsedUrl.hash="",parsedUrl.search="";return parsedUrl.toString()}async function hashContent(content){const hash=createHash("sha256");return hash.update(content),hash.digest("hex")}async function fetchAndHashContent(url,visitedUrls=new Set){if(visitedUrls.has(url))return"";visitedUrls.add(url);const content=(await axios.get(url,{responseType:"arraybuffer"})).data.toString("utf-8"),linkedResources=parseHtml(content).querySelectorAll('link[rel="stylesheet"], script[src]');return`${content}_${(await Promise.all(linkedResources.map((async resource=>{const resourceUrl=parseUrl(resource.getAttribute("href")||resource.getAttribute("src"),!0);let absoluteResourceUrl=resourceUrl.href;resourceUrl.hostname||(resourceUrl.path.startsWith("/")||url.endsWith("/")||(url+="/"),absoluteResourceUrl=`${url}${resourceUrl.path}`);const resourceContent=await fetchAndHashContent(absoluteResourceUrl,visitedUrls);return`${resourceUrl.path}_${resourceContent}`})))).join("_")}`}app.use(express.json()),app.use(rateLimit({windowMs:6e4,max:20})),app.get("/",((req,res)=>{res.send("Hello There!")}));const hashCache=new Map;async function fetchAndSaveAppHash(url,lastUpdated=Date.now()){const hashedContent=await fetchAndHashContent(url),hash=await hashContent(Buffer.from(hashedContent,"utf-8"));hashCache.set(url,{hash:hash,lastUpdated:lastUpdated})}app.post("/hash",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing in the request parameters"});Array.isArray(urls)||(urls=[urls]);const promises=urls.map((async url=>{const urlWithoutHashAndQuery=parseUrlWithoutHashAndQuery(url);let hash;const githubRepoRegex=/https?:\/\/([\w-]+)\.github\.io\/([\w-]+)/;if(githubRepoRegex.test(urlWithoutHashAndQuery)&&"ranchimall"===urlWithoutHashAndQuery.match(githubRepoRegex)[1])hashCache.has(urlWithoutHashAndQuery)||await fetchAndSaveAppHash(urlWithoutHashAndQuery),hash=hashCache.get(urlWithoutHashAndQuery).hash;else{const hashedContent=await fetchAndHashContent(urlWithoutHashAndQuery);hash=await hashContent(Buffer.from(hashedContent,"utf-8"))}return{url:url,hash:hash}})),results=await Promise.all(promises);res.json(results)}catch(error){res.status(500).json({error:error.message})}})),app.post("/gitwh",(async(req,res)=>{try{if(!req.headers["user-agent"].startsWith("GitHub-Hookshot/"))return;const{repository:{pushed_at:pushed_at,organization:organization,name:name,has_pages:has_pages}}=req.body;if(!has_pages)return;const url=`https://${organization}.github.io/${name}`;await fetchAndSaveAppHash(url,pushed_at),res.json({message:"success"})}catch(err){res.status(500).json({error:err.message})}})),app.listen(port,host,(()=>{console.log(`Server is running at http://${host}:${port}`)})),module.exports=app; \ No newline at end of file +require("dotenv").config();const express=require("express"),mongoose=require("mongoose"),cors=require("cors"),rateLimit=require("express-rate-limit"),path=require("path"),app=express(),PORT=process.env.PORT||3e3,HOST=process.env.HOST||"127.0.0.1";app.use(express.json()),app.use(cors()),app.use(rateLimit({windowMs:6e4,max:30})),mongoose.connect(`mongodb://${HOST}/price-history`);const db=mongoose.connection;db.on("error",console.error.bind(console,"connection error:")),db.once("open",(()=>{console.log("Connected to MongoDB")})),app.get("/",((req,res)=>{res.sendFile(path.join(__dirname,"./index.min.html"))}));const hash=require("./routes/hash");app.use("/hash",hash);const priceHistory=require("./routes/price-history");app.use("/price-history",priceHistory),app.listen(PORT,HOST,(()=>{console.log(`Server is running at http://${HOST}:${PORT}`)})),module.exports=app; \ No newline at end of file diff --git a/models/price-history.js b/models/price-history.js new file mode 100644 index 0000000..85344b2 --- /dev/null +++ b/models/price-history.js @@ -0,0 +1,22 @@ +const mongoose = require('mongoose'); +// scheme to store price history +const Schema = new mongoose.Schema({ + date: { + type: Date, + required: true, + unique: true + }, + asset: { + type: String, + required: true + }, + usd: { + type: Number, + required: true + }, + inr: { + type: Number, + required: true + } +}); +module.exports = mongoose.model('PriceHistory', Schema); \ No newline at end of file diff --git a/models/price-history.min.js b/models/price-history.min.js new file mode 100644 index 0000000..5ffdf7b --- /dev/null +++ b/models/price-history.min.js @@ -0,0 +1 @@ +const mongoose=require("mongoose"),Schema=new mongoose.Schema({date:{type:Date,required:!0,unique:!0},asset:{type:String,required:!0},usd:{type:Number,required:!0},inr:{type:Number,required:!0}});module.exports=mongoose.model("PriceHistory",Schema); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 04b2270..f90949d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,8 @@ "crypto": "^1.0.1", "express": "^4.18.2", "express-rate-limit": "^7.1.5", + "mongoose": "^8.0.4", + "node-cron": "^3.0.3", "node-html-parser": "^6.1.11" }, "devDependencies": { @@ -21,6 +23,36 @@ "nodemon": "^3.0.2" } }, + "node_modules/@mongodb-js/saslprep": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.3.tgz", + "integrity": "sha512-SyCxhJfmK6MoLNV5SbDpNdUy9SDv5H7y9/9rl3KpnwgTHWuNNMc87zWqbcIZXNWY+aUjxLGLEcvHoLagG4tWCg==", + "dependencies": { + "sparse-bitfield": "^3.0.3" + } + }, + "node_modules/@types/node": { + "version": "20.11.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.0.tgz", + "integrity": "sha512-o9bjXmDNcF7GbM4CNQpmi+TutCgap/K3w1JyKgxAjqx41zp9qlIAVFi0IhCNsJcXolEqLWhbFbEeL0PvYm4pcQ==", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==" + }, + "node_modules/@types/whatwg-url": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.2.tgz", + "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==", + "dependencies": { + "@types/node": "*", + "@types/webidl-conversions": "*" + } + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -137,6 +169,14 @@ "node": ">=8" } }, + "node_modules/bson": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.2.0.tgz", + "integrity": "sha512-ID1cI+7bazPDyL9wYy9GaQ8gEEohWvcUl/Yf0dIdutJxnmInEEyCsb4awy/OiBfall7zBA179Pahi3vCdFze3Q==", + "engines": { + "node": ">=16.20.1" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -763,6 +803,14 @@ "node": ">=0.12.0" } }, + "node_modules/kareem": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.5.1.tgz", + "integrity": "sha512-7jFxRVm+jD+rkq3kY0iZDJfsO2/t4BBPeEb2qKn2lR/9KhuksYk5hxzfRYWMPV8P/x2d0kHD306YyWLzjjH+uA==", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -783,6 +831,11 @@ "node": ">= 0.6" } }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==" + }, "node_modules/merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -838,6 +891,126 @@ "node": "*" } }, + "node_modules/mongodb": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.2.0.tgz", + "integrity": "sha512-d7OSuGjGWDZ5usZPqfvb36laQ9CPhnWkAGHT61x5P95p/8nMVeH8asloMwW6GcYFeB0Vj4CB/1wOTDG2RA9BFA==", + "dependencies": { + "@mongodb-js/saslprep": "^1.1.0", + "bson": "^6.2.0", + "mongodb-connection-string-url": "^2.6.0" + }, + "engines": { + "node": ">=16.20.1" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.188.0", + "@mongodb-js/zstd": "^1.1.0", + "gcp-metadata": "^5.2.0", + "kerberos": "^2.0.1", + "mongodb-client-encryption": ">=6.0.0 <7", + "snappy": "^7.2.2", + "socks": "^2.7.1" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.6.0.tgz", + "integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==", + "dependencies": { + "@types/whatwg-url": "^8.2.1", + "whatwg-url": "^11.0.0" + } + }, + "node_modules/mongoose": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.0.4.tgz", + "integrity": "sha512-wN9qvdevX3+922VnLT7CpaZRT3jmVCBOK2QMHMGeScQxDRnFMPpkuI9StEPpZo/3x8t+kbzH7F8RMPsyNwyM4w==", + "dependencies": { + "bson": "^6.2.0", + "kareem": "2.5.1", + "mongodb": "6.2.0", + "mpath": "0.9.0", + "mquery": "5.0.0", + "ms": "2.1.3", + "sift": "16.0.1" + }, + "engines": { + "node": ">=16.20.1" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mongoose" + } + }, + "node_modules/mongoose/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/mpath": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.9.0.tgz", + "integrity": "sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/mquery": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/mquery/-/mquery-5.0.0.tgz", + "integrity": "sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==", + "dependencies": { + "debug": "4.x" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/mquery/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/mquery/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -851,6 +1024,17 @@ "node": ">= 0.6" } }, + "node_modules/node-cron": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz", + "integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==", + "dependencies": { + "uuid": "8.3.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/node-html-parser": { "version": "6.1.11", "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.11.tgz", @@ -1021,6 +1205,14 @@ "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", "dev": true }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "engines": { + "node": ">=6" + } + }, "node_modules/qs": { "version": "6.11.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", @@ -1182,6 +1374,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/sift": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/sift/-/sift-16.0.1.tgz", + "integrity": "sha512-Wv6BjQ5zbhW7VFefWusVP33T/EM0vYikCaQ2qR8yULbsilAT8/wQaXvuQ3ptGLpoKx+lihJE3y2UTgKDyyNHZQ==" + }, "node_modules/simple-update-notifier": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", @@ -1194,6 +1391,14 @@ "node": ">=10" } }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "dependencies": { + "memory-pager": "^1.0.2" + } + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -1246,6 +1451,17 @@ "nodetouch": "bin/nodetouch.js" } }, + "node_modules/tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -1264,6 +1480,11 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -1280,6 +1501,14 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -1288,6 +1517,26 @@ "node": ">= 0.8" } }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "engines": { + "node": ">=12" + } + }, + "node_modules/whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "dependencies": { + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/package.json b/package.json index aef17c3..2216164 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "crypto": "^1.0.1", "express": "^4.18.2", "express-rate-limit": "^7.1.5", + "mongoose": "^8.0.4", + "node-cron": "^3.0.3", "node-html-parser": "^6.1.11" }, "devDependencies": { diff --git a/routes/hash.js b/routes/hash.js new file mode 100644 index 0000000..379cd17 --- /dev/null +++ b/routes/hash.js @@ -0,0 +1,118 @@ +const express = require('express'); +const router = express.Router(); +const axios = require('axios'); +const { createHash } = require('crypto'); +const { parse: parseUrl, URL } = require('url'); +const { parse: parseHtml } = require('node-html-parser'); +function addProtocolToUrl(url) { + if (!url.startsWith('http://') && !url.startsWith('https://')) { + url = 'https://' + url; + } + return url; +} + +function parseUrlWithoutHashAndQuery(fullUrl) { + fullUrl = addProtocolToUrl(fullUrl); + const parsedUrl = new URL(fullUrl); + + // Set the hash and search/query to empty strings + parsedUrl.hash = ''; + parsedUrl.search = ''; + + // Reconstruct the URL without hash and query + const urlWithoutHashAndQuery = parsedUrl.toString(); + + return urlWithoutHashAndQuery; +} +// hashContent function to hash the content of a file +async function hashContent(content) { + const hash = createHash('sha256'); + hash.update(content); + return hash.digest('hex'); +} + +// Recursive function to fetch and hash content, including linked resources +async function fetchAndHashContent(url, visitedUrls = new Set()) { + if (visitedUrls.has(url)) { + return ''; // Avoid fetching the same URL multiple times to prevent infinite loops + } + + visitedUrls.add(url); + const response = await axios.get(url, { responseType: 'arraybuffer' }); + const content = response.data.toString('utf-8'); + // Parse HTML content to identify linked resources + const root = parseHtml(content); + const linkedResources = root.querySelectorAll('link[rel="stylesheet"], script[src]'); + // Fetch and hash linked resources + const linkedResource = await Promise.all(linkedResources.map(async (resource) => { + const resourceUrl = parseUrl(resource.getAttribute('href') || resource.getAttribute('src'), true); + let absoluteResourceUrl = resourceUrl.href; + if (!resourceUrl.hostname) { + if (!resourceUrl.path.startsWith('/') && !url.endsWith('/')) + url += '/'; + absoluteResourceUrl = `${url}${resourceUrl.path}`; + } + const resourceContent = await fetchAndHashContent(absoluteResourceUrl, visitedUrls); + return `${resourceUrl.path}_${resourceContent}`; + })); + + // Combine the content and hashes of linked resources + return `${content}_${linkedResource.join('_')}`; +} + +const hashCache = new Map(); +// API endpoint to start the recursive download and hashing +router.post('/', async (req, res) => { + try { + let { urls } = req.body; + if (!urls) { + return res.status(400).json({ error: 'Missing in the request parameters' }); + } + if (!Array.isArray(urls)) + urls = [urls]; + const promises = urls.map(async (url) => { + const urlWithoutHashAndQuery = parseUrlWithoutHashAndQuery(url); + let hash; + // regex to identify owner and repo name from https://owner.github.io/repo-name + const githubRepoRegex = /https?:\/\/([\w-]+)\.github\.io\/([\w-]+)/; + if (githubRepoRegex.test(urlWithoutHashAndQuery) && urlWithoutHashAndQuery.match(githubRepoRegex)[1] === 'ranchimall') { + if (!hashCache.has(urlWithoutHashAndQuery)) { + await fetchAndSaveAppHash(urlWithoutHashAndQuery) + } + hash = hashCache.get(urlWithoutHashAndQuery).hash; + } else { + const hashedContent = await fetchAndHashContent(urlWithoutHashAndQuery); + hash = await hashContent(Buffer.from(hashedContent, 'utf-8')); + } + return { url, hash }; + }); + + const results = await Promise.all(promises); + res.json(results); + } catch (error) { + res.status(500).json({ error: error.message }); + } +}); +async function fetchAndSaveAppHash(url, lastUpdated = Date.now()) { + const hashedContent = await fetchAndHashContent(url); + const hash = await hashContent(Buffer.from(hashedContent, 'utf-8')); + hashCache.set(url, { hash, lastUpdated }); +} + +router.post('/gitwh', async (req, res) => { + try { + // ignore if request is not from github + if (!req.headers['user-agent'].startsWith('GitHub-Hookshot/')) + return res.json({ message: 'ignored' }); + const { repository: { pushed_at, organization, name, has_pages } } = req.body; + if (!has_pages) + return res.json({ message: 'ignored' }); + const url = `https://${organization}.github.io/${name}` + await fetchAndSaveAppHash(url, pushed_at) + res.json({ message: 'success' }); + } catch (err) { + res.status(500).json({ error: err.message }); + } +}) + +module.exports = router; \ No newline at end of file diff --git a/routes/hash.min.js b/routes/hash.min.js new file mode 100644 index 0000000..1fda5ad --- /dev/null +++ b/routes/hash.min.js @@ -0,0 +1 @@ +const express=require("express"),router=express.Router(),axios=require("axios"),{createHash:createHash}=require("crypto"),{parse:parseUrl,URL:URL}=require("url"),{parse:parseHtml}=require("node-html-parser");function addProtocolToUrl(url){return url.startsWith("http://")||url.startsWith("https://")||(url="https://"+url),url}function parseUrlWithoutHashAndQuery(fullUrl){fullUrl=addProtocolToUrl(fullUrl);const parsedUrl=new URL(fullUrl);parsedUrl.hash="",parsedUrl.search="";return parsedUrl.toString()}async function hashContent(content){const hash=createHash("sha256");return hash.update(content),hash.digest("hex")}async function fetchAndHashContent(url,visitedUrls=new Set){if(visitedUrls.has(url))return"";visitedUrls.add(url);const content=(await axios.get(url,{responseType:"arraybuffer"})).data.toString("utf-8"),linkedResources=parseHtml(content).querySelectorAll('link[rel="stylesheet"], script[src]');return`${content}_${(await Promise.all(linkedResources.map((async resource=>{const resourceUrl=parseUrl(resource.getAttribute("href")||resource.getAttribute("src"),!0);let absoluteResourceUrl=resourceUrl.href;resourceUrl.hostname||(resourceUrl.path.startsWith("/")||url.endsWith("/")||(url+="/"),absoluteResourceUrl=`${url}${resourceUrl.path}`);const resourceContent=await fetchAndHashContent(absoluteResourceUrl,visitedUrls);return`${resourceUrl.path}_${resourceContent}`})))).join("_")}`}const hashCache=new Map;async function fetchAndSaveAppHash(url,lastUpdated=Date.now()){const hashedContent=await fetchAndHashContent(url),hash=await hashContent(Buffer.from(hashedContent,"utf-8"));hashCache.set(url,{hash:hash,lastUpdated:lastUpdated})}router.post("/",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing in the request parameters"});Array.isArray(urls)||(urls=[urls]);const promises=urls.map((async url=>{const urlWithoutHashAndQuery=parseUrlWithoutHashAndQuery(url);let hash;const githubRepoRegex=/https?:\/\/([\w-]+)\.github\.io\/([\w-]+)/;if(githubRepoRegex.test(urlWithoutHashAndQuery)&&"ranchimall"===urlWithoutHashAndQuery.match(githubRepoRegex)[1])hashCache.has(urlWithoutHashAndQuery)||await fetchAndSaveAppHash(urlWithoutHashAndQuery),hash=hashCache.get(urlWithoutHashAndQuery).hash;else{const hashedContent=await fetchAndHashContent(urlWithoutHashAndQuery);hash=await hashContent(Buffer.from(hashedContent,"utf-8"))}return{url:url,hash:hash}})),results=await Promise.all(promises);res.json(results)}catch(error){res.status(500).json({error:error.message})}})),router.post("/gitwh",(async(req,res)=>{try{if(!req.headers["user-agent"].startsWith("GitHub-Hookshot/"))return res.json({message:"ignored"});const{repository:{pushed_at:pushed_at,organization:organization,name:name,has_pages:has_pages}}=req.body;if(!has_pages)return res.json({message:"ignored"});const url=`https://${organization}.github.io/${name}`;await fetchAndSaveAppHash(url,pushed_at),res.json({message:"success"})}catch(err){res.status(500).json({error:err.message})}})),module.exports=router; \ No newline at end of file diff --git a/routes/price-history.js b/routes/price-history.js new file mode 100644 index 0000000..dd2e622 --- /dev/null +++ b/routes/price-history.js @@ -0,0 +1,122 @@ +const express = require('express'); +const router = express.Router(); +const cron = require('node-cron'); + +const PriceHistory = require('../models/price-history'); + +function loadHistoricToDb() { + const now = parseInt(Date.now() / 1000); + Promise.all([ + fetch(`https://query1.finance.yahoo.com/v7/finance/download/BTC-USD?period1=1410912000&period2=${now}&interval=1d&events=history&includeAdjustedClose=true`).then((res) => res.text()), + fetch(`https://query1.finance.yahoo.com/v7/finance/download/BTC-INR?period1=1410912000&period2=${now}&interval=1d&events=history&includeAdjustedClose=true`).then((res) => res.text()), + ]) + .then(async ([usd, inr]) => { + const usdData = usd.split("\n").slice(1); + const inrData = inr.split("\n").slice(1); + const priceHistoryData = []; + for (let i = 0; i < usdData.length; i++) { + const [date, open, high, low, close, adjClose, volume] = usdData[i].split(","); + const [date2, open2, high2, low2, close2, adjClose2, volume2] = inrData[i].split(","); + priceHistoryData.push({ + date: new Date(date).getTime(), + asset: "btc", + usd: parseFloat(parseFloat(close).toFixed(2)), + inr: parseFloat(parseFloat(close2).toFixed(2)), + }); + } + // update many + await PriceHistory.deleteMany({ asset: 'btc' }); + await PriceHistory.insertMany(priceHistoryData); + }) + .catch((err) => { + console.log(err); + }) +} +loadHistoricToDb(); + +router.get("/", async (req, res) => { + console.log('price-history'); + try { + const { from, to, on, limit = 100, asset = 'btc', currency } = req.query; + const searchParams = { + asset + } + if (from) { + searchParams.date = { $gte: new Date(from).getTime() }; + } + if (to) { + searchParams.date = { ...searchParams.date, $lte: new Date(to).getTime() }; + } + if (on) { + searchParams.date = { $eq: new Date(on).getTime() }; + } + if (currency) { + searchParams[currency] = { $exists: true }; + } + const dataFormat = { _id: 0, __v: 0, asset: 0 }; + if (currency === 'inr') { + dataFormat.usd = 0; + } + if (currency === 'usd') { + dataFormat.inr = 0; + } + const priceHistory = await PriceHistory.find(searchParams, dataFormat) + .sort({ date: -1 }) + .limit(limit === 'all' ? 0 : parseInt(limit)); + res.json(priceHistory); + } catch (err) { + console.log(err); + res.status(500).json({ error: err }); + } +}) + +router.post("/", async (req, res) => { + try { + const { dates } = req.body; + if (!dates) { + return res.status(400).json({ error: 'dates is required' }); + } + if (!Array.isArray(dates)) { + return res.status(400).json({ error: 'dates must be an array' }); + } + const priceHistory = await PriceHistory.find({ date: { $in: dates } }, { _id: 0, __v: 0, asset: 0 }); + res.json(priceHistory); + } catch (err) { + console.log(err); + res.status(500).json({ error: err }); + } +}) + +cron.schedule('0 */4 * * *', async () => { + try { + // will return a csv file + const [usd, inr] = await Promise.all([ + fetch("https://query1.finance.yahoo.com/v7/finance/download/BTC-USD"). + then((res) => res.text()), + fetch("https://query1.finance.yahoo.com/v7/finance/download/BTC-INR"). + then((res) => res.text()) + ]); + + const usdData = usd.split("\n").slice(1); + const inrData = inr.split("\n").slice(1); + for (let i = 0; i < usdData.length; i++) { + const [date, open, high, low, close, adjClose, volume] = usdData[i].split(","); + const [date2, open2, high2, low2, close2, adjClose2, volume2] = inrData[i].split(","); + const priceHistoryData = { + date: new Date(date).getTime(), + asset: "btc", + usd: parseFloat(parseFloat(close).toFixed(2)), + inr: parseFloat(parseFloat(close2).toFixed(2)), + }; + await PriceHistory.findOneAndUpdate( + { date: priceHistoryData.date, asset: priceHistoryData.asset }, + priceHistoryData, + { upsert: true } + ); + } + } catch (err) { + console.log(err); + } +}) + +module.exports = router; \ No newline at end of file diff --git a/routes/price-history.min.js b/routes/price-history.min.js new file mode 100644 index 0000000..f97c0ad --- /dev/null +++ b/routes/price-history.min.js @@ -0,0 +1 @@ +const express=require("express"),router=express.Router(),cron=require("node-cron"),PriceHistory=require("../models/price-history");function loadHistoricToDb(){const now=parseInt(Date.now()/1e3);Promise.all([fetch(`https://query1.finance.yahoo.com/v7/finance/download/BTC-USD?period1=1410912000&period2=${now}&interval=1d&events=history&includeAdjustedClose=true`).then((res=>res.text())),fetch(`https://query1.finance.yahoo.com/v7/finance/download/BTC-INR?period1=1410912000&period2=${now}&interval=1d&events=history&includeAdjustedClose=true`).then((res=>res.text()))]).then((async([usd,inr])=>{const usdData=usd.split("\n").slice(1),inrData=inr.split("\n").slice(1),priceHistoryData=[];for(let i=0;i{console.log(err)}))}loadHistoricToDb(),router.get("/",(async(req,res)=>{console.log("price-history");try{const{from:from,to:to,on:on,limit:limit=100,asset:asset="btc",currency:currency}=req.query,searchParams={asset:asset};from&&(searchParams.date={$gte:new Date(from).getTime()}),to&&(searchParams.date={...searchParams.date,$lte:new Date(to).getTime()}),on&&(searchParams.date={$eq:new Date(on).getTime()}),currency&&(searchParams[currency]={$exists:!0});const dataFormat={_id:0,__v:0,asset:0};"inr"===currency&&(dataFormat.usd=0),"usd"===currency&&(dataFormat.inr=0);const priceHistory=await PriceHistory.find(searchParams,dataFormat).sort({date:-1}).limit("all"===limit?0:parseInt(limit));res.json(priceHistory)}catch(err){console.log(err),res.status(500).json({error:err})}})),router.post("/",(async(req,res)=>{try{const{dates:dates}=req.body;if(!dates)return res.status(400).json({error:"dates is required"});if(!Array.isArray(dates))return res.status(400).json({error:"dates must be an array"});const priceHistory=await PriceHistory.find({date:{$in:dates}},{_id:0,__v:0,asset:0});res.json(priceHistory)}catch(err){console.log(err),res.status(500).json({error:err})}})),cron.schedule("0 */4 * * *",(async()=>{try{const[usd,inr]=await Promise.all([fetch("https://query1.finance.yahoo.com/v7/finance/download/BTC-USD").then((res=>res.text())),fetch("https://query1.finance.yahoo.com/v7/finance/download/BTC-INR").then((res=>res.text()))]),usdData=usd.split("\n").slice(1),inrData=inr.split("\n").slice(1);for(let i=0;i