Skip to content

Commit

Permalink
merging hashing and price history APIs
Browse files Browse the repository at this point in the history
  • Loading branch information
@sairaj-mote
sairaj-mote committed Jan 12, 2024
1 parent ed41b6a commit a0c59e4
Show file tree
Hide file tree
Showing 12 changed files with 726 additions and 126 deletions.
181 changes: 181 additions & 0 deletions index.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
<!DOCTYPE html>
<html lang="en">

<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>RanchiMall Utility APIs</title>
<style>
body {
font-family: sans-serif;
}

h1 {
font-size: 2rem;
margin-bottom: 2rem;
}

ol li {
margin-bottom: 5rem;
}

a {
color: inherit;
}

table {
border-collapse: collapse;
}

table,
th,
td {
border: 1px solid black;
padding: 0.5rem;
}

code {
display: inline-block;
background-color: #eee;
padding: 0.3rem;
border-radius: 0.2rem;
font: monospace;
font-size: inherit;
}

@media (prefers-color-scheme: dark) {
body {
background-color: #222;
color: #eee;
}

table,
th,
td {
border-color: #eee;
}

code {
background-color: #333;
color: #eee;
}
}
</style>
</head>

<body>
<section style="padding:4vw;">
<h1>
Welcome to the RanchiMall Utility APIs!
</h1>
<h2>
Endpoints:
</h2>
<ol>
<li>
<h3>
<a href="/price-history">/price-history</a>
</h3>
<h4>
Query parameters:
</h4>
<table>
<thead>
<tr>
<th>Parameter</th>
<th>Required</th>
<th>Default</th>
<th>format | values</th>
</tr>
</thead>
<tbody>
<tr>
<td>from</td>
<td>No</td>
<td>None</td>
<td>YYYY-MM-DD</td>
</tr>
<tr>
<td>to</td>
<td>No</td>
<td>None</td>
<td>YYYY-MM-DD</td>
</tr>
<tr>
<td>on</td>
<td>No</td>
<td>None</td>
<td>YYYY-MM-DD</td>
</tr>
<tr>
<td>limit</td>
<td>No</td>
<td>100</td>
<td>all | &lt;number&gt;</td>
</tr>
<tr>
<td>asset</td>
<td>No</td>
<td>btc</td>
<td>btc</td>
</tr>
<tr>
<td>currency</td>
<td>No</td>
<td>All</td>
<td>usd | inr</td>
</tr>
</tbody>
</table>
<h4>
Example:
</h4>
<code>
/price-history?from=2020-01-01&to=2020-01-31
</code>
</li>
<li>
<h3>
<a href="/hash">/hash</a>
</h3>
<table>
<tbody>
<tr>
<td>Type</td>
<td>POST</td>
</tr>
<tr>
<td>Body</td>
<td>JSON</td>
</tr>
<tr>
<td>Body parameter</td>
<td>urls [Array]</td>
</tr>
</tbody>
</table>
<h4>
Example:
</h4>
<code>
fetch('https://utility-api.ranchimall.net/hash',{ <br>
&nbsp; method: 'POST',<br>
&nbsp; headers: {<br>
&nbsp; &nbsp; 'Content-Type': 'application/json'<br>
&nbsp; },<br>
&nbsp; body: JSON.stringify({ urls: [url] })<br>
}).then(res => res.json()).then(console.log)
<br>
<br>
Output: <br>
[{<br>
&nbsp; "url": url,<br>
&nbsp; "hash": hash<br>
}]
</code>
</li>
</ol>
</section>
</body>

</html>
152 changes: 27 additions & 125 deletions index.js
View file
Original file line number Diff line number Diff line change
@@ -1,155 +1,57 @@
require('dotenv').config();
const express = require('express');
const mongoose = require('mongoose');
const cors = require('cors');
const axios = require('axios');
const { createHash } = require('crypto');
const rateLimit = require('express-rate-limit');
const { parse: parseUrl, URL } = require('url');
const { parse: parseHtml } = require('node-html-parser');
const path = require('path');

// Set up the allowed domains (replace with your specific domains)
const allowedDomains = process.env.ALLOWED_DOMAINS.split(',');
// const allowedDomains = process.env.ALLOWED_DOMAINS.split(',');
const app = express();

const PORT = process.env.PORT || 3000;
const HOST = process.env.HOST || '127.0.0.1';

// Middleware to parse JSON requests
app.use(express.json());
// Middleware to enable CORS
// pass the cors options to the cors middleware to enable CORS for the allowed domains
// const corsOptions = {
// origin: allowedDomains,
// optionsSuccessStatus: 200, // Some legacy browsers (IE11, various SmartTVs) choke on 204
// }
app.use(cors());
const port = process.env.PORT || 3000;
const host = process.env.HOST || '0.0.0.0';

// Middleware to parse JSON requests
app.use(express.json());
// Middleware to enable CORS


app.use(
rateLimit({
windowMs: 1 * 60 * 1000, // 1 minute
max: 20, // limit each IP request per windowMs
max: 30, // limit each IP request per windowMs
})
);

app.get('/', (req, res) => {
res.send('Hello There!');
})
function addProtocolToUrl(url) {
if (!url.startsWith('http://') && !url.startsWith('https://')) {
url = 'https://' + url;
}
return url;
}

function parseUrlWithoutHashAndQuery(fullUrl) {
fullUrl = addProtocolToUrl(fullUrl);
const parsedUrl = new URL(fullUrl);

// Set the hash and search/query to empty strings
parsedUrl.hash = '';
parsedUrl.search = '';

// Reconstruct the URL without hash and query
const urlWithoutHashAndQuery = parsedUrl.toString();

return urlWithoutHashAndQuery;
}
// hashContent function to hash the content of a file
async function hashContent(content) {
const hash = createHash('sha256');
hash.update(content);
return hash.digest('hex');
}

// Recursive function to fetch and hash content, including linked resources
async function fetchAndHashContent(url, visitedUrls = new Set()) {
if (visitedUrls.has(url)) {
return ''; // Avoid fetching the same URL multiple times to prevent infinite loops
}

visitedUrls.add(url);
const response = await axios.get(url, { responseType: 'arraybuffer' });
const content = response.data.toString('utf-8');
// Parse HTML content to identify linked resources
const root = parseHtml(content);
const linkedResources = root.querySelectorAll('link[rel="stylesheet"], script[src]');
// Fetch and hash linked resources
const linkedResource = await Promise.all(linkedResources.map(async (resource) => {
const resourceUrl = parseUrl(resource.getAttribute('href') || resource.getAttribute('src'), true);
let absoluteResourceUrl = resourceUrl.href;
if (!resourceUrl.hostname) {
if (!resourceUrl.path.startsWith('/') && !url.endsWith('/'))
url += '/';
absoluteResourceUrl = `${url}${resourceUrl.path}`;
}
const resourceContent = await fetchAndHashContent(absoluteResourceUrl, visitedUrls);
return `${resourceUrl.path}_${resourceContent}`;
}));

// Combine the content and hashes of linked resources
return `${content}_${linkedResource.join('_')}`;
}

const hashCache = new Map();
// API endpoint to start the recursive download and hashing
app.post('/hash', async (req, res) => {
try {
let { urls } = req.body;
if (!urls) {
return res.status(400).json({ error: 'Missing <urls> in the request parameters' });
}
if (!Array.isArray(urls))
urls = [urls];
const promises = urls.map(async (url) => {
const urlWithoutHashAndQuery = parseUrlWithoutHashAndQuery(url);
let hash;
// regex to identify owner and repo name from https://owner.github.io/repo-name
const githubRepoRegex = /https?:\/\/([\w-]+)\.github\.io\/([\w-]+)/;
if (githubRepoRegex.test(urlWithoutHashAndQuery) && urlWithoutHashAndQuery.match(githubRepoRegex)[1] === 'ranchimall') {
if (!hashCache.has(urlWithoutHashAndQuery)) {
await fetchAndSaveAppHash(urlWithoutHashAndQuery)
}
hash = hashCache.get(urlWithoutHashAndQuery).hash;
} else {
const hashedContent = await fetchAndHashContent(urlWithoutHashAndQuery);
hash = await hashContent(Buffer.from(hashedContent, 'utf-8'));
}
return { url, hash };
});

const results = await Promise.all(promises);
res.json(results);
} catch (error) {
res.status(500).json({ error: error.message });
}
// connect to MongoDB
mongoose.connect(`mongodb://${HOST}/price-history`);
const db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', () => {
console.log('Connected to MongoDB');
});
async function fetchAndSaveAppHash(url, lastUpdated = Date.now()) {
const hashedContent = await fetchAndHashContent(url);
const hash = await hashContent(Buffer.from(hashedContent, 'utf-8'));
hashCache.set(url, { hash, lastUpdated });
}

app.post('/gitwh', async (req, res) => {
try {
// ignore if request is not from github
if (!req.headers['user-agent'].startsWith('GitHub-Hookshot/'))
return;
const { repository: { pushed_at, organization, name, has_pages } } = req.body;
if (!has_pages)
return;
const url = `https://${organization}.github.io/${name}`
await fetchAndSaveAppHash(url, pushed_at)
res.json({ message: 'success' });
} catch (err) {
res.status(500).json({ error: err.message });
}
app.get("/", (req, res) => {
res.sendFile(path.join(__dirname, './index.min.html'));
})
const hash = require('./routes/hash')
app.use("/hash", hash);
const priceHistory = require('./routes/price-history')
app.use("/price-history", priceHistory);

// Start the server
app.listen(port, host, () => {
console.log(`Server is running at http://${host}:${port}`);
app.listen(PORT, HOST, () => {
console.log(`Server is running at http://${HOST}:${PORT}`);
});
// TODO
//https://utility-api.ranchimall.net/hash/gitwh


// Export the Express API
module.exports = app;
1 change: 1 addition & 0 deletions index.min.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width,initial-scale=1"> <title>RanchiMall Utility APIs</title> <style>body{font-family:sans-serif}h1{font-size:2rem;margin-bottom:2rem}ol li{margin-bottom:5rem}a{color:inherit}table{border-collapse:collapse}table,td,th{border:1px solid #000;padding:.5rem}code{display:inline-block;background-color:#eee;padding:.3rem;border-radius:.2rem;font:monospace;font-size:inherit}@media (prefers-color-scheme:dark){body{background-color:#222;color:#eee}table,td,th{border-color:#eee}code{background-color:#333;color:#eee}}</style> </head> <body> <section style="padding:4vw"> <h1> Welcome to the RanchiMall Utility APIs! </h1> <h2> Endpoints: </h2> <ol> <li> <h3> <a href="/price-history">/price-history</a> </h3> <h4> Query parameters: </h4> <table> <thead> <tr> <th>Parameter</th> <th>Required</th> <th>Default</th> <th>format | values</th> </tr> </thead> <tbody> <tr> <td>from</td> <td>No</td> <td>None</td> <td>YYYY-MM-DD</td> </tr> <tr> <td>to</td> <td>No</td> <td>None</td> <td>YYYY-MM-DD</td> </tr> <tr> <td>on</td> <td>No</td> <td>None</td> <td>YYYY-MM-DD</td> </tr> <tr> <td>limit</td> <td>No</td> <td>100</td> <td>all | &lt;number&gt;</td> </tr> <tr> <td>asset</td> <td>No</td> <td>btc</td> <td>btc</td> </tr> <tr> <td>currency</td> <td>No</td> <td>All</td> <td>usd | inr</td> </tr> </tbody> </table> <h4> Example: </h4> <code> /price-history?from=2020-01-01&to=2020-01-31 </code> </li> <li> <h3> <a href="/hash">/hash</a> </h3> <table> <tbody> <tr> <td>Type</td> <td>POST</td> </tr> <tr> <td>Body</td> <td>JSON</td> </tr> <tr> <td>Body parameter</td> <td>urls [Array]</td> </tr> </tbody> </table> <h4> Example: </h4> <code> fetch('https://utility-api.ranchimall.net/hash',{ <br> &nbsp; method: 'POST',<br> &nbsp; headers: {<br> &nbsp; &nbsp; 'Content-Type': 'application/json'<br> &nbsp; },<br> &nbsp; body: JSON.stringify({ urls: [url] })<br> }).then(res => res.json()).then(console.log) <br> <br> Output: <br> [{<br> &nbsp; "url": url,<br> &nbsp; "hash": hash<br> }] </code> </li> </ol> </section>
2 changes: 1 addition & 1 deletion index.min.js
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 22 additions & 0 deletions models/price-history.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
const mongoose = require('mongoose');
// scheme to store price history
const Schema = new mongoose.Schema({
date: {
type: Date,
required: true,
unique: true
},
asset: {
type: String,
required: true
},
usd: {
type: Number,
required: true
},
inr: {
type: Number,
required: true
}
});
module.exports = mongoose.model('PriceHistory', Schema);
Loading

0 comments on commit a0c59e4

Please sign in to comment.