Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add allowedExtensions to bypass extensionsToIgnore list #185

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add allowedExtensions to bypass extensionsToIgnore list #185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
40f493b
Add allowedExtensions option
gsdriver Aug 23, 2019
33f69cc
Update readme
Aug 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,16 @@ app.use(require('prerender-node').blacklisted('^/search'));
app.use(require('prerender-node').blacklisted(['/search', '/users/.*/profile']));
```

### allowedExtensions

Allows a given file or files that would normally be ignored by prerender based on their extension to pass thru and be prerendered. Compares using regex, so be specific when possible. If allowedExtensions is applied, these files will be prerendered.
```js
app.use(require('prerender-node').allowedExtensions('sitemap.xml'));
```
```js
app.use(require('prerender-node').allowedExtensions(['favicon.ico', '/sitemap/*.xml']));
```

### beforeRender

This method is intended to be used for caching, but could be used to save analytics or anything else you need to do for each crawler request. If you return a string from beforeRender, the middleware will serve that to the crawler (with status `200`) instead of making a request to the prerender service. If you return an object the middleware will look for a `status` and `body` property (defaulting to `200` and `""` respectively) and serve those instead.
Expand Down
10 changes: 9 additions & 1 deletion index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,11 @@ prerender.blacklisted = function(blacklist) {
};


prerender.allowedExtensions = function(extensionsToAllow) {
prerender.extensionsToAllow = typeof extensionsToAllow === 'string' ? [extensionsToAllow] : extensionsToAllow;
return this;
}

prerender.shouldShowPrerenderedPage = function(req) {
var userAgent = req.headers['user-agent']
, bufferAgent = req.headers['x-bufferbot']
Expand All @@ -149,7 +154,10 @@ prerender.shouldShowPrerenderedPage = function(req) {
if(bufferAgent) isRequestingPrerenderedPage = true;

//if it is a bot and is requesting a resource...dont prerender
if(prerender.extensionsToIgnore.some(function(extension){return req.url.toLowerCase().indexOf(extension) !== -1;})) return false;
if(prerender.extensionsToIgnore.some(function(extension){return req.url.toLowerCase().indexOf(extension) !== -1;})){
//if there is a list of allowed files or extensions let them through
if(!Array.isArray(this.extensionsToAllow) || !this.extensionsToAllow.some(function(allowed){return (new RegExp(allowed)).test(req.url) === true;})) return false;
}

//if it is a bot and not requesting a resource and is not whitelisted...dont prerender
if(Array.isArray(this.whitelist) && this.whitelist.every(function(whitelisted){return (new RegExp(whitelisted)).test(req.url) === false;})) return false;
Expand Down
36 changes: 36 additions & 0 deletions test/index-spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,42 @@ describe ('Prerender', function(){

});

it('should return a prerendered response if the url is part of the regex specific allowedList', function(){
var req = { method: 'GET', url: '/favicon.ico', headers: { 'user-agent': bot }, connection: { encrypted: false } };

nock('https://service.prerender.io', {
reqheaders: {
'x-prerender-token': 'MY_TOKEN',
'Accept-Encoding': 'gzip'
}
})
.get('/http://google.com/favicon.ico')
.reply(200, '<html></html>');

res.end = sandbox.spy(function(){
assert.equal(next.callCount, 0);
assert.equal(res.writeHead.callCount, 1);
assert.equal(res.writeHead.getCall(0).args[0], 200);
assert.equal(res.end.callCount, 1);
assert.equal(res.end.getCall(0).args[0], '<html></html>');
done();
});

prerender.allowedExtensions('favicon.ico')(req, res, next);
delete prerender.extensionsToAllow;
});

it('should call next() if the url is not part of the regex specific allowedList', function(){
var req = { method: 'GET', url: '/favicon.ico', headers: { 'user-agent': bot }, connection: { encrypted: false } };

prerender.allowedExtensions(['sitemap/*.xml', '^/*.jpg'])(req, res, next);

delete prerender.extensionsToAllow;
assert.equal(next.callCount, 1);
assert.equal(res.writeHead.callCount, 0);
assert.equal(res.end.callCount, 0);
});

it('should return a prerendered response if a string is returned from beforeRender', function(){
var req = { method: 'GET', url: '/', headers: { 'user-agent': bot }, connection: { encrypted: false } };

Expand Down