Skip to content

Commit

Permalink
Merge pull request #23 from amplitude/batch_events
Browse files Browse the repository at this point in the history 
Add option to batch events
  • Loading branch information
@djih
djih committed Sep 2, 2015
2 parents 65eef18 + 603ea1d commit fd74808
Show file tree
Hide file tree
Showing 6 changed files with 164 additions and 22 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
## Unreleased

* Add option to batch events into a single request.

## 2.2.1 (Aug 13, 2015)

* Fix bug where multi-byte unicode characters were hashed improperly.
Expand Down
7 changes: 6 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,9 @@ You can configure Amplitude by passing an object as the third argument to the `i
// optional configuration options
saveEvents: true,
includeUtm: true,
includeReferrer: true
includeReferrer: true,
batchEvents: true,
eventUploadThreshold: 50
})

| option | description | default |
Expand All @@ -97,6 +99,9 @@ You can configure Amplitude by passing an object as the third argument to the `i
| uploadBatchSize | Maximum number of events to send to the server per request. | 100 |
| includeUtm | If `true`, finds utm parameters in the query string or the __utmz cookie, parses, and includes them as user propeties on all events uploaded. | `false` |
| includeReferrer | If `true`, includes `referrer` and `referring_domain` as user propeties on all events uploaded. | `false` |
| batchEvents | If `true`, events are batched together and uploaded only when the number of unsent events is greater than or equal to `eventUploadThreshold` or after `eventUploadPeriodMillis` milliseconds have passed since the first unsent event was logged. | `false` |
| eventUploadThreshold | Minimum number of events to batch together per request if `batchEvents` is `true`. | 30 |
| eventUploadPeriodMillis | Amount of time in milliseconds that the SDK waits before uploading events if `batchEvents` is `true`. | 30*1000 |


# Advanced #
Expand Down
41 changes: 32 additions & 9 deletions amplitude.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,10 @@ var DEFAULT_OPTIONS = {
saveEvents: true,
sessionTimeout: 30 * 60 * 1000,
unsentKey: 'amplitude_unsent',
uploadBatchSize: 100
uploadBatchSize: 100,
batchEvents: false,
eventUploadThreshold: 30,
eventUploadPeriodMillis: 30 * 1000 // 30s
};
var LocalStorageKeys = {
LAST_EVENT_ID: 'amplitude_lastEventId',
Expand Down Expand Up @@ -188,11 +191,16 @@ Amplitude.prototype.init = function(apiKey, opt_userId, opt_config) {
if (opt_config.includeReferrer !== undefined) {
this.options.includeReferrer = !!opt_config.includeReferrer;
}
if (opt_config.batchEvents !== undefined) {
this.options.batchEvents = !!opt_config.batchEvents;
}
this.options.platform = opt_config.platform || this.options.platform;
this.options.language = opt_config.language || this.options.language;
this.options.sessionTimeout = opt_config.sessionTimeout || this.options.sessionTimeout;
this.options.uploadBatchSize = opt_config.uploadBatchSize || this.options.uploadBatchSize;
this.options.eventUploadThreshold = opt_config.eventUploadThreshold || this.options.eventUploadThreshold;
this.options.savedMaxCount = opt_config.savedMaxCount || this.options.savedMaxCount;
this.options.eventUploadPeriodMillis = opt_config.eventUploadPeriodMillis || this.options.eventUploadPeriodMillis;
}

Cookie.options({
Expand Down Expand Up @@ -222,9 +230,8 @@ Amplitude.prototype.init = function(apiKey, opt_userId, opt_config) {
}
}
}
if (this._unsentEvents.length > 0) {
this.sendEvents();
}

this._sendEventsIfReady();

if (this.options.includeUtm) {
this._initUtmData();
Expand Down Expand Up @@ -255,6 +262,23 @@ Amplitude.prototype.nextEventId = function() {
return this._eventId;
};

Amplitude.prototype._sendEventsIfReady = function() {
if (this._unsentEvents.length === 0) {
return;
}

if (!this.options.batchEvents) {
this.sendEvents();
return;
}

if (this._unsentEvents.length >= this.options.eventUploadThreshold) {
this.sendEvents();
} else {
setTimeout(this.sendEvents.bind(this), this.options.eventUploadPeriodMillis);
}
};

var _loadCookieData = function(scope) {
var cookieData = Cookie.get(scope.options.cookieName);
if (cookieData) {
Expand Down Expand Up @@ -477,7 +501,7 @@ Amplitude.prototype._logEvent = function(eventType, eventProperties, apiProperti
this.saveEvents();
}

this.sendEvents();
this._sendEventsIfReady();

return eventId;
} catch (e) {
Expand Down Expand Up @@ -524,7 +548,7 @@ Amplitude.prototype.removeEvents = function (maxEventId) {
};

Amplitude.prototype.sendEvents = function() {
if (!this._sending && !this.options.optOut) {
if (!this._sending && !this.options.optOut && this._unsentEvents.length > 0) {
this._sending = true;
var url = ('https:' === window.location.protocol ? 'https' : 'http') + '://' +
this.options.apiEndpoint + '/';
Expand Down Expand Up @@ -557,9 +581,8 @@ Amplitude.prototype.sendEvents = function() {
}

// Send more events if any queued during previous send.
if (scope._unsentEvents.length > 0) {
scope.sendEvents();
}
scope._sendEventsIfReady();

} else if (status === 413) {
//log('request too large');
// Can't even get this one massive event through. Drop it.
Expand Down
4 changes: 2 additions & 2 deletions amplitude.min.js
View file

Large diffs are not rendered by default.

41 changes: 32 additions & 9 deletions src/amplitude.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,10 @@ var DEFAULT_OPTIONS = {
saveEvents: true,
sessionTimeout: 30 * 60 * 1000,
unsentKey: 'amplitude_unsent',
uploadBatchSize: 100
uploadBatchSize: 100,
batchEvents: false,
eventUploadThreshold: 30,
eventUploadPeriodMillis: 30 * 1000 // 30s
};
var LocalStorageKeys = {
LAST_EVENT_ID: 'amplitude_lastEventId',
Expand Down Expand Up @@ -76,11 +79,16 @@ Amplitude.prototype.init = function(apiKey, opt_userId, opt_config) {
if (opt_config.includeReferrer !== undefined) {
this.options.includeReferrer = !!opt_config.includeReferrer;
}
if (opt_config.batchEvents !== undefined) {
this.options.batchEvents = !!opt_config.batchEvents;
}
this.options.platform = opt_config.platform || this.options.platform;
this.options.language = opt_config.language || this.options.language;
this.options.sessionTimeout = opt_config.sessionTimeout || this.options.sessionTimeout;
this.options.uploadBatchSize = opt_config.uploadBatchSize || this.options.uploadBatchSize;
this.options.eventUploadThreshold = opt_config.eventUploadThreshold || this.options.eventUploadThreshold;
this.options.savedMaxCount = opt_config.savedMaxCount || this.options.savedMaxCount;
this.options.eventUploadPeriodMillis = opt_config.eventUploadPeriodMillis || this.options.eventUploadPeriodMillis;
}

Cookie.options({
Expand Down Expand Up @@ -110,9 +118,8 @@ Amplitude.prototype.init = function(apiKey, opt_userId, opt_config) {
}
}
}
if (this._unsentEvents.length > 0) {
this.sendEvents();
}

this._sendEventsIfReady();

if (this.options.includeUtm) {
this._initUtmData();
Expand Down Expand Up @@ -143,6 +150,23 @@ Amplitude.prototype.nextEventId = function() {
return this._eventId;
};

Amplitude.prototype._sendEventsIfReady = function() {
if (this._unsentEvents.length === 0) {
return;
}

if (!this.options.batchEvents) {
this.sendEvents();
return;
}

if (this._unsentEvents.length >= this.options.eventUploadThreshold) {
this.sendEvents();
} else {
setTimeout(this.sendEvents.bind(this), this.options.eventUploadPeriodMillis);
}
};

var _loadCookieData = function(scope) {
var cookieData = Cookie.get(scope.options.cookieName);
if (cookieData) {
Expand Down Expand Up @@ -365,7 +389,7 @@ Amplitude.prototype._logEvent = function(eventType, eventProperties, apiProperti
this.saveEvents();
}

this.sendEvents();
this._sendEventsIfReady();

return eventId;
} catch (e) {
Expand Down Expand Up @@ -412,7 +436,7 @@ Amplitude.prototype.removeEvents = function (maxEventId) {
};

Amplitude.prototype.sendEvents = function() {
if (!this._sending && !this.options.optOut) {
if (!this._sending && !this.options.optOut && this._unsentEvents.length > 0) {
this._sending = true;
var url = ('https:' === window.location.protocol ? 'https' : 'http') + '://' +
this.options.apiEndpoint + '/';
Expand Down Expand Up @@ -445,9 +469,8 @@ Amplitude.prototype.sendEvents = function() {
}

// Send more events if any queued during previous send.
if (scope._unsentEvents.length > 0) {
scope.sendEvents();
}
scope._sendEventsIfReady();

} else if (status === 413) {
//log('request too large');
// Can't even get this one massive event through. Drop it.
Expand Down
91 changes: 90 additions & 1 deletion test/amplitude.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,12 +130,16 @@ describe('Amplitude', function() {

describe('logEvent', function() {

var clock;

beforeEach(function() {
clock = sinon.useFakeTimers();
amplitude.init(apiKey);
});

afterEach(function() {
reset();
clock.restore();
});

it('should send request', function() {
Expand Down Expand Up @@ -260,7 +264,7 @@ describe('Amplitude', function() {
assert.deepEqual(amplitude2._unsentEvents, []);
});

it('should batch events sent', function() {
it('should limit events sent', function() {
amplitude.init(apiKey, null, {uploadBatchSize: 10});

amplitude._sending = true;
Expand All @@ -287,6 +291,91 @@ describe('Amplitude', function() {
assert.deepEqual(events[5].event_properties, {index: 100});
});

it('should batch events sent', function() {
var eventUploadPeriodMillis = 10*1000;
amplitude.init(apiKey, null, {
batchEvents: true,
eventUploadThreshold: 10,
eventUploadPeriodMillis: eventUploadPeriodMillis
});

for (var i = 0; i < 15; i++) {
amplitude.logEvent('Event', {index: i});
}

assert.lengthOf(server.requests, 1);
var events = JSON.parse(querystring.parse(server.requests[0].requestBody).e);
assert.lengthOf(events, 10);
assert.deepEqual(events[0].event_properties, {index: 0});
assert.deepEqual(events[9].event_properties, {index: 9});

server.respondWith('success');
server.respond();

assert.lengthOf(server.requests, 1);
var unsentEvents = amplitude._unsentEvents;
assert.lengthOf(unsentEvents, 5);
assert.deepEqual(unsentEvents[4].event_properties, {index: 14});

// remaining 5 events should be sent by the delayed sendEvent call
clock.tick(eventUploadPeriodMillis);
assert.lengthOf(server.requests, 2);
server.respondWith('success');
server.respond();
assert.lengthOf(amplitude._unsentEvents, 0);
var events = JSON.parse(querystring.parse(server.requests[1].requestBody).e);
assert.lengthOf(events, 5);
assert.deepEqual(events[4].event_properties, {index: 14});
});

it('should send events after a delay', function() {
var eventUploadPeriodMillis = 10*1000;
amplitude.init(apiKey, null, {
batchEvents: true,
eventUploadThreshold: 2,
eventUploadPeriodMillis: eventUploadPeriodMillis
});
amplitude.logEvent('Event');

// saveEvent should not have been called yet
assert.lengthOf(amplitude._unsentEvents, 1);
assert.lengthOf(server.requests, 0);

// saveEvent should be called after delay
clock.tick(eventUploadPeriodMillis);
assert.lengthOf(server.requests, 1);
server.respondWith('success');
server.respond();
var events = JSON.parse(querystring.parse(server.requests[0].requestBody).e);
assert.lengthOf(events, 1);
assert.deepEqual(events[0].event_type, 'Event');
});

it('should not send events after a delay if no events to send', function() {
var eventUploadPeriodMillis = 10*1000;
amplitude.init(apiKey, null, {
batchEvents: true,
eventUploadThreshold: 2,
eventUploadPeriodMillis: eventUploadPeriodMillis
});
amplitude.logEvent('Event1');
amplitude.logEvent('Event2');

// saveEvent triggered by 2 event batch threshold
assert.lengthOf(amplitude._unsentEvents, 2);
assert.lengthOf(server.requests, 1);
server.respondWith('success');
server.respond();
var events = JSON.parse(querystring.parse(server.requests[0].requestBody).e);
assert.lengthOf(events, 2);
assert.deepEqual(events[1].event_type, 'Event2');

// saveEvent should be called after delay, but no request made
assert.lengthOf(amplitude._unsentEvents, 0);
clock.tick(eventUploadPeriodMillis);
assert.lengthOf(server.requests, 1);
})

it('should back off on 413 status', function() {
amplitude.init(apiKey, null, {uploadBatchSize: 10});

Expand Down

0 comments on commit fd74808

Please sign in to comment.