-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.js
81 lines (75 loc) · 2.85 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
'use strict';
const ElasticsearchStream = require('./lib/elasticsearch-stream');
const redis = require('redis');
const path = require('path');
const fs = require('fs');
const fse = require('fs-extra');
const uuid = require('uuid');
const esConf = require('co-config/es.js');
const coSelect = {
init () {
this.redisHost = process.env.REDIS_HOST || 'localhost';
this.redisPort = process.env.REDIS_PORT || 6379;
this.pubClient = redis.createClient({
'host': this.redisHost,
'port': this.redisPort
});
this.CONDITOR_SESSION = process.env.ISTEX_SESSION || 'TEST_1970-01-01-00-00-00';
this.MODULEROOT = process.env.MODULEROOT || __dirname;
this.redisKey = this.CONDITOR_SESSION + ':co-select';
const elasticUrl = esConf.host;
this.elasticsearchStream = new ElasticsearchStream({ elasticUrl });
return this;
},
doTheJob (docObject, next) {
this.elasticsearchStream.elasticIndex = docObject.elasticIndex || esConf.index;
this.elasticsearchStream.elasticReq = docObject.elasticReq || '*';
let count = 0;
let myDocObjectFilePath = this.getWhereIWriteMyFiles(uuid.v4() + '.json', 'out');
let directoryOfMyFile = myDocObjectFilePath.substr(0, myDocObjectFilePath.lastIndexOf('/'));
fse.ensureDirSync(directoryOfMyFile);
let writableStream = fs.createWriteStream(myDocObjectFilePath);
this.elasticsearchStream.on('data', (data) => {
count++;
writableStream.write(JSON.stringify(data) + '\n');
this.pubClient.hincrby('Module:' + this.redisKey, 'outDocObject', 1);
if (count === 100) {
count = 0;
writableStream.end();
this.pubClient.hincrby('Module:' + this.redisKey, 'out', 1);
this.pubClient.publish(this.redisKey + ':out', path.basename(myDocObjectFilePath));
myDocObjectFilePath = this.getWhereIWriteMyFiles(uuid.v4() + '.json', 'out');
directoryOfMyFile = myDocObjectFilePath.substr(0, myDocObjectFilePath.lastIndexOf('/'));
fse.ensureDirSync(directoryOfMyFile);
writableStream = fs.createWriteStream(myDocObjectFilePath);
}
});
this.elasticsearchStream.on('end', () => {
if (count !== 0) {
this.pubClient.hincrby('Module:' + this.redisKey, 'out', 1);
this.pubClient.publish(this.redisKey + ':out', path.basename(myDocObjectFilePath));
}
const error = new Error('The first docObject goes into error so as not to pollute the processing chain.');
next(error, docObject);
});
this.elasticsearchStream.on('error', (error) => {
next(error, docObject);
});
},
finalJob (docObjects, done) {
this.pubClient.quit();
done();
},
getWhereIWriteMyFiles (file, dirOutOrErr) {
return path.join(
this.MODULEROOT,
dirOutOrErr,
this.CONDITOR_SESSION,
file[0],
file[1],
file[2],
file
);
}
};
module.exports = coSelect.init();