diff --git a/src/run-es.js b/src/run-es.js index 0187a05..35ebfaf 100644 --- a/src/run-es.js +++ b/src/run-es.js @@ -1,4 +1,3 @@ -const fs = require('fs'); const nconf = require('nconf'); const esUtils = require('./v2/es-utils'); const semver = require('semver'); @@ -18,13 +17,13 @@ class Elastic { await this.es.waitPort(60000, this.env.ES_HOST, this.env.ES_PORT); await this.es.healthy('60s', 'yellow'); await this.es.info(); - // Get versions of API and KIBANA + check if update is needed - nconf.set('env:CURR_VERSION', semver.valid(await this.es.getTmplVer(nconf.get('env:ES_MAJOR'))) || null); + // Get API version + check if template needs update + nconf.set('env:CURR_VERSION', semver.valid(await this.es.getTemplateVersion(nconf.get('env:ES_MAJOR'))) || null); if (nconf.get('env:CURR_VERSION')) { if (await this.checkUpgrade() === true) { await this.upgrade(); } else { - this.es.logElastic('info', `[UPDATE] API and KIBANA are up-to-date!`); + this.es.logElastic('info', `[UPDATE] API and ELK are up-to-date!`); } } else { // New installation - put templates in place before anything else! @@ -49,21 +48,6 @@ class Elastic { Object.keys(this.templates[`elk${nconf.get('env:ES_MAJOR')}`]).forEach(async templ => { await this.es.putTemplate(templ, this.templates[`elk${nconf.get('env:ES_MAJOR')}`][templ]); }); - // if (nconf.get('env:KB_MAJOR') < 6 && nconf.get('env:ES_MAJOR') > 5) { - // // This is a 5.x -> 6.x upgrade! Run reindex jobs! - // this.es.logElastic( - // 'info', - // `[UPGRADE] upgrading indices from [${nconf.get('env:KB_MAJOR')}] to [${nconf.get('env:ES_MAJOR')}]!` - // ); - // const indexDay = new Date().toISOString().slice(0, 10).replace(/-/g, '.'); - // await this.upgradeReindex(this.env.KB_INDEX); - // await this.upgradeReindex(`${this.env.INDEX_PERF}-${indexDay}`); - // await this.upgradeReindex(`${this.env.INDEX_RES}-${indexDay}`); - // await this.upgradeReindex(`${this.env.INDEX_ERR}-${indexDay}`); - // } - // // Always run the regular checks - // await this.es.kbImport(JSON.parse(this.importFile('./.kibana_items.json'))); - // await this.es.defaultIndex(this.env.INDEX_PERF + '*', this.env.ES_VERSION); this.es.logElastic('info', `[UPDATE] completed successfully!`); } catch (err) { if (err) { @@ -72,42 +56,6 @@ class Elastic { } } - async upgradeReindex(index) { - try { - const indexSettings = await this.es.getSettings(index); - if (!indexSettings.hasOwnProperty(index)) return; - const indexVer = indexSettings[index].settings.index.version.created_string || '0'; - // Make sure we don't attempt reindex of v6.x indices! - if (parseInt(indexVer.substr(0, 1), 10) > 5) return; - - const srcConvert = (index === this.env.KB_INDEX) ? 'ctx._source=[ctx._type:ctx._source];' : ''; - await this.es.reindex( - index, '-v6', - { - source: srcConvert + 'ctx._source.type=ctx._type;ctx._id=ctx._type + ":" + ctx._id;ctx._type="doc";', - lang: 'painless' - } - ); - } catch (err) { - if (err) { - this.es.logElastic('warn', `[REINDEX] skip reindex [${index}] - ${err.displayName || 'unknown'}!`); - return; - } - } - } - - importFile(file) { - let importFile = fs.readFileSync(file, 'utf8'); - const replText = this.env.KB_RENAME; - if (replText && typeof replText === 'string') { - importFile = importFile.replace(/TIMINGS/g, replText.toUpperCase()); - } - if (nconf.get('env:ES_MAJOR') > 5) { - importFile = importFile.replace(/_type:navtiming/g, '(_type:navtiming OR type:navtiming)'); - } - return importFile; - } - async checkUpgrade() { nconf.set('env:NEW_VERSION', semver.valid(nconf.get('env:APP_VERSION')) || nconf.get('env:CURR_VERSION')); nconf.set('env:KB_VERSION', await this.es.getKBVer()); diff --git a/src/v2/es-utils.js b/src/v2/es-utils.js index e95ef52..bfba01a 100644 --- a/src/v2/es-utils.js +++ b/src/v2/es-utils.js @@ -99,113 +99,46 @@ class ESClass { return; } - async defaultIndex(name) { - let id = this.env.ES_VERSION; - let body = { defaultIndex: name }; - let type = 'config'; - if (parseInt(this.env.ES_MAJOR, 10) > 5) { - id = 'config:' + this.env.ES_VERSION; - type = (parseInt(this.env.ES_MAJOR, 10) > 6) ? '_doc' : 'doc'; - body = { - type: 'config', - config: { - defaultIndex: name - } - }; - } - await this.client.index({ index: (this.env.KB_INDEX || '.kibana'), type: type, id: id, body: body }); - this.logElastic('info', `[DEFAULT] Default Index set to [${name}]`); - } - - async delIndex(index) { - await this.client - .indices - .delete({ index: index }); - this.logElastic('info', `[DELETE] successfully deleted index [${index}]!`); - } - - async delDocById(index, type, id) { - if (await this.client - .exists({ index: index, type: type, id: id }) - ) { - return await this.client - .delete({ index: index, type: type, id: id }); - } - return { _id: id, result: 'not_exists', statusCode: 200, reason: 'Does not exist' }; - } - - async reindex(src, suffix, script) { - const dst = src + suffix; - const opts = { body: { source: { index: src }, dest: { index: dst } } }; - if (script && typeof script === 'object') { - opts.body.script = script; - } - await this.client.reindex(opts); - await this.delIndex(src); - await this.putAlias(dst, src); - this.logElastic('info', `[REINDEX] successfully reindexed [${src}] to [${dst}]!`); - } - async exists(index, type, id) { - return await this.client - .exists({ - index: index, - type: (this.env.ES_MAJOR > 5) ? 'doc' : type, - id: id - }); + const existsBody = { + index: index, + id: id + }; + if (this.env.ES_MAJOR < 7) { + existsBody.type = (this.env.ES_MAJOR < 6) ? type : 'doc'; + } + return await this.client.exists(existsBody); } async search(index, type, body) { - return await this.client - .search({ - index: index, - type: (this.env.ES_MAJOR > 5) ? 'doc' : type, - body: body - }); - } - - async getSettings(index) { - const response = await this.client - .indices - .getSettings({ index: index, includeDefaults: true, human: true }); - return response; - } - - async putAlias(index, alias) { - await this.client - .indices - .putAlias({ index: index, name: alias }); - this.logElastic('info', `[ALIAS] successfully added alias [${alias}] to [${index}]!`); - } - - async getAlias(alias) { - return await this.client - .indices - .getAlias({ name: alias }); + const searchBody = { + index: index, + body: body + }; + if (this.env.ES_MAJOR < 7) { + searchBody.type = (this.env.ES_MAJOR < 6) ? type : 'doc'; + } + return await this.client.search(searchBody); } async getKBVer() { let index; try { - index = await this.getAlias((this.env.KB_INDEX || '.kibana')); + index = await this.client.indices.getAlias({ name: (this.env.KB_INDEX || '.kibana') }); } catch (err) { if (err) index = '.kibana'; } index = Object.keys(index.body)[0]; - if (await this.client - .indices - .exists({ index: index }) + if (await this.client.indices.exists({ index: index }) ) { - const settings = await this.client - .indices - .getSettings({ index: index, includeDefaults: true, human: true }); + const settings = await this.client.indices.getSettings({ index: index, includeDefaults: true, human: true }); return (settings.body[index].settings.index.version.created_string || '0'); } return '0'; } - async getTmplVer(esVersion) { + async getTemplateVersion(esVersion) { try { const currTemplate = await this.getTemplate(this.env.INDEX_PERF); if (currTemplate.body) { @@ -232,15 +165,14 @@ class ESClass { } async index(index, type, id, body) { - if (parseInt(this.env.ES_MAJOR, 10) > 5) { - type = (parseInt(this.env.ES_MAJOR, 10) > 6) ? '_doc' : 'doc'; - } const sendBody = { index: index, body: body, - ...parseInt(this.env.ES_MAJOR, 10) < 7 && { type: type }, ...id && { id: id } }; + if (this.env.ES_MAJOR < 7) { + sendBody.type = (this.env.ES_MAJOR < 6) ? type : 'doc'; + } try { return await this.client.index(sendBody); } catch (err) { @@ -276,51 +208,6 @@ class ESClass { return response.successful > 0; } - - async kbImport(importJson) { - for (const index of Object.keys(importJson)) { - let item = importJson[index]; - const title = item._source.title; - if (item._source === 'delete_this') { - this.checkImportErrors( - await this.delDocById((this.env.KB_INDEX || '.kibana'), item._type, item._id), - 'delete [' + item._type + ']', - item._id - ); - } else { - if (item._type === 'index-pattern' && item._id === 'cicd-perf*') { - const apiHost = (this.env.HTTP_PORT !== 80) ? this.env.HOST + ':' + this.env.HTTP_PORT : this.env.HOST; - item._source.fieldFormatMap = item._source.fieldFormatMap.replace('__api__hostname', apiHost.toLowerCase()); - } - item = this.upgradeConvert(item); - this.checkImportErrors( - await this.index((this.env.KB_INDEX || '.kibana'), item._type, item._id, item._source), - 'import [' + item._type + ']', title - ); - } - } - this.logElastic('info', `[IMPORT] Created/updated ${Object.keys(importJson).length} Kibana item(s)!`); - } - - upgradeConvert(item) { - if (parseInt(nconf.get('env:ES_MAJOR'), 10) > 5) { - item._id = item._type + ':' + item._id; - item._type + ':' + item._id; - const _sourceTmp = {}; - _sourceTmp[item._type] = item._source; - item._source = _sourceTmp; - } - return item; - } - - async checkImportErrors(response, job, item) { - if (response.hasOwnProperty('error')) { - this.logElastic('error', `[IMPORT] ${job} - item: ${item} - ERROR! - reason: ${response.error.reason}`); - } else { - this.logElastic('debug', `[IMPORT] ${job} - item: ${item} - ${response.result.toUpperCase() || 'SUCCESS'}!`); - } - } - logElastic(level, msg) { logger[level](`Elasticsearch - UTILS - ${msg}`); } diff --git a/src/v2/perf-utils.js b/src/v2/perf-utils.js index 2ee4de8..3b57f17 100644 --- a/src/v2/perf-utils.js +++ b/src/v2/perf-utils.js @@ -490,9 +490,6 @@ class PUClass { async getResources(req, cb) { // Collect POST data if (this.env.useES === false) { - // Send something back to the user - // const err = new Error('Resources not available - ELK could not be reached or is not configured'); - // err.status = 400; return cb(null, { status: 200, kibana_host: (this.env.ES_PROTOCOL || 'http') + '://' + this.env.KB_HOST,