Skip to content

Commit

Permalink
fix: get resources returning empty response
Browse files Browse the repository at this point in the history 
chore: remove obsolete update/reindex code
  • Loading branch information
@verkurkie
verkurkie committed Feb 21, 2021
1 parent d0607a6 commit 9b1f6a9
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 194 deletions.
58 changes: 3 additions & 55 deletions src/run-es.js
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
const fs = require('fs');
const nconf = require('nconf');
const esUtils = require('./v2/es-utils');
const semver = require('semver');
Expand All @@ -18,13 +17,13 @@ class Elastic {
await this.es.waitPort(60000, this.env.ES_HOST, this.env.ES_PORT);
await this.es.healthy('60s', 'yellow');
await this.es.info();
// Get versions of API and KIBANA + check if update is needed
nconf.set('env:CURR_VERSION', semver.valid(await this.es.getTmplVer(nconf.get('env:ES_MAJOR'))) || null);
// Get API version + check if template needs update
nconf.set('env:CURR_VERSION', semver.valid(await this.es.getTemplateVersion(nconf.get('env:ES_MAJOR'))) || null);
if (nconf.get('env:CURR_VERSION')) {
if (await this.checkUpgrade() === true) {
await this.upgrade();
} else {
this.es.logElastic('info', `[UPDATE] API and KIBANA are up-to-date!`);
this.es.logElastic('info', `[UPDATE] API and ELK are up-to-date!`);
}
} else {
// New installation - put templates in place before anything else!
Expand All @@ -49,21 +48,6 @@ class Elastic {
Object.keys(this.templates[`elk${nconf.get('env:ES_MAJOR')}`]).forEach(async templ => {
await this.es.putTemplate(templ, this.templates[`elk${nconf.get('env:ES_MAJOR')}`][templ]);
});
// if (nconf.get('env:KB_MAJOR') < 6 && nconf.get('env:ES_MAJOR') > 5) {
// // This is a 5.x -> 6.x upgrade! Run reindex jobs!
// this.es.logElastic(
// 'info',
// `[UPGRADE] upgrading indices from [${nconf.get('env:KB_MAJOR')}] to [${nconf.get('env:ES_MAJOR')}]!`
// );
// const indexDay = new Date().toISOString().slice(0, 10).replace(/-/g, '.');
// await this.upgradeReindex(this.env.KB_INDEX);
// await this.upgradeReindex(`${this.env.INDEX_PERF}-${indexDay}`);
// await this.upgradeReindex(`${this.env.INDEX_RES}-${indexDay}`);
// await this.upgradeReindex(`${this.env.INDEX_ERR}-${indexDay}`);
// }
// // Always run the regular checks
// await this.es.kbImport(JSON.parse(this.importFile('./.kibana_items.json')));
// await this.es.defaultIndex(this.env.INDEX_PERF + '*', this.env.ES_VERSION);
this.es.logElastic('info', `[UPDATE] completed successfully!`);
} catch (err) {
if (err) {
Expand All @@ -72,42 +56,6 @@ class Elastic {
}
}

async upgradeReindex(index) {
try {
const indexSettings = await this.es.getSettings(index);
if (!indexSettings.hasOwnProperty(index)) return;
const indexVer = indexSettings[index].settings.index.version.created_string || '0';
// Make sure we don't attempt reindex of v6.x indices!
if (parseInt(indexVer.substr(0, 1), 10) > 5) return;

const srcConvert = (index === this.env.KB_INDEX) ? 'ctx._source=[ctx._type:ctx._source];' : '';
await this.es.reindex(
index, '-v6',
{
source: srcConvert + 'ctx._source.type=ctx._type;ctx._id=ctx._type + ":" + ctx._id;ctx._type="doc";',
lang: 'painless'
}
);
} catch (err) {
if (err) {
this.es.logElastic('warn', `[REINDEX] skip reindex [${index}] - ${err.displayName || 'unknown'}!`);
return;
}
}
}

importFile(file) {
let importFile = fs.readFileSync(file, 'utf8');
const replText = this.env.KB_RENAME;
if (replText && typeof replText === 'string') {
importFile = importFile.replace(/TIMINGS/g, replText.toUpperCase());
}
if (nconf.get('env:ES_MAJOR') > 5) {
importFile = importFile.replace(/_type:navtiming/g, '(_type:navtiming OR type:navtiming)');
}
return importFile;
}

async checkUpgrade() {
nconf.set('env:NEW_VERSION', semver.valid(nconf.get('env:APP_VERSION')) || nconf.get('env:CURR_VERSION'));
nconf.set('env:KB_VERSION', await this.es.getKBVer());
Expand Down
159 changes: 23 additions & 136 deletions src/v2/es-utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,113 +99,46 @@ class ESClass {
return;
}

async defaultIndex(name) {
let id = this.env.ES_VERSION;
let body = { defaultIndex: name };
let type = 'config';
if (parseInt(this.env.ES_MAJOR, 10) > 5) {
id = 'config:' + this.env.ES_VERSION;
type = (parseInt(this.env.ES_MAJOR, 10) > 6) ? '_doc' : 'doc';
body = {
type: 'config',
config: {
defaultIndex: name
}
};
}
await this.client.index({ index: (this.env.KB_INDEX || '.kibana'), type: type, id: id, body: body });
this.logElastic('info', `[DEFAULT] Default Index set to [${name}]`);
}

async delIndex(index) {
await this.client
.indices
.delete({ index: index });
this.logElastic('info', `[DELETE] successfully deleted index [${index}]!`);
}

async delDocById(index, type, id) {
if (await this.client
.exists({ index: index, type: type, id: id })
) {
return await this.client
.delete({ index: index, type: type, id: id });
}
return { _id: id, result: 'not_exists', statusCode: 200, reason: 'Does not exist' };
}

async reindex(src, suffix, script) {
const dst = src + suffix;
const opts = { body: { source: { index: src }, dest: { index: dst } } };
if (script && typeof script === 'object') {
opts.body.script = script;
}
await this.client.reindex(opts);
await this.delIndex(src);
await this.putAlias(dst, src);
this.logElastic('info', `[REINDEX] successfully reindexed [${src}] to [${dst}]!`);
}

async exists(index, type, id) {
return await this.client
.exists({
index: index,
type: (this.env.ES_MAJOR > 5) ? 'doc' : type,
id: id
});
const existsBody = {
index: index,
id: id
};
if (this.env.ES_MAJOR < 7) {
existsBody.type = (this.env.ES_MAJOR < 6) ? type : 'doc';
}
return await this.client.exists(existsBody);
}

async search(index, type, body) {
return await this.client
.search({
index: index,
type: (this.env.ES_MAJOR > 5) ? 'doc' : type,
body: body
});
}

async getSettings(index) {
const response = await this.client
.indices
.getSettings({ index: index, includeDefaults: true, human: true });
return response;
}

async putAlias(index, alias) {
await this.client
.indices
.putAlias({ index: index, name: alias });
this.logElastic('info', `[ALIAS] successfully added alias [${alias}] to [${index}]!`);
}

async getAlias(alias) {
return await this.client
.indices
.getAlias({ name: alias });
const searchBody = {
index: index,
body: body
};
if (this.env.ES_MAJOR < 7) {
searchBody.type = (this.env.ES_MAJOR < 6) ? type : 'doc';
}
return await this.client.search(searchBody);
}

async getKBVer() {
let index;
try {
index = await this.getAlias((this.env.KB_INDEX || '.kibana'));
index = await this.client.indices.getAlias({ name: (this.env.KB_INDEX || '.kibana') });
} catch (err) {
if (err) index = '.kibana';
}
index = Object.keys(index.body)[0];

if (await this.client
.indices
.exists({ index: index })
if (await this.client.indices.exists({ index: index })
) {
const settings = await this.client
.indices
.getSettings({ index: index, includeDefaults: true, human: true });
const settings = await this.client.indices.getSettings({ index: index, includeDefaults: true, human: true });
return (settings.body[index].settings.index.version.created_string || '0');
}
return '0';
}

async getTmplVer(esVersion) {
async getTemplateVersion(esVersion) {
try {
const currTemplate = await this.getTemplate(this.env.INDEX_PERF);
if (currTemplate.body) {
Expand All @@ -232,15 +165,14 @@ class ESClass {
}

async index(index, type, id, body) {
if (parseInt(this.env.ES_MAJOR, 10) > 5) {
type = (parseInt(this.env.ES_MAJOR, 10) > 6) ? '_doc' : 'doc';
}
const sendBody = {
index: index,
body: body,
...parseInt(this.env.ES_MAJOR, 10) < 7 && { type: type },
...id && { id: id }
};
if (this.env.ES_MAJOR < 7) {
sendBody.type = (this.env.ES_MAJOR < 6) ? type : 'doc';
}
try {
return await this.client.index(sendBody);
} catch (err) {
Expand Down Expand Up @@ -276,51 +208,6 @@ class ESClass {
return response.successful > 0;
}


async kbImport(importJson) {
for (const index of Object.keys(importJson)) {
let item = importJson[index];
const title = item._source.title;
if (item._source === 'delete_this') {
this.checkImportErrors(
await this.delDocById((this.env.KB_INDEX || '.kibana'), item._type, item._id),
'delete [' + item._type + ']',
item._id
);
} else {
if (item._type === 'index-pattern' && item._id === 'cicd-perf*') {
const apiHost = (this.env.HTTP_PORT !== 80) ? this.env.HOST + ':' + this.env.HTTP_PORT : this.env.HOST;
item._source.fieldFormatMap = item._source.fieldFormatMap.replace('__api__hostname', apiHost.toLowerCase());
}
item = this.upgradeConvert(item);
this.checkImportErrors(
await this.index((this.env.KB_INDEX || '.kibana'), item._type, item._id, item._source),
'import [' + item._type + ']', title
);
}
}
this.logElastic('info', `[IMPORT] Created/updated ${Object.keys(importJson).length} Kibana item(s)!`);
}

upgradeConvert(item) {
if (parseInt(nconf.get('env:ES_MAJOR'), 10) > 5) {
item._id = item._type + ':' + item._id;
item._type + ':' + item._id;
const _sourceTmp = {};
_sourceTmp[item._type] = item._source;
item._source = _sourceTmp;
}
return item;
}

async checkImportErrors(response, job, item) {
if (response.hasOwnProperty('error')) {
this.logElastic('error', `[IMPORT] ${job} - item: ${item} - ERROR! - reason: ${response.error.reason}`);
} else {
this.logElastic('debug', `[IMPORT] ${job} - item: ${item} - ${response.result.toUpperCase() || 'SUCCESS'}!`);
}
}

logElastic(level, msg) {
logger[level](`Elasticsearch - UTILS - ${msg}`);
}
Expand Down
3 changes: 0 additions & 3 deletions src/v2/perf-utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,9 +490,6 @@ class PUClass {
async getResources(req, cb) {
// Collect POST data
if (this.env.useES === false) {
// Send something back to the user
// const err = new Error('Resources not available - ELK could not be reached or is not configured');
// err.status = 400;
return cb(null, {
status: 200,
kibana_host: (this.env.ES_PROTOCOL || 'http') + '://' + this.env.KB_HOST,
Expand Down

0 comments on commit 9b1f6a9

Please sign in to comment.