Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🗑️ remove next prefix #165

Merged
merged 1 commit into from
Jul 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions admin/checkAliasWithRelease.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,7 @@ const rAllAliases = await client.cat.aliases({
assert(rAllAliases.statusCode === 200);

const allAliases = rAllAliases.body;
const hasNext = allAliases.some(x => x.alias.includes('next_'));
const clinicalAliases = allAliases
.filter(cbKeepClinicalIndicesOnly)
.filter(x => (hasNext ? x.alias.includes('next_') : x));
const clinicalAliases = allAliases.filter(cbKeepClinicalIndicesOnly);

const aliasToReleases = clinicalAliases.reduce((xs, x) => {
const r = 're' + x.index.split('_re_')[1];
Expand All @@ -34,9 +31,9 @@ const { all, ...entities } = aliasToReleases;
console.log(`\n`);

//not the best test but it should suffice
const ok = hasNext ? all.length === 1 : all.length <= 2
const ok = all.length <= 2;
if (!ok) {
console.warn('Check if the clinical aliases are ok - There might be a problem')
console.warn('Check if the clinical aliases are ok - There might be a problem');
}
console.log(`Release(s) found: ${all}`);
console.log(entities);
12 changes: 6 additions & 6 deletions admin/confKfNext.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,31 @@
"indices": [
{
"graphqlField": "biospecimen",
"esIndex": "next_biospecimen_centric"
"esIndex": "biospecimen_centric"
},
{
"graphqlField": "file",
"esIndex": "next_file_centric"
"esIndex": "file_centric"
},
{
"graphqlField": "genes",
"esIndex": "next_gene_centric"
"esIndex": "gene_centric"
},
{
"graphqlField": "members",
"esIndex": "members-public"
},
{
"graphqlField": "participant",
"esIndex": "next_participant_centric"
"esIndex": "participant_centric"
},
{
"graphqlField": "study",
"esIndex": "next_study_centric"
"esIndex": "study_centric"
},
{
"graphqlField": "variants",
"esIndex": "next_variant_centric"
"esIndex": "variant_centric"
}
],
"extendedMappingMutations": [
Expand Down
28 changes: 3 additions & 25 deletions admin/postReleaseIndicesToAliasesHelper.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
*
* npm run post-re-alias-helper -- release:re_test_023 action:remove
*
* npm run post-re-alias-helper -- release:re_test_023 action:remove test
* npm run post-re-alias-helper -- release:re_test_023 action:remove
*
* */
import assert from 'node:assert/strict';
Expand All @@ -31,9 +31,6 @@ assert(
}"`,
);

const testArgument = args.find(a => a.startsWith('test')) ?? '';
const isTest = !!testArgument;

const userReadline = readline.createInterface({
input: process.stdin,
output: process.stdout,
Expand Down Expand Up @@ -86,37 +83,18 @@ await displayIndicesQuestion();

const INDICES_PREFIXES = ['biospecimen_centric', 'participant_centric', 'study_centric', 'file_centric'];

const allAliases = await client.cat.aliases({
h: 'alias',
format: 'json',
});

//At this point, KF next uses the prefix next.
const isNextFormat = allAliases.body
.map(x => x.alias)
.some(
x =>
x.startsWith('next_study_centric') ||
x.startsWith('next_participant_centric') ||
x.startsWith('next_biospecimen_centric') ||
x.startsWith('next_variant_centric') ||
x.startsWith('next_gene_centric') ||
x.startsWith('next_file_centric'),
);

const actions = releaseIndices.reduce((xs, x) => {
const prefix = INDICES_PREFIXES.find(p => x.startsWith(p));
if (!prefix) {
// Must never happen
return xs;
}
const alias = isNextFormat ? `next_${prefix}` : prefix;
return [
...xs,
{
[aliasAction]: {
index: x,
alias: isTest ? `${alias}_test` : alias,
alias: prefix,
},
},
];
Expand All @@ -126,7 +104,7 @@ assert(actions.length === releaseIndices.length);
const allIndexTargetsCorrectAlias = actions.every(a => {
const index = a[aliasAction].index;
const alias = a[aliasAction].alias;
return index.startsWith(alias.replace('next_', ''));
return index.startsWith(alias);
});
assert(allIndexTargetsCorrectAlias);

Expand Down
50 changes: 15 additions & 35 deletions admin/run.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// docker run -u node -it --rm --network host -v ${PWD}:/code --workdir /code node:20-alpine3.18 sh
// examples: npm run admin-project p:include e:include test OR npm run admin-project p:next_prd (will default to kf)
// examples: npm run admin-project p:include e:include OR npm run admin-project p:prd (will default to kf)
//kf-api-arranger/node_modules/@arranger/mapping-utils/dist/extendMapping.js change rangeStep: ['double', 'float', 'half_float', 'scaled_float'].includes(type) ? 0.01 : 1 to rangeStep: 1.0
/* eslint-disable no-console */
import 'regenerator-runtime/runtime.js';
Expand All @@ -10,7 +10,6 @@ import { countNOfDocs, createIndexIfNeeded } from '../dist/src/esUtils.js';
import { ArrangerApi } from './arrangerApi.mjs';
import { projectsConfig } from './projectsConfig.mjs';
import { esHost } from '../dist/src/env.js';
import { clinicalIndexStems, isClinicalIndex } from './releaseStatsUtils.mjs';

const hasProjectArrangerMetadataIndex = async (esClient, projectName) => {
const r = await esClient.indices.exists({
Expand Down Expand Up @@ -40,7 +39,7 @@ const ENV = {
const args = process.argv.slice(2);

//de-hardcode when possible
const SUPPORTED_PROJECT_NAMES = ['next_prd', 'next_qa', 'include', 'test'];
const SUPPORTED_PROJECT_NAMES = ['prd', 'qa', 'include'];
const projectArg = args.find(a => a.startsWith('p:'))?.split(':')[1];
if (!projectArg || !SUPPORTED_PROJECT_NAMES.some(sp => sp === projectArg)) {
console.warn(
Expand Down Expand Up @@ -73,35 +72,16 @@ console.info(`admin-project-script - Project value is=${envVal}`);
console.debug(`admin-project-script - Reaching to ElasticSearch at ${esHost}`);
const client = await EsInstance.default.getInstance();

const isTest = args.some(a => a === 'test');
const appendTestIfNeeded = x => (isTest && isClinicalIndex(x) ? `${x}_test` : x);
if (isTest) {
const allAliases = await client.cat.aliases({
h: 'alias',
format: 'json',
});
const clinicalTestAliases = allAliases.body
.filter(x => isClinicalIndex(x.alias) && x.alias.endsWith('_test'))
.map(x => x.alias);
if (!clinicalTestAliases || !clinicalIndexStems.every(s => clinicalTestAliases.some(x => x.includes(s)))) {
console.debug(
`admin-project-script - Terminating. When creating a test, all clinical entities must be aliased with _test suffix`,
);
console.debug(`admin-project-script - received test aliases: `);
console.debug(clinicalTestAliases);
process.exit(1);
}
}
//values are hardcoded for now, but as soon as possible, we should use env var from env.ts
const kfNext = [
'next_participant_centric',
'next_study_centric',
'next_biospecimen_centric',
'next_file_centric',
'next_variant_centric',
'next_gene_centric',
const kf = [
'participant_centric',
'study_centric',
'biospecimen_centric',
'file_centric',
'variant_centric',
'gene_centric',
'members-public',
].map(x => (isTest && isClinicalIndex(x) ? appendTestIfNeeded(x) : x));
];

const include = [
'participant_centric',
Expand All @@ -110,10 +90,10 @@ const include = [
'file_centric',
'variant_centric',
'gene_centric',
].map(x => (isTest && isClinicalIndex(x) ? appendTestIfNeeded(x) : x));
];

const envToIndicesPrefixes = {
kf: kfNext,
kf: kf,
include: include,
};

Expand All @@ -134,7 +114,7 @@ const projectName = projectArg;
const allProjectsConf = projectsConfig(projectName, envVal);

const projectsConf = allProjectsConf
.map(x => ({ ...x, indices: x.indices.map(i => ({ ...i, esIndex: appendTestIfNeeded(i.esIndex) })) }))
.map(x => ({ ...x, indices: x.indices.map(i => ({ ...i, esIndex: i.esIndex })) }))
.filter(p => {
const indicesInConf = p.indices.map(i => i.esIndex);
// indices in conf are the same as target indices from env vars?
Expand Down Expand Up @@ -200,9 +180,9 @@ if (updateConditions.every(b => b)) {
);
if (nOfDocsInProjectMetadata === projectConf.indices.length) {
console.debug(`admin-project-script - Applying extended mapping mutations.`);
console.time("fixExtendedMapping")
console.time('fixExtendedMapping');
await ArrangerApi.fixExtendedMapping(client, projectConf.extendedMappingMutations);
console.timeEnd("fixExtendedMapping")
console.timeEnd('fixExtendedMapping');
}
}

Expand Down
113 changes: 0 additions & 113 deletions admin/suggestStepFnPayload.mjs

This file was deleted.

Loading