From e69c4f9ff6209558390ade91be0689c0fd3a1e5e Mon Sep 17 00:00:00 2001 From: Amndeep Singh Mann Date: Mon, 1 May 2023 14:41:31 -0400 Subject: [PATCH 01/14] first pass on supplement tags - write is just a copy, read has been edited Signed-off-by: Amndeep Singh Mann --- src/commands/supplement/tags/read.ts | 34 ++++++++++++++++++ src/commands/supplement/tags/write.ts | 52 +++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 src/commands/supplement/tags/read.ts create mode 100644 src/commands/supplement/tags/write.ts diff --git a/src/commands/supplement/tags/read.ts b/src/commands/supplement/tags/read.ts new file mode 100644 index 000000000..1e5ff22bc --- /dev/null +++ b/src/commands/supplement/tags/read.ts @@ -0,0 +1,34 @@ +import {Command, Flags} from '@oclif/core' +import {ExecJSON, ProfileJSON} from 'inspecjs' +import fs from 'fs' + +export default class ReadTags extends Command { + static usage = 'supplement tags read -i [-o ] [-c control-id ...]' + + static description = 'Read the `tags` attribute in a given Heimdall Data Format or InSpec Profile JSON file and send it to stdout or write it to a file' + + static examples = ['saf supplement tags read -i hdf.json -o tag.json'] + + static flags = { + help: Flags.help({char: 'h'}), + input: Flags.string({char: 'i', required: true, description: 'An input HDF or profile file'}), + output: Flags.string({char: 'o', description: 'An output `tags` JSON file (otherwise the data is sent to stdout)'}), + controls: Flags.string({char: 'c', description: 'The id of the control whose tags will be extracted', multiple: true}), + } + + async run() { + const {flags} = await this.parse(ReadTags) + + const input: ExecJSON.Execution | ProfileJSON.Profile = JSON.parse(fs.readFileSync(flags.input, 'utf8')) + + const extractTags = (profile: ExecJSON.Profile | ProfileJSON.Profile) => profile.controls.map(control => control.tags) + + const tags = Object.hasOwn(input, 'profiles') ? (input as ExecJSON.Execution).profiles.map(profile => extractTags(profile)) : extractTags(input as ProfileJSON.Profile) + + if (flags.output) { + fs.writeFileSync(flags.output, JSON.stringify(tags, null, 2)) + } else { + process.stdout.write(JSON.stringify(tags, null, 2)) + } + } +} diff --git a/src/commands/supplement/tags/write.ts b/src/commands/supplement/tags/write.ts new file mode 100644 index 000000000..7d4c1425d --- /dev/null +++ b/src/commands/supplement/tags/write.ts @@ -0,0 +1,52 @@ +import {Command, Flags} from '@oclif/core' +import {ExecJSON} from 'inspecjs' +import fs from 'fs' + +export default class WritePassthrough extends Command { + static usage = 'supplement passthrough write -i (-f | -d ) [-o ]' + + static summary = 'Overwrite the `passthrough` attribute in a given HDF file with the provided `passthrough` JSON data' + + static description = 'Passthrough data can be any context/structure. See sample ideas at https://github.com/mitre/saf/wiki/Supplement-HDF-files-with-additional-information-(ex.-%60passthrough%60,-%60target%60)' + + static examples = [ + 'saf supplement passthrough write -i hdf.json -d \'{"a": 5}\'', + 'saf supplement passthrough write -i hdf.json -f passthrough.json -o new-hdf.json', + ] + + static flags = { + help: Flags.help({char: 'h'}), + input: Flags.string({char: 'i', required: true, description: 'An input Heimdall Data Format file'}), + passthroughFile: Flags.string({char: 'f', exclusive: ['passthroughData'], description: 'An input passthrough-data file (can contain any valid JSON); this flag or `passthroughData` must be provided'}), + passthroughData: Flags.string({char: 'd', exclusive: ['passthroughFile'], description: 'Input passthrough-data (can be any valid JSON); this flag or `passthroughFile` must be provided'}), + output: Flags.string({char: 'o', description: 'An output Heimdall Data Format JSON file (otherwise the input file is overwritten)'}), + } + + async run() { + const {flags} = await this.parse(WritePassthrough) + + const input: ExecJSON.Execution & {passthrough?: unknown} = JSON.parse(fs.readFileSync(flags.input, 'utf8')) + const output: string = flags.output || flags.input + + let passthrough: unknown + if (flags.passthroughFile) { + try { + passthrough = JSON.parse(fs.readFileSync(flags.passthroughFile, 'utf8')) + } catch (error: unknown) { + throw new Error(`Couldn't parse passthrough data: ${error}`) + } + } else if (flags.passthroughData) { + try { + passthrough = JSON.parse(flags.passthroughData) + } catch { + passthrough = flags.passthroughData + } + } else { + throw new Error('One out of passthroughFile or passthroughData must be passed') + } + + input.passthrough = passthrough + + fs.writeFileSync(output, JSON.stringify(input, null, 2)) + } +} From b317727262145895ad004476e4e86fd937f2d4cb Mon Sep 17 00:00:00 2001 From: Gavin Mason <75515923+GavMason@users.noreply.github.com> Date: Tue, 2 May 2023 08:47:27 -0400 Subject: [PATCH 02/14] Added filtering on controls (-c) for Read Tags Signed-off-by: Gavin Mason <75515923+GavMason@users.noreply.github.com> --- src/commands/supplement/tags/read.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/commands/supplement/tags/read.ts b/src/commands/supplement/tags/read.ts index 1e5ff22bc..bf5b05979 100644 --- a/src/commands/supplement/tags/read.ts +++ b/src/commands/supplement/tags/read.ts @@ -25,7 +25,14 @@ export default class ReadTags extends Command { const tags = Object.hasOwn(input, 'profiles') ? (input as ExecJSON.Execution).profiles.map(profile => extractTags(profile)) : extractTags(input as ProfileJSON.Profile) - if (flags.output) { + if (flags.controls) { + const filterTags: any[][] = tags.map(tags => tags.filter((tag: { gid: string }) => flags.controls?.includes(tag.gid))) + if (flags.output) { + fs.writeFileSync(flags.output, JSON.stringify(filterTags, null, 2)) + } else { + process.stdout.write(JSON.stringify(filterTags, null, 2)) + } + } else if (flags.output) { fs.writeFileSync(flags.output, JSON.stringify(tags, null, 2)) } else { process.stdout.write(JSON.stringify(tags, null, 2)) From 99598a9210d1d5b2545b2dac9a0f74a4f1bbd893 Mon Sep 17 00:00:00 2001 From: Amndeep Singh Mann Date: Tue, 2 May 2023 13:59:45 -0400 Subject: [PATCH 03/14] put filter in the correct location and added another example Signed-off-by: Amndeep Singh Mann --- src/commands/supplement/tags/read.ts | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/commands/supplement/tags/read.ts b/src/commands/supplement/tags/read.ts index bf5b05979..eeab16fb1 100644 --- a/src/commands/supplement/tags/read.ts +++ b/src/commands/supplement/tags/read.ts @@ -7,7 +7,7 @@ export default class ReadTags extends Command { static description = 'Read the `tags` attribute in a given Heimdall Data Format or InSpec Profile JSON file and send it to stdout or write it to a file' - static examples = ['saf supplement tags read -i hdf.json -o tag.json'] + static examples = ['saf supplement tags read -i hdf.json -o tag.json', 'saf supplement tags read -i hdf.json -o tag.json -c V-00001 V-00002'] static flags = { help: Flags.help({char: 'h'}), @@ -21,18 +21,11 @@ export default class ReadTags extends Command { const input: ExecJSON.Execution | ProfileJSON.Profile = JSON.parse(fs.readFileSync(flags.input, 'utf8')) - const extractTags = (profile: ExecJSON.Profile | ProfileJSON.Profile) => profile.controls.map(control => control.tags) + const extractTags = (profile: ExecJSON.Profile | ProfileJSON.Profile) => (profile.controls as Array).filter(control => flags.controls ? flags.controls.includes(control.id) : true).map(control => control.tags) const tags = Object.hasOwn(input, 'profiles') ? (input as ExecJSON.Execution).profiles.map(profile => extractTags(profile)) : extractTags(input as ProfileJSON.Profile) - if (flags.controls) { - const filterTags: any[][] = tags.map(tags => tags.filter((tag: { gid: string }) => flags.controls?.includes(tag.gid))) - if (flags.output) { - fs.writeFileSync(flags.output, JSON.stringify(filterTags, null, 2)) - } else { - process.stdout.write(JSON.stringify(filterTags, null, 2)) - } - } else if (flags.output) { + if (flags.output) { fs.writeFileSync(flags.output, JSON.stringify(tags, null, 2)) } else { process.stdout.write(JSON.stringify(tags, null, 2)) From 52d727c1c0de21806b4c407cdd1f9c3387ea23a4 Mon Sep 17 00:00:00 2001 From: Gavin Mason <75515923+GavMason@users.noreply.github.com> Date: Wed, 3 May 2023 12:13:05 -0400 Subject: [PATCH 04/14] Added tags write functionality Signed-off-by: Gavin Mason <75515923+GavMason@users.noreply.github.com> --- src/commands/supplement/tags/write.ts | 50 +++++++++++++++++---------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/src/commands/supplement/tags/write.ts b/src/commands/supplement/tags/write.ts index 7d4c1425d..5c6fbc8df 100644 --- a/src/commands/supplement/tags/write.ts +++ b/src/commands/supplement/tags/write.ts @@ -2,51 +2,63 @@ import {Command, Flags} from '@oclif/core' import {ExecJSON} from 'inspecjs' import fs from 'fs' -export default class WritePassthrough extends Command { - static usage = 'supplement passthrough write -i (-f | -d ) [-o ]' +export default class WriteTags extends Command { + static usage = 'supplement tags write -i (-f | -d ) [-o ]' - static summary = 'Overwrite the `passthrough` attribute in a given HDF file with the provided `passthrough` JSON data' + static summary = 'Overwrite the `tags` attribute in a given HDF file with the provided `tags` JSON data' - static description = 'Passthrough data can be any context/structure. See sample ideas at https://github.com/mitre/saf/wiki/Supplement-HDF-files-with-additional-information-(ex.-%60passthrough%60,-%60target%60)' + static description = 'Tags data can be any context/structure. See sample ideas at https://github.com/mitre/saf/wiki/Supplement-HDF-files-with-additional-information-(ex.-%60tags%60,-%60target%60)' static examples = [ - 'saf supplement passthrough write -i hdf.json -d \'{"a": 5}\'', - 'saf supplement passthrough write -i hdf.json -f passthrough.json -o new-hdf.json', + 'saf supplement tags write -i hdf.json -d \'{"a": 5}\'', + 'saf supplement tags write -i hdf.json -f tags.json -o new-hdf.json', ] static flags = { help: Flags.help({char: 'h'}), input: Flags.string({char: 'i', required: true, description: 'An input Heimdall Data Format file'}), - passthroughFile: Flags.string({char: 'f', exclusive: ['passthroughData'], description: 'An input passthrough-data file (can contain any valid JSON); this flag or `passthroughData` must be provided'}), - passthroughData: Flags.string({char: 'd', exclusive: ['passthroughFile'], description: 'Input passthrough-data (can be any valid JSON); this flag or `passthroughFile` must be provided'}), + tagsFile: Flags.string({char: 'f', exclusive: ['tagsData'], description: 'An input tags-data file (can contain any valid JSON); this flag or `tagsData` must be provided'}), + tagsData: Flags.string({char: 'd', exclusive: ['tagsFile'], description: 'Input tags-data (can be any valid JSON); this flag or `tagsFile` must be provided'}), output: Flags.string({char: 'o', description: 'An output Heimdall Data Format JSON file (otherwise the input file is overwritten)'}), } async run() { - const {flags} = await this.parse(WritePassthrough) + const {flags} = await this.parse(WriteTags) - const input: ExecJSON.Execution & {passthrough?: unknown} = JSON.parse(fs.readFileSync(flags.input, 'utf8')) + const input: ExecJSON.Execution & {tags?: unknown} = JSON.parse(fs.readFileSync(flags.input, 'utf8')) const output: string = flags.output || flags.input - let passthrough: unknown - if (flags.passthroughFile) { + let tags: any + if (flags.tagsFile) { try { - passthrough = JSON.parse(fs.readFileSync(flags.passthroughFile, 'utf8')) + tags = JSON.parse(fs.readFileSync(flags.tagsFile, 'utf8')) } catch (error: unknown) { - throw new Error(`Couldn't parse passthrough data: ${error}`) + throw new Error(`Couldn't parse tags data: ${error}`) } - } else if (flags.passthroughData) { + } else if (flags.tagsData) { try { - passthrough = JSON.parse(flags.passthroughData) + tags = JSON.parse(flags.tagsData) } catch { - passthrough = flags.passthroughData + tags = flags.tagsData } } else { - throw new Error('One out of passthroughFile or passthroughData must be passed') + throw new Error('One out of tagsFile or tagsData must be passed') } - input.passthrough = passthrough + // Check for num of keys and type of objects + if (Object.keys(input.profiles[0].controls).length !== Object.keys(tags[0]).length || typeof input.profiles[0].controls !== typeof tags[0]) { + throw new TypeError('Structure of tags data is invalid') + } + + // Overwrite tags + input.profiles[0].controls.forEach(control => { + const matchingTag = tags[0].find((tag: { gid: string }) => tag.gid === control.id) + if (matchingTag !== undefined) { + control.tags = matchingTag + } + }) fs.writeFileSync(output, JSON.stringify(input, null, 2)) + console.log('Tags successfully overwritten') } } From 22fbc18a377ab92fa542ae6d2c8ef4401bb82094 Mon Sep 17 00:00:00 2001 From: Gavin Mason <75515923+GavMason@users.noreply.github.com> Date: Thu, 4 May 2023 12:35:48 -0400 Subject: [PATCH 05/14] Updated tags write command to handle only HDF/InspecProfile inputs and fixed descriptions Signed-off-by: Gavin Mason <75515923+GavMason@users.noreply.github.com> --- src/commands/supplement/tags/write.ts | 54 +++++++++++++++++---------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/src/commands/supplement/tags/write.ts b/src/commands/supplement/tags/write.ts index 5c6fbc8df..f0d5d5e39 100644 --- a/src/commands/supplement/tags/write.ts +++ b/src/commands/supplement/tags/write.ts @@ -1,31 +1,32 @@ import {Command, Flags} from '@oclif/core' -import {ExecJSON} from 'inspecjs' +import {ExecJSON, ProfileJSON} from 'inspecjs' import fs from 'fs' export default class WriteTags extends Command { - static usage = 'supplement tags write -i (-f | -d ) [-o ]' + static usage = 'supplement tags write -i (-f | -d ) [-o ]' - static summary = 'Overwrite the `tags` attribute in a given HDF file with the provided `tags` JSON data' + static description = 'Overwrite the `tags` attribute in a given Heimdall Data Format or InSpec Profile JSON file and overwrite original file or optionally write it to a new file' - static description = 'Tags data can be any context/structure. See sample ideas at https://github.com/mitre/saf/wiki/Supplement-HDF-files-with-additional-information-(ex.-%60tags%60,-%60target%60)' + static summary = 'Tags data can be either a Heimdall Data Format or InSpec Profile JSON file. See sample ideas at https://github.com/mitre/saf/wiki/Supplement-HDF-files-with-additional-information-(ex.-%60tags%60,-%60target%60)' static examples = [ - 'saf supplement tags write -i hdf.json -d \'{"a": 5}\'', + 'saf supplement tags write -i hdf.json -d \'[[{"a": 5}]]\'', 'saf supplement tags write -i hdf.json -f tags.json -o new-hdf.json', ] static flags = { help: Flags.help({char: 'h'}), - input: Flags.string({char: 'i', required: true, description: 'An input Heimdall Data Format file'}), - tagsFile: Flags.string({char: 'f', exclusive: ['tagsData'], description: 'An input tags-data file (can contain any valid JSON); this flag or `tagsData` must be provided'}), - tagsData: Flags.string({char: 'd', exclusive: ['tagsFile'], description: 'Input tags-data (can be any valid JSON); this flag or `tagsFile` must be provided'}), - output: Flags.string({char: 'o', description: 'An output Heimdall Data Format JSON file (otherwise the input file is overwritten)'}), + input: Flags.string({char: 'i', required: true, description: 'An input HDF or profile file'}), + tagsFile: Flags.string({char: 'f', exclusive: ['tagsData'], description: 'An input tags-data file (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsData` must be provided'}), + tagsData: Flags.string({char: 'd', exclusive: ['tagsFile'], description: 'Input tags-data (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsFile` must be provided'}), + output: Flags.string({char: 'o', description: 'An output file that matches structure of input file (otherwise the input file is overwritten)'}), } async run() { const {flags} = await this.parse(WriteTags) - const input: ExecJSON.Execution & {tags?: unknown} = JSON.parse(fs.readFileSync(flags.input, 'utf8')) + const input: ExecJSON.Execution | ProfileJSON.Profile = JSON.parse(fs.readFileSync(flags.input, 'utf8')) + const output: string = flags.output || flags.input let tags: any @@ -45,18 +46,31 @@ export default class WriteTags extends Command { throw new Error('One out of tagsFile or tagsData must be passed') } - // Check for num of keys and type of objects - if (Object.keys(input.profiles[0].controls).length !== Object.keys(tags[0]).length || typeof input.profiles[0].controls !== typeof tags[0]) { - throw new TypeError('Structure of tags data is invalid') - } + if (Object.hasOwn(input, 'profiles')) { + if (Object.keys((input as ExecJSON.Execution).profiles).length !== Object.keys(tags).length) { + throw new TypeError('Structure of tags data is invalid') + } + + for (const profile of (input as ExecJSON.Execution).profiles) { + for (const control of profile.controls) { + const matchingTag = tags[0].find((tag: { gid: string }) => tag.gid === control.id) + if (matchingTag !== undefined) { + control.tags = matchingTag + } + } + } + } else { + if (Object.keys((input as ProfileJSON.Profile).controls).length !== Object.keys(tags).length) { + throw new TypeError('Structure of tags data is invalid') + } - // Overwrite tags - input.profiles[0].controls.forEach(control => { - const matchingTag = tags[0].find((tag: { gid: string }) => tag.gid === control.id) - if (matchingTag !== undefined) { - control.tags = matchingTag + for (const control of (input as ProfileJSON.Profile).controls) { + const matchingTag = tags[0].find((tag: { gid: string }) => tag.gid === control.id) + if (matchingTag !== undefined) { + control.tags = matchingTag + } } - }) + } fs.writeFileSync(output, JSON.stringify(input, null, 2)) console.log('Tags successfully overwritten') From 4baa6effe6d947992b9862e67c9d7a07cd8aae95 Mon Sep 17 00:00:00 2001 From: Gavin Mason <75515923+GavMason@users.noreply.github.com> Date: Thu, 4 May 2023 12:47:57 -0400 Subject: [PATCH 06/14] Small fix for tags write functionality for HDF type inputs Signed-off-by: Gavin Mason <75515923+GavMason@users.noreply.github.com> --- src/commands/supplement/tags/write.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/commands/supplement/tags/write.ts b/src/commands/supplement/tags/write.ts index f0d5d5e39..9c858b82d 100644 --- a/src/commands/supplement/tags/write.ts +++ b/src/commands/supplement/tags/write.ts @@ -51,9 +51,10 @@ export default class WriteTags extends Command { throw new TypeError('Structure of tags data is invalid') } - for (const profile of (input as ExecJSON.Execution).profiles) { + for (const [i, profile] of (input as ExecJSON.Execution).profiles.entries()) { + const currTags = tags[i] for (const control of profile.controls) { - const matchingTag = tags[0].find((tag: { gid: string }) => tag.gid === control.id) + const matchingTag = currTags.find((tag: { gid: string }) => tag.gid === control.id) if (matchingTag !== undefined) { control.tags = matchingTag } @@ -65,7 +66,7 @@ export default class WriteTags extends Command { } for (const control of (input as ProfileJSON.Profile).controls) { - const matchingTag = tags[0].find((tag: { gid: string }) => tag.gid === control.id) + const matchingTag = tags.find((tag: { gid: string }) => tag.gid === control.id) if (matchingTag !== undefined) { control.tags = matchingTag } From 4eeb9523827f2dcdd5d372ca3ed5a3d248f6efc8 Mon Sep 17 00:00:00 2001 From: Gavin Mason <75515923+GavMason@users.noreply.github.com> Date: Mon, 8 May 2023 16:02:43 -0400 Subject: [PATCH 07/14] Added write command functionality Signed-off-by: Gavin Mason <75515923+GavMason@users.noreply.github.com> --- src/commands/supplement/tags/write.ts | 47 +++++++++++++++++---------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/commands/supplement/tags/write.ts b/src/commands/supplement/tags/write.ts index 9c858b82d..37320d7fd 100644 --- a/src/commands/supplement/tags/write.ts +++ b/src/commands/supplement/tags/write.ts @@ -12,6 +12,7 @@ export default class WriteTags extends Command { static examples = [ 'saf supplement tags write -i hdf.json -d \'[[{"a": 5}]]\'', 'saf supplement tags write -i hdf.json -f tags.json -o new-hdf.json', + 'saf supplement tags write -i hdf.json -f tags.json -o new-hdf.json -c "V-000001', ] static flags = { @@ -20,6 +21,7 @@ export default class WriteTags extends Command { tagsFile: Flags.string({char: 'f', exclusive: ['tagsData'], description: 'An input tags-data file (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsData` must be provided'}), tagsData: Flags.string({char: 'd', exclusive: ['tagsFile'], description: 'Input tags-data (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsFile` must be provided'}), output: Flags.string({char: 'o', description: 'An output file that matches structure of input file (otherwise the input file is overwritten)'}), + controls: Flags.string({char: 'c', description: 'The id of the control whose tags will be extracted', multiple: true}), } async run() { @@ -46,34 +48,45 @@ export default class WriteTags extends Command { throw new Error('One out of tagsFile or tagsData must be passed') } - if (Object.hasOwn(input, 'profiles')) { - if (Object.keys((input as ExecJSON.Execution).profiles).length !== Object.keys(tags).length) { + const overwriteTags = (profile: ExecJSON.Profile | ProfileJSON.Profile, tags: any) => { + // Filter our controls + const filteredControls = (profile.controls as Array)?.filter(control => flags.controls ? flags.controls.includes(control.id) : true) + + // Check shape + console.log(profile.controls.length) + console.log(tags.length) + if (!flags.controls && profile.controls.length !== tags.length) { throw new TypeError('Structure of tags data is invalid') } - for (const [i, profile] of (input as ExecJSON.Execution).profiles.entries()) { - const currTags = tags[i] - for (const control of profile.controls) { - const matchingTag = currTags.find((tag: { gid: string }) => tag.gid === control.id) - if (matchingTag !== undefined) { - control.tags = matchingTag + // Overwrite tags + const updatedControls = profile.controls.map((control: any, index: number) => { + if (filteredControls.includes(control)) { + return { + ...control, + tags: tags[index], } } + + return control + }) + return updatedControls + } + + if (Object.hasOwn(input, 'profiles')) { + for (const [i, profile] of (input as ExecJSON.Execution).profiles.entries()) { + const updatedControls = overwriteTags(profile, tags[i]) + + profile.controls = updatedControls } } else { - if (Object.keys((input as ProfileJSON.Profile).controls).length !== Object.keys(tags).length) { - throw new TypeError('Structure of tags data is invalid') - } + const updatedControls = overwriteTags((input as ProfileJSON.Profile), tags); - for (const control of (input as ProfileJSON.Profile).controls) { - const matchingTag = tags.find((tag: { gid: string }) => tag.gid === control.id) - if (matchingTag !== undefined) { - control.tags = matchingTag - } - } + (input as ProfileJSON.Profile).controls = updatedControls } fs.writeFileSync(output, JSON.stringify(input, null, 2)) console.log('Tags successfully overwritten') } } + From 617d459b64b41e639e3f981c1bbee7bf0aeee222 Mon Sep 17 00:00:00 2001 From: Gavin Mason <75515923+GavMason@users.noreply.github.com> Date: Tue, 9 May 2023 15:57:29 -0400 Subject: [PATCH 08/14] Fixed non-type requested changes Signed-off-by: Gavin Mason <75515923+GavMason@users.noreply.github.com> --- src/commands/supplement/tags/write.ts | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/src/commands/supplement/tags/write.ts b/src/commands/supplement/tags/write.ts index 37320d7fd..c317ebf23 100644 --- a/src/commands/supplement/tags/write.ts +++ b/src/commands/supplement/tags/write.ts @@ -51,38 +51,23 @@ export default class WriteTags extends Command { const overwriteTags = (profile: ExecJSON.Profile | ProfileJSON.Profile, tags: any) => { // Filter our controls const filteredControls = (profile.controls as Array)?.filter(control => flags.controls ? flags.controls.includes(control.id) : true) - // Check shape - console.log(profile.controls.length) - console.log(tags.length) - if (!flags.controls && profile.controls.length !== tags.length) { + if (filteredControls.length !== tags.length) { throw new TypeError('Structure of tags data is invalid') } // Overwrite tags - const updatedControls = profile.controls.map((control: any, index: number) => { - if (filteredControls.includes(control)) { - return { - ...control, - tags: tags[index], - } - } - - return control - }) - return updatedControls + for (const [index, control] of filteredControls.entries()) { + control.tags = tags[index] + } } if (Object.hasOwn(input, 'profiles')) { for (const [i, profile] of (input as ExecJSON.Execution).profiles.entries()) { - const updatedControls = overwriteTags(profile, tags[i]) - - profile.controls = updatedControls + overwriteTags(profile, tags[i]) } } else { - const updatedControls = overwriteTags((input as ProfileJSON.Profile), tags); - - (input as ProfileJSON.Profile).controls = updatedControls + overwriteTags((input as ProfileJSON.Profile), tags) } fs.writeFileSync(output, JSON.stringify(input, null, 2)) From 1c4029f12f11dc00b9fe0196fc36421ebe1d78ce Mon Sep 17 00:00:00 2001 From: Gavin Mason <75515923+GavMason@users.noreply.github.com> Date: Wed, 10 May 2023 09:32:29 -0400 Subject: [PATCH 09/14] Added type fixes for requested changes Signed-off-by: Gavin Mason <75515923+GavMason@users.noreply.github.com> --- src/commands/supplement/tags/write.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/commands/supplement/tags/write.ts b/src/commands/supplement/tags/write.ts index c317ebf23..341eb65cb 100644 --- a/src/commands/supplement/tags/write.ts +++ b/src/commands/supplement/tags/write.ts @@ -31,7 +31,7 @@ export default class WriteTags extends Command { const output: string = flags.output || flags.input - let tags: any + let tags: ExecJSON.Control[][] | ProfileJSON.Control[] | string if (flags.tagsFile) { try { tags = JSON.parse(fs.readFileSync(flags.tagsFile, 'utf8')) @@ -48,10 +48,11 @@ export default class WriteTags extends Command { throw new Error('One out of tagsFile or tagsData must be passed') } - const overwriteTags = (profile: ExecJSON.Profile | ProfileJSON.Profile, tags: any) => { + const overwriteTags = (profile: ExecJSON.Profile | ProfileJSON.Profile, tags: ExecJSON.Control[] | ProfileJSON.Control[]) => { // Filter our controls const filteredControls = (profile.controls as Array)?.filter(control => flags.controls ? flags.controls.includes(control.id) : true) // Check shape + console.log() if (filteredControls.length !== tags.length) { throw new TypeError('Structure of tags data is invalid') } @@ -64,10 +65,10 @@ export default class WriteTags extends Command { if (Object.hasOwn(input, 'profiles')) { for (const [i, profile] of (input as ExecJSON.Execution).profiles.entries()) { - overwriteTags(profile, tags[i]) + overwriteTags(profile, tags[i] as ExecJSON.Control[]) } } else { - overwriteTags((input as ProfileJSON.Profile), tags) + overwriteTags((input as ProfileJSON.Profile), (tags as ProfileJSON.Control[])) } fs.writeFileSync(output, JSON.stringify(input, null, 2)) From a13185cbe718ab89329b6e13910ae52ad772d375 Mon Sep 17 00:00:00 2001 From: Gavin Mason <75515923+GavMason@users.noreply.github.com> Date: Wed, 10 May 2023 09:34:10 -0400 Subject: [PATCH 10/14] Cleaned up code Signed-off-by: Gavin Mason <75515923+GavMason@users.noreply.github.com> --- src/commands/supplement/tags/write.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/commands/supplement/tags/write.ts b/src/commands/supplement/tags/write.ts index 341eb65cb..5776a7e72 100644 --- a/src/commands/supplement/tags/write.ts +++ b/src/commands/supplement/tags/write.ts @@ -52,7 +52,6 @@ export default class WriteTags extends Command { // Filter our controls const filteredControls = (profile.controls as Array)?.filter(control => flags.controls ? flags.controls.includes(control.id) : true) // Check shape - console.log() if (filteredControls.length !== tags.length) { throw new TypeError('Structure of tags data is invalid') } From e01549362b3aa3625f1bf4c5cac324460743278e Mon Sep 17 00:00:00 2001 From: Gavin Mason <75515923+GavMason@users.noreply.github.com> Date: Thu, 11 May 2023 09:06:36 -0400 Subject: [PATCH 11/14] Fix flag mismatch Signed-off-by: Gavin Mason <75515923+GavMason@users.noreply.github.com> --- src/commands/supplement/tags/write.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/supplement/tags/write.ts b/src/commands/supplement/tags/write.ts index 5776a7e72..f98465689 100644 --- a/src/commands/supplement/tags/write.ts +++ b/src/commands/supplement/tags/write.ts @@ -18,8 +18,8 @@ export default class WriteTags extends Command { static flags = { help: Flags.help({char: 'h'}), input: Flags.string({char: 'i', required: true, description: 'An input HDF or profile file'}), - tagsFile: Flags.string({char: 'f', exclusive: ['tagsData'], description: 'An input tags-data file (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsData` must be provided'}), - tagsData: Flags.string({char: 'd', exclusive: ['tagsFile'], description: 'Input tags-data (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsFile` must be provided'}), + tagsFile: Flags.string({char: 'f', exclusive: ['tagsFile'], description: 'An input tags-data file (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsData` must be provided'}), + tagsData: Flags.string({char: 'd', exclusive: ['tagsData'], description: 'Input tags-data (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsFile` must be provided'}), output: Flags.string({char: 'o', description: 'An output file that matches structure of input file (otherwise the input file is overwritten)'}), controls: Flags.string({char: 'c', description: 'The id of the control whose tags will be extracted', multiple: true}), } From 24e4a3e92afe7fbe07a6a5e0b94651e32ce8c950 Mon Sep 17 00:00:00 2001 From: Gavin Mason <75515923+GavMason@users.noreply.github.com> Date: Thu, 11 May 2023 09:10:14 -0400 Subject: [PATCH 12/14] Undo last commit Signed-off-by: Gavin Mason <75515923+GavMason@users.noreply.github.com> --- src/commands/supplement/tags/write.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/supplement/tags/write.ts b/src/commands/supplement/tags/write.ts index f98465689..5776a7e72 100644 --- a/src/commands/supplement/tags/write.ts +++ b/src/commands/supplement/tags/write.ts @@ -18,8 +18,8 @@ export default class WriteTags extends Command { static flags = { help: Flags.help({char: 'h'}), input: Flags.string({char: 'i', required: true, description: 'An input HDF or profile file'}), - tagsFile: Flags.string({char: 'f', exclusive: ['tagsFile'], description: 'An input tags-data file (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsData` must be provided'}), - tagsData: Flags.string({char: 'd', exclusive: ['tagsData'], description: 'Input tags-data (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsFile` must be provided'}), + tagsFile: Flags.string({char: 'f', exclusive: ['tagsData'], description: 'An input tags-data file (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsData` must be provided'}), + tagsData: Flags.string({char: 'd', exclusive: ['tagsFile'], description: 'Input tags-data (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsFile` must be provided'}), output: Flags.string({char: 'o', description: 'An output file that matches structure of input file (otherwise the input file is overwritten)'}), controls: Flags.string({char: 'c', description: 'The id of the control whose tags will be extracted', multiple: true}), } From fe7155b1540639e8c5c8a930eebbdfe45b89c176 Mon Sep 17 00:00:00 2001 From: Gavin Mason <75515923+GavMason@users.noreply.github.com> Date: Thu, 11 May 2023 15:45:15 -0400 Subject: [PATCH 13/14] Added tags extend functionality Signed-off-by: Gavin Mason <75515923+GavMason@users.noreply.github.com> --- src/commands/supplement/tags/extend.ts | 94 ++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/commands/supplement/tags/extend.ts diff --git a/src/commands/supplement/tags/extend.ts b/src/commands/supplement/tags/extend.ts new file mode 100644 index 000000000..1254315b2 --- /dev/null +++ b/src/commands/supplement/tags/extend.ts @@ -0,0 +1,94 @@ +import {Command, Flags} from '@oclif/core' +import {ExecJSON, ProfileJSON} from 'inspecjs' +import fs from 'fs' +import csvParse from 'csv-parse' + +export default class WriteTags extends Command { + static usage = 'supplement tags extend -i (-f | -d ) [-o ]' + + static description = 'Extends the `tags` attribute in a given Heimdall Data Format or InSpec Profile JSON file and overwrite original file or optionally output it to a new file' + + static summary = 'Tags data can be either be a CSV file or JSON data. See sample ideas at https://github.com/mitre/saf/wiki/Supplement-HDF-files-with-additional-information-(ex.-%60tags%60,-%60target%60)' + + static examples = [ + 'saf supplement tags extend -i hdf.json -d \'[[{"a": 5}]]\'', + 'saf supplement tags extend -i hdf.json -f tags.csv -o new-hdf.json', + 'saf supplement tags extend -i hdf.json -f tags.csv -o new-hdf.json -c "V-000001', + ] + + static flags = { + help: Flags.help({char: 'h'}), + input: Flags.string({char: 'i', required: true, description: 'An input HDF or profile file'}), + tagsFile: Flags.string({char: 'f', exclusive: ['tagsData'], description: 'An input tags-data file (can contain CSV file)); this flag or `tagsData` must be provided'}), + tagsData: Flags.string({char: 'd', exclusive: ['tagsFile'], description: 'Input tags-data (can contain JSON that matches structure of tags in input file(HDF or profile)); this flag or `tagsFile` must be provided'}), + output: Flags.string({char: 'o', description: 'An output file that matches structure of input file (otherwise the input file is overwritten)'}), + controls: Flags.string({char: 'c', description: 'The id of the control whose tags will be extracted', multiple: true}), + } + + async run() { + const {flags} = await this.parse(WriteTags) + + const input: ExecJSON.Execution | ProfileJSON.Profile = JSON.parse(fs.readFileSync(flags.input, 'utf8')) + + const output: string = flags.output || flags.input + + let CCItags: object | string + if (flags.tagsFile) { + try { + const fileContent = fs.readFileSync(flags.tagsFile, 'utf8') + csvParse(fileContent, {columns: true, delimiter: ','}, (err, output) => { + if (err) { + throw new Error(`CSV parse error ${err}`) + } + + CCItags = JSON.parse(JSON.stringify(output)) + processParsedData(CCItags) + }) + } catch (error: unknown) { + throw new Error(`Couldn't parse tags data: ${error}`) + } + } else if (flags.tagsData) { + try { + CCItags = JSON.parse(flags.tagsData) + } catch { + CCItags = flags.tagsData + } + + processParsedData(CCItags) + } else { + throw new Error('One out of tagsFile or tagsData must be passed') + } + + const extendTags = (profile: ExecJSON.Profile | ProfileJSON.Profile, CCItags: []) => { + // Filter our controls + const filteredControls = (profile.controls as Array)?.filter(control => flags.controls ? flags.controls.includes(control.id) : true) + for (const tag of filteredControls.map(control => control.tags)) { + if (tag.cci) { + const cms_ars5_ce: string[] = [] + for (const cci of tag.cci) { + const matchingTag = CCItags.find((currTag: { cci: any }) => currTag.cci.replace(/\s/g, '').includes(cci)) + if (matchingTag && matchingTag['cms-ars5-ce'] !== '') { + cms_ars5_ce.push(matchingTag['cms-ars5-ce']) + } + } + + if (cms_ars5_ce.length !== 0) + tag.cms_ars5_ce = cms_ars5_ce + } + } + } + + function processParsedData(CCItags: any) { + if (Object.hasOwn(input, 'profiles')) { + for (const [i, profile] of (input as ExecJSON.Execution).profiles.entries()) { + extendTags(profile, CCItags) + } + } else { + extendTags((input as ProfileJSON.Profile), CCItags) + } + + fs.writeFileSync(output, JSON.stringify(input, null, 2)) + console.log('Tags successfully extended') + } + } +} From fabaf53d516cd8c925b5557f2fdf52e882a6be0f Mon Sep 17 00:00:00 2001 From: Gavin Mason <75515923+GavMason@users.noreply.github.com> Date: Fri, 12 May 2023 07:40:39 -0400 Subject: [PATCH 14/14] Added dev notes/TODOs Signed-off-by: Gavin Mason <75515923+GavMason@users.noreply.github.com> --- src/commands/supplement/tags/extend.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/commands/supplement/tags/extend.ts b/src/commands/supplement/tags/extend.ts index 1254315b2..58b28ea75 100644 --- a/src/commands/supplement/tags/extend.ts +++ b/src/commands/supplement/tags/extend.ts @@ -33,6 +33,7 @@ export default class WriteTags extends Command { const output: string = flags.output || flags.input let CCItags: object | string + // TODO: Make more generic if (flags.tagsFile) { try { const fileContent = fs.readFileSync(flags.tagsFile, 'utf8') @@ -42,6 +43,7 @@ export default class WriteTags extends Command { } CCItags = JSON.parse(JSON.stringify(output)) + // TODO: Right now passing into function. When parsing csv add proper await function processParsedData(CCItags) }) } catch (error: unknown) { @@ -66,6 +68,7 @@ export default class WriteTags extends Command { if (tag.cci) { const cms_ars5_ce: string[] = [] for (const cci of tag.cci) { + // TODO: Currently striping whitespace might not need this const matchingTag = CCItags.find((currTag: { cci: any }) => currTag.cci.replace(/\s/g, '').includes(cci)) if (matchingTag && matchingTag['cms-ars5-ce'] !== '') { cms_ars5_ce.push(matchingTag['cms-ars5-ce'])