diff --git a/app/api/definitions/paths/stix-bundles-paths.yml b/app/api/definitions/paths/stix-bundles-paths.yml index aaacf87a..03ad60ea 100644 --- a/app/api/definitions/paths/stix-bundles-paths.yml +++ b/app/api/definitions/paths/stix-bundles-paths.yml @@ -44,6 +44,14 @@ paths: schema: type: boolean default: false + - name: includeMissingAttackId + in: query + description: | + Whether to include objects that should have an ATT&CK ID set but do not. + schema: + type: boolean + default: false + responses: '200': description: 'An exported stix bundle.' diff --git a/app/controllers/stix-bundles-controller.js b/app/controllers/stix-bundles-controller.js index 43f86441..74d6b52c 100644 --- a/app/controllers/stix-bundles-controller.js +++ b/app/controllers/stix-bundles-controller.js @@ -12,7 +12,8 @@ exports.exportBundle = async function(req, res) { domain: req.query.domain, state: req.query.state, includeRevoked: req.query.includeRevoked, - includeDeprecated: req.query.includeDeprecated + includeDeprecated: req.query.includeDeprecated, + includeMissingAttackId: req.query.includeMissingAttackId }; try { diff --git a/app/lib/linkById.js b/app/lib/linkById.js new file mode 100644 index 00000000..215a80cc --- /dev/null +++ b/app/lib/linkById.js @@ -0,0 +1,93 @@ +'use strict'; + +const AttackObject = require('../models/attack-object-model'); + +// Default implmentation. Retrieves the attack object from the database. +async function getAttackObjectFromDatabase(attackId) { + const attackObject = await AttackObject + .findOne({ 'workspace.attack_id': attackId }) + .sort('-stix.modified') + .lean() + .exec(); + + return attackObject; +} +exports.getAttackObjectFromDatabase = getAttackObjectFromDatabase; + +const sourceNames = ['mitre-attack', 'mitre-mobile-attack', 'mobile-attack', 'mitre-ics-attack']; +function attackReference(externalReferences) { + if (Array.isArray(externalReferences) && externalReferences.length > 0) { + return externalReferences.find(ref => sourceNames.includes(ref.source_name)); + } + else { + return null; + } +} + +const linkByIdRegex = /\(LinkById: ([A-Z]+[0-9]+(\.[0-9]+)?)\)/g; +async function convertLinkById(text, getAttackObject) { + if (text) { + let convertedText = ''; + let lastIndex = 0; + + const matches = text.matchAll(linkByIdRegex); + for (const match of matches) { + const prefix = text.slice(lastIndex, match.index); + const attackId = match[1]; + const citation = { + name: 'linked object not found', + url: '' + } + + if (attackId) { + const attackObject = await getAttackObject(attackId); + if (attackObject) { + citation.name = attackObject.stix.name; + const reference = attackReference(attackObject?.stix.external_references); + if (reference) { + citation.url = reference.url; + } + } + } + + convertedText = convertedText.concat(prefix, `[${ citation.name }](${ citation.url })`); + lastIndex = match.index + match[0].length; + } + + const postText = text.slice(lastIndex); + convertedText = convertedText.concat(postText); + + return convertedText; + } + else { + return text; + } +} + +async function convertExternalReferencesWithLinkById(externalReferences, getAttackObject) { + if (Array.isArray(externalReferences)) { + for (const externalReference of externalReferences) { + externalReference.description = await convertLinkById(externalReference.description, getAttackObject); + } + } +} + + +// If provided, getAttackObject() must be an async function with the signature: +// getAttackObject(attackId) returning an attack object +async function convertLinkByIdTags (stixObject, getAttackObject) { + if (!(getAttackObject instanceof Function)) { + getAttackObject = getAttackObjectFromDatabase; + } + + if (stixObject) { + stixObject.description = await convertLinkById(stixObject.description, getAttackObject); + + if (stixObject.type === 'attack-pattern') { + stixObject.x_mitre_detection = await convertLinkById(stixObject.x_mitre_detection, getAttackObject); + } + + await convertExternalReferencesWithLinkById(stixObject.external_references, getAttackObject); + } +} +exports.convertLinkByIdTags = convertLinkByIdTags; diff --git a/app/services/stix-bundles-service.js b/app/services/stix-bundles-service.js index dc7697e3..6fb197bd 100644 --- a/app/services/stix-bundles-service.js +++ b/app/services/stix-bundles-service.js @@ -5,7 +5,6 @@ const uuid = require('uuid'); const AttackObject = require('../models/attack-object-model'); -const Group = require('../models/group-model'); const Matrix = require('../models/matrix-model'); const Mitigation = require('../models/mitigation-model'); const Note = require('../models/note-model'); @@ -13,13 +12,16 @@ const Relationship = require('../models/relationship-model'); const Software = require('../models/software-model'); const Tactic = require('../models/tactic-model'); const Technique = require('../models/technique-model'); + const systemConfigurationService = require('./system-configuration-service'); +const linkById = require('../lib/linkById'); const errors = { notFound: 'Domain not found', }; exports.errors = errors; +// Retrieve the attack object from the database using its STIX ID async function getAttackObject(stixId) { const attackObject = await AttackObject .findOne({ 'stix.id': stixId }) @@ -30,7 +32,34 @@ async function getAttackObject(stixId) { return attackObject; } +const attackIdObjectTypes = ['intrusion-set', 'malware', 'tool', 'attack-pattern', 'course-of-action', 'x-mitre-data_source']; +function requiresAttackId(attackObject) { + return attackIdObjectTypes.includes(attackObject?.stix.type); +} + +const sourceNames = ['mitre-attack', 'mitre-mobile-attack', 'mobile-attack', 'mitre-ics-attack']; +function hasAttackId(attackObject) { + const externalReferences = attackObject?.stix.external_references; + if (Array.isArray(externalReferences) && externalReferences.length > 0) { + const mitreAttackReference = externalReferences.find(ref => sourceNames.includes(ref.source_name)); + if (mitreAttackReference?.external_id) { + return true; + } + } + + return false; +} + exports.exportBundle = async function(options) { + // The attackObjectMap maps attack IDs to attack objects and is used to make the LinkById conversion + // more efficient. + const attackObjectMap = new Map(); + function addAttackObjectToMap(attackObject) { + if (attackObject?.workspace.attack_id) { + attackObjectMap.set(attackObject.workspace.attack_id, attackObject); + } + } + // Create the bundle to hold the exported objects const bundle = { type: 'bundle', @@ -75,7 +104,6 @@ exports.exportBundle = async function(options) { ]; // Retrieve the primary objects - const domainGroups = await Group.aggregate(aggregation); const domainMitigations = await Mitigation.aggregate(aggregation); const domainSoftware = await Software.aggregate(aggregation); const domainTactics = await Tactic.aggregate(aggregation); @@ -87,7 +115,7 @@ exports.exportBundle = async function(options) { const allMatrices = await Matrix.aggregate(matrixAggregation); const domainMatrices = allMatrices.filter(matrix => matrix?.stix?.external_references.length && matrix.stix.external_references[0].external_id === options.domain); - const primaryObjects = [...domainGroups, ...domainMatrices, ...domainMitigations, ...domainSoftware, ...domainTactics, ...domainTechniques]; + let primaryObjects = [...domainMatrices, ...domainMitigations, ...domainSoftware, ...domainTactics, ...domainTechniques]; // No primary objects means that the domain doesn't exist // Return an empty bundle @@ -95,9 +123,15 @@ exports.exportBundle = async function(options) { return bundle; } + // Remove any primary objects that don't have an ATT&CK ID + if (!options.includeMissingAttackId) { + primaryObjects = primaryObjects.filter(o => hasAttackId(o)); + } + // Put the primary objects in the bundle for (const primaryObject of primaryObjects) { bundle.objects.push(primaryObject.stix); + addAttackObjectToMap(primaryObject); } // Get the relationships that point at primary objects (removing duplicates) @@ -147,6 +181,14 @@ exports.exportBundle = async function(options) { bundle.objects.push(relationship.stix); } + function secondaryObjectIsValid(secondaryObject) { + // The object must exist and at least one of these conditions must apply: + // 1) The request allows objects that are missing an ATT&CK ID + // 2) The objects is a type that doesn't require an ATT&CK ID + // 3) The object has an ATT&CK ID + return (secondaryObject && (options.includeMissingAttackId || !requiresAttackId(secondaryObject) || hasAttackId(secondaryObject))); + } + // Get the secondary objects (additional objects pointed to by a relationship) const secondaryObjects = []; const dataComponents = new Map(); @@ -154,13 +196,10 @@ exports.exportBundle = async function(options) { // Check source_ref if (!objectsMap.has(relationship.stix.source_ref)) { const secondaryObject = await getAttackObject(relationship.stix.source_ref); - if (secondaryObject) { + if (secondaryObjectIsValid(secondaryObject)) { secondaryObjects.push(secondaryObject); objectsMap.set(secondaryObject.stix.id, true); } - else { - console.log(`Could not find secondary object with id ${ relationship.stix.source_ref }`); - } // Save data components for later if (relationship.stix.relationship_type === 'detects') { @@ -171,19 +210,21 @@ exports.exportBundle = async function(options) { // Check target_ref if (!objectsMap.has(relationship.stix.target_ref)) { const secondaryObject = await getAttackObject(relationship.stix.target_ref); - if (secondaryObject) { + if (secondaryObjectIsValid(secondaryObject)) { secondaryObjects.push(secondaryObject); objectsMap.set(secondaryObject.stix.id, true); } - else { - console.log(`Could not find secondary object with id ${ relationship.stix.target_ref }`); - } } } // Put the secondary objects in the bundle for (const secondaryObject of secondaryObjects) { + // Groups need to have the domain added to x_mitre_domains + if (secondaryObject.stix.type === 'intrusion-set') { + secondaryObject.stix.x_mitre_domains = [ options.domain ]; + } bundle.objects.push(secondaryObject.stix); + addAttackObjectToMap(secondaryObject); } // Data components have already been added to the bundle because they're referenced in a relationship @@ -199,8 +240,11 @@ exports.exportBundle = async function(options) { const dataSources = new Map(); for (const dataSourceId of dataSourceIds.keys()) { const dataSource = await getAttackObject(dataSourceId); - bundle.objects.push(dataSource.stix); - dataSources.set(dataSourceId, dataSource.stix); + if (options.includeMissingAttackId || hasAttackId(dataSource)) { + bundle.objects.push(dataSource.stix); + dataSources.set(dataSourceId, dataSource.stix); + addAttackObjectToMap(dataSource); + } } // Create a map of techniques detected by data components @@ -317,6 +361,22 @@ exports.exportBundle = async function(options) { bundle.objects.push(note.stix); } + // Create the function to be used by the LinkById conversion process + // Note that using this map instead of database retrieval results in a + // dramatic performance improvement. + const getAttackObjectFromMap = async function (attackId) { + let attackObject = attackObjectMap.get(attackId); + if (!attackObject) { + attackObject = await linkById.getAttackObjectFromDatabase(attackId); + } + return attackObject; + } + + // Convert LinkById tags into markdown citations + for (const attackObject of bundle.objects) { + await linkById.convertLinkByIdTags(attackObject, getAttackObjectFromMap); + } + // Make a list of identities referenced const identitiesMap = new Map(); for (const bundleObject of bundle.objects) { diff --git a/app/tests/api/stix-bundles/stix-bundles.spec.js b/app/tests/api/stix-bundles/stix-bundles.spec.js index 90f639ac..e22b9bf4 100644 --- a/app/tests/api/stix-bundles/stix-bundles.spec.js +++ b/app/tests/api/stix-bundles/stix-bundles.spec.js @@ -27,9 +27,7 @@ const initialObjectData = { spec_version: '2.1', type: 'x-mitre-collection', description: 'This is a collection.', - external_references: [ - {source_name: 'source-1', external_id: 's1'} - ], + external_references: [], object_marking_refs: ['marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'], created_by_ref: "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", x_mitre_contents: [ @@ -148,7 +146,7 @@ const initialObjectData = { type: 'attack-pattern', description: 'This is a technique.', external_references: [ - { source_name: 'source-1', external_id: 's1' }, + { source_name: 'mitre-attack', external_id: 'T1' }, { source_name: 'attack-pattern-1 source', description: 'this is a source description'} ], object_marking_refs: [ 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168' ], @@ -173,7 +171,7 @@ const initialObjectData = { type: 'attack-pattern', description: 'This is a technique.', external_references: [ - { source_name: 'source-1', external_id: 's1' }, + { source_name: 'mitre-attack', external_id: 'T1' }, { source_name: 'attack-pattern-1 source', description: 'this is a source description'} ], object_marking_refs: [ 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168' ], @@ -197,7 +195,7 @@ const initialObjectData = { type: 'attack-pattern', description: 'This is another technique.', external_references: [ - { source_name: 'source-1', external_id: 's1' }, + { source_name: 'mitre-attack', external_id: 'T1' }, { source_name: 'attack-pattern-2 source', description: 'this is a source description 2'} ], object_marking_refs: [ 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168' ], @@ -222,7 +220,7 @@ const initialObjectData = { type: 'attack-pattern', description: 'This is another technique.', external_references: [ - { source_name: 'source-1', external_id: 's1' }, + { source_name: 'mitre-attack', external_id: 'T1' }, { source_name: 'attack-pattern-2 source', description: 'this is a source description 2'} ], object_marking_refs: [ 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168' ], @@ -244,7 +242,7 @@ const initialObjectData = { name: "mitigation-1", description: "This is a mitigation", external_references: [ - { source_name: 'source-1', external_id: 's1' } + { source_name: 'mitre-attack', external_id: 'M1' } ], object_marking_refs: [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" @@ -262,7 +260,7 @@ const initialObjectData = { name: "mitigation-2", description: "This is a mitigation that isn't in the contents", external_references: [ - { source_name: 'source-1', external_id: 's1' } + { source_name: 'mitre-attack', external_id: 'M1' } ], object_marking_refs: [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" @@ -281,7 +279,7 @@ const initialObjectData = { name: "software-1", description: "This is a software with an alias", external_references: [ - { source_name: 'source-1', external_id: 's1' }, + { source_name: 'mitre-attack', external_id: 'S1' }, { source_name: 'malware-1 source', description: 'this is a source description'}, { source_name: 'xyzzy', description: '(Citation: Adventure 1975)'} ], @@ -372,9 +370,6 @@ const initialObjectData = { 'Author 1', 'Author 2' ], - external_references: [ - { source_name: 'source-1', external_id: 's1' } - ], object_marking_refs: [ 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168' ], created_by_ref: "identity--6444f546-6900-4456-b3b1-015c88d70dab", object_refs: [ 'malware--04227b24-7817-4de1-9050-b7b1b57f5866' ], @@ -389,7 +384,10 @@ const initialObjectData = { object_marking_refs: [ 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168' ], created_by_ref: "identity--6444f546-6900-4456-b3b1-015c88d70dab", modified: "2020-04-12T15:44:47.629Z", - created: "2019-10-22T00:14:20.652Z" + created: "2019-10-22T00:14:20.652Z", + external_references: [ + {source_name: 'mitre-attack', external_id: 'DS1'} + ], }, { type: 'x-mitre-data-source', @@ -399,7 +397,10 @@ const initialObjectData = { object_marking_refs: [ 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168' ], created_by_ref: "identity--6444f546-6900-4456-b3b1-015c88d70dab", modified: "2020-04-12T15:44:47.629Z", - created: "2019-10-22T00:14:20.652Z" + created: "2019-10-22T00:14:20.652Z", + external_references: [ + {source_name: 'mitre-attack', external_id: 'DS1'} + ], }, { type: 'x-mitre-data-component', diff --git a/package-lock.json b/package-lock.json index 7ac2cc6d..d73a7422 100644 --- a/package-lock.json +++ b/package-lock.json @@ -975,9 +975,9 @@ "dev": true }, "async": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.1.tgz", - "integrity": "sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg==" + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz", + "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==" }, "async-await-retry": { "version": "1.2.3", diff --git a/package.json b/package.json index c0f05e9f..a33b47ce 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ }, "dependencies": { "@apidevtools/json-schema-ref-parser": "^9.0.9", - "async": "^3.2.1", + "async": "^3.2.3", "async-await-retry": "^1.2.3", "body-parser": "^1.19.0", "compression": "^1.7.4",