Skip to content

Commit

Permalink
Merge pull request #170 from center-for-threat-informed-defense/feature/
Browse files Browse the repository at this point in the history 
#133-export-groups

Modify the STIX bundle export
  • Loading branch information
@ElJocko
ElJocko authored Apr 8, 2022
2 parents 2f62a8e + 69257a0 commit e8356b9
Show file tree
Hide file tree
Showing 7 changed files with 196 additions and 33 deletions.
8 changes: 8 additions & 0 deletions app/api/definitions/paths/stix-bundles-paths.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,14 @@ paths:
schema:
type: boolean
default: false
- name: includeMissingAttackId
in: query
description: |
Whether to include objects that should have an ATT&CK ID set but do not.
schema:
type: boolean
default: false

responses:
'200':
description: 'An exported stix bundle.'
Expand Down
3 changes: 2 additions & 1 deletion app/controllers/stix-bundles-controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ exports.exportBundle = async function(req, res) {
domain: req.query.domain,
state: req.query.state,
includeRevoked: req.query.includeRevoked,
includeDeprecated: req.query.includeDeprecated
includeDeprecated: req.query.includeDeprecated,
includeMissingAttackId: req.query.includeMissingAttackId
};

try {
Expand Down
93 changes: 93 additions & 0 deletions app/lib/linkById.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
'use strict';

const AttackObject = require('../models/attack-object-model');

// Default implmentation. Retrieves the attack object from the database.
async function getAttackObjectFromDatabase(attackId) {
const attackObject = await AttackObject
.findOne({ 'workspace.attack_id': attackId })
.sort('-stix.modified')
.lean()
.exec();

return attackObject;
}
exports.getAttackObjectFromDatabase = getAttackObjectFromDatabase;

const sourceNames = ['mitre-attack', 'mitre-mobile-attack', 'mobile-attack', 'mitre-ics-attack'];
function attackReference(externalReferences) {
if (Array.isArray(externalReferences) && externalReferences.length > 0) {
return externalReferences.find(ref => sourceNames.includes(ref.source_name));
}
else {
return null;
}
}

const linkByIdRegex = /\(LinkById: ([A-Z]+[0-9]+(\.[0-9]+)?)\)/g;
async function convertLinkById(text, getAttackObject) {
if (text) {
let convertedText = '';
let lastIndex = 0;

const matches = text.matchAll(linkByIdRegex);
for (const match of matches) {
const prefix = text.slice(lastIndex, match.index);
const attackId = match[1];
const citation = {
name: 'linked object not found',
url: ''
}

if (attackId) {
const attackObject = await getAttackObject(attackId);
if (attackObject) {
citation.name = attackObject.stix.name;
const reference = attackReference(attackObject?.stix.external_references);
if (reference) {
citation.url = reference.url;
}
}
}

convertedText = convertedText.concat(prefix, `[${ citation.name }](${ citation.url })`);
lastIndex = match.index + match[0].length;
}

const postText = text.slice(lastIndex);
convertedText = convertedText.concat(postText);

return convertedText;
}
else {
return text;
}
}

async function convertExternalReferencesWithLinkById(externalReferences, getAttackObject) {
if (Array.isArray(externalReferences)) {
for (const externalReference of externalReferences) {
externalReference.description = await convertLinkById(externalReference.description, getAttackObject);
}
}
}


// If provided, getAttackObject() must be an async function with the signature:
// getAttackObject(attackId) returning an attack object
async function convertLinkByIdTags (stixObject, getAttackObject) {
if (!(getAttackObject instanceof Function)) {
getAttackObject = getAttackObjectFromDatabase;
}

if (stixObject) {
stixObject.description = await convertLinkById(stixObject.description, getAttackObject);

if (stixObject.type === 'attack-pattern') {
stixObject.x_mitre_detection = await convertLinkById(stixObject.x_mitre_detection, getAttackObject);
}

await convertExternalReferencesWithLinkById(stixObject.external_references, getAttackObject);
}
}
exports.convertLinkByIdTags = convertLinkByIdTags;
86 changes: 73 additions & 13 deletions app/services/stix-bundles-service.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,23 @@
const uuid = require('uuid');

const AttackObject = require('../models/attack-object-model');
const Group = require('../models/group-model');
const Matrix = require('../models/matrix-model');
const Mitigation = require('../models/mitigation-model');
const Note = require('../models/note-model');
const Relationship = require('../models/relationship-model');
const Software = require('../models/software-model');
const Tactic = require('../models/tactic-model');
const Technique = require('../models/technique-model');

const systemConfigurationService = require('./system-configuration-service');
const linkById = require('../lib/linkById');

const errors = {
notFound: 'Domain not found',
};
exports.errors = errors;

// Retrieve the attack object from the database using its STIX ID
async function getAttackObject(stixId) {
const attackObject = await AttackObject
.findOne({ 'stix.id': stixId })
Expand All @@ -30,7 +32,34 @@ async function getAttackObject(stixId) {
return attackObject;
}

const attackIdObjectTypes = ['intrusion-set', 'malware', 'tool', 'attack-pattern', 'course-of-action', 'x-mitre-data_source'];
function requiresAttackId(attackObject) {
return attackIdObjectTypes.includes(attackObject?.stix.type);
}

const sourceNames = ['mitre-attack', 'mitre-mobile-attack', 'mobile-attack', 'mitre-ics-attack'];
function hasAttackId(attackObject) {
const externalReferences = attackObject?.stix.external_references;
if (Array.isArray(externalReferences) && externalReferences.length > 0) {
const mitreAttackReference = externalReferences.find(ref => sourceNames.includes(ref.source_name));
if (mitreAttackReference?.external_id) {
return true;
}
}

return false;
}

exports.exportBundle = async function(options) {
// The attackObjectMap maps attack IDs to attack objects and is used to make the LinkById conversion
// more efficient.
const attackObjectMap = new Map();
function addAttackObjectToMap(attackObject) {
if (attackObject?.workspace.attack_id) {
attackObjectMap.set(attackObject.workspace.attack_id, attackObject);
}
}

// Create the bundle to hold the exported objects
const bundle = {
type: 'bundle',
Expand Down Expand Up @@ -75,7 +104,6 @@ exports.exportBundle = async function(options) {
];

// Retrieve the primary objects
const domainGroups = await Group.aggregate(aggregation);
const domainMitigations = await Mitigation.aggregate(aggregation);
const domainSoftware = await Software.aggregate(aggregation);
const domainTactics = await Tactic.aggregate(aggregation);
Expand All @@ -87,17 +115,23 @@ exports.exportBundle = async function(options) {
const allMatrices = await Matrix.aggregate(matrixAggregation);
const domainMatrices = allMatrices.filter(matrix => matrix?.stix?.external_references.length && matrix.stix.external_references[0].external_id === options.domain);

const primaryObjects = [...domainGroups, ...domainMatrices, ...domainMitigations, ...domainSoftware, ...domainTactics, ...domainTechniques];
let primaryObjects = [...domainMatrices, ...domainMitigations, ...domainSoftware, ...domainTactics, ...domainTechniques];

// No primary objects means that the domain doesn't exist
// Return an empty bundle
if (primaryObjects.length === 0) {
return bundle;
}

// Remove any primary objects that don't have an ATT&CK ID
if (!options.includeMissingAttackId) {
primaryObjects = primaryObjects.filter(o => hasAttackId(o));
}

// Put the primary objects in the bundle
for (const primaryObject of primaryObjects) {
bundle.objects.push(primaryObject.stix);
addAttackObjectToMap(primaryObject);
}

// Get the relationships that point at primary objects (removing duplicates)
Expand Down Expand Up @@ -147,20 +181,25 @@ exports.exportBundle = async function(options) {
bundle.objects.push(relationship.stix);
}

function secondaryObjectIsValid(secondaryObject) {
// The object must exist and at least one of these conditions must apply:
// 1) The request allows objects that are missing an ATT&CK ID
// 2) The objects is a type that doesn't require an ATT&CK ID
// 3) The object has an ATT&CK ID
return (secondaryObject && (options.includeMissingAttackId || !requiresAttackId(secondaryObject) || hasAttackId(secondaryObject)));
}

// Get the secondary objects (additional objects pointed to by a relationship)
const secondaryObjects = [];
const dataComponents = new Map();
for (const relationship of relationships) {
// Check source_ref
if (!objectsMap.has(relationship.stix.source_ref)) {
const secondaryObject = await getAttackObject(relationship.stix.source_ref);
if (secondaryObject) {
if (secondaryObjectIsValid(secondaryObject)) {
secondaryObjects.push(secondaryObject);
objectsMap.set(secondaryObject.stix.id, true);
}
else {
console.log(`Could not find secondary object with id ${ relationship.stix.source_ref }`);
}

// Save data components for later
if (relationship.stix.relationship_type === 'detects') {
Expand All @@ -171,19 +210,21 @@ exports.exportBundle = async function(options) {
// Check target_ref
if (!objectsMap.has(relationship.stix.target_ref)) {
const secondaryObject = await getAttackObject(relationship.stix.target_ref);
if (secondaryObject) {
if (secondaryObjectIsValid(secondaryObject)) {
secondaryObjects.push(secondaryObject);
objectsMap.set(secondaryObject.stix.id, true);
}
else {
console.log(`Could not find secondary object with id ${ relationship.stix.target_ref }`);
}
}
}

// Put the secondary objects in the bundle
for (const secondaryObject of secondaryObjects) {
// Groups need to have the domain added to x_mitre_domains
if (secondaryObject.stix.type === 'intrusion-set') {
secondaryObject.stix.x_mitre_domains = [ options.domain ];
}
bundle.objects.push(secondaryObject.stix);
addAttackObjectToMap(secondaryObject);
}

// Data components have already been added to the bundle because they're referenced in a relationship
Expand All @@ -199,8 +240,11 @@ exports.exportBundle = async function(options) {
const dataSources = new Map();
for (const dataSourceId of dataSourceIds.keys()) {
const dataSource = await getAttackObject(dataSourceId);
bundle.objects.push(dataSource.stix);
dataSources.set(dataSourceId, dataSource.stix);
if (options.includeMissingAttackId || hasAttackId(dataSource)) {
bundle.objects.push(dataSource.stix);
dataSources.set(dataSourceId, dataSource.stix);
addAttackObjectToMap(dataSource);
}
}

// Create a map of techniques detected by data components
Expand Down Expand Up @@ -317,6 +361,22 @@ exports.exportBundle = async function(options) {
bundle.objects.push(note.stix);
}

// Create the function to be used by the LinkById conversion process
// Note that using this map instead of database retrieval results in a
// dramatic performance improvement.
const getAttackObjectFromMap = async function (attackId) {
let attackObject = attackObjectMap.get(attackId);
if (!attackObject) {
attackObject = await linkById.getAttackObjectFromDatabase(attackId);
}
return attackObject;
}

// Convert LinkById tags into markdown citations
for (const attackObject of bundle.objects) {
await linkById.convertLinkByIdTags(attackObject, getAttackObjectFromMap);
}

// Make a list of identities referenced
const identitiesMap = new Map();
for (const bundleObject of bundle.objects) {
Expand Down
Loading

0 comments on commit e8356b9

Please sign in to comment.