Skip to content

Commit

Permalink
feat(qseow): Always look for circular task chains in qseow task-vis
Browse files Browse the repository at this point in the history
… command

Implements ptarmiganlabs#597
  • Loading branch information
mountaindude committed Jan 3, 2025
1 parent 042891a commit 1320df2
Show file tree
Hide file tree
Showing 9 changed files with 206 additions and 88 deletions.
46 changes: 37 additions & 9 deletions src/lib/cmd/qseow/vistask.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import sea from 'node:sea';

import { appVersion, logger, setLoggingLevel, isSea, execPath, verifyFileSystemExists, verifySeaAssetExists } from '../../../globals.js';
import { QlikSenseTasks } from '../../task/class_alltasks.js';
import { findCircularTaskChains } from '../../task/find_circular_task_chain.js';
import { catchLog } from '../../util/log.js';

// js: 'application/javascript',
const MIME_TYPES = {
Expand Down Expand Up @@ -550,10 +552,6 @@ export async function visTask(options) {
// Filters above are additive, i.e. all tasks that match any of the filters are included in the network diagram.
// Make sure to de-duplicate root tasks.

// Arrays to keep track of which nodes in task model to visualize
// const nodesToVisualize = [];
// const edgesToVisualize = [];

// If no task id or tag filters specified, visualize all nodes in task model
if (!options.taskId && !options.taskTag) {
// No task id filters specified
Expand Down Expand Up @@ -587,15 +585,45 @@ export async function visTask(options) {

// Get all nodes that are children of the root nodes
const { nodes, edges, tasks } = await qlikSenseTasks.getNodesAndEdgesFromRootNodes(rootNodes);
// nodesToVisualize.push(...nodes);
// edgesToVisualize.push(...edges);
// tasksToVisualize.push(...tasks);

// taskNetwork = { nodes: nodesToVisualize, edges: edgesToVisualize, tasks)

taskNetwork = { nodes, edges, tasks };
}

// Look for circular task chains in the task network
logger.info('');
logger.info('Looking for circular task chains in the task network');

try {
const circularTaskChains = findCircularTaskChains(taskNetwork, logger);

// Errros?
if (circularTaskChains === false) {
return false;
}

// De-duplicate circular task chains (where fromTask.id and toTask.id matches in two different chains).
const deduplicatedCircularTaskChain = circularTaskChains.filter((chain, index, self) => {
return self.findIndex((c) => c.fromTask.id === chain.fromTask.id && c.toTask.id === chain.toTask.id) === index;
});

// Log circular task chains, if any were found.
if (deduplicatedCircularTaskChain?.length > 0) {
logger.warn('');
logger.warn(`Found ${deduplicatedCircularTaskChain.length} circular task chains in task model`);
for (const chain of deduplicatedCircularTaskChain) {
logger.warn(`Circular task chain:`);

logger.warn(` From task : [${chain.fromTask.id}] "${chain.fromTask.taskName}"`);
logger.warn(` To task : [${chain.toTask.id}] "${chain.toTask.taskName}"`);
}
} else {
logger.info('No circular task chains found in task model');
}
} catch (error) {
catchLog('FIND CIRCULAR TASK CHAINS', error);
return false;
}

// Add additional values to Handlebars template
templateData.visTaskHost = options.visHost;
templateData.visTaskPort = options.visPort;
Expand Down
36 changes: 6 additions & 30 deletions src/lib/task/class_alltasks.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import axios from 'axios';
import { v4 as uuidv4, validate } from 'uuid';

import { logger } from '../../globals.js';
import { setupQrsConnection } from '../util/qseow/qrs.js';
Expand Down Expand Up @@ -38,7 +37,6 @@ export class QlikSenseTasks {
this.taskIdMap = new Map();

// Data structure to keep track of which up-tree nodes a node is connected to in a task tree or network
this.taskCyclicVisited = new Set();
this.taskCyclicStack = new Set();

// Data structure to keep track of which nodes have been visited when looking for root nodes
Expand All @@ -63,18 +61,6 @@ export class QlikSenseTasks {
}
}

// Function to determine if a task tree is cyclic
// Uses a depth-first search algorithm to determine if a task tree is cyclic
isTaskTreeCyclic(task) {
if (this.taskCyclicVisited.has(task)) {
return true;
}

this.taskCyclicVisited.add(task);

return false;
}

getTask(taskId) {
if (taskId === undefined || taskId === null) {
return false;
Expand Down Expand Up @@ -115,22 +101,6 @@ export class QlikSenseTasks {
findRootNodes(node) {
const result = extFindRootNodes(this, node, logger);

// logger.verbose(`Root node count: ${result.length}`);
// Log root node and name
// for (const rootNode of result) {
// // Meta node?
// if (rootNode.metaNode === true) {
// // Reload task?
// if (rootNode.taskType === 'reloadTask') {
// logger.verbose(
// `Meta node: metanode type=${rootNode.metaNodeType} id=[${rootNode.id}] task type=${rootNode.taskType} task name="${rootNode.completeSchemaEvent.reloadTask.name}"`
// );
// }
// } else {
// logger.verbose(`Root node: [${rootNode.id}] "${rootNode.taskName}"`);
// }
// }

return result;
}

Expand Down Expand Up @@ -403,6 +373,12 @@ export class QlikSenseTasks {
);
});

// De-duplicate edges.
// Edges are identical if they connect the same two nodes, i.e. have the same from and to properties.
edgesFound = edgesFound.filter((edge, index, self) => {
return index === self.findIndex((t) => t.from === edge.from && t.to === edge.to);
});

return { nodes: nodesFound, edges: edgesFound, tasks: tasksFound };
}
}
125 changes: 125 additions & 0 deletions src/lib/task/find_circular_task_chain.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
import { catchLog } from '../util/log.js';

/**
* Recursively detect circular task chains starting from a given task node.
* For the provided node, get all downstream nodes and associated edges, then recursively investigate each of those downstream nodes.
* Examine the network top down, from current node to downstream node(s).
*
* @param {object} taskNetwork - Complete task network object with properties:
* - nodes: An array of task nodes.
* - edges: An array of task edges.
* - tasks: An array of task objects.
* @param {object} node - Task node object to start the search from.
* @param {Set} visitedNodes - Set of node IDs that have been visited.
* @param {object} logger - Logger object for logging information.
*
* @returns {Array} An array of objects, each with properties (which are all objects):
* - fromTask: The source node where the circular dependency was detected.
* - toTask: The target node where the circular dependency was detected.
* - edge: The edge connecting the two tasks.
*
* Returns false if something went wrong.
*/
function recursiveFindCircularTaskChain(taskNetwork, node, visitedNodes, logger) {
try {
const circularTaskChain = [];

// Get downstream nodes. Downstream nodes are identified by node.id === edge.from.
// The downstream node id is then found in edge.to.
const downstreamEdges = taskNetwork.edges.filter((edge) => {
return edge.from === node.id;
});

// Any downstream edges found?
if (downstreamEdges?.length > 0) {
for (const downstreamEdge of downstreamEdges) {
// Get downstream node
const downstreamNode = taskNetwork.nodes.find((n) => n.id === downstreamEdge.to);
// If the task network is correctly defined in nodes and edges, the downstream node should always be found.
// If not, there is an error in the task network definition.
if (downstreamNode === undefined) {
logger.error(`DOWNSTREAM NODE NOT FOUND: ${downstreamEdge.to}`);
continue;
}

// Check if the downstream node has been visited before.
// If so, a circular dependency has been detected.
if (visitedNodes.has(downstreamNode.id)) {
circularTaskChain.push({ fromTask: node, toTask: downstreamNode, edge: downstreamEdge });
return circularTaskChain;
}

// Add downstream node to visited nodes.
// visitedNodes.add(downstreamNode.id);
visitedNodes.add(node.id);

// Recursively investigate downstream node.
const result = recursiveFindCircularTaskChain(taskNetwork, downstreamNode, visitedNodes, logger);

if (result?.length > 0) {
circularTaskChain.push(...result);
}
}
}

return circularTaskChain;
} catch (err) {
catchLog('FIND CIRCULAR TASK CHAINS', err);
return false;
}
}

/**
* Detect circular task chains within the task network.
*
* @param {object} taskNetwork - Task network object with properties:
* - nodes: An array of task nodes.
* - edges: An array of task edges.
* - tasks: An array of task objects.
* @param {object} logger - Logger object for logging information.
* @returns {Array} An array of objects representing circular task chains, each with properties:
* - fromTask: The source node where the circular dependency was detected.
* - toTask: The target node where the circular dependency was detected.
* - edge: The edge connecting the two tasks.
*
* Returns false if something went wrong.
*/
export function findCircularTaskChains(taskNetwork, logger) {
const circularTaskChains = [];

try {
// Get all root nodes in task network.
// Root nodes are nodes meeting either of the following criteria:
// - node's isTopLevelNode property is true.
const rootNodes = taskNetwork.nodes.filter((node) => node.isTopLevelNode);

if (!rootNodes) {
logger.error('Could not find root nodes in task model');
return false;
}

// Log number of root nodes found.
logger.info(`Found ${rootNodes.length} root nodes in task model`);

// For each root node, find circular task chains.
// Add all found circular task chains to the circularTaskChains array
for (const rootNode of rootNodes) {
// Returns array of zero of more objects describing circular task chains, or false if something went wrong.
const result = recursiveFindCircularTaskChain(taskNetwork, rootNode, new Set(), logger);

if (result) {
circularTaskChains.push(...result);
} else {
logger.error('Error when looking for circular task chains in task model');
return false;
}
}

return circularTaskChains;
} catch (err) {
catchLog('FIND CIRCULAR TASK CHAINS', err);
return false;
}

return circularTaskChains;
}
2 changes: 2 additions & 0 deletions src/lib/task/find_root_nodes.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { catchLog } from '../util/log.js';

export function extFindRootNodes(_, node, logger, visitedNodes = new Set()) {
const rootNodes = [];

Expand Down
1 change: 1 addition & 0 deletions src/lib/task/get_task_model_from_file.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { mapTaskType } from '../util/qseow/lookups.js';
import { catchLog } from '../util/log.js';

export async function extGetTaskModelFromFile(_, tasksFromFile, tagsExisting, cpExisting, options, logger) {
return new Promise(async (resolve, reject) => {
Expand Down
Loading

0 comments on commit 1320df2

Please sign in to comment.