Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Data Analytics code generation #1872

Draft
wants to merge 27 commits into
base: main
Choose a base branch
from
Draft

Data Analytics code generation #1872

Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
9b8d384
Generate data per realm and create seperate data files + added handli…
t-scholtz Jun 18, 2024
36593ce
Added simple testing for new Python code generation feauture
t-scholtz Jun 20, 2024
7e25e5f
Updated formatting to be more align to style guide
t-scholtz Jun 20, 2024
391f3aa
More stlye fixes
t-scholtz Jun 20, 2024
8452924
Move tests to different spot so that they should now run during testing
t-scholtz Jun 24, 2024
471d180
updating ESlint test formatting
t-scholtz Jun 24, 2024
28cd3e2
Fixed tests
t-scholtz Jun 24, 2024
eae401a
Fixed tests
t-scholtz Jun 24, 2024
ac273d2
Testing tests
t-scholtz Jun 24, 2024
818c992
Updated style + troubleshooting
t-scholtz Jun 24, 2024
3863dc1
Linter stuff
t-scholtz Jun 24, 2024
104e9f4
Linter stuff
t-scholtz Jun 25, 2024
a0be3e8
Updated tests
t-scholtz Jun 25, 2024
4b63858
Updated tests
t-scholtz Jun 26, 2024
39cd9ea
Updated tests
t-scholtz Jun 26, 2024
479bcc8
Updated tests
t-scholtz Jun 26, 2024
d53e9a2
Updated tests
t-scholtz Jun 26, 2024
250003a
Updated tests
t-scholtz Jun 26, 2024
ffcc190
Updated tests
t-scholtz Jun 26, 2024
86480c7
Updated tests
t-scholtz Jun 26, 2024
a60e89a
Responding to requested changes
t-scholtz Aug 2, 2024
174f3fa
Fixing Linter Error
t-scholtz Aug 2, 2024
c692994
Fixing Linter Error
t-scholtz Aug 2, 2024
86b1a04
Fixing Linter Error
t-scholtz Aug 2, 2024
8accca2
Fixing Linter Error
t-scholtz Aug 2, 2024
9952bed
Fixing Linter Error
t-scholtz Aug 2, 2024
a9a02f4
Fixing Linter Error
t-scholtz Aug 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 138 additions & 1 deletion html/gui/js/modules/metric_explorer/MetricExplorer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,124 @@ Ext.apply(XDMoD.Module.MetricExplorer, {

return result;
};
const generatePythonCode = function (config) {
t-scholtz marked this conversation as resolved.
Show resolved Hide resolved
let duration;
if (config.timeframe_label === 'User Defined' && config.start_date && config.end_date) {
duration = `${config.start_date} , ${config.end_date}`;
} else if (config.timeframe_label) {
duration = config.timeframe_label;
} else {
duration = 'Previous Month';
}
const dataType = config.timeseries ? 'timeseries' : 'aggregate';
const aggregationUnit = config.aggregation_unit || 'Auto';
const swapXY = config.swap_xy;
let filters = '';
const filterDict = {};
let subTitle = '';
let dataCalls = 'import pandas as pd\n\n# Call to Data Analytics Framework requesting data \nwith dw:';

for (let i = 0; i < config.global_filters.total; i += 1) {
const { dimension_id: id, value_name: value } = config.global_filters.data[i];
if (filterDict[id]) {
filterDict[id].push(value);
} else {
filterDict[id] = [value];
}
}

for (const id in filterDict) {
t-scholtz marked this conversation as resolved.
Show resolved Hide resolved
if (Object.prototype.hasOwnProperty.call(filterDict, id)) {
const values = filterDict[id].join("', '");
filters += `\n\t\t'${id}': ('${values}'),`;
subTitle += `${id}: ${values.replace(/'/g, '')}`;
}
}

for (let i = 0; i < config.data_series.total; i += 1) {
const {
realm = 'Jobs',
metric = 'CPU Hours: Total',
group_by: dimension = 'none',
log_scale: logScale,
display_type: displayType
} = config.data_series.data[i];
let graphType = displayType || 'line';
let lineShape = '';

if (graphType === 'column') {
graphType = 'bar';
lineShape = "barmode='group',";
} else if (graphType === 'spline') {
graphType = 'line';
lineShape = "\nline_shape='spline',";
} else if (graphType === 'line' && dataType === 'aggregate' && dimension === 'none') {
graphType = 'scatter';
} else if (graphType === 'areaspline') {
graphType = 'area';
lineShape = "\nline_shape='spline',";
}

dataCalls += `\n\tdata_${i} = dw.get_data(`;
dataCalls += `\n\t\tduration=('${duration}'),`;
dataCalls += `\n\t\trealm='${realm}',`;
dataCalls += `\n\t\tmetric='${metric}',`;
dataCalls += `\n\t\tdimension='${dimension}',`;
dataCalls += `\n\t\tfilters={${filters}},`;
dataCalls += `\n\t\tdataset_type='${dataType}',`;
dataCalls += `\n\t\taggregation_unit='${aggregationUnit}',`;
dataCalls += '\n\t)\n';

if (dataType === 'aggregate') {
dataCalls += '\n# Process the data series, combine the lower values into a single Other category, and change to series to a dataframe';
dataCalls += `\n\ttop_ten=data_${i}.nlargest(10)`;
if (graphType === 'pie') {
dataCalls += `\n\tif(data_${i}.size > 10):`;
dataCalls += `\n\t\tothers_sum=data_${i}[~data_${i}.isin(top_ten)].sum()`;
dataCalls += `\n\t\tdata_${i} = top_ten.combine_first(pd.Series({'Other ' + String(data_${i}.size - 10): others_sum}))\n`;
} else {
dataCalls += `\n\tdata_${i} = top_ten`;
}
dataCalls += `\n\tdata_${i} = data_${i}.to_frame()`;
dataCalls += `\n\tcolumns_list = data_${i}.columns.tolist()`;
} else {
dataCalls += '\n# Limit the number of data items/source to at most 10 and sort by descending';
dataCalls += `\n\tcolumns_list = data_${i}.columns.tolist()`;
dataCalls += '\n\tif (columns_list.length > 10):';
dataCalls += `\n\t\tcolumn_sums = data_${i}.sum()`;
dataCalls += '\n\t\ttop_ten_columns = column_sums.nlargest(10).index.tolist()';
dataCalls += `\n\t\tdata_${i} = data_${i}[top_ten_columns]\n`;
}

let axis = '';
if (swapXY && graphType !== 'pie') {
dataCalls += `\n\tdata_${i} = data_${i}.reset_index()`;
axis = `\n\t\ty= data_${i}.columns[0],\n\t\tx= data_${i}.columns[1:],`;
} else {
axis = `\n\t\tlabels={"value": dw.describe_metrics('${realm}').loc['${metric}', 'label']},`;
}

dataCalls += '# Format and draw the graph to the screen\n';
dataCalls += `\n\tplot = px.${graphType}`;
dataCalls += `(\n\t\tdata_${i},`;
if (graphType === 'pie') {
dataCalls += '\n\t\tvalues= columns_list[0],';
dataCalls += `\n\t\tnames= data_${i}.index,`;
}
dataCalls += axis;
dataCalls += `\n\t\ttitle='${config.title || 'Untitled Query'}${subTitle ? `&lt;br&gt;&lt;sup&gt;${subTitle}&lt;/sup&gt` : ''},`;
if (logScale) {
dataCalls += `\n\t\tlog_${swapXY ? 'x' : 'y'}=True,`;
}
dataCalls += `\n\t${lineShape})`;
dataCalls += '\n\tplot.update_layout(';
dataCalls += '\n\t\txaxis_automargin=True,';
dataCalls += '\n\t)';
dataCalls += '\n\tplot.show()\n';
}
return dataCalls;
};

const chartJSON = JSON.stringify(filterConfigForExport(instance.getConfig()), null, 4);
menu.add({
text: 'View chart json',
Expand Down Expand Up @@ -439,6 +557,25 @@ Ext.apply(XDMoD.Module.MetricExplorer, {
win.show();
}
});
menu.add({
text: 'View python code',
iconCls: 'custom_chart',
handler: () => {
const win = new Ext.Window({
title: 'API Code',
width: 800,
height: 600,
layout: 'fit',
autoScroll: true,
closeAction: 'destroy',
items: [{
autoScroll: true,
html: `<pre>Python API code \n************************************************\n<code>${generatePythonCode(instance.getConfig())} </code>\n************************************************<br></br>The link to the data analytisc API can be found <a href="https://github.com/ubccr/xdmod-data" target="_blank">here</a><br></br>Infomation about the Plotly Express Libary can be found <a href="https://plotly.com/python/plotly-express/" target="_blank">here</a><br></br>Example XDmod API Notebooks can be found <a href="https://github.com/ubccr/xdmod-notebooks" target="_blank">here</a></pre>`
}]
});
win.show();
}
});
const chartLayoutJSON = JSON.stringify(instance.plotlyPanel.chartOptions.layout, null, 4);
menu.add({
text: 'View Plotly JS chart layout',
Expand Down Expand Up @@ -2089,7 +2226,7 @@ Ext.extend(XDMoD.Module.MetricExplorer, XDMoD.PortalModule, {
chartLinkButton: true

},

t-scholtz marked this conversation as resolved.
Show resolved Hide resolved
show_filters: true,
show_warnings: true,
font_size: 3,
Expand Down
12 changes: 12 additions & 0 deletions tests/ui/test/specs/xdmod/metricExplorer.js
View file
Open in desktop
t-scholtz marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,18 @@ describe('Metric Explorer', function metricExplorer() {
it('Undo Trend Line looks the same as previous run', function () {
me.checkChart(chartName, 'Node Hours: Total', expected.legend);
});
it('Open Context Menu', () => {
const elems = browser.elements('//div[@id="metric_explorer"]//div[contains(@class, "plot-container")]//*[local-name() = "svg"]/*[name()="g" and contains(@class, "draglayer")]//*[contains(@class, "xy")]//*[contains(@class, "nsewdrag drag")]');
elems.value[0].doubleClick();
browser.waitForVisible(me.selectors.chart.contextMenu.menuByTitle('Chart Options:'));
});
it('Press Genererate code', () => {
browser.click(me.selectors.chart.contextMenu.menuItemByText('Chart Options:', 'View python code'));
});
it('Check for code pop up', () => {
browser.waitUntilNotExist(me.selectors.chart.contextMenu.menuItemByText('Chart Options:', 'View python code'));
browser.waitForVisible('//div[contains(@class, "x-window x-resizable-pinned")]//div[contains(@class, "x-window-tl")]//span[contains(text(),"API Code")]', 10000);
});
});
/* The following tests are disabled until such a time as they can be changed to work
* reliably without browser.pause()
Expand Down