Skip to content

Commit

Permalink
Merge pull request #1499 from Esri/webtool-id-check
Browse files Browse the repository at this point in the history 
Webtool id check
  • Loading branch information
@jmhauck
jmhauck authored Sep 11, 2024
2 parents f0e6257 + 7f30bc2 commit d57eb99
Show file tree
Hide file tree
Showing 12 changed files with 225 additions and 250 deletions.
4 changes: 4 additions & 0 deletions packages/common/src/resources/getItemResourcesPaths.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,10 @@ export function getItemResourcesPaths(
}
}

// GP Services
if (itemTemplate.type === "Geoprocessing Service") {
return res.indexOf("webtoolDefinition") > -1 || res.indexOf(".json") < 0;
}
return result;
});
// create the filePaths
Expand Down
32 changes: 0 additions & 32 deletions packages/common/src/restHelpersGet.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,38 +233,6 @@ export function getFilenameFromUrl(url: string): string {
return iFilenameStart < iParamsStart ? url.substring(iFilenameStart, iParamsStart) : "";
}

/**
* Verify if id is a valid item
*
* @param id potential item id
* @param srcAuthentication Credentials for requests to source items
* @returns boolean
*/
export function isItem(id: string, srcAuthentication: UserSession): Promise<boolean> {
return new Promise<boolean>((resolve) => {
getItemBase(id, srcAuthentication).then(
() => resolve(true),
() => resolve(false),
);
});
}

/**
* Verify if id is a valid group
*
* @param id potential group id
* @param srcAuthentication Credentials for requests to source items
* @returns boolean
*/
export function isGroup(id: string, srcAuthentication: UserSession): Promise<boolean> {
return new Promise<boolean>((resolve) => {
getGroupBase(id, srcAuthentication).then(
() => resolve(true),
() => resolve(false),
);
});
}

/**
* Gets the primary information of an AGO group.
*
Expand Down
17 changes: 16 additions & 1 deletion packages/common/test/mocks/templates.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -975,7 +975,7 @@ export function getItemTemplateData(type: string): any {
trusted: true,
},
cell_type: "code",
source: "{{3b927de78a784a5aa3981469d85cf45d.itemId}}",
source: "3b927de78a784a5aa3981469d85cf45d",
execution_count: null,
outputs: [],
},
Expand Down Expand Up @@ -1292,6 +1292,21 @@ export function getItemTemplateResourcesAsSourceFiles(type: string, itemId: stri
folder: itemId + "/info",
filename: "webtoolDefinition.json",
},
{
itemId,
file: utils.getSampleImageAsFile("ago_downloaded.png"),
folder: "_info_thumbnail",
filename: "ago_downloaded.png",
},
{
itemId,
file: generalHelpers.jsonToFile(
hasService ? sampleWebToolJson : emptySampleWebToolJson,
"webtoolService.json",
),
folder: itemId + "/info",
filename: "webtoolService.json",
},
];
break;
}
Expand Down
52 changes: 52 additions & 0 deletions packages/common/test/resources/getItemResourcesPaths.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,58 @@ describe("getItemResourcesPaths :: ", () => {
});
});

it("filters out geoprocessing service resources", () => {
const getResSpy = spyOn(restHelpersModule, "getItemResources").and.resolveTo({
total: 4,
start: 1,
num: 0,
nextStart: -1,
resources: [
{
resource: "some-image.jpeg",
created: 1591306005000,
size: 207476,
access: "inherit",
},
{
resource: "webtool.json",
created: 1591306005000,
size: 13850,
access: "inherit",
},
],
});

const itemTemplate: IItemTemplate = templates.getItemTemplateSkeleton();
itemTemplate.itemId = "bc3";
itemTemplate.type = "Geoprocessing Service";

return getItemResourcesPaths(itemTemplate, "4de", MOCK_USER_SESSION, 1).then((response) => {
expect(Array.isArray(response)).withContext("should return an array").toBe(true);
expect(response.length).withContext("filter out webtool.json").toBe(2);

expect(response).toEqual(
[
{
itemId: "bc3",
url: "https://myorg.maps.arcgis.com/sharing/rest/content/items/bc3/resources/some-image.jpeg",
folder: "bc3",
filename: "some-image.jpeg",
},
{
itemId: "bc3",
url: "https://myorg.maps.arcgis.com/sharing/rest/content/items/bc3/info/metadata/metadata.xml",
folder: "bc3_info_metadata",
filename: "metadata.xml",
},
],
"should return full path of the file in the storage item",
);
expect(getResSpy.calls.count()).withContext("should get resources").toBe(1);
expect(getResSpy.calls.argsFor(0)[0]).withContext("should get resources for template item").toBe("bc3");
});
});

describe("getItemResourcesPaths, template version 0", () => {
it("can get item resources paths for quick capture project", async () => {
const itemTemplate: IItemTemplate = templates.getItemTemplateSkeleton();
Expand Down
76 changes: 26 additions & 50 deletions packages/creator/src/createItemTemplate.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ import {
getItemResourcesFilesFromPaths,
getItemResourcesPaths,
hasDatasource,
isItem,
isGroup,
jsonToFile,
replaceTemplate,
sanitizeJSON,
Expand Down Expand Up @@ -230,6 +228,8 @@ export function createItemTemplate(
resourceItemFiles.map((file: ISourceFile) => file.folder + "/" + file.filename),
);

itemTemplate = postProcessGPResources(itemTemplate);

// Set the value keyed by the id to the created template, replacing the placeholder template
replaceTemplate(existingTemplates, itemTemplate.itemId, itemTemplate);

Expand Down Expand Up @@ -295,6 +295,22 @@ export function createItemTemplate(
});
}

/**
* Remove all *.json resources from Geoprocessing Service
* This needs to be done after fetched so we can read from one of the files
*
* @param template The current template
* @returns The updated template
*/
export function postProcessGPResources(template: IItemTemplate): IItemTemplate {
if (template.type === "Geoprocessing Service") {
template.resources = template.resources.filter((r) => {
return r.indexOf(".json") < 0;
});
}
return template;
}

/**
* Templatizes field references within specific template types.
* Currently only handles web applications
Expand Down Expand Up @@ -537,59 +553,19 @@ export function _templatizeResources(
synchronizePromises.push(
new Promise((resolve) => {
// Read the file
// eslint-disable-next-line @typescript-eslint/no-floating-promises
blobToJson(rootFileResource.file).then((fileJson) => {
if (rootFileResource.filename.indexOf("webtoolDefinition") > -1) {
if (rootFileResource.filename.indexOf("webtoolDefinition") > -1) {
// eslint-disable-next-line @typescript-eslint/no-floating-promises
blobToJson(rootFileResource.file).then((fileJson) => {
itemTemplate.data = {
name: fileJson.jsonProperties.tasks[0].name,
notebookId: fileJson.jsonProperties.notebookId,
timeoutInMinutes: fileJson.jsonProperties.timeoutInMinutes,
};
}

const idTest: RegExp = /[0-9A-F]{32}/gim;
let dataString = JSON.stringify(fileJson);
if (fileJson && idTest.test(dataString)) {
const ids: string[] = dataString.match(idTest) as string[];
const promises = [];
const idLookup = [];
ids.forEach((id) => {
if (idLookup.indexOf(id) === -1) {
idLookup.push(id);
promises.push(isItem(id, srcAuthentication));
idLookup.push(id);
promises.push(isGroup(id, srcAuthentication));
}
});

// eslint-disable-next-line @typescript-eslint/no-floating-promises
Promise.all(promises).then((results) => {
const verifiedIds = [];
results.forEach((isValid, i) => {
if (isValid) {
const id: string = idLookup[i];
if (id && verifiedIds.indexOf(id) < 0) {
// templatize the itemId--but only once per unique id
const regEx = new RegExp(id, "gm");
dataString = dataString.replace(regEx, "{{" + id + ".itemId}}");

// update the dependencies
if (itemTemplate.dependencies.indexOf(id) === -1) {
itemTemplate.dependencies.push(id);
}
verifiedIds.push(id);
}
}
});
const updatedFileJson = JSON.parse(dataString);
rootFileResource.file = jsonToFile(updatedFileJson, rootFileResource.filename);
resolve(null);
});
} else {
const updatedFileJson = JSON.parse(dataString);
rootFileResource.file = jsonToFile(updatedFileJson, rootFileResource.filename);
resolve(null);
}
});
});
} else {
resolve(null);
}
}),
);
});
Expand Down
4 changes: 4 additions & 0 deletions packages/creator/src/helpers/add-content-to-solution.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import { getProp, getWithDefault } from "@esri/hub-common";
import { createItemTemplate, postProcessFieldReferences } from "../createItemTemplate";
import * as form from "@esri/solution-form";
import { getDataFilesFromTemplates, removeDataFilesFromTemplates } from "./template";
import { notebookProcessor } from "@esri/solution-simple-types";

// ------------------------------------------------------------------------------------------------------------------ //

Expand Down Expand Up @@ -185,6 +186,9 @@ export function addContentToSolution(
solutionTemplates = _postProcessIgnoredItems(solutionTemplates, templateDictionary);
const templateIds = solutionTemplates.map((template) => template.itemId);

// check notebooks data for any item or group references
notebookProcessor.postProcessNotebookTemplates(solutionTemplates, templateDictionary);

// Extract resource data files from templates
resourceItemFiles = resourceItemFiles.concat(getDataFilesFromTemplates(solutionTemplates));

Expand Down
35 changes: 1 addition & 34 deletions packages/creator/test/createItemTemplate.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -901,39 +901,6 @@ describe("Module `createItemTemplate`", () => {
expect(stylesRootJson).toEqual(templates.sampleStylesRootTemplatizedJson);
});

it("handles a Geoprocessing Service resources list", async () => {
const itemTemplate: common.IItemTemplate = templates.getItemTemplate("Geoprocessing Service");
const resourceItemFiles: common.ISourceFile[] = templates.getItemTemplateResourcesAsSourceFiles(
"Geoprocessing Service",
itemTemplate.itemId,
);

const idA = "aaaeefbeb43245ccbe00a948e87ccdfa";
const idB = "aaa637ded3a74a7f9c2325a043f59fb6";

const fsResponse = mockItems.getAGOLItem("Feature Service");
const grpResponse = mockItems.getAGOLGroup(idA);

fetchMock
.get(
utils.PORTAL_SUBSET.restUrl + `/content/items/${idA}?f=json&token=fake-token`,
mockItems.get400FailureResponse(),
)
.get(utils.PORTAL_SUBSET.restUrl + `/community/groups/${idA}?f=json&token=fake-token`, grpResponse)
.get(utils.PORTAL_SUBSET.restUrl + `/content/items/${idB}?f=json&token=fake-token`, fsResponse)
.get(
utils.PORTAL_SUBSET.restUrl + `/community/groups/${idB}?f=json&token=fake-token`,
mockItems.get400FailureResponse(),
);

await createItemTemplate._templatizeResources(itemTemplate, resourceItemFiles, MOCK_USER_SESSION);
expect(resourceItemFiles.length).toEqual(1);

// Check file contents
const infoRootJson = await common.blobToJson(resourceItemFiles[0].file);
expect(infoRootJson).toEqual(templates.sampleWebToolTemplatizedJson);
});

it("handles a Geoprocessing Service with no item ids", async () => {
const itemTemplate: common.IItemTemplate = templates.getItemTemplate("Geoprocessing Service");
const resourceItemFiles: common.ISourceFile[] = templates.getItemTemplateResourcesAsSourceFiles(
Expand All @@ -943,7 +910,7 @@ describe("Module `createItemTemplate`", () => {
);

await createItemTemplate._templatizeResources(itemTemplate, resourceItemFiles, MOCK_USER_SESSION);
expect(resourceItemFiles.length).toEqual(1);
expect(resourceItemFiles.length).toEqual(3);
});
});
});
Expand Down
5 changes: 4 additions & 1 deletion packages/simple-types/src/helpers/convert-item-to-template.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@ export function convertItemToTemplate(
// Fetch all of the resources to get the config
resourcesPromise = common.getItemResourcesFiles(itemTemplate.itemId, srcAuthentication);
break;
case "Geoprocessing Service":
itemTemplate.item.url = undefined;
break;
}

// Errors are handled as resolved empty values; this means that there's no `reject` clause to handle, hence:
Expand Down Expand Up @@ -121,7 +124,7 @@ export function convertItemToTemplate(
dashboard.convertItemToTemplate(itemTemplate, templateDictionary);
break;
case "Notebook":
templateModifyingPromise = notebook.convertNotebookToTemplate(itemTemplate, srcAuthentication);
templateModifyingPromise = notebook.convertNotebookToTemplate(itemTemplate);
break;
case "Oriented Imagery Catalog":
templateModifyingPromise = oic.convertItemToTemplate(itemTemplate, destAuthentication, srcAuthentication);
Expand Down
Loading

0 comments on commit d57eb99

Please sign in to comment.