Merge pull request #1501 from Esri/gp-service

Gp service updates

packages/common/src/index.ts

@@ -45,6 +45,7 @@ export * from "./sharing";
 export * from "./templatization";
 export * from "./trackingHelpers";
 export * from "./velocityHelpers";
+export * from "./webtoolHelpers";
 export * from "./workflowHelpers";
 export * from "./workforceHelpers";
 export * from "./zip-utils";

packages/common/src/resources/getItemResourcesPaths.ts

@@ -76,10 +76,6 @@ export function getItemResourcesPaths(
           }
         }
 
-        // GP Services
-        if (itemTemplate.type === "Geoprocessing Service") {
-          return res.indexOf("webtoolDefinition") > -1 || res.indexOf(".json") < 0;
-        }
         return result;
       });
     // create the filePaths

packages/common/src/webtoolHelpers.ts

@@ -0,0 +1,112 @@
+/** @license
+ * Copyright 2024 Esri
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { IItemTemplate } from "./interfaces";
+import { globalStringReplace } from "./generalHelpers";
+
+/**
+ * Store any web tool urls in the templateDictionary so we can use them to search other items
+ * after they have been converted to templates
+ *
+ * @param template the current template
+ * @param templateDictionary Hash of facts: folder id, org URL, adlib replacements
+ *
+ */
+export function preProcessWebTool(template: IItemTemplate, templateDictionary: any): void {
+  if (template.type === "Geoprocessing Service") {
+    const url = template.item.origUrl;
+    const urlVar = `{{${template.itemId}.url}}`;
+    templateDictionary[url] = urlVar;
+  }
+}
+
+/**
+ * Use any stored GPServer urls to search all other templates for potential references
+ * This will allow us to replace the base server name as will as the itemId
+ *
+ * @param templates the list of all the templates in the solution
+ * @param templateDictionary Hash of facts: folder id, org URL, adlib replacements
+ *
+ * @returns potentially updated list of all the templates in the solution
+ */
+export function postProcessWebToolReferences(templates: IItemTemplate[], templateDictionary: any): IItemTemplate[] {
+  const gpServerUrls = Object.keys(templateDictionary).filter((k) => k.indexOf("GPServer") > -1);
+
+  if (gpServerUrls.length > 0) {
+    gpServerUrls.forEach((url) => {
+      const itemId = templateDictionary[url].replace("{{", "").replace(".url}}", "");
+
+      _globalTemplatize(templates, url, templateDictionary[url], itemId);
+
+      // handle items that already have a templatized itemId in the url
+      const idTest: RegExp = /[0-9A-F]{32}/gim;
+      const templatizedUrl = url.replace(idTest, `{{${itemId}.itemId}}`);
+      _globalTemplatize(templates, templatizedUrl, templateDictionary[url], itemId);
+    });
+  }
+  return templates;
+}
+
+/**
+ * Use any stored GPServer urls to search all other templates for potential references
+ * This will allow us to replace the base server name as will as the itemId
+ *
+ * @param templates the list of all the templates in the solution
+ * @param orgUrl the item url of the GPServer
+ * @param templatizedUrl the templatized GPServer Url
+ * @param orgItemId the item id of the GPServer
+ *
+ * @returns potentially updated list of all the templates in the solution
+ */
+export function _globalTemplatize(
+  templates: IItemTemplate[],
+  orgUrl: string,
+  templatizedUrl: string,
+  orgItemId: string,
+): IItemTemplate[] {
+  // Cycle through each of the items in the template and scan the `item` and `data` sections of each for replacements
+  templates.forEach((template: IItemTemplate) => {
+    const itemString = JSON.stringify(template.item);
+    const dataString = JSON.stringify(template.data);
+
+    globalStringReplace(template.item, new RegExp(orgUrl, "gi"), templatizedUrl);
+    globalStringReplace(template.data, new RegExp(orgUrl, "gi"), templatizedUrl);
+
+    _updateDependencies(template, itemString, dataString, orgItemId);
+  });
+  return templates;
+}
+
+/**
+ * Update the templates dependencies if we can detect differences after we try and replace a GPServer url
+ *
+ * @param template the current template
+ * @param itemString stringified version of the the templates item before we replaced anything
+ * @param dataString stringified version of the the templates data before we replaced anything
+ * @param id the current item Id of the GPServer
+ *
+ */
+export function _updateDependencies(template: IItemTemplate, itemString: string, dataString: string, id: string): void {
+  const hasItemDepdendency = template.dependencies.indexOf(id) > -1;
+
+  if (itemString && itemString !== JSON.stringify(template.item) && !hasItemDepdendency) {
+    template.dependencies.push(id);
+  }
+
+  if (dataString && dataString !== JSON.stringify(template.data) && !hasItemDepdendency) {
+    template.dependencies.push(id);
+  }
+}

packages/common/test/resources/getItemResourcesPaths.test.ts

@@ -299,58 +299,6 @@ describe("getItemResourcesPaths :: ", () => {
     });
   });
 
-  it("filters out geoprocessing service resources", () => {
-    const getResSpy = spyOn(restHelpersModule, "getItemResources").and.resolveTo({
-      total: 4,
-      start: 1,
-      num: 0,
-      nextStart: -1,
-      resources: [
-        {
-          resource: "some-image.jpeg",
-          created: 1591306005000,
-          size: 207476,
-          access: "inherit",
-        },
-        {
-          resource: "webtool.json",
-          created: 1591306005000,
-          size: 13850,
-          access: "inherit",
-        },
-      ],
-    });
-
-    const itemTemplate: IItemTemplate = templates.getItemTemplateSkeleton();
-    itemTemplate.itemId = "bc3";
-    itemTemplate.type = "Geoprocessing Service";
-
-    return getItemResourcesPaths(itemTemplate, "4de", MOCK_USER_SESSION, 1).then((response) => {
-      expect(Array.isArray(response)).withContext("should return an array").toBe(true);
-      expect(response.length).withContext("filter out webtool.json").toBe(2);
-
-      expect(response).toEqual(
-        [
-          {
-            itemId: "bc3",
-            url: "https://myorg.maps.arcgis.com/sharing/rest/content/items/bc3/resources/some-image.jpeg",
-            folder: "bc3",
-            filename: "some-image.jpeg",
-          },
-          {
-            itemId: "bc3",
-            url: "https://myorg.maps.arcgis.com/sharing/rest/content/items/bc3/info/metadata/metadata.xml",
-            folder: "bc3_info_metadata",
-            filename: "metadata.xml",
-          },
-        ],
-        "should return full path of the file in the storage item",
-      );
-      expect(getResSpy.calls.count()).withContext("should get resources").toBe(1);
-      expect(getResSpy.calls.argsFor(0)[0]).withContext("should get resources for template item").toBe("bc3");
-    });
-  });
-
   describe("getItemResourcesPaths, template version 0", () => {
     it("can get item resources paths for quick capture project", async () => {
       const itemTemplate: IItemTemplate = templates.getItemTemplateSkeleton();

packages/common/test/webtoolHelpers.test.ts

@@ -0,0 +1,49 @@
+/** @license
+ * Copyright 2024 Esri
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Provides tests for functions involving deployment of workflow items via the REST API.
+ */
+
+import * as templates from "../../common/test/mocks/templates";
+import * as webtoolHelpers from "../src/webtoolHelpers";
+
+// ------------------------------------------------------------------------------------------------------------------ //
+
+describe("Module `webtoolHelpers`", () => {
+  describe("postProcessWebToolReferences", () => {
+    it("will globally templatize GPServer references", () => {
+      const url = "http://local/GPServer";
+      const url2 = "http://local2/GPServer";
+      const templateDictionary = {
+        "http://local/GPServer": "{{xxx1f09e3867449d94bc21033032da7f}}.url",
+        "http://local2/GPServer": "{{xxx2f09e3867449d94bc21033032da7f}}.url"
+      };
+
+      const notebookTemplate = templates.getItemTemplate("Notebook");
+      notebookTemplate.data.cells.push({
+        url
+      });
+      notebookTemplate.item.snippet = url2;
+
+      webtoolHelpers.postProcessWebToolReferences([notebookTemplate], templateDictionary);
+
+      expect(notebookTemplate.data.cells[1].url).toBe("{{xxx1f09e3867449d94bc21033032da7f}}.url");
+      expect(notebookTemplate.item.snippet).toBe("{{xxx2f09e3867449d94bc21033032da7f}}.url");
+      expect(notebookTemplate.dependencies.length).toBe(2);
+    });
+  });
+});

packages/creator/src/createItemTemplate.ts

@@ -223,13 +223,13 @@ export function createItemTemplate(
                       // Perform any custom processing needed on resource files
                       await _templatizeResources(itemTemplate, resourceItemFiles, srcAuthentication);
 
+                      resourceItemFiles = postProcessResourceFiles(itemTemplate, resourceItemFiles);
+
                       // update the template's resources
                       itemTemplate.resources = itemTemplate.resources.concat(
                         resourceItemFiles.map((file: ISourceFile) => file.folder + "/" + file.filename),
                       );
 
-                      itemTemplate = postProcessGPResources(itemTemplate);
-
                       // Set the value keyed by the id to the created template, replacing the placeholder template
                       replaceTemplate(existingTemplates, itemTemplate.itemId, itemTemplate);
 
@@ -296,19 +296,15 @@ export function createItemTemplate(
 }
 
 /**
- * Remove all *.json resources from Geoprocessing Service
- * This needs to be done after fetched so we can read from one of the files
+ * Remove webtoolDefinition resource from Geoprocessing Service
+ * This needs to be done after fetched so we can read from the file before we remove it
  *
  * @param template The current template
+ * @param files The list of resource files for the given template
  * @returns The updated template
  */
-export function postProcessGPResources(template: IItemTemplate): IItemTemplate {
-  if (template.type === "Geoprocessing Service") {
-    template.resources = template.resources.filter((r) => {
-      return r.indexOf(".json") < 0;
-    });
-  }
-  return template;
+export function postProcessResourceFiles(template: IItemTemplate, files: ISourceFile[]): ISourceFile[] {
+  return template.type === "Geoprocessing Service" ? files.filter((f) => f.filename.indexOf("webtool") < 0) : files;
 }
 
 /**
@@ -556,9 +552,14 @@ export function _templatizeResources(
           if (rootFileResource.filename.indexOf("webtoolDefinition") > -1) {
             // eslint-disable-next-line @typescript-eslint/no-floating-promises
             blobToJson(rootFileResource.file).then((fileJson) => {
+              const notebookId = fileJson.jsonProperties.notebookId;
+              if (itemTemplate.dependencies.indexOf(notebookId) < 0) {
+                itemTemplate.dependencies.push(notebookId);
+              }
+
               itemTemplate.data = {
                 name: fileJson.jsonProperties.tasks[0].name,
-                notebookId: fileJson.jsonProperties.notebookId,
+                notebookId,
                 timeoutInMinutes: fileJson.jsonProperties.timeoutInMinutes,
               };
               resolve(null);

packages/creator/src/helpers/add-content-to-solution.ts

@@ -36,6 +36,7 @@ import {
   replaceInTemplate,
   SItemProgressStatus,
   copyFilesToStorageItem,
+  postProcessWebToolReferences,
   postProcessWorkforceTemplates,
   UNREACHABLE,
   updateItem,
@@ -189,6 +190,8 @@ export function addContentToSolution(
           // check notebooks data for any item or group references
           notebookProcessor.postProcessNotebookTemplates(solutionTemplates, templateDictionary);
 
+          solutionTemplates = postProcessWebToolReferences(solutionTemplates, templateDictionary);
+
           // Extract resource data files from templates
           resourceItemFiles = resourceItemFiles.concat(getDataFilesFromTemplates(solutionTemplates));
 

packages/simple-types/src/helpers/convert-item-to-template.ts

@@ -82,16 +82,15 @@ export function convertItemToTemplate(
         // Fetch all of the resources to get the config
         resourcesPromise = common.getItemResourcesFiles(itemTemplate.itemId, srcAuthentication);
         break;
-      case "Geoprocessing Service":
-        itemTemplate.item.url = undefined;
-        break;
     }
 
     // Errors are handled as resolved empty values; this means that there's no `reject` clause to handle, hence:
     // eslint-disable-next-line @typescript-eslint/no-floating-promises
     Promise.all([dataPromise, relatedPromise, resourcesPromise]).then((responses) => {
       const [itemDataResponse, relatedItemsResponse, resourcesResponse] = responses;
 
+      common.preProcessWebTool(itemTemplate, templateDictionary);
+
       // need to pre-process for velocity urls before they could be templatized by other processors
       itemTemplate.data = common.updateVelocityReferences(itemDataResponse, itemInfo.type, templateDictionary);
       const relationships = relatedItemsResponse;