INT-11156: performance upgrade

src/configuration-manager/client.ts

@@ -102,38 +102,42 @@ class MicrosoftConfigurationManagerClient {
     let offset = 0;
     let hasMoreRecords = true;
 
-    while (hasMoreRecords) {
-      const query = buildApplicationDeviceTargetingList(
-        this.dbName,
-        offset,
-        pageSize,
-      );
-      const result = await this.wrapWithRequestFailedHandler(
-        () => this.connection.query(query),
-        {
-          logger,
-          query,
-        },
-      );
-
-      const records = result.recordset;
-
-      logger.info(`Proccesed ${records.length} records from query`);
-
-      if (records.length > 0) {
-        await pMap(
-          records,
-          async (record) => {
-            await iteratee(record);
-          },
+    try {
+      while (hasMoreRecords) {
+        const query = buildApplicationDeviceTargetingList(
+          this.dbName,
+          offset,
+          pageSize,
+        );
+        const result = await this.wrapWithRequestFailedHandler(
+          () => this.connection.query(query),
           {
-            concurrency: 2,
+            logger,
+            query,
           },
         );
-        offset += pageSize;
-      } else {
-        hasMoreRecords = false;
+
+        const records = result.recordset;
+
+        logger.info(`Proccesed ${records.length} records from query`);
+
+        if (records.length > 0) {
+          await pMap(
+            records,
+            async (record) => {
+              await iteratee(record);
+            },
+            {
+              concurrency: 2,
+            },
+          );
+          offset += pageSize;
+        } else {
+          hasMoreRecords = false;
+        }
       }
+    } catch (err) {
+      logger.info(`Received SQL error when processing query: ${err}`);
     }
   }
 
@@ -171,7 +175,7 @@ class MicrosoftConfigurationManagerClient {
   async listCollectionSubscriptions<T>(
     tableName: string,
     iteratee: ResourceIteratee<T>,
-    pageSize: number = 200,
+    pageSize: number = 600,
   ) {
     let offset = 0;
     let hasMoreRecords = true;

src/configuration-manager/queries.ts

@@ -182,6 +182,147 @@ function buildDeviceCollectionQuery(
   WHERE Sub.RowNum BETWEEN ${offset} AND ${offset + limit - 1}`;
 }
 
+// INT-11156 (gastonyelmini): commenting out this query. We are overfetching data when we just need the MachineID.
+
+// function buildCollectionSubscriptionQuery(
+//   dbName: string,
+//   tableName: string,
+//   offset: number,
+//   limit: number,
+// ) {
+//   validateDbName(dbName);
+
+//   return `SELECT [MachineID]
+//       ,[ArchitectureKey]
+//       ,[Name]
+//       ,[SMSID]
+//       ,[SiteCode]
+//       ,[Domain]
+//       ,[ClientEdition]
+//       ,[ClientType]
+//       ,[ClientVersion]
+//       ,[IsDirect]
+//       ,[IsAssigned]
+//       ,[IsClient]
+//       ,[IsVirtualMachine]
+//       ,[IsAOACCapable]
+//       ,[IsObsolete]
+//       ,[IsActive]
+//       ,[IsDecommissioned]
+//       ,[WipeStatus]
+//       ,[RetireStatus]
+//       ,[SyncNowStatus]
+//       ,[LastSyncNowRequest]
+//       ,[ManagementAuthority]
+//       ,[AMTStatus]
+//       ,[AMTFullVersion]
+//       ,[DeviceOwner]
+//       ,[DeviceCategory]
+//       ,[SuppressAutoProvision]
+//       ,[IsApproved]
+//       ,[IsBlocked]
+//       ,[IsAlwaysInternet]
+//       ,[IsInternetEnabled]
+//       ,[ClientCertType]
+//       ,[UserName]
+//       ,[LastClientCheckTime]
+//       ,[ClientCheckPass]
+//       ,[ADSiteName]
+//       ,[UserDomainName]
+//       ,[ADLastLogonTime]
+//       ,[ClientRemediationSuccess]
+//       ,[ClientActiveStatus]
+//       ,[LastStatusMessage]
+//       ,[LastPolicyRequest]
+//       ,[LastDDR]
+//       ,[LastHardwareScan]
+//       ,[LastSoftwareScan]
+//       ,[LastMPServerName]
+//       ,[LastActiveTime]
+//       ,[CP_Status]
+//       ,[CP_LatestProcessingAttempt]
+//       ,[CP_LastInstallationError]
+//       ,[EAS_DeviceID]
+//       ,[DeviceOS]
+//       ,[DeviceOSBuild]
+//       ,[DeviceType]
+//       ,[ExchangeServer]
+//       ,[ExchangeOrganization]
+//       ,[PolicyApplicationStatus]
+//       ,[LastSuccessSyncTimeUTC]
+//       ,[PhoneNumber]
+//       ,[DeviceAccessState]
+//       ,[DeviceThreatLevel]
+//       ,[CoManaged]
+//       ,[PasscodeResetState]
+//       ,[PasscodeResetStateTimeStamp]
+//       ,[RemoteLockState]
+//       ,[RemoteLockStateTimeStamp]
+//       ,[ActivationLockBypassState]
+//       ,[ActivationLockBypassStateTimeStamp]
+//       ,[ActivationLockState]
+//       ,[IsSupervised]
+//       ,[EP_DeploymentState]
+//       ,[EP_DeploymentErrorCode]
+//       ,[EP_DeploymentDescription]
+//       ,[EP_PolicyApplicationState]
+//       ,[EP_PolicyApplicationErrorCode]
+//       ,[EP_PolicyApplicationDescription]
+//       ,[EP_Enabled]
+//       ,[EP_ClientVersion]
+//       ,[EP_ProductStatus]
+//       ,[EP_EngineVersion]
+//       ,[EP_AntivirusEnabled]
+//       ,[EP_AntivirusSignatureVersion]
+//       ,[EP_AntivirusSignatureUpdateDateTime]
+//       ,[EP_AntispywareEnabled]
+//       ,[EP_AntispywareSignatureVersion]
+//       ,[EP_AntispywareSignatureUpdateDateTime]
+//       ,[EP_LastFullScanDateTimeStart]
+//       ,[EP_LastFullScanDateTimeEnd]
+//       ,[EP_LastQuickScanDateTimeStart]
+//       ,[EP_LastQuickScanDateTimeEnd]
+//       ,[EP_InfectionStatus]
+//       ,[EP_PendingFullScan]
+//       ,[EP_PendingReboot]
+//       ,[EP_PendingManualSteps]
+//       ,[EP_PendingOfflineScan]
+//       ,[EP_LastInfectionTime]
+//       ,[EP_LastThreatName]
+//       ,[CNIsOnline]
+//       ,[CNLastOnlineTime]
+//       ,[CNLastOfflineTime]
+//       ,[ClientState]
+//       ,[CNAccessMP]
+//       ,[CNIsOnInternet]
+//       ,[Unknown]
+//       ,[ATP_LastConnected]
+//       ,[ATP_SenseIsRunning]
+//       ,[ATP_OnboardingState]
+//       ,[ATP_OrgId]
+//       ,[CA_IsCompliant]
+//       ,[CA_ComplianceSetTime]
+//       ,[CA_ComplianceEvalTime]
+//       ,[CA_ErrorDetails]
+//       ,[CA_ErrorLocation]
+//       ,[AADTenantID]
+//       ,[AADDeviceID]
+//       ,[SerialNumber]
+//       ,[IMEI]
+//       ,[PrimaryUser]
+//       ,[CurrentlogonUser]
+//       ,[LastLogonUser]
+//       ,[MACAddress]
+//       ,[SMBIOSGUID]
+//       ,[IsMDMActive]
+//       ,[SenseID]
+//       ,[BoundaryGroups]
+//       ,[LastFUErrorDetail]
+//   FROM [${dbName}].[dbo].[${tableName}]
+//   ORDER BY [MachineID]
+//   OFFSET ${offset} ROWS FETCH NEXT ${limit} ROWS ONLY`;
+// }
+
 function buildCollectionSubscriptionQuery(
   dbName: string,
   tableName: string,
@@ -190,133 +331,7 @@ function buildCollectionSubscriptionQuery(
 ) {
   validateDbName(dbName);
 
-  return `SELECT [MachineID]
-      ,[ArchitectureKey]
-      ,[Name]
-      ,[SMSID]
-      ,[SiteCode]
-      ,[Domain]
-      ,[ClientEdition]
-      ,[ClientType]
-      ,[ClientVersion]
-      ,[IsDirect]
-      ,[IsAssigned]
-      ,[IsClient]
-      ,[IsVirtualMachine]
-      ,[IsAOACCapable]
-      ,[IsObsolete]
-      ,[IsActive]
-      ,[IsDecommissioned]
-      ,[WipeStatus]
-      ,[RetireStatus]
-      ,[SyncNowStatus]
-      ,[LastSyncNowRequest]
-      ,[ManagementAuthority]
-      ,[AMTStatus]
-      ,[AMTFullVersion]
-      ,[DeviceOwner]
-      ,[DeviceCategory]
-      ,[SuppressAutoProvision]
-      ,[IsApproved]
-      ,[IsBlocked]
-      ,[IsAlwaysInternet]
-      ,[IsInternetEnabled]
-      ,[ClientCertType]
-      ,[UserName]
-      ,[LastClientCheckTime]
-      ,[ClientCheckPass]
-      ,[ADSiteName]
-      ,[UserDomainName]
-      ,[ADLastLogonTime]
-      ,[ClientRemediationSuccess]
-      ,[ClientActiveStatus]
-      ,[LastStatusMessage]
-      ,[LastPolicyRequest]
-      ,[LastDDR]
-      ,[LastHardwareScan]
-      ,[LastSoftwareScan]
-      ,[LastMPServerName]
-      ,[LastActiveTime]
-      ,[CP_Status]
-      ,[CP_LatestProcessingAttempt]
-      ,[CP_LastInstallationError]
-      ,[EAS_DeviceID]
-      ,[DeviceOS]
-      ,[DeviceOSBuild]
-      ,[DeviceType]
-      ,[ExchangeServer]
-      ,[ExchangeOrganization]
-      ,[PolicyApplicationStatus]
-      ,[LastSuccessSyncTimeUTC]
-      ,[PhoneNumber]
-      ,[DeviceAccessState]
-      ,[DeviceThreatLevel]
-      ,[CoManaged]
-      ,[PasscodeResetState]
-      ,[PasscodeResetStateTimeStamp]
-      ,[RemoteLockState]
-      ,[RemoteLockStateTimeStamp]
-      ,[ActivationLockBypassState]
-      ,[ActivationLockBypassStateTimeStamp]
-      ,[ActivationLockState]
-      ,[IsSupervised]
-      ,[EP_DeploymentState]
-      ,[EP_DeploymentErrorCode]
-      ,[EP_DeploymentDescription]
-      ,[EP_PolicyApplicationState]
-      ,[EP_PolicyApplicationErrorCode]
-      ,[EP_PolicyApplicationDescription]
-      ,[EP_Enabled]
-      ,[EP_ClientVersion]
-      ,[EP_ProductStatus]
-      ,[EP_EngineVersion]
-      ,[EP_AntivirusEnabled]
-      ,[EP_AntivirusSignatureVersion]
-      ,[EP_AntivirusSignatureUpdateDateTime]
-      ,[EP_AntispywareEnabled]
-      ,[EP_AntispywareSignatureVersion]
-      ,[EP_AntispywareSignatureUpdateDateTime]
-      ,[EP_LastFullScanDateTimeStart]
-      ,[EP_LastFullScanDateTimeEnd]
-      ,[EP_LastQuickScanDateTimeStart]
-      ,[EP_LastQuickScanDateTimeEnd]
-      ,[EP_InfectionStatus]
-      ,[EP_PendingFullScan]
-      ,[EP_PendingReboot]
-      ,[EP_PendingManualSteps]
-      ,[EP_PendingOfflineScan]
-      ,[EP_LastInfectionTime]
-      ,[EP_LastThreatName]
-      ,[CNIsOnline]
-      ,[CNLastOnlineTime]
-      ,[CNLastOfflineTime]
-      ,[ClientState]
-      ,[CNAccessMP]
-      ,[CNIsOnInternet]
-      ,[Unknown]
-      ,[ATP_LastConnected]
-      ,[ATP_SenseIsRunning]
-      ,[ATP_OnboardingState]
-      ,[ATP_OrgId]
-      ,[CA_IsCompliant]
-      ,[CA_ComplianceSetTime]
-      ,[CA_ComplianceEvalTime]
-      ,[CA_ErrorDetails]
-      ,[CA_ErrorLocation]
-      ,[AADTenantID]
-      ,[AADDeviceID]
-      ,[SerialNumber]
-      ,[IMEI]
-      ,[PrimaryUser]
-      ,[CurrentlogonUser]
-      ,[LastLogonUser]
-      ,[MACAddress]
-      ,[SMBIOSGUID]
-      ,[IsMDMActive]
-      ,[SenseID]
-      ,[BoundaryGroups]
-      ,[LastFUErrorDetail]
-  FROM [${dbName}].[dbo].[${tableName}]
+  return `SELECT [MachineID] FROM [${dbName}].[dbo].[${tableName}]
   ORDER BY [MachineID]
   OFFSET ${offset} ROWS FETCH NEXT ${limit} ROWS ONLY`;
 }

src/steps/applications/converters.ts

@@ -54,12 +54,14 @@ export function createLegacyApplicationEntity(application: any): Entity {
 }
 
 export function createDeviceApplicationRelationship(
-  device: Entity,
-  application: Entity,
+  deviceKey: string,
+  applicationKey: string,
 ): Relationship {
   return createDirectRelationship({
     _class: RelationshipClass.INSTALLED,
-    from: device,
-    to: application,
+    fromKey: deviceKey,
+    fromType: Entities.DEVICE._type,
+    toKey: applicationKey,
+    toType: Entities.APPLICATION._type,
   });
 }