From ff02b90b10d3362507cabc0b4cf81bdce5c94577 Mon Sep 17 00:00:00 2001 From: Gaston Yelmini Date: Thu, 4 Jul 2024 11:07:21 -0300 Subject: [PATCH] INT-11156: performance upgrade --- src/configuration-manager/client.ts | 62 +++--- src/configuration-manager/queries.ts | 269 ++++++++++++++------------- src/steps/applications/converters.ts | 10 +- src/steps/applications/index.ts | 26 +-- src/steps/collections/converters.ts | 8 +- src/steps/collections/index.ts | 9 +- 6 files changed, 205 insertions(+), 179 deletions(-) diff --git a/src/configuration-manager/client.ts b/src/configuration-manager/client.ts index 7764b28..ecde10f 100644 --- a/src/configuration-manager/client.ts +++ b/src/configuration-manager/client.ts @@ -102,38 +102,42 @@ class MicrosoftConfigurationManagerClient { let offset = 0; let hasMoreRecords = true; - while (hasMoreRecords) { - const query = buildApplicationDeviceTargetingList( - this.dbName, - offset, - pageSize, - ); - const result = await this.wrapWithRequestFailedHandler( - () => this.connection.query(query), - { - logger, - query, - }, - ); - - const records = result.recordset; - - logger.info(`Proccesed ${records.length} records from query`); - - if (records.length > 0) { - await pMap( - records, - async (record) => { - await iteratee(record); - }, + try { + while (hasMoreRecords) { + const query = buildApplicationDeviceTargetingList( + this.dbName, + offset, + pageSize, + ); + const result = await this.wrapWithRequestFailedHandler( + () => this.connection.query(query), { - concurrency: 2, + logger, + query, }, ); - offset += pageSize; - } else { - hasMoreRecords = false; + + const records = result.recordset; + + logger.info(`Proccesed ${records.length} records from query`); + + if (records.length > 0) { + await pMap( + records, + async (record) => { + await iteratee(record); + }, + { + concurrency: 2, + }, + ); + offset += pageSize; + } else { + hasMoreRecords = false; + } } + } catch (err) { + logger.info(`Received SQL error when processing query: ${err}`); } } @@ -171,7 +175,7 @@ class MicrosoftConfigurationManagerClient { async listCollectionSubscriptions( tableName: string, iteratee: ResourceIteratee, - pageSize: number = 200, + pageSize: number = 600, ) { let offset = 0; let hasMoreRecords = true; diff --git a/src/configuration-manager/queries.ts b/src/configuration-manager/queries.ts index 2aaf79a..eecd175 100644 --- a/src/configuration-manager/queries.ts +++ b/src/configuration-manager/queries.ts @@ -182,6 +182,147 @@ function buildDeviceCollectionQuery( WHERE Sub.RowNum BETWEEN ${offset} AND ${offset + limit - 1}`; } +// INT-11156 (gastonyelmini): commenting out this query. We are overfetching data when we just need the MachineID. + +// function buildCollectionSubscriptionQuery( +// dbName: string, +// tableName: string, +// offset: number, +// limit: number, +// ) { +// validateDbName(dbName); + +// return `SELECT [MachineID] +// ,[ArchitectureKey] +// ,[Name] +// ,[SMSID] +// ,[SiteCode] +// ,[Domain] +// ,[ClientEdition] +// ,[ClientType] +// ,[ClientVersion] +// ,[IsDirect] +// ,[IsAssigned] +// ,[IsClient] +// ,[IsVirtualMachine] +// ,[IsAOACCapable] +// ,[IsObsolete] +// ,[IsActive] +// ,[IsDecommissioned] +// ,[WipeStatus] +// ,[RetireStatus] +// ,[SyncNowStatus] +// ,[LastSyncNowRequest] +// ,[ManagementAuthority] +// ,[AMTStatus] +// ,[AMTFullVersion] +// ,[DeviceOwner] +// ,[DeviceCategory] +// ,[SuppressAutoProvision] +// ,[IsApproved] +// ,[IsBlocked] +// ,[IsAlwaysInternet] +// ,[IsInternetEnabled] +// ,[ClientCertType] +// ,[UserName] +// ,[LastClientCheckTime] +// ,[ClientCheckPass] +// ,[ADSiteName] +// ,[UserDomainName] +// ,[ADLastLogonTime] +// ,[ClientRemediationSuccess] +// ,[ClientActiveStatus] +// ,[LastStatusMessage] +// ,[LastPolicyRequest] +// ,[LastDDR] +// ,[LastHardwareScan] +// ,[LastSoftwareScan] +// ,[LastMPServerName] +// ,[LastActiveTime] +// ,[CP_Status] +// ,[CP_LatestProcessingAttempt] +// ,[CP_LastInstallationError] +// ,[EAS_DeviceID] +// ,[DeviceOS] +// ,[DeviceOSBuild] +// ,[DeviceType] +// ,[ExchangeServer] +// ,[ExchangeOrganization] +// ,[PolicyApplicationStatus] +// ,[LastSuccessSyncTimeUTC] +// ,[PhoneNumber] +// ,[DeviceAccessState] +// ,[DeviceThreatLevel] +// ,[CoManaged] +// ,[PasscodeResetState] +// ,[PasscodeResetStateTimeStamp] +// ,[RemoteLockState] +// ,[RemoteLockStateTimeStamp] +// ,[ActivationLockBypassState] +// ,[ActivationLockBypassStateTimeStamp] +// ,[ActivationLockState] +// ,[IsSupervised] +// ,[EP_DeploymentState] +// ,[EP_DeploymentErrorCode] +// ,[EP_DeploymentDescription] +// ,[EP_PolicyApplicationState] +// ,[EP_PolicyApplicationErrorCode] +// ,[EP_PolicyApplicationDescription] +// ,[EP_Enabled] +// ,[EP_ClientVersion] +// ,[EP_ProductStatus] +// ,[EP_EngineVersion] +// ,[EP_AntivirusEnabled] +// ,[EP_AntivirusSignatureVersion] +// ,[EP_AntivirusSignatureUpdateDateTime] +// ,[EP_AntispywareEnabled] +// ,[EP_AntispywareSignatureVersion] +// ,[EP_AntispywareSignatureUpdateDateTime] +// ,[EP_LastFullScanDateTimeStart] +// ,[EP_LastFullScanDateTimeEnd] +// ,[EP_LastQuickScanDateTimeStart] +// ,[EP_LastQuickScanDateTimeEnd] +// ,[EP_InfectionStatus] +// ,[EP_PendingFullScan] +// ,[EP_PendingReboot] +// ,[EP_PendingManualSteps] +// ,[EP_PendingOfflineScan] +// ,[EP_LastInfectionTime] +// ,[EP_LastThreatName] +// ,[CNIsOnline] +// ,[CNLastOnlineTime] +// ,[CNLastOfflineTime] +// ,[ClientState] +// ,[CNAccessMP] +// ,[CNIsOnInternet] +// ,[Unknown] +// ,[ATP_LastConnected] +// ,[ATP_SenseIsRunning] +// ,[ATP_OnboardingState] +// ,[ATP_OrgId] +// ,[CA_IsCompliant] +// ,[CA_ComplianceSetTime] +// ,[CA_ComplianceEvalTime] +// ,[CA_ErrorDetails] +// ,[CA_ErrorLocation] +// ,[AADTenantID] +// ,[AADDeviceID] +// ,[SerialNumber] +// ,[IMEI] +// ,[PrimaryUser] +// ,[CurrentlogonUser] +// ,[LastLogonUser] +// ,[MACAddress] +// ,[SMBIOSGUID] +// ,[IsMDMActive] +// ,[SenseID] +// ,[BoundaryGroups] +// ,[LastFUErrorDetail] +// FROM [${dbName}].[dbo].[${tableName}] +// ORDER BY [MachineID] +// OFFSET ${offset} ROWS FETCH NEXT ${limit} ROWS ONLY`; +// } + function buildCollectionSubscriptionQuery( dbName: string, tableName: string, @@ -190,133 +331,7 @@ function buildCollectionSubscriptionQuery( ) { validateDbName(dbName); - return `SELECT [MachineID] - ,[ArchitectureKey] - ,[Name] - ,[SMSID] - ,[SiteCode] - ,[Domain] - ,[ClientEdition] - ,[ClientType] - ,[ClientVersion] - ,[IsDirect] - ,[IsAssigned] - ,[IsClient] - ,[IsVirtualMachine] - ,[IsAOACCapable] - ,[IsObsolete] - ,[IsActive] - ,[IsDecommissioned] - ,[WipeStatus] - ,[RetireStatus] - ,[SyncNowStatus] - ,[LastSyncNowRequest] - ,[ManagementAuthority] - ,[AMTStatus] - ,[AMTFullVersion] - ,[DeviceOwner] - ,[DeviceCategory] - ,[SuppressAutoProvision] - ,[IsApproved] - ,[IsBlocked] - ,[IsAlwaysInternet] - ,[IsInternetEnabled] - ,[ClientCertType] - ,[UserName] - ,[LastClientCheckTime] - ,[ClientCheckPass] - ,[ADSiteName] - ,[UserDomainName] - ,[ADLastLogonTime] - ,[ClientRemediationSuccess] - ,[ClientActiveStatus] - ,[LastStatusMessage] - ,[LastPolicyRequest] - ,[LastDDR] - ,[LastHardwareScan] - ,[LastSoftwareScan] - ,[LastMPServerName] - ,[LastActiveTime] - ,[CP_Status] - ,[CP_LatestProcessingAttempt] - ,[CP_LastInstallationError] - ,[EAS_DeviceID] - ,[DeviceOS] - ,[DeviceOSBuild] - ,[DeviceType] - ,[ExchangeServer] - ,[ExchangeOrganization] - ,[PolicyApplicationStatus] - ,[LastSuccessSyncTimeUTC] - ,[PhoneNumber] - ,[DeviceAccessState] - ,[DeviceThreatLevel] - ,[CoManaged] - ,[PasscodeResetState] - ,[PasscodeResetStateTimeStamp] - ,[RemoteLockState] - ,[RemoteLockStateTimeStamp] - ,[ActivationLockBypassState] - ,[ActivationLockBypassStateTimeStamp] - ,[ActivationLockState] - ,[IsSupervised] - ,[EP_DeploymentState] - ,[EP_DeploymentErrorCode] - ,[EP_DeploymentDescription] - ,[EP_PolicyApplicationState] - ,[EP_PolicyApplicationErrorCode] - ,[EP_PolicyApplicationDescription] - ,[EP_Enabled] - ,[EP_ClientVersion] - ,[EP_ProductStatus] - ,[EP_EngineVersion] - ,[EP_AntivirusEnabled] - ,[EP_AntivirusSignatureVersion] - ,[EP_AntivirusSignatureUpdateDateTime] - ,[EP_AntispywareEnabled] - ,[EP_AntispywareSignatureVersion] - ,[EP_AntispywareSignatureUpdateDateTime] - ,[EP_LastFullScanDateTimeStart] - ,[EP_LastFullScanDateTimeEnd] - ,[EP_LastQuickScanDateTimeStart] - ,[EP_LastQuickScanDateTimeEnd] - ,[EP_InfectionStatus] - ,[EP_PendingFullScan] - ,[EP_PendingReboot] - ,[EP_PendingManualSteps] - ,[EP_PendingOfflineScan] - ,[EP_LastInfectionTime] - ,[EP_LastThreatName] - ,[CNIsOnline] - ,[CNLastOnlineTime] - ,[CNLastOfflineTime] - ,[ClientState] - ,[CNAccessMP] - ,[CNIsOnInternet] - ,[Unknown] - ,[ATP_LastConnected] - ,[ATP_SenseIsRunning] - ,[ATP_OnboardingState] - ,[ATP_OrgId] - ,[CA_IsCompliant] - ,[CA_ComplianceSetTime] - ,[CA_ComplianceEvalTime] - ,[CA_ErrorDetails] - ,[CA_ErrorLocation] - ,[AADTenantID] - ,[AADDeviceID] - ,[SerialNumber] - ,[IMEI] - ,[PrimaryUser] - ,[CurrentlogonUser] - ,[LastLogonUser] - ,[MACAddress] - ,[SMBIOSGUID] - ,[IsMDMActive] - ,[SenseID] - ,[BoundaryGroups] - ,[LastFUErrorDetail] - FROM [${dbName}].[dbo].[${tableName}] + return `SELECT [MachineID] FROM [${dbName}].[dbo].[${tableName}] ORDER BY [MachineID] OFFSET ${offset} ROWS FETCH NEXT ${limit} ROWS ONLY`; } diff --git a/src/steps/applications/converters.ts b/src/steps/applications/converters.ts index 837db80..3ed6c4d 100644 --- a/src/steps/applications/converters.ts +++ b/src/steps/applications/converters.ts @@ -54,12 +54,14 @@ export function createLegacyApplicationEntity(application: any): Entity { } export function createDeviceApplicationRelationship( - device: Entity, - application: Entity, + deviceKey: string, + applicationKey: string, ): Relationship { return createDirectRelationship({ _class: RelationshipClass.INSTALLED, - from: device, - to: application, + fromKey: deviceKey, + fromType: Entities.DEVICE._type, + toKey: applicationKey, + toType: Entities.APPLICATION._type, }); } diff --git a/src/steps/applications/index.ts b/src/steps/applications/index.ts index 0b1b0f4..d50e2e7 100644 --- a/src/steps/applications/index.ts +++ b/src/steps/applications/index.ts @@ -58,11 +58,14 @@ export async function fetchLegacyApplications({ const applicationEntity = await jobState.addEntity( legacyApplicationEntity, ); - const deviceID = createDeviceKey(application.ResourceID?.toString()); - const deviceEntity = await jobState.findEntity(deviceID); - if (applicationEntity && deviceEntity) { + const deviceKey = createDeviceKey(application.ResourceID?.toString()); + + if (applicationEntity._key && deviceKey) { await jobState.addRelationship( - createDeviceApplicationRelationship(deviceEntity, applicationEntity), + createDeviceApplicationRelationship( + deviceKey, + applicationEntity._key, + ), ); } } @@ -96,20 +99,21 @@ export async function buildApplicationRelationships({ await client.listApplicationDeviceTargets( async (application: any) => { - const appID = createApplicationKey(application.CIGUID?.toString()); - const deviceID = createDeviceKey(application.ResourceID?.toString()); - const applicationEntity = await jobState.findEntity(appID); - const deviceEntity = await jobState.findEntity(deviceID); - if (applicationEntity && deviceEntity) { + const applicationKey = createApplicationKey( + application.CIGUID?.toString(), + ); + const deviceKey = createDeviceKey(application.ResourceID?.toString()); + + if (applicationKey && deviceKey) { const deviceApplicationRelationship = - createDeviceApplicationRelationship(deviceEntity, applicationEntity); + createDeviceApplicationRelationship(deviceKey, applicationKey); if (!jobState.hasKey(deviceApplicationRelationship._key)) { await jobState.addRelationship(deviceApplicationRelationship); } } }, - 600, + 300, logger, ); } diff --git a/src/steps/collections/converters.ts b/src/steps/collections/converters.ts index 57b2377..6ce41b6 100644 --- a/src/steps/collections/converters.ts +++ b/src/steps/collections/converters.ts @@ -35,11 +35,13 @@ export function createDeviceCollectionEntity(collection: any): Entity { export function createCollectorDeviceRelationship( collector: Entity, - device: Entity, + deviceKey: string, ): Relationship { return createDirectRelationship({ _class: RelationshipClass.HAS, - from: collector, - to: device, + fromKey: collector._key, + fromType: collector._type, + toKey: deviceKey, + toType: Entities.DEVICE._type, }); } diff --git a/src/steps/collections/index.ts b/src/steps/collections/index.ts index 87127af..387ab2f 100644 --- a/src/steps/collections/index.ts +++ b/src/steps/collections/index.ts @@ -69,14 +69,13 @@ export async function buildCollectionsRelationships({ await client.listCollectionSubscriptions( collectionRawData.ResultTableName, async (subscription: any) => { - const deviceEntity = await jobState.findEntity( - createDeviceKey(subscription.MachineID), - ); - if (deviceEntity) { + const deviceKey = createDeviceKey(subscription.MachineID); + + if (deviceKey && subscription.MachineID) { await jobState.addRelationship( createCollectorDeviceRelationship( collectionEntity, - deviceEntity, + createDeviceKey(subscription.MachineID), ), ); }