Add flag to skip inserting notes into the database (#42)

* Add flag to skip inserting notes into the database * Update Sources/XCMetricsClient/Mobius/Domain/MetricsUploaderLoopTypes.swift Co-authored-by: Bartosz Polaczyk <[email protected]> Co-authored-by: Erick Camacho <[email protected]> Co-authored-by: Bartosz Polaczyk <[email protected]>
spotify · Apr 29, 2021 · 14a5d12 · 14a5d12
1 parent 0f449cc
commit 14a5d12
Show file tree

Hide file tree

Showing 24 changed files with 165 additions and 75 deletions.
diff --git a/Examples/BasicApp/BasicApp.xcodeproj/xcshareddata/xcschemes/BasicApp.xcscheme b/Examples/BasicApp/BasicApp.xcodeproj/xcshareddata/xcschemes/BasicApp.xcscheme
@@ -11,7 +11,7 @@
             ActionType = "Xcode.IDEStandardExecutionActionsCore.ExecutionActionType.ShellScriptAction">
             <ActionContent
                title = "Run Script"
-               scriptText = "${SRCROOT}/../../XCMetricsLauncher ${SRCROOT}/../../.build/release/XCMetrics --name BasicApp --buildDir ${BUILD_DIR} --serviceURL http://localhost:8080/v1/metrics&#10;&#10;&#10;">
+               scriptText = "${SRCROOT}/../../XCMetricsLauncher ${SRCROOT}/../../.build/release/XCMetrics --name BasicApp --buildDir ${BUILD_DIR} --isCI false --skipNotes false --serviceURL http://localhost:8080/v1/metrics&#10;&#10;&#10;">
                <EnvironmentBuildable>
                   <BuildableReference
                      BuildableIdentifier = "primary"

diff --git a/Package.resolved b/Package.resolved
@@ -42,8 +42,8 @@
         "repositoryURL": "https://github.com/krzyzanowskim/CryptoSwift.git",
         "state": {
           "branch": null,
-          "revision": "af1b58fc569bfde777462349b9f7314b61762be0",
-          "version": "1.3.2"
+          "revision": "e2bc81be54d71d566a52ca17c3983d141c30aa70",
+          "version": "1.3.3"
         }
       },
       {
@@ -384,8 +384,8 @@
         "repositoryURL": "https://github.com/spotify/xclogparser",
         "state": {
           "branch": null,
-          "revision": "ab85951bc54bfcbae27b2b1f22097e955f8e41e5",
-          "version": "0.2.24"
+          "revision": "171247366dd1bcd9bf61f9699867b18282eea842",
+          "version": "0.2.27"
         }
       },
       {

diff --git a/Package.swift b/Package.swift
@@ -15,7 +15,7 @@ let package = Package(
         .library(name: "XCMetricsUtils", targets: ["XCMetricsUtils"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/spotify/xclogparser", from: "0.2.24"),
+        .package(url: "https://github.com/spotify/xclogparser", from: "0.2.27"),
         .package(url: "https://github.com/apple/swift-package-manager.git", .exact("0.3.0")),
         .package(url: "https://github.com/grpc/grpc-swift.git", .exact("1.0.0-alpha.9")),
         .package(url: "https://github.com/apple/swift-nio.git", from: "2.23.0"),
@@ -25,7 +25,6 @@ let package = Package(
         .package(url: "https://github.com/krzyzanowskim/CryptoSwift.git", from: "1.3.0"),
         .package(url: "https://github.com/jpsim/Yams.git", from: "3.0.0"),
         .package(url: "https://github.com/apple/swift-argument-parser", from: "0.1.0"),
-
         .package(url: "https://github.com/vapor/vapor.git", from: "4.0.0"),
         .package(url: "https://github.com/vapor/fluent.git", from: "4.0.0"),
         .package(url: "https://github.com/vapor/fluent-postgres-driver.git", from: "2.1.0"),
@@ -38,7 +37,18 @@ let package = Package(
     targets: [
         .target(
             name: "XCMetricsClient",
-            dependencies: ["XCLogParser", "XCMetricsProto", "XCMetricsUtils", .product(name: "Utility", package: "SwiftPM"), "GRPC", "NIO", "NIOHTTP2", "MobiusCore", "MobiusExtras", "CryptoSwift", "Yams",  "ArgumentParser"]
+            dependencies: ["XCLogParser",
+                           "XCMetricsProto",
+                           "XCMetricsUtils",
+                           "GRPC",
+                           "NIO",
+                           "NIOHTTP2",
+                           "MobiusCore",
+                           "MobiusExtras",
+                           "CryptoSwift",
+                           "Yams",
+                           "ArgumentParser",
+                           "XCMetricsCommon"]
         ),
         .target(
             name: "XCMetricsPlugins",
@@ -56,6 +66,10 @@ let package = Package(
             name: "XCMetricsApp",
             dependencies: ["XCMetricsClient"]
         ),
+        .target(
+            name: "XCMetricsCommon",
+            dependencies: []
+        ),
        .target(
             name: "XCMetricsBackendLib",
             dependencies: [
@@ -69,6 +83,7 @@ let package = Package(
                 .product(name: "CryptoSwift", package: "CryptoSwift"),
                 .product(name: "GoogleCloudKit", package: "google-cloud-kit"),
                 .product(name: "S3", package: "AWSSDKSwift"),
+                "XCMetricsCommon"
             ],
             swiftSettings: [
                 // Enable better optimizations when building in Release configuration. Despite the use of
@@ -80,7 +95,7 @@ let package = Package(
         .target(name: "XCMetricsBackend", dependencies: [.target(name: "XCMetricsBackendLib")]),
         .testTarget(
             name: "XCMetricsTests",
-            dependencies: ["XCMetricsClient", "XCMetricsProto", .product(name: "Utility", package: "SwiftPM"), "MobiusTest"]
+            dependencies: ["XCMetricsClient", "XCMetricsProto", "MobiusTest", .product(name: "Utility", package: "SwiftPM")]
         ),
         .testTarget(
             name: "XCMetricsPluginsTests",

diff --git a/Sources/XCMetricsBackendLib/Config/Configuration.swift b/Sources/XCMetricsBackendLib/Config/Configuration.swift
@@ -5,7 +5,7 @@ class Configuration {
 
     /// If "1", the logs will be processed Asynchronously, it will need a `REDIS_HOST` to be defined
     /// Turn it off in environments where Async processing is not available like in Cloud Run
-    lazy var useAsyncLogProcessing: Bool = {
+    lazy var useAsyncLogProcessing: Bool = {        
         return (Environment.get("XCMETRICS_USE_ASYNC_LOG_PROCESSING") ?? "1" ) == "1"
     }()
 

diff --git a/Sources/XCMetricsBackendLib/UploadMetrics/LogProcessing/LogParser.swift b/Sources/XCMetricsBackendLib/UploadMetrics/LogProcessing/LogParser.swift
@@ -48,10 +48,15 @@ struct LogParser {
         userId: String,
         userIdSHA256: String,
         isCI: Bool,
-        sleepTime: Int?
+        sleepTime: Int?,
+        skipNotes: Bool?
     ) throws -> BuildMetrics {
         let activityLog = try ActivityParser().parseActivityLogInURL(url, redacted: true, withoutBuildSpecificInformation: true)
-        let buildSteps = try ParserBuildSteps(machineName: machineName, omitWarningsDetails: false).parse(activityLog: activityLog).flatten()
+        let buildSteps = try ParserBuildSteps(machineName: machineName,
+                                              omitWarningsDetails: false,
+                                              omitNotesDetails: skipNotes ?? false)
+            .parse(activityLog: activityLog)
+            .flatten()
         return toBuildMetrics(
             buildSteps,
             projectName: projectName,

diff --git a/Sources/XCMetricsBackendLib/UploadMetrics/LogProcessing/MetricsProcessor.swift b/Sources/XCMetricsBackendLib/UploadMetrics/LogProcessing/MetricsProcessor.swift
@@ -17,7 +17,8 @@ struct MetricsProcessor {
             userId: userId,
             userIdSHA256: userIdSHA256,
             isCI: isCI,
-            sleepTime: metricsRequest.extraInfo.sleepTime
+            sleepTime: metricsRequest.extraInfo.sleepTime,
+            skipNotes: metricsRequest.extraInfo.skipNotes
         )
     }
 

diff --git a/Sources/XCMetricsBackendLib/UploadMetrics/Model/UploadMetricsModel.swift b/Sources/XCMetricsBackendLib/UploadMetrics/Model/UploadMetricsModel.swift
@@ -19,6 +19,7 @@
 
 import Foundation
 import Vapor
+import XCMetricsCommon
 
 /// Content of a Multipart request to upload Metrics
 final class UploadMetricsPayload: Content {
@@ -47,7 +48,7 @@ final class UploadMetricsRequest: Codable {
     let logURL: URL
 
     /// Extra data needed
-    let extraInfo: ExtraInfo
+    let extraInfo: UploadRequestExtraInfo
 
     /// Build host information
     let buildHost: BuildHost
@@ -59,7 +60,7 @@ final class UploadMetricsRequest: Codable {
     let buildMetadata: BuildMetadata?
 
     init(logURL : URL,
-         extraInfo: ExtraInfo,
+         extraInfo: UploadRequestExtraInfo,
          buildHost: BuildHost,
          xcodeVersion: XcodeVersion?,
          buildMetadata: BuildMetadata?) {
@@ -78,7 +79,7 @@ final class UploadMetricsRequest: Codable {
     convenience init?(logURL: URL, payload: UploadMetricsPayload) throws {
         let decoder = JSONDecoder()
 
-        let extraInfo = try decoder.decode(ExtraInfo.self, from: payload.extraInfo.xcm_onlyJsonContent())
+        let extraInfo = try decoder.decode(UploadRequestExtraInfo.self, from: payload.extraInfo.xcm_onlyJsonContent())
         let buildHost = try decoder.decode(BuildHost.self, from: payload.buildHost.xcm_onlyJsonContent())
         let xcodeVersion: XcodeVersion?
         if let xcodeVersionData = payload.xcodeVersion?.xcm_onlyJsonContent() {
@@ -101,25 +102,6 @@ final class UploadMetricsRequest: Codable {
     }
 }
 
-/// Data needed from the client that did the build
-final class ExtraInfo: Codable {
-
-    /// Name of the Xcode project
-    let projectName: String
-
-    /// Name of the host where the build was done
-    let machineName: String
-
-    /// Name of the user that did the build
-    let user: String
-
-    /// True if the build was performed on a continuous integration machine, false otherwise.
-    let isCI: Bool
-
-    /// The last time the host went to sleep as reported by sysctl's `kern.sleeptime` property.
-    let sleepTime: Int?
-}
-
 extension ByteBuffer {
 
     /// Removes the Content-Type part of an `octet-stream` request content, leaving only the actual file content

diff --git a/Sources/XCMetricsClient/Log Management/LogParser.swift b/Sources/XCMetricsClient/Log Management/LogParser.swift
@@ -72,7 +72,10 @@ class LogParserImplementation: LogParser {
         dispatchQueue.async {
             do {
                 let activityLog = try ActivityParser().parseActivityLogInURL(logURL, redacted: true, withoutBuildSpecificInformation: true)
-                let buildSteps = try ParserBuildSteps(machineName: self.machineNameReader.machineName, omitWarningsDetails: false).parse(activityLog: activityLog).flatten()
+                let buildSteps = try ParserBuildSteps(machineName: self.machineNameReader.machineName,
+                                                      omitWarningsDetails: false,
+                                                      omitNotesDetails: false)
+                    .parse(activityLog: activityLog).flatten()
                 let uploadRequest = self.parseBuildSteps(buildSteps, projectName: projectName, isCI: isCI, userID: userID)
                 completion(.success(uploadRequest))
             } catch {

diff --git a/Sources/XCMetricsClient/Mobius/Domain/MetricsUploaderControllerFactory.swift b/Sources/XCMetricsClient/Mobius/Domain/MetricsUploaderControllerFactory.swift
@@ -34,7 +34,8 @@ enum ControllerFactory {
             serviceURL: serviceURL,
             timeout: command.timeout,
             isCI: command.isCI,
-            plugins: plugins
+            plugins: plugins,
+            skipNotes: command.skipNotes
         )
         let initEffect = MetricsUploaderEffect.findLogs(buildDirectory: model.buildDirectory, timeout: model.timeout)
         let logManager = LogManagerImplementation(projectName: model.projectName)

diff --git a/Sources/XCMetricsClient/Mobius/Domain/MetricsUploaderLogic.swift b/Sources/XCMetricsClient/Mobius/Domain/MetricsUploaderLogic.swift
@@ -82,7 +82,11 @@ enum MetricsUploaderLogic {
         })
 
         if !uploadRequests.isEmpty {
-            effects.append(.uploadLogs(serviceURL: model.serviceURL, projectName: model.projectName, isCI: model.isCI, logs: uploadRequests))
+            effects.append(.uploadLogs(serviceURL: model.serviceURL,
+                                       projectName: model.projectName,
+                                       isCI: model.isCI,
+                                       skipNotes: model.skipNotes,
+                                       logs: uploadRequests))
         }
         let updatedModel = model.withChanged(
             parsedRequests: model.parsedRequests.union(cachedUploadRequest.prefix(maximumNumberOfParsedRequestsToSend)),
@@ -94,6 +98,7 @@ enum MetricsUploaderLogic {
                 serviceURL: model.serviceURL,
                 projectName: model.projectName,
                 isCI: model.isCI,
+                skipNotes: model.skipNotes,
                 logs: updatedModel.parsedRequests
             )])
         }
@@ -111,6 +116,7 @@ enum MetricsUploaderLogic {
                 serviceURL: model.serviceURL,
                 projectName: model.projectName,
                 isCI: model.isCI,
+                skipNotes: model.skipNotes,
                 logs: updatedModel.parsedRequests
             )
         ])

diff --git a/Sources/XCMetricsClient/Mobius/Domain/MetricsUploaderLoopTypes.swift b/Sources/XCMetricsClient/Mobius/Domain/MetricsUploaderLoopTypes.swift
@@ -40,6 +40,8 @@ struct MetricsUploaderModel: Equatable, Hashable {
     let parsedRequests: Set<MetricsUploadRequest>
     /// Number of scheduled parsing logs in progress
     let awaitingParsingResultsCount: Int
+    /// If true, the Notes found in the log won't be inserted into the database
+    let skipNotes: Bool
 
     init(
         buildDirectory: String,
@@ -49,7 +51,8 @@ struct MetricsUploaderModel: Equatable, Hashable {
         isCI: Bool,
         plugins: [XCMetricsPlugin],
         parsedRequests: Set<MetricsUploadRequest> = Set(),
-        awaitingParsingLogResponses: Int = 0
+        awaitingParsingLogResponses: Int = 0,
+        skipNotes: Bool = false
     ) {
         self.buildDirectory = buildDirectory
         self.projectName = projectName
@@ -59,6 +62,7 @@ struct MetricsUploaderModel: Equatable, Hashable {
         self.isCI = isCI
         self.parsedRequests = parsedRequests
         self.awaitingParsingResultsCount = awaitingParsingLogResponses
+        self.skipNotes = skipNotes
     }
 
     init() {
@@ -70,6 +74,7 @@ struct MetricsUploaderModel: Equatable, Hashable {
         self.isCI = false
         self.parsedRequests = []
         self.awaitingParsingResultsCount = 0
+        self.skipNotes = false
     }
 }
 
@@ -84,6 +89,7 @@ extension MetricsUploaderModel: CustomDebugStringConvertible {
         isCI: \(isCI),
         parsedRequests: \(parsedRequests.count),
         awaitingParsingLogResponses: \(awaitingParsingResultsCount)
+        skipNotes: \(skipNotes)
         """
     }
 }
@@ -148,7 +154,7 @@ enum MetricsUploaderEffect: Hashable {
     /// Executes the custom plugins configured if any to add more data to the build.
     case executePlugins(request: MetricsUploadRequest, plugins: [XCMetricsPlugin])
     /// Uploads the given log upload requests to the specified backend service.
-    case uploadLogs(serviceURL: URL, projectName: String, isCI: Bool, logs: Set<MetricsUploadRequest>)
+    case uploadLogs(serviceURL: URL, projectName: String, isCI: Bool, skipNotes: Bool, logs: Set<MetricsUploadRequest>)
     /// Uploaded logs should be renamed to signal their status and differentiate them from logs yet to be uploaded.
     case tagLogsAsUploaded(logs: Set<URL>)
     /// Logs failed to upload are saved to disk in order to preserve the metadata collected (the actual xcactivitylog is always kept on disk for 7 days).

diff --git a/Sources/XCMetricsClient/Mobius/Effect Handlers/DumpParser/JSONMetricsParserFactory.swift b/Sources/XCMetricsClient/Mobius/Effect Handlers/DumpParser/JSONMetricsParserFactory.swift
@@ -91,7 +91,8 @@ class JSONMetricsParserFactory: MetricsParserFactory {
                              compilationEndTimestamp: representation.fetch("compilationEndTimestamp"),
                              compilationDuration: representation.fetch("compilationDuration"),
                              clangTimeTraceFile: nil,
-                             linkerStatistics: nil)
+                             linkerStatistics: nil,
+                             swiftTypeCheckTimes: nil)
             return BuildInfo(step: step,
                              projectName: representation.fetch("projectName"),
                              userID: representation.fetch("userID"))

diff --git a/Sources/XCMetricsClient/Mobius/Effect Handlers/DumpParser/SchemaTypeBuilders.swift b/Sources/XCMetricsClient/Mobius/Effect Handlers/DumpParser/SchemaTypeBuilders.swift
@@ -76,7 +76,8 @@ extension BuildStep {
                          compilationEndTimestamp: compilationEndTimestamp ?? self.compilationEndTimestamp,
                          compilationDuration: compilationDuration ?? self.compilationDuration,
                          clangTimeTraceFile: nil,
-                         linkerStatistics: nil)
+                         linkerStatistics: nil,
+                         swiftTypeCheckTimes: nil)
     }
 }
 

diff --git a/Sources/XCMetricsClient/Mobius/Effect Handlers/UploadMetricsEffectHandler.swift b/Sources/XCMetricsClient/Mobius/Effect Handlers/UploadMetricsEffectHandler.swift
@@ -30,12 +30,14 @@ struct UploadMetricsEffectHandler: EffectHandler {
         self.metricsPublisher = metricsPublisher
     }
 
-    func handle(_ effectParameters: (serviceURL: URL, projectName: String, isCI: Bool, logs: Set<MetricsUploadRequest>), _ callback: EffectCallback<MetricsUploaderEvent>) -> Disposable {
+    func handle(_ effectParameters: (serviceURL: URL, projectName: String, isCI: Bool, skipNotes: Bool,
+                                     logs: Set<MetricsUploadRequest>), _ callback: EffectCallback<MetricsUploaderEvent>) -> Disposable {
         log("Started uploading metrics.")
         metricsPublisher.uploadMetrics(
             serviceURL: effectParameters.serviceURL,
             projectName: effectParameters.projectName,
             isCI: effectParameters.isCI,
+            skipNotes: effectParameters.skipNotes,
             uploadRequests: effectParameters.logs) { successfulURLs, failedURLs in
             var effects = [MetricsUploaderEvent]()
             // Handle failed log uploads. Skip if empty.

diff --git a/Sources/XCMetricsClient/Network/MetricsPublisherService.swift b/Sources/XCMetricsClient/Network/MetricsPublisherService.swift
@@ -30,10 +30,13 @@ protocol MetricsPublisherService {
     /// - Parameter uploadRequests: The upload requests to be sent to the backend service.
     /// - Parameter completion: The result is successfull if no error occurred. The .success enum case contains the URLs of the uploaded metrics.
     /// - Parameter projectName: The name of the project
+    /// - Parameter isCI: Boolean. If XCMetrics is running in CI or note
+    /// - Parameter skipNotes: Boolean. If the Notes found in the log won't be inserted in the database
     func uploadMetrics(
         serviceURL: URL,
         projectName: String,
         isCI: Bool,
+        skipNotes: Bool,
         uploadRequests: Set<MetricsUploadRequest>,
         completion: @escaping (_ successfulURLs: Set<URL>, _ failedURLs: [URL: Data]) -> Void
     )

diff --git a/Sources/XCMetricsClient/Network/MetricsPublisherServiceHTTP.swift b/Sources/XCMetricsClient/Network/MetricsPublisherServiceHTTP.swift
@@ -39,6 +39,7 @@ public class MetricsPublisherServiceHTTP: MetricsPublisherService {
         serviceURL: URL,
         projectName: String,
         isCI: Bool,
+        skipNotes: Bool,
         uploadRequests: Set<MetricsUploadRequest>,
         completion: @escaping (_ successfulURLs: Set<URL>, _ failedURLs: [URL: Data]) -> Void
     ) {
@@ -49,7 +50,7 @@ public class MetricsPublisherServiceHTTP: MetricsPublisherService {
         for uploadRequest in uploadRequests {
             self.dispatchGroup.enter()
 
-            self.uploadLog(uploadRequest, to: serviceURL, projectName: projectName, isCI: isCI) { (result: Result<Void, LogUploadError>) in
+            self.uploadLog(uploadRequest, to: serviceURL, projectName: projectName, isCI: isCI, skipNotes: skipNotes) { (result: Result<Void, LogUploadError>) in
                 switch result {
                 case .success:
                     successfulURLsLock.lock()
@@ -81,6 +82,7 @@ public class MetricsPublisherServiceHTTP: MetricsPublisherService {
         to requestUrl: URL,
         projectName: String,
         isCI: Bool,
+        skipNotes: Bool,
         completion: @escaping (Result<Void, LogUploadError>) -> Void
     ) {
         /// We send the unencrypted machine name, the backend will decide if is going to store it encrypted or not
@@ -91,7 +93,8 @@ public class MetricsPublisherServiceHTTP: MetricsPublisherService {
                            url: requestUrl,
                            machineName: machineName,
                            projectName: projectName,
-                           isCI: isCI).build()
+                           isCI: isCI,
+                           skipNotes: skipNotes).build()
 
             getURLSession().dataTask(with: request) { (data, response, error) in
                 defer {