Added arch in metrics

Signed-off-by: munishchouhan <[email protected]>
seqeralabs · Jan 31, 2025 · b0c8bfb · b0c8bfb
1 parent facbf99
commit b0c8bfb
Show file tree

Hide file tree

Showing 16 changed files with 364 additions and 123 deletions.
diff --git a/docs/metrics.mdx b/docs/metrics.mdx
@@ -15,6 +15,15 @@ These are stored using the following keys:
 - `builds/d/YYYY-MM-DD`
 - `builds/o/<org>`
 - `builds/o/<org>/d/YYYY-MM-DD`
+- `pulls/a/<arch>/d/YYYY-MM-DD`
+- `pulls/o/<org>/a/<arch>`
+- `pulls/o/<org>/a/<arch>/d/YYYY-MM-DD`
+- `fusion/a/<arch>/d/YYYY-MM-DD`
+- `fusion/o/<org>/a/<arch>`
+- `fusion/o/<org>/a/<arch>/d/YYYY-MM-DD`
+- `builds/a/<arch>/d/YYYY-MM-DD`
+- `builds/o/<org>/a/<arch>`
+- `builds/o/<org>/a/<arch>/d/YYYY-MM-DD`
 
 ## Functionality
 

diff --git a/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy b/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy
@@ -18,7 +18,6 @@
 
 package io.seqera.wave.controller
 
-import java.util.regex.Pattern
 import javax.annotation.Nullable
 
 import groovy.transform.CompileStatic
@@ -35,10 +34,8 @@ import io.micronaut.scheduling.annotation.ExecuteOn
 import io.micronaut.security.annotation.Secured
 import io.micronaut.security.authentication.AuthorizationException
 import io.micronaut.security.rules.SecurityRule
-import io.seqera.wave.exception.BadRequestException
 import io.seqera.wave.service.metric.MetricsConstants
 import io.seqera.wave.service.metric.MetricsService
-
 import jakarta.inject.Inject
 
 import static io.micronaut.http.HttpHeaders.WWW_AUTHENTICATE
@@ -58,59 +55,88 @@ class MetricsController {
     @Inject
     private MetricsService metricsService
 
-    static final private Pattern DATE_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}");
-
+    @Deprecated
     @Get(uri = "/v1alpha2/metrics/builds", produces = MediaType.APPLICATION_JSON)
     HttpResponse<?> getBuildsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org) {
         if(!date && !org)
             return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_BUILDS))
-        validateQueryParams(date)
         return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_BUILDS, date, org))
     }
 
+    @Deprecated
     @Get(uri = "/v1alpha2/metrics/pulls", produces = MediaType.APPLICATION_JSON)
     HttpResponse<?> getPullsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org) {
         if(!date && !org)
             return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_PULLS))
-        validateQueryParams(date)
         return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_PULLS, date, org))
     }
 
+    @Deprecated
     @Get(uri = "/v1alpha2/metrics/fusion/pulls", produces = MediaType.APPLICATION_JSON)
     HttpResponse<?> getFusionPullsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org) {
         if(!date && !org)
             return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_FUSION))
-        validateQueryParams(date)
         return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_FUSION, date, org))
 
     }
 
+    @Deprecated
     @Get(uri = "/v1alpha2/metrics/mirrors", produces = MediaType.APPLICATION_JSON)
     HttpResponse<?> getMirrorsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org) {
         if(!date && !org)
             return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_MIRRORS))
-        validateQueryParams(date)
         return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_MIRRORS, date, org))
     }
 
+    @Deprecated
     @Get(uri = "/v1alpha2/metrics/scans", produces = MediaType.APPLICATION_JSON)
     HttpResponse<?> getScansMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org) {
         if(!date && !org)
             return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_SCANS))
-        validateQueryParams(date)
         return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_SCANS, date, org))
     }
 
+    @Get(uri = "/v1alpha3/metrics/builds", produces = MediaType.APPLICATION_JSON)
+    HttpResponse<?> getBuildsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org, @Nullable @QueryValue String arch) {
+        if(!date && !org)
+            return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_BUILDS, arch))
+        return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_BUILDS, date, org, arch))
+    }
+
+    @Get(uri = "/v1alpha3/metrics/pulls", produces = MediaType.APPLICATION_JSON)
+    HttpResponse<?> getPullsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org, @Nullable @QueryValue String arch) {
+        if(!date && !org)
+            return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_PULLS, arch))
+        return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_PULLS, date, org, arch))
+    }
+
+    @Get(uri = "/v1alpha3/metrics/fusion/pulls", produces = MediaType.APPLICATION_JSON)
+    HttpResponse<?> getFusionPullsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org, @Nullable @QueryValue String arch) {
+        if(!date && !org)
+            return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_FUSION, arch))
+        return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_FUSION, date, org, arch))
+
+    }
+
+    @Get(uri = "/v1alpha3/metrics/mirrors", produces = MediaType.APPLICATION_JSON)
+    HttpResponse<?> getMirrorsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org, @Nullable @QueryValue String arch) {
+        if(!date && !org)
+            return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_MIRRORS, arch))
+        return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_MIRRORS, date, org, arch))
+    }
+
+    @Get(uri = "/v1alpha3/metrics/scans", produces = MediaType.APPLICATION_JSON)
+    HttpResponse<?> getScansMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org, @Nullable @QueryValue String arch) {
+        if(!date && !org)
+            return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_SCANS, arch))
+        return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_SCANS, date, org, arch))
+    }
+
 
     @Error(exception = AuthorizationException.class)
     HttpResponse<?> handleAuthorizationException() {
         return HttpResponse.unauthorized()
                 .header(WWW_AUTHENTICATE, "Basic realm=Wave Authentication")
     }
 
-    static void validateQueryParams(String date) {
-        if(date && !DATE_PATTERN.matcher(date).matches()) {
-            throw new BadRequestException('date format should be yyyy-MM-dd')
-        }
-    }
 }
diff --git a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy
@@ -31,10 +31,10 @@ class ContainerPlatform {
 
     public static final ContainerPlatform DEFAULT = new ContainerPlatform(DEFAULT_OS, DEFAULT_ARCH)
 
-    private static List<String> ARM64 = ['arm64', 'aarch64']
+    final static List<String> ARM64 = ['arm64', 'aarch64']
     private static List<String> V8 = ['8','v8']
-    private static List<String> AMD64 = ['amd64', 'x86_64', 'x86-64']
-    private static List<String> ALLOWED_ARCH = AMD64 + ARM64 + ['arm']
+    final static List<String> AMD64 = ['amd64', 'x86_64', 'x86-64']
+    final static List<String> ALLOWED_ARCH = AMD64 + ARM64 + ['arm']
     public static final String DEFAULT_ARCH = 'amd64'
     public static final String DEFAULT_OS = 'linux'
 

diff --git a/src/main/groovy/io/seqera/wave/filter/MetricsQueryParamValidationFilter.groovy b/src/main/groovy/io/seqera/wave/filter/MetricsQueryParamValidationFilter.groovy
@@ -0,0 +1,40 @@
+package io.seqera.wave.filter
+
+import java.util.regex.Pattern
+
+import groovy.transform.CompileStatic
+import io.micronaut.context.annotation.Requires
+import io.micronaut.http.HttpRequest
+import io.micronaut.http.MutableHttpResponse
+import io.micronaut.http.annotation.Filter
+import io.micronaut.http.filter.HttpServerFilter
+import io.micronaut.http.filter.ServerFilterChain
+import io.seqera.wave.core.ContainerPlatform
+import io.seqera.wave.exception.BadRequestException
+import org.reactivestreams.Publisher
+
+@CompileStatic
+@Filter(["/v1alpha2/metrics/**", "/v1alpha3/metrics/**/**"])
+@Requires(property = 'wave.metrics.enabled', value = 'true')
+class MetricsQueryParamValidationFilter implements HttpServerFilter {
+
+    static final private Pattern DATE_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}")
+
+    @Override
+    Publisher<MutableHttpResponse<?>> doFilter(HttpRequest<?> request, ServerFilterChain chain) {
+
+        Map<String, List<String>> queryParams = request.getParameters().asMap()
+
+        final String date = queryParams.get("date")?[0]
+        final String arch = queryParams.get("arch")?[0]
+
+        if(date && !DATE_PATTERN.matcher(date).matches()) {
+            throw new BadRequestException('date format should be yyyy-MM-dd')
+        } else if (arch && !ContainerPlatform.ALLOWED_ARCH.contains(arch)) {
+            throw new BadRequestException('arch should be one of ' + ContainerPlatform.ALLOWED_ARCH)
+        }
+
+        return chain.proceed(request)
+    }
+}
+
diff --git a/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy b/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy
@@ -83,10 +83,11 @@ class PullMetricsRequestsFilter implements HttpServerFilter {
         final contentType = response.headers.get(HttpHeaders.CONTENT_TYPE)
         if( contentType && contentType in MANIFEST_TYPES ) {
             final route = routeHelper.parse(request.path)
-            CompletableFuture.runAsync(() -> metricsService.incrementPullsCounter(route.identity), executor)
+            final arch = route.request.platform.arch
+            CompletableFuture.runAsync(() -> metricsService.incrementPullsCounter(route.identity, arch), executor)
             final version = route.request?.containerConfig?.fusionVersion()
             if (version) {
-                CompletableFuture.runAsync(() -> metricsService.incrementFusionPullsCounter(route.identity), executor)
+                CompletableFuture.runAsync(() -> metricsService.incrementFusionPullsCounter(route.identity, arch), executor)
             }
         }
     }

diff --git a/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy
@@ -207,7 +207,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
 
         //increment metrics
         CompletableFuture
-                .runAsync(() -> metricsService.incrementBuildsCounter(request.identity), executor)
+                .runAsync(() -> metricsService.incrementBuildsCounter(request.identity, request.platform.arch), executor)
 
         // launch the build async
         CompletableFuture

diff --git a/src/main/groovy/io/seqera/wave/service/metric/MetricsConstants.groovy b/src/main/groovy/io/seqera/wave/service/metric/MetricsConstants.groovy
@@ -18,6 +18,8 @@
 
 package io.seqera.wave.service.metric
 
+import io.seqera.wave.core.ContainerPlatform
+
 /**
  * Metric constants
  *
@@ -39,4 +41,10 @@ interface MetricsConstants {
 
     static final public String PREFIX_DAY =  'd'
 
+    static final public String PREFIX_ARCH =  'a'
+
+    static final public String ARM64 =  'arm64'
+
+    static final public String AMD64 =  'amd64'
+
 }
diff --git a/src/main/groovy/io/seqera/wave/service/metric/MetricsService.groovy b/src/main/groovy/io/seqera/wave/service/metric/MetricsService.groovy
@@ -18,6 +18,7 @@
 
 package io.seqera.wave.service.metric
 
+import io.seqera.wave.service.metric.model.GetOrgArchCountResponse
 import io.seqera.wave.service.metric.model.GetOrgCountResponse
 import io.seqera.wave.tower.PlatformId
 /**
@@ -31,35 +32,35 @@ interface MetricsService {
      *
      * @param seqera platform id
      */
-    void incrementFusionPullsCounter(PlatformId platformId)
+    void incrementFusionPullsCounter(PlatformId platformId, String arch)
 
     /**
      * increment wave builds count
      *
      * @param seqera platform id
      */
-    void incrementBuildsCounter(PlatformId platformId)
+    void incrementBuildsCounter(PlatformId platformId, String arch)
 
     /**
      * increment wave pulls count
      *
      * @param seqera platform id
      */
-    void incrementPullsCounter(PlatformId platformId)
+    void incrementPullsCounter(PlatformId platformId, String arch)
 
     /**
      * increment wave mirrors count
      *
      * @param seqera platform id
      */
-    void incrementMirrorsCounter(PlatformId platformId)
+    void incrementMirrorsCounter(PlatformId platformId, String arch)
 
     /**
      * increment wave scans count
      *
      * @param seqera platform id
      */
-    void incrementScansCounter(PlatformId platformId)
+    void incrementScansCounter(PlatformId platformId, String arch)
 
     /**
      * Get counts of all organisations
@@ -76,4 +77,20 @@ interface MetricsService {
      * @return GetOrgCountResponse
      */
     GetOrgCountResponse getOrgCount(String metric, String date, String org)
+
+    /**
+     * Get counts by organisations or by date or by arch or by all
+     *
+     * @param metric
+     * @return GetOrgCountResponse
+     */
+    GetOrgArchCountResponse getOrgCount(String metric, String date, String org, String arch)
+
+    /**
+     * Get counts of all organisations by arch
+     *
+     * @param metric
+     * @return GetOrgCountResponse
+     */
+    GetOrgArchCountResponse getAllOrgCount(String metric, String arch)
 }