Merge branch 'master' into technical/upgrade-js-deps

.github/workflows/test.yml

@@ -34,7 +34,7 @@ jobs:
         distribution: 'temurin'
         java-version: '11'
     #test DK
-    - uses: coursier/cache-action@v5
+    - uses: coursier/cache-action@v6
     - name: test
       id: test
       run: |

daikoku/app/controllers/AdminApiController.scala

@@ -95,6 +95,10 @@ class StateController(DaikokuAction: DaikokuAction,
                     tenant.copy(tenantMode = TenantMode.Maintenance.some)))
               }
               _ <- removeAllUserSessions(ctx)
+              _ <- env.dataStore.notificationRepo
+                .forAllTenant()
+                .delete(
+                  Json.obj("status.status" -> Json.obj("$ne" -> "Pending")))
               _ <- postgresStore.checkDatabase()
               source = env.dataStore.exportAsStream(pretty = false)
               _ <- postgresStore.importFromStream(source)

daikoku/app/domain/json.scala

@@ -109,7 +109,8 @@ object json {
             clientSecret = (json \ "clientSecret")
               .asOpt[String]
               .getOrElse("admin-api-apikey-secret"),
-            elasticConfig = (json \ "elasticConfig").asOpt(ElasticAnalyticsConfig.format)
+            elasticConfig =
+              (json \ "elasticConfig").asOpt(ElasticAnalyticsConfig.format)
           )
         )
       } recover {
@@ -122,7 +123,10 @@ object json {
       "host" -> o.host,
       "clientId" -> o.clientId,
       "clientSecret" -> o.clientSecret,
-      "elasticConfig" -> o.elasticConfig.map(ElasticAnalyticsConfig.format.writes).getOrElse(JsNull).as[JsValue]
+      "elasticConfig" -> o.elasticConfig
+        .map(ElasticAnalyticsConfig.format.writes)
+        .getOrElse(JsNull)
+        .as[JsValue]
     )
   }
 

daikoku/app/env/evolutions.scala

@@ -893,6 +893,51 @@ object evolution_1630 extends EvolutionScript {
     }
 }
 
+
+object evolution_1634 extends EvolutionScript {
+  override def version: String = "16.3.4"
+
+  override def script: (
+    Option[DatastoreId],
+      DataStore,
+      Materializer,
+      ExecutionContext,
+      OtoroshiClient
+    ) => Future[Done] =
+    (
+      _: Option[DatastoreId],
+      dataStore: DataStore,
+      mat: Materializer,
+      ec: ExecutionContext,
+      _: OtoroshiClient
+    ) => {
+      AppLogger.info(s"Begin evolution $version - update all consumptions to set state")
+
+      implicit val executionContext: ExecutionContext = ec
+      implicit val materializer: Materializer = mat
+
+      dataStore.consumptionRepo.forAllTenant().streamAllRaw()
+        .mapAsync(10)(consumption => {
+          (consumption \ "state").asOpt(json.ApiKeyConsumptionStateFormat) match {
+            case Some(_) => FastFuture.successful(())
+            case None =>
+              val from = (consumption \ "from").as(json.DateTimeFormat)
+              val to = (consumption \ "to").as(json.DateTimeFormat)
+              val id = (consumption \ "_id").as[String]
+
+              val state = if (from.plusDays(1).equals(to))
+                ApiKeyConsumptionState.Completed
+              else
+                ApiKeyConsumptionState.InProgress
+
+              dataStore.consumptionRepo.forAllTenant()
+                .save(Json.obj("_id" -> id), consumption.as[JsObject] + ("state" -> json.ApiKeyConsumptionStateFormat.writes(state)))
+          }
+        })
+        .runWith(Sink.ignore)
+    }
+}
+
 object evolutions {
   val list: List[EvolutionScript] =
     List(
@@ -908,7 +953,8 @@ object evolutions {
       evolution_1612_c,
       evolution_1613,
       evolution_1613_b,
-      evolution_1630
+      evolution_1630,
+      evolution_1634
     )
   def run(
       dataStore: DataStore,

daikoku/app/utils/otoroshi.scala

@@ -6,10 +6,19 @@ import cats.data.EitherT
 import cats.implicits.catsSyntaxOptionId
 import controllers.AppError
 import controllers.AppError.OtoroshiError
-import fr.maif.otoroshi.daikoku.audit.{ElasticReadsAnalytics, ElasticWritesAnalytics}
+import fr.maif.otoroshi.daikoku.audit.{
+  ElasticReadsAnalytics,
+  ElasticWritesAnalytics
+}
 import fr.maif.otoroshi.daikoku.audit.config.ElasticAnalyticsConfig
 import fr.maif.otoroshi.daikoku.domain.json.ActualOtoroshiApiKeyFormat
-import fr.maif.otoroshi.daikoku.domain.{ActualOtoroshiApiKey, ApiSubscription, OtoroshiSettings, Tenant, json}
+import fr.maif.otoroshi.daikoku.domain.{
+  ActualOtoroshiApiKey,
+  ApiSubscription,
+  OtoroshiSettings,
+  Tenant,
+  json
+}
 import fr.maif.otoroshi.daikoku.env.Env
 import fr.maif.otoroshi.daikoku.logger.AppLogger
 import play.api.libs.json._
@@ -291,7 +300,9 @@ class OtoroshiClient(env: Env) {
     }
   }
 
-  def getSubscriptionLastUsage(subscriptions: Seq[ApiSubscription])(implicit otoroshiSettings: OtoroshiSettings, tenant: Tenant): EitherT[Future, JsArray, JsArray] = {
+  def getSubscriptionLastUsage(subscriptions: Seq[ApiSubscription])(
+      implicit otoroshiSettings: OtoroshiSettings,
+      tenant: Tenant): EitherT[Future, JsArray, JsArray] = {
     otoroshiSettings.elasticConfig match {
       case Some(config) =>
         new ElasticReadsAnalytics(config, env)
@@ -300,68 +311,83 @@ class OtoroshiClient(env: Env) {
               "bool" -> Json.obj(
                 "filter" -> Json.arr(
                   Json.obj("terms" -> Json.obj(
-                    "identity.identity" -> JsArray(subscriptions.map(_.apiKey.clientId).map(JsString))
+                    "identity.identity" -> JsArray(
+                      subscriptions.map(_.apiKey.clientId).map(JsString))
                   ))
                 )
               )
             ),
-            "aggs" -> Json.obj(
-              "lastUsages" -> Json.obj(
-                "terms" -> Json.obj(
-                  "field" -> "identity.identity"
-                ),
-                "aggs" -> Json.obj(
-                  "latest" -> Json.obj(
-                    "top_hits" -> Json.obj(
-                      "size" -> 1,
-                      "sort" -> Json.arr(Json.obj(
-                        "@timestamp" -> Json.obj(
-                          "order" -> "desc"
-                        )
-                      ))
-                    )
+            "aggs" -> Json.obj("lastUsages" -> Json.obj(
+              "terms" -> Json.obj(
+                "field" -> "identity.identity"
+              ),
+              "aggs" -> Json.obj(
+                "latest" -> Json.obj(
+                  "top_hits" -> Json.obj(
+                    "size" -> 1,
+                    "sort" -> Json.arr(Json.obj(
+                      "@timestamp" -> Json.obj(
+                        "order" -> "desc"
+                      )
+                    ))
                   )
                 )
-              )),
+              )
+            )),
             "size" -> 0
           ))
           .map(resp => {
-            val buckets = (resp \ "aggregations" \ "lastUsages" \ "buckets").as[JsArray]
+            val buckets =
+              (resp \ "aggregations" \ "lastUsages" \ "buckets").as[JsArray]
             JsArray(buckets.value.map(agg => {
               val key = (agg \ "key").as[String]
-              val lastUsage = (agg \ "latest" \ "hits" \ "hits").as[JsArray].value.head
+              val lastUsage =
+                (agg \ "latest" \ "hits" \ "hits").as[JsArray].value.head
               val date = (lastUsage \ "_source" \ "@timestamp").as[JsValue]
 
               Json.obj(
                 "clientName" -> key,
                 "date" -> date,
-                "subscription" -> subscriptions.find(_.apiKey.clientId == key).map(_.id.asJson).getOrElse(JsNull).as[JsValue]
+                "subscription" -> subscriptions
+                  .find(_.apiKey.clientId == key)
+                  .map(_.id.asJson)
+                  .getOrElse(JsNull)
+                  .as[JsValue]
               )
             }))
           })
           .leftMap(e => {
             AppLogger.error(e.getErrorMessage())
             Json.arr()
           })
-      case None => for {
-        elasticConfig <- EitherT.fromOptionF(getElasticConfig(), Json.arr())
-        updatedSettings = otoroshiSettings.copy(elasticConfig = elasticConfig.some)
-        updatedTenant = tenant.copy(otoroshiSettings = tenant.otoroshiSettings.filter(_.id != otoroshiSettings.id) + updatedSettings)
-        _ <- EitherT.liftF(env.dataStore.tenantRepo.save(updatedTenant))
-        r <- getSubscriptionLastUsage(subscriptions)(updatedSettings, updatedTenant)
-      } yield r
+      case None =>
+        for {
+          elasticConfig <- EitherT.fromOptionF(getElasticConfig(), Json.arr())
+          updatedSettings = otoroshiSettings.copy(
+            elasticConfig = elasticConfig.some)
+          updatedTenant = tenant.copy(
+            otoroshiSettings = tenant.otoroshiSettings.filter(
+              _.id != otoroshiSettings.id) + updatedSettings)
+          _ <- EitherT.liftF(env.dataStore.tenantRepo.save(updatedTenant))
+          r <- getSubscriptionLastUsage(subscriptions)(updatedSettings,
+                                                       updatedTenant)
+        } yield r
     }
   }
 
-  private def getElasticConfig()(implicit otoroshiSettings: OtoroshiSettings): Future[Option[ElasticAnalyticsConfig]] = {
-    client(s"/api/globalconfig").get().map(resp => {
-      if (resp.status == 200) {
-        val config = resp.json.as[JsObject]
-        val elasticReadConfig = (config \ "elasticReadsConfig").asOpt(ElasticAnalyticsConfig.format)
-        elasticReadConfig
-      } else {
-        None
-      }
-    })
+  private def getElasticConfig()(implicit otoroshiSettings: OtoroshiSettings)
+    : Future[Option[ElasticAnalyticsConfig]] = {
+    client(s"/api/globalconfig")
+      .get()
+      .map(resp => {
+        if (resp.status == 200) {
+          val config = resp.json.as[JsObject]
+          val elasticReadConfig =
+            (config \ "elasticReadsConfig").asOpt(ElasticAnalyticsConfig.format)
+          elasticReadConfig
+        } else {
+          None
+        }
+      })
   }
 }

daikoku/build.sbt

@@ -5,7 +5,7 @@ import com.typesafe.sbt.packager.docker.DockerPermissionStrategy
 name := """daikoku"""
 organization := "fr.maif.otoroshi"
 maintainer := "[email protected]"
-packageName in Universal := "daikoku"
+Universal / packageName := "daikoku"
 
 scalaVersion := "2.13.1"
 
@@ -20,9 +20,9 @@ lazy val root = (project in file("."))
     buildInfoPackage := "daikoku"
   )
 
-javaOptions in Test += "-Dconfig.file=conf/application.test.conf"
+Test / javaOptions += "-Dconfig.file=conf/application.test.conf"
 
-assemblyMergeStrategy in assembly := {
+assembly / assemblyMergeStrategy := {
   case PathList("org", "apache", "commons", "logging", xs @ _*) =>
     MergeStrategy.first
   case PathList(ps @ _*) if ps.contains("module-info.class") =>
@@ -38,7 +38,7 @@ assemblyMergeStrategy in assembly := {
   case "META-INF/mailcap.default"   => MergeStrategy.last
   case "META-INF/mimetypes.default" => MergeStrategy.last
   case x =>
-    val oldStrategy = (assemblyMergeStrategy in assembly).value
+    val oldStrategy = (assembly / assemblyMergeStrategy).value
     oldStrategy(x)
 }
 
@@ -120,11 +120,11 @@ PlayKeys.devSettings := Seq("play.server.http.port" -> "9000")
 
 /// ASSEMBLY CONFIG
 
-mainClass in assembly := Some("play.core.server.ProdServerStart")
-test in assembly := {}
-assemblyJarName in assembly := "daikoku.jar"
-fullClasspath in assembly += Attributed.blank(PlayKeys.playPackageAssets.value)
-assemblyMergeStrategy in assembly := {
+assembly / mainClass := Some("play.core.server.ProdServerStart")
+assembly / test := {}
+assembly / assemblyJarName := "daikoku.jar"
+assembly / fullClasspath += Attributed.blank(PlayKeys.playPackageAssets.value)
+assembly / assemblyMergeStrategy := {
   //case PathList("META-INF", xs @ _*) => MergeStrategy.discard
   case PathList("javax", xs @ _*) =>
     MergeStrategy.first
@@ -142,31 +142,31 @@ assemblyMergeStrategy in assembly := {
   case PathList(ps @ _*) if ps.contains("buildinfo") =>
     MergeStrategy.discard
   case o =>
-    val oldStrategy = (assemblyMergeStrategy in assembly).value
+    val oldStrategy = (assembly / assemblyMergeStrategy).value
     oldStrategy(o)
 }
 
 lazy val packageAll = taskKey[Unit]("PackageAll")
 packageAll := {
-  (dist in Compile).value
-  (assembly in Compile).value
+  (Compile / dist).value
+  (Compile / assembly).value
 }
 
 /// DOCKER CONFIG
 
 dockerExposedPorts := Seq(
   8080
 )
-packageName in Docker := "daikoku"
-maintainer in Docker := "MAIF OSS Team <[email protected]>"
+Docker / packageName := "daikoku"
+Docker / maintainer := "MAIF OSS Team <[email protected]>"
 dockerBaseImage := "eclipse-temurin:11.0.13_8-jre-focal"
 dockerUsername := Some("maif")
 dockerUpdateLatest := true
 dockerCommands := dockerCommands.value.filterNot {
   case ExecCmd("CMD", args @ _*) => true
   case cmd                       => false
 }
-dockerPackageMappings in Docker += (baseDirectory.value / "docker" / "start.sh") -> "/opt/docker/bin/start.sh"
+Docker / dockerPackageMappings += (baseDirectory.value / "docker" / "start.sh") -> "/opt/docker/bin/start.sh"
 dockerEntrypoint := Seq("/opt/docker/bin/start.sh")
 dockerUpdateLatest := true
 

daikoku/conf/base.conf

@@ -157,6 +157,8 @@ http.port = 8080
 http.port = ${?PORT}
 https.port = disabled
 https.port = ${?HTTPS_PORT}
+http2.enabled = no
+http2.enabled = ${?HTTP2_ENABLED}
 
 play {
   application.loader = "fr.maif.otoroshi.daikoku.DaikokuLoader"

daikoku/javascript/src/services/index.ts

@@ -1165,7 +1165,7 @@ export const importApiPages = (
     method: 'PUT',
     body: JSON.stringify({
       pages,
-      linked
+      linked,
     }),
   });
 
@@ -1181,7 +1181,7 @@ export const importPlanPages = (
     method: 'PUT',
     body: JSON.stringify({
       pages,
-      linked
+      linked,
     }),
   });
 
@@ -1947,12 +1947,15 @@ export const fetchInvoices = (teamId: string, apiId: string, planId: string, cal
   customFetch(`/api/teams/${teamId}/apis/${apiId}/plan/${planId}/invoices?callback=${callback}`);
 
 export type ILastUsage = {
-  clientName: string
-  date: number
-  subscription: string
-}
-export const getSubscriptionsLastUsages = (teamId: string, subscriptions: Array<string>): PromiseWithError<Array<ILastUsage>> =>
+  clientName: string;
+  date: number;
+  subscription: string;
+};
+export const getSubscriptionsLastUsages = (
+  teamId: string,
+  subscriptions: Array<string>
+): PromiseWithError<Array<ILastUsage>> =>
   customFetch(`/api/teams/${teamId}/subscriptions/_lastUsage`, {
     method: 'POST',
-    body: JSON.stringify({subscriptions})
-  })
+    body: JSON.stringify({ subscriptions }),
+  });