diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d6175e132..9e515ea0d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ jobs: distribution: 'temurin' java-version: '11' #test DK - - uses: coursier/cache-action@v5 + - uses: coursier/cache-action@v6 - name: test id: test run: | diff --git a/daikoku/app/controllers/AdminApiController.scala b/daikoku/app/controllers/AdminApiController.scala index 21d5b8233..11d63eed2 100644 --- a/daikoku/app/controllers/AdminApiController.scala +++ b/daikoku/app/controllers/AdminApiController.scala @@ -95,6 +95,10 @@ class StateController(DaikokuAction: DaikokuAction, tenant.copy(tenantMode = TenantMode.Maintenance.some))) } _ <- removeAllUserSessions(ctx) + _ <- env.dataStore.notificationRepo + .forAllTenant() + .delete( + Json.obj("status.status" -> Json.obj("$ne" -> "Pending"))) _ <- postgresStore.checkDatabase() source = env.dataStore.exportAsStream(pretty = false) _ <- postgresStore.importFromStream(source) diff --git a/daikoku/app/domain/json.scala b/daikoku/app/domain/json.scala index a75b14f00..6ae0b21b0 100644 --- a/daikoku/app/domain/json.scala +++ b/daikoku/app/domain/json.scala @@ -109,7 +109,8 @@ object json { clientSecret = (json \ "clientSecret") .asOpt[String] .getOrElse("admin-api-apikey-secret"), - elasticConfig = (json \ "elasticConfig").asOpt(ElasticAnalyticsConfig.format) + elasticConfig = + (json \ "elasticConfig").asOpt(ElasticAnalyticsConfig.format) ) ) } recover { @@ -122,7 +123,10 @@ object json { "host" -> o.host, "clientId" -> o.clientId, "clientSecret" -> o.clientSecret, - "elasticConfig" -> o.elasticConfig.map(ElasticAnalyticsConfig.format.writes).getOrElse(JsNull).as[JsValue] + "elasticConfig" -> o.elasticConfig + .map(ElasticAnalyticsConfig.format.writes) + .getOrElse(JsNull) + .as[JsValue] ) } diff --git a/daikoku/app/env/evolutions.scala b/daikoku/app/env/evolutions.scala index e12054425..47e56d906 100644 --- a/daikoku/app/env/evolutions.scala +++ b/daikoku/app/env/evolutions.scala @@ -893,6 +893,51 @@ object evolution_1630 extends EvolutionScript { } } + +object evolution_1634 extends EvolutionScript { + override def version: String = "16.3.4" + + override def script: ( + Option[DatastoreId], + DataStore, + Materializer, + ExecutionContext, + OtoroshiClient + ) => Future[Done] = + ( + _: Option[DatastoreId], + dataStore: DataStore, + mat: Materializer, + ec: ExecutionContext, + _: OtoroshiClient + ) => { + AppLogger.info(s"Begin evolution $version - update all consumptions to set state") + + implicit val executionContext: ExecutionContext = ec + implicit val materializer: Materializer = mat + + dataStore.consumptionRepo.forAllTenant().streamAllRaw() + .mapAsync(10)(consumption => { + (consumption \ "state").asOpt(json.ApiKeyConsumptionStateFormat) match { + case Some(_) => FastFuture.successful(()) + case None => + val from = (consumption \ "from").as(json.DateTimeFormat) + val to = (consumption \ "to").as(json.DateTimeFormat) + val id = (consumption \ "_id").as[String] + + val state = if (from.plusDays(1).equals(to)) + ApiKeyConsumptionState.Completed + else + ApiKeyConsumptionState.InProgress + + dataStore.consumptionRepo.forAllTenant() + .save(Json.obj("_id" -> id), consumption.as[JsObject] + ("state" -> json.ApiKeyConsumptionStateFormat.writes(state))) + } + }) + .runWith(Sink.ignore) + } +} + object evolutions { val list: List[EvolutionScript] = List( @@ -908,7 +953,8 @@ object evolutions { evolution_1612_c, evolution_1613, evolution_1613_b, - evolution_1630 + evolution_1630, + evolution_1634 ) def run( dataStore: DataStore, diff --git a/daikoku/app/utils/otoroshi.scala b/daikoku/app/utils/otoroshi.scala index 32e24f055..a1de670bb 100644 --- a/daikoku/app/utils/otoroshi.scala +++ b/daikoku/app/utils/otoroshi.scala @@ -6,10 +6,19 @@ import cats.data.EitherT import cats.implicits.catsSyntaxOptionId import controllers.AppError import controllers.AppError.OtoroshiError -import fr.maif.otoroshi.daikoku.audit.{ElasticReadsAnalytics, ElasticWritesAnalytics} +import fr.maif.otoroshi.daikoku.audit.{ + ElasticReadsAnalytics, + ElasticWritesAnalytics +} import fr.maif.otoroshi.daikoku.audit.config.ElasticAnalyticsConfig import fr.maif.otoroshi.daikoku.domain.json.ActualOtoroshiApiKeyFormat -import fr.maif.otoroshi.daikoku.domain.{ActualOtoroshiApiKey, ApiSubscription, OtoroshiSettings, Tenant, json} +import fr.maif.otoroshi.daikoku.domain.{ + ActualOtoroshiApiKey, + ApiSubscription, + OtoroshiSettings, + Tenant, + json +} import fr.maif.otoroshi.daikoku.env.Env import fr.maif.otoroshi.daikoku.logger.AppLogger import play.api.libs.json._ @@ -291,7 +300,9 @@ class OtoroshiClient(env: Env) { } } - def getSubscriptionLastUsage(subscriptions: Seq[ApiSubscription])(implicit otoroshiSettings: OtoroshiSettings, tenant: Tenant): EitherT[Future, JsArray, JsArray] = { + def getSubscriptionLastUsage(subscriptions: Seq[ApiSubscription])( + implicit otoroshiSettings: OtoroshiSettings, + tenant: Tenant): EitherT[Future, JsArray, JsArray] = { otoroshiSettings.elasticConfig match { case Some(config) => new ElasticReadsAnalytics(config, env) @@ -300,42 +311,48 @@ class OtoroshiClient(env: Env) { "bool" -> Json.obj( "filter" -> Json.arr( Json.obj("terms" -> Json.obj( - "identity.identity" -> JsArray(subscriptions.map(_.apiKey.clientId).map(JsString)) + "identity.identity" -> JsArray( + subscriptions.map(_.apiKey.clientId).map(JsString)) )) ) ) ), - "aggs" -> Json.obj( - "lastUsages" -> Json.obj( - "terms" -> Json.obj( - "field" -> "identity.identity" - ), - "aggs" -> Json.obj( - "latest" -> Json.obj( - "top_hits" -> Json.obj( - "size" -> 1, - "sort" -> Json.arr(Json.obj( - "@timestamp" -> Json.obj( - "order" -> "desc" - ) - )) - ) + "aggs" -> Json.obj("lastUsages" -> Json.obj( + "terms" -> Json.obj( + "field" -> "identity.identity" + ), + "aggs" -> Json.obj( + "latest" -> Json.obj( + "top_hits" -> Json.obj( + "size" -> 1, + "sort" -> Json.arr(Json.obj( + "@timestamp" -> Json.obj( + "order" -> "desc" + ) + )) ) ) - )), + ) + )), "size" -> 0 )) .map(resp => { - val buckets = (resp \ "aggregations" \ "lastUsages" \ "buckets").as[JsArray] + val buckets = + (resp \ "aggregations" \ "lastUsages" \ "buckets").as[JsArray] JsArray(buckets.value.map(agg => { val key = (agg \ "key").as[String] - val lastUsage = (agg \ "latest" \ "hits" \ "hits").as[JsArray].value.head + val lastUsage = + (agg \ "latest" \ "hits" \ "hits").as[JsArray].value.head val date = (lastUsage \ "_source" \ "@timestamp").as[JsValue] Json.obj( "clientName" -> key, "date" -> date, - "subscription" -> subscriptions.find(_.apiKey.clientId == key).map(_.id.asJson).getOrElse(JsNull).as[JsValue] + "subscription" -> subscriptions + .find(_.apiKey.clientId == key) + .map(_.id.asJson) + .getOrElse(JsNull) + .as[JsValue] ) })) }) @@ -343,25 +360,34 @@ class OtoroshiClient(env: Env) { AppLogger.error(e.getErrorMessage()) Json.arr() }) - case None => for { - elasticConfig <- EitherT.fromOptionF(getElasticConfig(), Json.arr()) - updatedSettings = otoroshiSettings.copy(elasticConfig = elasticConfig.some) - updatedTenant = tenant.copy(otoroshiSettings = tenant.otoroshiSettings.filter(_.id != otoroshiSettings.id) + updatedSettings) - _ <- EitherT.liftF(env.dataStore.tenantRepo.save(updatedTenant)) - r <- getSubscriptionLastUsage(subscriptions)(updatedSettings, updatedTenant) - } yield r + case None => + for { + elasticConfig <- EitherT.fromOptionF(getElasticConfig(), Json.arr()) + updatedSettings = otoroshiSettings.copy( + elasticConfig = elasticConfig.some) + updatedTenant = tenant.copy( + otoroshiSettings = tenant.otoroshiSettings.filter( + _.id != otoroshiSettings.id) + updatedSettings) + _ <- EitherT.liftF(env.dataStore.tenantRepo.save(updatedTenant)) + r <- getSubscriptionLastUsage(subscriptions)(updatedSettings, + updatedTenant) + } yield r } } - private def getElasticConfig()(implicit otoroshiSettings: OtoroshiSettings): Future[Option[ElasticAnalyticsConfig]] = { - client(s"/api/globalconfig").get().map(resp => { - if (resp.status == 200) { - val config = resp.json.as[JsObject] - val elasticReadConfig = (config \ "elasticReadsConfig").asOpt(ElasticAnalyticsConfig.format) - elasticReadConfig - } else { - None - } - }) + private def getElasticConfig()(implicit otoroshiSettings: OtoroshiSettings) + : Future[Option[ElasticAnalyticsConfig]] = { + client(s"/api/globalconfig") + .get() + .map(resp => { + if (resp.status == 200) { + val config = resp.json.as[JsObject] + val elasticReadConfig = + (config \ "elasticReadsConfig").asOpt(ElasticAnalyticsConfig.format) + elasticReadConfig + } else { + None + } + }) } } diff --git a/daikoku/build.sbt b/daikoku/build.sbt index 9bdb9ca7b..c57c6b922 100644 --- a/daikoku/build.sbt +++ b/daikoku/build.sbt @@ -5,7 +5,7 @@ import com.typesafe.sbt.packager.docker.DockerPermissionStrategy name := """daikoku""" organization := "fr.maif.otoroshi" maintainer := "oss@maif.fr" -packageName in Universal := "daikoku" +Universal / packageName := "daikoku" scalaVersion := "2.13.1" @@ -20,9 +20,9 @@ lazy val root = (project in file(".")) buildInfoPackage := "daikoku" ) -javaOptions in Test += "-Dconfig.file=conf/application.test.conf" +Test / javaOptions += "-Dconfig.file=conf/application.test.conf" -assemblyMergeStrategy in assembly := { +assembly / assemblyMergeStrategy := { case PathList("org", "apache", "commons", "logging", xs @ _*) => MergeStrategy.first case PathList(ps @ _*) if ps.contains("module-info.class") => @@ -38,7 +38,7 @@ assemblyMergeStrategy in assembly := { case "META-INF/mailcap.default" => MergeStrategy.last case "META-INF/mimetypes.default" => MergeStrategy.last case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value + val oldStrategy = (assembly / assemblyMergeStrategy).value oldStrategy(x) } @@ -120,11 +120,11 @@ PlayKeys.devSettings := Seq("play.server.http.port" -> "9000") /// ASSEMBLY CONFIG -mainClass in assembly := Some("play.core.server.ProdServerStart") -test in assembly := {} -assemblyJarName in assembly := "daikoku.jar" -fullClasspath in assembly += Attributed.blank(PlayKeys.playPackageAssets.value) -assemblyMergeStrategy in assembly := { +assembly / mainClass := Some("play.core.server.ProdServerStart") +assembly / test := {} +assembly / assemblyJarName := "daikoku.jar" +assembly / fullClasspath += Attributed.blank(PlayKeys.playPackageAssets.value) +assembly / assemblyMergeStrategy := { //case PathList("META-INF", xs @ _*) => MergeStrategy.discard case PathList("javax", xs @ _*) => MergeStrategy.first @@ -142,14 +142,14 @@ assemblyMergeStrategy in assembly := { case PathList(ps @ _*) if ps.contains("buildinfo") => MergeStrategy.discard case o => - val oldStrategy = (assemblyMergeStrategy in assembly).value + val oldStrategy = (assembly / assemblyMergeStrategy).value oldStrategy(o) } lazy val packageAll = taskKey[Unit]("PackageAll") packageAll := { - (dist in Compile).value - (assembly in Compile).value + (Compile / dist).value + (Compile / assembly).value } /// DOCKER CONFIG @@ -157,8 +157,8 @@ packageAll := { dockerExposedPorts := Seq( 8080 ) -packageName in Docker := "daikoku" -maintainer in Docker := "MAIF OSS Team " +Docker / packageName := "daikoku" +Docker / maintainer := "MAIF OSS Team " dockerBaseImage := "eclipse-temurin:11.0.13_8-jre-focal" dockerUsername := Some("maif") dockerUpdateLatest := true @@ -166,7 +166,7 @@ dockerCommands := dockerCommands.value.filterNot { case ExecCmd("CMD", args @ _*) => true case cmd => false } -dockerPackageMappings in Docker += (baseDirectory.value / "docker" / "start.sh") -> "/opt/docker/bin/start.sh" +Docker / dockerPackageMappings += (baseDirectory.value / "docker" / "start.sh") -> "/opt/docker/bin/start.sh" dockerEntrypoint := Seq("/opt/docker/bin/start.sh") dockerUpdateLatest := true diff --git a/daikoku/conf/base.conf b/daikoku/conf/base.conf index c6fd19603..36ae0c98a 100644 --- a/daikoku/conf/base.conf +++ b/daikoku/conf/base.conf @@ -157,6 +157,8 @@ http.port = 8080 http.port = ${?PORT} https.port = disabled https.port = ${?HTTPS_PORT} +http2.enabled = no +http2.enabled = ${?HTTP2_ENABLED} play { application.loader = "fr.maif.otoroshi.daikoku.DaikokuLoader" diff --git a/daikoku/javascript/src/services/index.ts b/daikoku/javascript/src/services/index.ts index 95dff27d3..7374a684b 100644 --- a/daikoku/javascript/src/services/index.ts +++ b/daikoku/javascript/src/services/index.ts @@ -1165,7 +1165,7 @@ export const importApiPages = ( method: 'PUT', body: JSON.stringify({ pages, - linked + linked, }), }); @@ -1181,7 +1181,7 @@ export const importPlanPages = ( method: 'PUT', body: JSON.stringify({ pages, - linked + linked, }), }); @@ -1947,12 +1947,15 @@ export const fetchInvoices = (teamId: string, apiId: string, planId: string, cal customFetch(`/api/teams/${teamId}/apis/${apiId}/plan/${planId}/invoices?callback=${callback}`); export type ILastUsage = { - clientName: string - date: number - subscription: string -} -export const getSubscriptionsLastUsages = (teamId: string, subscriptions: Array): PromiseWithError> => + clientName: string; + date: number; + subscription: string; +}; +export const getSubscriptionsLastUsages = ( + teamId: string, + subscriptions: Array +): PromiseWithError> => customFetch(`/api/teams/${teamId}/subscriptions/_lastUsage`, { method: 'POST', - body: JSON.stringify({subscriptions}) - }) + body: JSON.stringify({ subscriptions }), + }); diff --git a/daikoku/javascript/src/types/api.ts b/daikoku/javascript/src/types/api.ts index 0cb399ca5..42a3ff17c 100644 --- a/daikoku/javascript/src/types/api.ts +++ b/daikoku/javascript/src/types/api.ts @@ -332,7 +332,7 @@ export interface IDocPage { remoteContentEnabled: boolean; remoteContentUrl: string | null; remoteContentHeaders: object; - linked?: boolean + linked?: boolean; } export interface IOtoroshiApiKey { diff --git a/daikoku/project/plugins.sbt b/daikoku/project/plugins.sbt index ef1fd9318..d19118df5 100644 --- a/daikoku/project/plugins.sbt +++ b/daikoku/project/plugins.sbt @@ -1,4 +1,4 @@ -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.13") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.19") addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.3") addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "1.15") addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.12") diff --git a/daikoku/test/daikoku/EnvironmentDisplayMode.scala b/daikoku/test/daikoku/EnvironmentDisplayMode.scala index ec4f1ff7f..9692af0f4 100644 --- a/daikoku/test/daikoku/EnvironmentDisplayMode.scala +++ b/daikoku/test/daikoku/EnvironmentDisplayMode.scala @@ -41,7 +41,7 @@ class EnvironmentDisplayMode() autoRotation = Some(false) ) - "a usage plan" must { + "a usage plan" must { "have a custom name as an avalaible environment" in { val api = generateApi("0", tenant.id, teamOwnerId, Seq.empty).api diff --git a/daikoku/version.sbt b/daikoku/version.sbt index 6d095df3f..440ac21ac 100644 --- a/daikoku/version.sbt +++ b/daikoku/version.sbt @@ -1 +1 @@ -version in ThisBuild := "16.4.0-dev" +ThisBuild / version := "16.4.0-dev" diff --git a/docs/manual/about.html b/docs/manual/about.html index ad24c3bee..45f08defe 100644 --- a/docs/manual/about.html +++ b/docs/manual/about.html @@ -30,7 +30,7 @@ Daikoku
-16.3.3 +16.3.4