Merge pull request #231 from hmrc/feature/ITSASU-1369

[ITSASU-1369] Convert throttling repository to hmrc mongo.

README.md

@@ -45,6 +45,20 @@ To connect to the mongo db provided by docker (recommended) please use
 docker exec -it mongo-db mongosh
 ```
 
+# Throttling
+
+Throttling is managed via a record per minute per throttle point, in Mongo.  The current value can be viewed at 
+
+``` 
+http://localhost:9560/income-tax-subscription/test-only/throttle?throttle=<throttleId>
+```
+
+and the limit can be set by using the 'copy as cUrl' facility, and use using POST to:
+
+``` 
+http://localhost:9560/income-tax-subscription/test-only/throttle?throttle=<throttleId>&value=<throttleMaxValue>
+```
+
 ### License
 
 This code is open source software licensed under the [Apache 2.0 License]("http://www.apache.org/licenses/LICENSE-2.0.html")

app/config/MicroserviceAppConfig.scala

@@ -40,6 +40,8 @@ trait AppConfig {
   val paperlessPreferencesExpirySeconds: Int
   val desAuthorisationToken: String
   val desEnvironmentHeader: (String, String)
+
+  val mongoUri: String
 }
 
 @Singleton
@@ -76,4 +78,6 @@ class MicroserviceAppConfig @Inject()(servicesConfig: ServicesConfig, val config
   def businessSubscribeUrl(nino: String): String = s"$desURL/income-tax-self-assessment/nino/$nino/business"
 
   def propertySubscribeUrl(nino: String): String = s"$desURL/income-tax-self-assessment/nino/$nino/properties"
+
+  lazy val mongoUri: String = loadConfig("mongodb.uri")
 }

app/controllers/throttle/ThrottlingController.scala

@@ -31,13 +31,16 @@ import scala.util.Try
 class ThrottlingController @Inject()(throttlingRepository: ThrottlingRepository,
                                      servicesConfig: ServicesConfig,
                                      cc: ControllerComponents
-                                  ) extends BackendController(cc) with Logging {
+                                    ) extends BackendController(cc) with Logging {
 
   def throttled(throttleId: String): Action[AnyContent] = Action.async { _ =>
     val throttleKey = s"throttle.$throttleId.max"
-    Try {
+    val configValue: Option[Int] = Try {
       servicesConfig.getInt(throttleKey)
-    }.toOption match {
+    }.toOption
+    val configOrProperties: Option[Int] = sys.props.get(throttleKey).map(v => v.toInt) orElse configValue
+
+    configOrProperties match {
       case None =>
         logger.warn(s"No throttle max found for $throttleKey in config")
         Future.successful(BadRequest)

app/repositories/ThrottlingRepository.scala

@@ -16,19 +16,25 @@
 
 package repositories
 
-import config.MicroserviceAppConfig
-import config.featureswitch.FeatureSwitching
+import com.mongodb.client.model.{FindOneAndUpdateOptions, IndexOptions, ReturnDocument}
+import config.AppConfig
+import org.bson.Document
+import org.bson.conversions.Bson
+import org.mongodb.scala.model.IndexModel
+import org.mongodb.scala.result.InsertOneResult
+import org.mongodb.scala.{Observable, SingleObservable}
 import play.api.libs.json.{Format, JsObject, JsValue, Json}
-import play.modules.reactivemongo.ReactiveMongoComponent
-import reactivemongo.api.commands.FindAndModifyCommand
-import reactivemongo.api.indexes.{Index, IndexType}
-import reactivemongo.bson.{BSONDocument, BSONObjectID}
+import reactivemongo.bson.BSONDocument
 import reactivemongo.play.json.ImplicitBSONHandlers._
-import uk.gov.hmrc.mongo.ReactiveRepository
+import repositories.ThrottlingRepository._
+import uk.gov.hmrc.mongo.MongoComponent
+import uk.gov.hmrc.mongo.play.json.PlayMongoRepository
 
 import java.time.Instant
+import java.util.concurrent.TimeUnit
 import javax.inject.{Inject, Singleton}
 import scala.concurrent.{ExecutionContext, Future}
+import scala.language.implicitConversions
 
 @Singleton
 class InstantProvider @Inject()() {
@@ -38,21 +44,27 @@ class InstantProvider @Inject()() {
 }
 
 @Singleton
-class ThrottlingRepository @Inject()(mongo: ReactiveMongoComponent,
-                                     val appConfig: MicroserviceAppConfig,
-                                     instantProvider: InstantProvider)(implicit ec: ExecutionContext)
+class ThrottlingRepositoryConfig @Inject()(val appConfig: AppConfig) {
 
-  extends ReactiveRepository[JsObject, BSONObjectID](
-    "throttling",
-    mongo.mongoConnector.db,
-    implicitly[Format[JsObject]],
-    implicitly[Format[BSONObjectID]]
-  ) with FeatureSwitching {
+  private val ttlLengthSeconds = appConfig.timeToLiveSecondsSaveAndRetrieve
 
-  val throttleIdKey = "throttleId"
-  val timecodeKey = "timecode"
-  val countKey = "count"
-  val lastUpdatedTimestampKey = "lastUpdatedTimestamp"
+  def mongoComponent: MongoComponent = MongoComponent(appConfig.mongoUri)
+
+  def indexes: Seq[IndexModel] =
+    Seq(ttlIndex(ttlLengthSeconds)) ++
+      Seq(idTimecodeIndex).map(toIndexModel)
+}
+
+@Singleton
+class ThrottlingRepository @Inject()(config: ThrottlingRepositoryConfig, instantProvider: InstantProvider)(implicit ec: ExecutionContext)
+
+  extends PlayMongoRepository[JsObject](
+    collectionName = "throttling",
+    mongoComponent = config.mongoComponent,
+    domainFormat = implicitly[Format[JsObject]],
+    indexes = config.indexes,
+    replaceIndexes = true
+  ) {
 
   private def timecode(time: Long) = time / 60000
 
@@ -74,24 +86,85 @@ class ThrottlingRepository @Inject()(mongo: ReactiveMongoComponent,
     Json.obj(f"$$set" -> set, f"$$inc" -> inc)
   }
 
-  private def resultToThrottleCount: FindAndModifyCommand.Result[collection.pack.type] => Int = {
-    _.result[JsObject] match {
+    private def resultToThrottleCount: Option[JsValue] => Int = {
       case Some(json) => (json \ countKey).as[Int]
       case None => 0
     }
-  }
 
   def checkThrottle(id: String): Future[Int] = {
     val time: Long = instantProvider.getInstantNowMilli
+    val eventualMaybeValue = findAndUpdate(query(id, time), update(id, time))
+    eventualMaybeValue map resultToThrottleCount
+  }
+
+  def stateOfThrottle(id: String): Future[(Int, Long)] = {
+    val time: Long = instantProvider.getInstantNowMilli
+    val eventualMaybeValue = find(query(id, time), None).map(l => l.headOption)
+    eventualMaybeValue map resultToThrottleCount map (c => (c, timecode(time)))
+  }
+
+  private def findAndUpdate(selector: JsObject, update: JsObject): Future[Option[JsValue]] =
+    collection.findOneAndUpdate(selector, update, findOneAndUpdateOptions).toFuture().map(asOption)
+
+  def insert(document: JsObject): Future[InsertOneResult] = collection.insertOne(document).toFuture
+
+  def drop(): Future[Void] = collection.drop().toFuture
+
+  private val findOneAndUpdateOptions: FindOneAndUpdateOptions = new FindOneAndUpdateOptions()
+    .upsert(true)
+    .returnDocument(ReturnDocument.AFTER)
+
+  def find(selector: JsObject, projection: Option[JsObject]): Future[List[JsObject]] = {
+    collection
+      .find(selector)
+      .projection(projection.map(toBson).getOrElse(removeIdProjection))
+      .toFuture()
+      .map(_.toList)
+  }
 
-    findAndUpdate(
-      query = query(id, time),
-      update = update(id, time),
-      fetchNewObject = true,
-      upsert = true
-    ) map resultToThrottleCount
+  val _Id = "_id"
+  private val removeIdProjection = toBson(Json.obj(_Id -> 0))
+}
+
+
+object ThrottlingRepository {
+
+  case class Index(
+                    key: Seq[(String, Json.JsValueWrapper)],
+                    name: Option[String],
+                    unique: Boolean,
+                    dropDups: Boolean,
+                    sparse: Boolean,
+                    version: Option[Any],
+                    options:BSONDocument
+                  )
+
+  object IndexType {
+    def Ascending: Int = 1
+
+    def Descending: Int = -1
   }
 
+  implicit def asOption(o: JsObject): Option[JsValue] = o.result.toOption.flatMap(Option(_))
+
+  implicit def toBson(doc: JsObject): Bson = Document.parse(doc.toString())
+
+  implicit def toFuture[T](observable: SingleObservable[T]): Future[T] = observable.toFuture()
+
+  implicit def toFuture[T](observable: Observable[T]): Future[Seq[T]] = observable.toFuture()
+
+  implicit def toIndexModel(index: Index): IndexModel = new IndexModel(
+    Json.obj(index.key: _*),
+    new IndexOptions()
+      .name(index.name.get)
+      .unique(index.unique)
+      .sparse(index.sparse))
+
+  val throttleIdKey = "throttleId"
+  val timecodeKey = "timecode"
+  val countKey = "count"
+  val lastUpdatedTimestampKey = "lastUpdatedTimestamp"
+
 
   val idTimecodeIndex: Index =
     Index(
@@ -100,18 +173,20 @@ class ThrottlingRepository @Inject()(mongo: ReactiveMongoComponent,
         timecodeKey -> IndexType.Ascending
       ),
       name = Some("idTimecodeIndex"),
-      unique = true
+      unique = true,
+      dropDups = false,
+      sparse = true,
+      version = None,
+      options = BSONDocument()
     )
 
-  lazy val ttlIndex: Index = Index(
-    Seq((lastUpdatedTimestampKey, IndexType.Ascending)),
-    name = Some("throttleExpiryIndex"),
-    options = BSONDocument("expireAfterSeconds" -> appConfig.timeToLiveSeconds)
+  def ttlIndex(ttlLengthSeconds: Long): IndexModel = new IndexModel(
+    Json.obj(lastUpdatedTimestampKey -> IndexType.Ascending),
+    new IndexOptions()
+      .name("selfEmploymentsDataExpires")
+      .unique(false)
+      .sparse(false)
+      .expireAfter(ttlLengthSeconds, TimeUnit.SECONDS)
   )
 
-  collection.indexesManager.ensure(idTimecodeIndex)
-  collection.indexesManager.ensure(ttlIndex)
-
-}
-
-
+}

app/testonly/controllers/throttle/ThrottleController.scala

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2022 HM Revenue & Customs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package testonly.controllers.throttle
+
+import play.api.i18n.MessagesApi
+import play.api.mvc.ControllerComponents
+import repositories.ThrottlingRepository
+import uk.gov.hmrc.play.bootstrap.backend.controller.BackendController
+import uk.gov.hmrc.play.bootstrap.config.ServicesConfig
+
+import javax.inject.Inject
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.Try
+
+class ThrottleController @Inject()(
+                                    override val messagesApi: MessagesApi,
+                                    cc: ControllerComponents,
+                                    throttlingRepository: ThrottlingRepository,
+                                    val servicesConfig: ServicesConfig
+                                  )(
+                                    implicit ec: ExecutionContext)
+  extends BackendController(cc) {
+
+  lazy val get = Action.async { implicit req =>
+    req.getQueryString("throttle") match {
+      case Some(throttleId) =>
+        val throttleKey = getThrottleKey(throttleId)
+        val configValueMaybe: Option[Int] = Try {
+          servicesConfig.getInt(throttleKey)
+        }.toOption
+        val propsValueMaybe = sys.props.get(throttleKey).map(v => v.toInt)
+        val throttleValMaybe: Option[Int] = propsValueMaybe orElse configValueMaybe
+        throttleValMaybe match {
+          case Some(throttleVal) =>
+            throttlingRepository.stateOfThrottle(throttleId)
+              .map(currentValues => Ok(s"$throttleKey is set to ${currentValues._1} out of $throttleVal (at ${currentValues._2})"))
+          case _ => Future.successful(BadRequest(s"Throttle id $throttleId has no configured value"))
+        }
+      case _ => Future.successful(BadRequest(s"No throttle id provided"))
+    }
+  }
+
+  private def getThrottleKey(throttleId: String) = {
+    s"throttle.$throttleId.max"
+  }
+
+  lazy val update = Action.async { implicit req =>
+    val throttleIdMaybe = req.getQueryString("throttle")
+    val newThrottleValMaybe = req.getQueryString("value")
+    Future.successful(
+      (throttleIdMaybe, newThrottleValMaybe) match {
+        case (Some(throttleId), Some(newThrottleVal)) =>
+          val throttleKey = getThrottleKey(throttleId)
+          sys.props += (throttleKey -> newThrottleVal)
+          Ok(throttleKey + " has been set to " + newThrottleVal)
+        case _ => BadRequest(s"Throttle id given as $throttleIdMaybe, throttleVal given as $newThrottleValMaybe")
+      }
+    )
+  }
+}

conf/testOnlyDoNotUseInAppConf.routes

@@ -12,5 +12,8 @@ GET         /income-tax-subscription/client-matching/test-only/reset-agent-locko
 GET         /income-tax-subscription/test-only/feature-switch                             testonly.controllers.featureswitch.FeatureSwitchController.get
 POST        /income-tax-subscription/test-only/feature-switch                             testonly.controllers.featureswitch.FeatureSwitchController.update
 
+GET         /income-tax-subscription/test-only/throttle                                   testonly.controllers.throttle.ThrottleController.get
+POST        /income-tax-subscription/test-only/throttle                                   testonly.controllers.throttle.ThrottleController.update
+
 # Add all the application routes to the prod.routes file
 ->          /                                                                             prod.Routes