Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

893 allow to wipe a tenant before data import #908

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

893 allow to wipe a tenant before data import #908

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2fbd4d2
feat#893: add wipeData toggle button
helakaraa Nov 15, 2024
ecc4588
fix: prevent importing v2 without import file
ptitFicus Nov 19, 2024
1676984
feat#893: Add Info tooltip for wipeData, add wipeData to importTenan…
helakaraa Nov 27, 2024
249aad6
feat#893: Heading Modification + renaming the import data V1
helakaraa Dec 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 51 additions & 33 deletions app/fr/maif/izanami/datastores/ImportExportDatastore.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import fr.maif.izanami.datastores.ImportExportDatastore.TableMetadata
import fr.maif.izanami.env.Env
import fr.maif.izanami.env.pgimplicits.EnhancedRow
import fr.maif.izanami.errors.{InternalServerError, IzanamiError, PartialImportFailure}
import fr.maif.izanami.models.{ExportedType, KeyRightType, ProjectRightType, WebhookRightType}
import fr.maif.izanami.models.{ExportedType, KeyRightType, ProjectRightType, TenantCreationRequest, WebhookRightType}
import fr.maif.izanami.models.ExportedType.exportedTypeToString
import fr.maif.izanami.utils.Datastore
import fr.maif.izanami.utils.syntax.implicits.BetterJsValue
Expand All @@ -13,7 +13,7 @@ import fr.maif.izanami.web.ImportController.ImportConflictStrategy
import fr.maif.izanami.web.{ExportController, ImportController}
import io.vertx.core.json.JsonArray
import io.vertx.sqlclient.SqlConnection
import play.api.libs.json.{JsObject, Json}
import play.api.libs.json.{JsObject, JsString, Json}

import java.util.concurrent.atomic.AtomicInteger
import scala.concurrent.Future
Expand Down Expand Up @@ -47,8 +47,10 @@ class ImportExportDatastore(val env: Env) extends Datastore {

def importTenantData(
tenant: String,
user: String,
entries: Map[ExportedType, Seq[JsObject]],
conflictStrategy: ImportConflictStrategy
conflictStrategy: ImportConflictStrategy,
wipeData: Boolean
): Future[Either[IzanamiError, Unit]] = {

Future
Expand All @@ -70,38 +72,55 @@ class ImportExportDatastore(val env: Env) extends Datastore {
.foldLeft(Future.successful(Right(())): Future[Either[Map[ExportedType, Seq[JsObject]], Unit]])(
(agg, t) => {
agg.flatMap(previousResult => {
val f = conflictStrategy match {
case ImportController.MergeOverwrite => importTenantDataWithMergeOnConflict(tenant, t._1, t._2)
case ImportController.Skip => importTenantDataWithSkipOnConflict(tenant, t._1, t._2)
case ImportController.Fail if previousResult.isRight =>
importTenantDataWithFailOnConflict(tenant, t._1, t._2, conn)
case _ => Future.successful(Right(()))
}
if(wipeData) {
env.datastores.tenants.deleteTenant(tenant, user).flatMap {
case Left(err) => Future.successful(Left(Map(t._3 -> Seq(JsObject(Seq("error" -> JsString(err.toString)))))))
case Right(_) =>
env.datastores.tenants.createTenant(TenantCreationRequest(tenant), user).flatMap {
case Left(err) => Future.successful(Left(Map(t._3 -> Seq(JsObject(Seq("error" -> JsString(err.toString)))))))
case Right(_) => importTenantDataWithSkipOnConflict(tenant, t._1, t._2).map {
case Left(failedJsons) =>
Left(Map(t._3 -> failedJsons))
case Right(_) =>
Right(())
}
}
}
} else {
val f = conflictStrategy match {
case ImportController.MergeOverwrite => importTenantDataWithMergeOnConflict(tenant, t._1, t._2)
case ImportController.Skip => importTenantDataWithSkipOnConflict(tenant, t._1, t._2)
case ImportController.Fail if previousResult.isRight =>
importTenantDataWithFailOnConflict(tenant, t._1, t._2, conn)
case _ => Future.successful(Right(()))

f
.flatMap(e => {
updateUserTenantRightIfNeeded(
tenant,
entries,
if (conflictStrategy == ImportController.Fail) Some(conn) else None
)
.map(_ => e)
})
.map(either =>
either.left.map(jsons =>
jsons.map(json => Json.obj("row" -> json, "_type" -> exportedTypeToString(t._3)))
}
f
.flatMap(e => {
updateUserTenantRightIfNeeded(
tenant,
entries,
if (conflictStrategy == ImportController.Fail) Some(conn) else None
)
.map(_ => e)
})
.map(either =>
either.left.map(jsons =>
jsons.map(json => Json.obj("row" -> json, "_type" -> exportedTypeToString(t._3)))
)
)
)
.map(either =>
(either, previousResult) match {
case (Left(err), Left(errs)) => {
Left(errs + (t._3 -> err))
.map(either =>
(either, previousResult) match {
case (Left(err), Left(errs)) => {
Left(errs + (t._3 -> err))
}
case (Left(err), Right(_)) => Left(Map(t._3 -> err))
case (Right(_), Right(_)) => Right(())
case (Right(_), Left(errs)) => Left(errs)
}
case (Left(err), Right(_)) => Left(Map(t._3 -> err))
case (Right(_), Right(_)) => Right(())
case (Right(_), Left(errs)) => Left(errs)
}
)
)
}

})
}
)
Expand All @@ -110,7 +129,6 @@ class ImportExportDatastore(val env: Env) extends Datastore {
}
})
}

def importTenantDataWithMergeOnConflict(
tenant: String,
metadata: TableMetadata,
Expand Down
34 changes: 19 additions & 15 deletions app/fr/maif/izanami/web/ImportController.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ class ImportController(
version: Int,
tenant: String,
conflict: String,
wipeData: Option[Boolean],
timezone: Option[String],
deduceProject: Boolean,
create: Option[Boolean],
Expand All @@ -219,7 +220,7 @@ class ImportController(
})
.getOrElse(BadRequest(Json.obj("message" -> "Missing timezone")).future)
} else if (version == 2) {
importV2Data(request, tenant, conflict)
importV2Data(request, tenant, conflict, wipeData)
} else {
BadRequest(Json.obj("message" -> s"Invalid version: $version")).future
}
Expand All @@ -228,12 +229,14 @@ class ImportController(
def importV2Data(
request: UserNameRequest[MultipartFormData[Files.TemporaryFile]],
tenant: String,
conflict: String
conflict: String,
wipeData: Option[Boolean]
): Future[Result] = {
val files: Map[String, URI] = request.body.files.map(f => (f.key, f.ref.path.toUri)).toMap
(for (
conflictStrategy <- ImportController.parseStrategy(conflict);
uri <- files.get("export")
uri <- files.get("export");
wipeData <- wipeData
) yield {
Try(scala.io.Source.fromFile(uri)).toEither.left
.map(ex => {
Expand All @@ -258,19 +261,20 @@ class ImportController(
.map(m => fixImportDataIfNeeded(tenant, m))
.map {
case (messages, data) => {
env.datastores.exportDatastore
.importTenantData(tenant, data, conflictStrategy)
.map {
case Left(PartialImportFailure(failedElements)) =>
Conflict(
Json.obj(
"messages" -> Json.toJson(messages),
"conflicts" -> Json.toJson(failedElements.values.flatten)
env.datastores.exportDatastore
.importTenantData(tenant, request.user, data, conflictStrategy, wipeData)
.map {
case Left(PartialImportFailure(failedElements)) =>
Conflict(
Json.obj(
"messages" -> Json.toJson(messages),
"conflicts" -> Json.toJson(failedElements.values.flatten)
)
)
)
case Right(_) => Ok(Json.obj("messages" -> Json.toJson(messages)))
case Left(err) => err.toHttpResponse
}
case Right(_) => Ok(Json.obj("messages" -> Json.toJson(messages)))
case Left(err) => err.toHttpResponse
}

}
}
}).toRight(Future.successful(BadRequest(Json.obj("message" -> "Missing export file")))).flatten.fold(r => r, r => r)
Expand Down
2 changes: 1 addition & 1 deletion conf/routes
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ PUT /api/admin/tenants/:tenant/local-scripts/:script
DELETE /api/admin/local-scripts/_cache fr.maif.izanami.web.PluginController.clearWasmCache()
DELETE /api/admin/tenants/:tenant/local-scripts/:script fr.maif.izanami.web.PluginController.deleteScript(tenant: String, script: String)

POST /api/admin/tenants/:tenant/_import fr.maif.izanami.web.ImportController.importData(version: Int, tenant: String, conflict: String, timezone: Option[String], deduceProject: Boolean ?= false, create: Option[Boolean], project: Option[String], projectPartSize: Option[Int], inlineScript: Option[Boolean])
POST /api/admin/tenants/:tenant/_import fr.maif.izanami.web.ImportController.importData(version: Int, tenant: String, conflict: String,wipeData : Option[Boolean], timezone: Option[String], deduceProject: Boolean ?= false, create: Option[Boolean], project: Option[String], projectPartSize: Option[Int], inlineScript: Option[Boolean])
GET /api/admin/tenants/:tenant/_import/:id fr.maif.izanami.web.ImportController.readImportStatus(tenant: String, id: String)
DELETE /api/admin/tenants/:tenant/_import/:id fr.maif.izanami.web.ImportController.deleteImportStatus(tenant: String, id: String)

Expand Down
Loading