Skip to content

Commit

Permalink
Importing Resource Container from Zip File (#423)
Browse files Browse the repository at this point in the history
* Refactor to support loading of zip RC's. No Tree support yet.

* Creation of OtterTree from zip file works, several issues remain

1. Collection slugs are incorrect
2. chapters, verses, and body/notes are being put into the tree backwards

* Polished and moved ZipEntryTreeBuilder to new package

* Update CollectionRepository.kt

* Two menu options to import from zip or folder

* PR Comment

* Update travis.yml to clone dev branch of kotlin-resource-container instead of master

* Use ResourceContainer's new autoclose feature.

* Corrected mispelled package name

* PR Comment- categories and versification

* PR Comments- mispelling and categories and versification
  • Loading branch information
KJoslyn authored and jsarabia committed Mar 4, 2019
1 parent 869e495 commit 88c5482
Show file tree
Hide file tree
Showing 8 changed files with 152 additions and 60 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ cache:
- $HOME/.gradle/caches/
- $HOME/.gradle/wrapper
install:
- git clone --branch=master https://github.com/WycliffeAssociates/kotlin-resource-container.git ../kotlin-resource-container
- git clone --branch=dev https://github.com/WycliffeAssociates/kotlin-resource-container.git ../kotlin-resource-container
- git clone https://github.com/WycliffeAssociates/8woc2018-common.git ../8woc2018-common
- export BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; else echo $TRAVIS_PULL_REQUEST_BRANCH; fi)
- echo "TRAVIS_BRANCH=$TRAVIS_BRANCH, PR=$PR, BRANCH=$BRANCH"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package org.wycliffeassociates.otter.jvm.app.ui.inject

import org.wycliffeassociates.otter.jvm.device.audio.injection.DaggerAudioComponent
import org.wycliffeassociates.otter.jvm.device.audioplugin.injection.DaggerAudioPluginComponent
import org.wycliffeassociates.otter.jvm.domain.resourcecontainer.project.ZipEntryTreeBuilder
import org.wycliffeassociates.otter.jvm.persistence.injection.DaggerPersistenceComponent
import org.wycliffeassociates.otter.jvm.persistence.repositories.*
import org.wycliffeassociates.otter.jvm.persistence.repositories.mapping.LanguageMapper
Expand Down Expand Up @@ -36,4 +37,6 @@ class Injector : Component(), ScopedInstance {
get() = audioComponent.injectPlayer()

val audioPluginRegistrar = audioPluginComponent.injectRegistrar()

val zipEntryTreeBuilder = ZipEntryTreeBuilder
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package org.wycliffeassociates.otter.jvm.app.ui.menu.view

import com.github.thomasnield.rxkotlinfx.toObservable
import de.jensd.fx.glyphs.materialicons.MaterialIcon
import de.jensd.fx.glyphs.materialicons.MaterialIconView
import javafx.application.Platform
import javafx.event.EventHandler
import javafx.scene.control.Menu
import javafx.scene.control.MenuBar
import javafx.scene.control.MenuItem
import javafx.scene.control.ToggleGroup
import javafx.stage.FileChooser
import org.wycliffeassociates.otter.jvm.app.theme.AppStyles
import org.wycliffeassociates.otter.jvm.app.ui.addplugin.view.AddPluginView
import org.wycliffeassociates.otter.jvm.app.ui.menu.viewmodel.MainMenuViewModel
Expand All @@ -19,27 +20,42 @@ class MainMenu : MenuBar() {

private val viewModel: MainMenuViewModel = find()

private fun Menu.importMenuItem(message: String): MenuItem {
return item(message) {
graphic = MainMenuStyles.importIcon("20px")
val dialog = progressdialog {
text = message
graphic = MainMenuStyles.importIcon("60px")
root.addClass(AppStyles.progressDialog)
}
viewModel.showImportDialogProperty.onChange {
Platform.runLater { if (it) dialog.open() else dialog.close() }
}
}
}

init {
importStylesheet<MainMenuStyles>()
with(this) {
menu(messages["file"]) {
item(messages["importResource"]) {
graphic = MainMenuStyles.importIcon("20px")
val dialog = progressdialog {
text = messages["importResource"]
graphic = MainMenuStyles.importIcon( "60px")
root.addClass(AppStyles.progressDialog)
}
viewModel.showImportDialogProperty.onChange {
Platform.runLater { if (it) dialog.open() else dialog.close() }
}
action {
val file = chooseDirectory(messages["importResourceTip"])
file?.let {
viewModel.importContainerDirectory(file)
importMenuItem(messages["importResourceFromFolder"])
.setOnAction {
val file = chooseDirectory(messages["importResourceFromFolder"])
file?.let {
viewModel.importContainerDirectory(file)
}
}
importMenuItem(messages["importResourceFromZip"])
.setOnAction {
val file = chooseFile(
messages["importResourceFromZip"],
arrayOf(FileChooser.ExtensionFilter("Zip files (*.zip)", "*.zip")),
FileChooserMode.Single
).firstOrNull()
file?.let {
viewModel.importContainerDirectory(file)
}
}
}
}
}
menu(messages["audioPlugins"]) {
onShowing = EventHandler {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ class MainMenuViewModel : ViewModel() {
private val resourceContainerRepository = injector.resourceContainerRepository
private val directoryProvider = injector.directoryProvider
private val pluginRepository = injector.pluginRepository
private val zipEntryTreeBuilder = injector.zipEntryTreeBuilder

val editorPlugins: ObservableList<AudioPluginData> = FXCollections.observableArrayList<AudioPluginData>()
val recorderPlugins: ObservableList<AudioPluginData> = FXCollections.observableArrayList<AudioPluginData>()
Expand All @@ -34,7 +35,8 @@ class MainMenuViewModel : ViewModel() {
fun importContainerDirectory(dir: File) {
val importer = ImportResourceContainer(
resourceContainerRepository,
directoryProvider
directoryProvider,
zipEntryTreeBuilder
)
showImportDialogProperty.value = true
importer.import(dir)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package org.wycliffeassociates.otter.jvm.domain.resourcecontainer.project

import org.wycliffeassociates.otter.common.collections.tree.OtterTree
import org.wycliffeassociates.otter.common.collections.tree.OtterTreeNode
import org.wycliffeassociates.otter.common.domain.resourcecontainer.project.IZipEntryTreeBuilder
import org.wycliffeassociates.otter.common.domain.resourcecontainer.project.OtterFile
import org.wycliffeassociates.otter.common.domain.resourcecontainer.project.OtterZipFile.Companion.otterFileZ
import java.io.IOException
import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.Files
import java.nio.file.SimpleFileVisitor
import java.nio.file.FileSystem
import java.nio.file.Paths
import java.nio.file.Path
import java.nio.file.FileSystems
import java.nio.file.FileVisitResult
import java.util.zip.ZipFile
import java.util.ArrayDeque

object ZipEntryTreeBuilder : IZipEntryTreeBuilder {

private fun createZipFileSystem(zipFilename: String): FileSystem {
val path = Paths.get(zipFilename)
return FileSystems.newFileSystem(path, null)
}

override fun buildOtterFileTree(zipFile: ZipFile, projectPath: String): OtterTree<OtterFile> {
var treeRoot: OtterTree<OtterFile>? = null
val treeCursor = ArrayDeque<OtterTree<OtterFile>>()
createZipFileSystem(zipFile.name).use { zipFileSystem ->

val projectRoot = zipFileSystem.getPath(projectPath)
val sep = zipFileSystem.separator

Files.walkFileTree(projectRoot, object : SimpleFileVisitor<Path>() {
@Throws(IOException::class)
override fun visitFile(file: Path,
attrs: BasicFileAttributes): FileVisitResult {
val filepath = file.toString().substringAfter(sep)
val entry = zipFile.getEntry(filepath)
val otterZipFile = otterFileZ(filepath, zipFile, sep, treeCursor.peek()?.value, entry)
treeCursor.peek()?.addChild(OtterTreeNode(otterZipFile))
return FileVisitResult.CONTINUE
}

@Throws(IOException::class)
override fun postVisitDirectory(dir: Path?, exc: IOException?): FileVisitResult {
treeRoot = treeCursor.pop()
return FileVisitResult.CONTINUE
}

@Throws(IOException::class)
override fun preVisitDirectory(dir: Path,
attrs: BasicFileAttributes): FileVisitResult {
val newDirNode = OtterTree(
otterFileZ(dir.toString(), zipFile, zipFileSystem.separator, treeCursor.peek()?.value)
)
treeCursor.peek()?.addChild(newDirNode)
treeCursor.push(newDirNode)
return FileVisitResult.CONTINUE
}
})
return treeRoot ?: OtterTree(otterFileZ(zipFile.name, zipFile, zipFileSystem.separator))
}
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,17 @@ class CollectionRepository(
// 2. Load the resource container
val metadata = project.resourceContainer
if (metadata != null) {
val container = ResourceContainer.load(metadata.path)
// 3. Remove the project from the manifest
container.manifest.projects = container.manifest.projects.filter { it.identifier != project.slug }
// 4a. If the manifest has more projects, write out the new manifest
if (container.manifest.projects.isNotEmpty()) {
container.writeManifest()
} else {
// 4b. If the manifest has no projects left, delete the RC folder and the metadata from the database
metadata.path.deleteRecursively()
metadataDao.delete(metadataMapper.mapToEntity(metadata))
ResourceContainer.load(metadata.path).use { container ->
// 3. Remove the project from the manifest
container.manifest.projects = container.manifest.projects.filter { it.identifier != project.slug }
// 4a. If the manifest has more projects, write out the new manifest
if (container.manifest.projects.isNotEmpty()) {
container.writeManifest()
} else {
// 4b. If the manifest has no projects left, delete the RC folder and the metadata from the database
metadata.path.deleteRecursively()
metadataDao.delete(metadataMapper.mapToEntity(metadata))
}
}
}
}.andThen(
Expand Down Expand Up @@ -212,6 +213,7 @@ class CollectionRepository(
dublinCore.mapToMetadata(File("."), targetLanguage),
metadata
)
// TODO 2/14/19: Move create to ResourceContainer to be able to create a zip resource container?
val container = ResourceContainer.create(directory) {
// Set up the manifest
manifest = Manifest(
Expand Down Expand Up @@ -240,16 +242,17 @@ class CollectionRepository(

val metadataEntity = if (matches.isEmpty()) {
// This combination of identifier and language does not already exist; create it
val container = createResourceContainer(source, language)
// Convert DublinCore to ResourceMetadata
val metadata = container.manifest.dublinCore
.mapToMetadata(container.dir, language)
createResourceContainer(source, language).use { container ->
// Convert DublinCore to ResourceMetadata
val metadata = container.manifest.dublinCore
.mapToMetadata(container.file, language)

// Insert ResourceMetadata into database
val entity = metadataMapper.mapToEntity(metadata)
entity.derivedFromFk = source.resourceContainer?.id
entity.id = metadataDao.insert(entity, dsl)
/* return@if */ entity
// Insert ResourceMetadata into database
val entity = metadataMapper.mapToEntity(metadata)
entity.derivedFromFk = source.resourceContainer?.id
entity.id = metadataDao.insert(entity, dsl)
/* return@if */ entity
}
} else {
// Use the existing metadata
/* return@if */ matches.first()
Expand All @@ -273,27 +276,26 @@ class CollectionRepository(

// Add a project to the container if necessary
// Load the existing resource container and see if we need to add another project
val container = ResourceContainer.load(File(metadataEntity.path))
if (container.manifest.projects.none { it.identifier == source.slug }) {
container.manifest.projects = container.manifest.projects.plus(
project {
sort = if (metadataEntity.subject.toLowerCase() == "bible"
&& projectEntity.sort > 39) {
projectEntity.sort + 1
} else {
projectEntity.sort
ResourceContainer.load(File(metadataEntity.path)).use { container ->
if (container.manifest.projects.none { it.identifier == source.slug }) {
container.manifest.projects = container.manifest.projects.plus(
project {
sort = if (metadataEntity.subject.toLowerCase() == "bible"
&& projectEntity.sort > 39) {
projectEntity.sort + 1
} else {
projectEntity.sort
}
identifier = projectEntity.slug
path = "./${projectEntity.slug}"
// This title will not be localized into the target language
title = projectEntity.title
// Unable to get categories and versification from the source collection
}
identifier = projectEntity.slug
path = "./${projectEntity.slug}"
// This title will not be localized into the target language
title = projectEntity.title
// Unable to get these fields from the source collection
categories = listOf()
versification = ""
}
)
// Update the container
container.write()
)
// Update the container
container.write()
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ class ResourceContainerRepository(
return Completable.fromAction {
database.transaction { dsl ->
val language = LanguageMapper().mapFromEntity(languageDao.fetchBySlug(languageSlug, dsl))
val metadata = dublinCore.mapToMetadata(rc.dir, language)
val metadata = dublinCore.mapToMetadata(rc.file, language)
val dublinCoreFk = resourceMetadataDao.insert(ResourceMetadataMapper().mapToEntity(metadata), dsl)

val relatedDublinCoreIds: List<Int> =
Expand Down
2 changes: 2 additions & 0 deletions src/main/resources/Messages_en.properties
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ finish=FINISH
refresh = Refresh
file = File
importResource = Import Resource Container
importResourceFromFolder = Import Resource Container from Folder
importResourceFromZip = Import Resource Container from Zip File
importResourceTip = Please Select Resource Container to Import
importError = Unable to Import Resource Container
importErrorInvalidRc = Invalid resource container
Expand Down

0 comments on commit 88c5482

Please sign in to comment.