Skip to content

Commit

Permalink
Merge pull request #22 from at88mph/download-options-fix
Browse files Browse the repository at this point in the history 
feat: Download options fix
  • Loading branch information
@at88mph
at88mph authored Dec 3, 2024
2 parents d968d10 + 4e46500 commit 9bfeda1
Show file tree
Hide file tree
Showing 19 changed files with 702 additions and 697 deletions.
10 changes: 10 additions & 0 deletions .dockerignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
Dockerfile*
bin
build
.idea
.vscode
.gradle
.github
.gitlab-ci.yml
.gitignore
*.md
16 changes: 14 additions & 2 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
FROM images.opencadc.org/library/cadc-tomcat:1.2.1
FROM eclipse-temurin:11-alpine AS base

COPY build/libs/storage.war /usr/share/tomcat/webapps/
FROM base AS builder

COPY . /storage-ui
WORKDIR /storage-ui
RUN apk --no-cache add git \
&& git fetch origin main \
&& ./gradlew -i clean spotlessCheck build test --no-daemon

FROM images.opencadc.org/library/cadc-tomcat:1.3 AS production

RUN mkdir -p /usr/share/tomcat/config

COPY --from=builder /storage-ui/build/libs/storage.war /usr/share/tomcat/webapps/
2 changes: 1 addition & 1 deletion VERSION
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
## deployable containers have a semantic and build tag
# semantic version tag: major.minor
# build version tag: timestamp
VERSION="1.1.8"
VERSION="1.2.0"
TAGS="${VERSION} ${VERSION}-$(date -u +"%Y%m%dT%H%M%S")"
unset VERSION
73 changes: 73 additions & 0 deletions build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
plugins {
id 'war'
id "com.github.node-gradle.node" version "3.0.1"
id 'com.diffplug.spotless' version '6.25.0'

// IntelliJ IDEA plugin here to allow integration tests to appear properly in IDEs.
id 'idea'
id 'jacoco'
id 'org.jetbrains.dokka' version '1.6.0'
}

repositories {
Expand Down Expand Up @@ -38,13 +42,82 @@ dependencies {
implementation 'org.restlet.jee:org.restlet.ext.json:[2.4.3,2.4.99)'

testImplementation 'junit:junit:[4.12,5.0)'
testImplementation 'org.xmlunit:xmlunit-core:2.10.0'
testImplementation 'org.xmlunit:xmlunit-assertj:2.10.0'
testImplementation 'org.opencadc:cadc-web-test:[2.1.0,3.0.0)'
testImplementation 'org.mockito:mockito-core:[3.9.0,4.0.0)'
testImplementation 'org.seleniumhq.selenium:selenium-java:[3.14,4.0)'
}

sourceCompatibility = 11

spotless {
// optional: only format files which have changed since origin/main
ratchetFrom 'origin/main'

java {
// Pass spotless:on or spotless:off
toggleOffOn()

// Use the default importOrder configuration
importOrder()
// Remove unused imports
removeUnusedImports()
// Google Java Format, Android Open Source Project style which uses 4 spaces for indentation
palantirJavaFormat('2.50.0').formatJavadoc(true)
// Format annotations on a single line
formatAnnotations()
}
format 'misc', {
target '*.gradle'
trimTrailingWhitespace()
indentWithSpaces(4)
endWithNewline()
}
}
check.dependsOn spotlessCheck

// Create Java Code Coverage Reports
jacocoTestReport {
reports {
xml.enabled true
html.enabled true
}
}
check.dependsOn jacocoTestReport

// Create JavaDoc
javadoc {
destinationDir = file("${buildDir}/docs/javadoc")
}

// Create Java Documentation using Dokka for Github Markdown and HTML
tasks.dokkaGfm.configure {
outputDirectory.set(file("${buildDir}/docs/dokka/gfm"))
dokkaSourceSets {
register("main") {
sourceRoots.from(file("src/main/java"))
}
}
}

tasks.dokkaHtml.configure {
outputDirectory.set(file("${buildDir}/docs/dokka/html"))
dokkaSourceSets {
register("main") {
sourceRoots.from(file("src/main/java"))
}
configureEach {
jdkVersion.set(11)
sourceLink {
localDirectory.set(file("src/main/java"))
remoteUrl.set("https://github.com/opencadc/storage-ui/tree/main/src/main/java")
}
}
}
}


war {
archiveName 'storage.war'
}
Expand Down
146 changes: 65 additions & 81 deletions src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,15 +74,24 @@
import ca.nrc.cadc.net.ResourceNotFoundException;
import ca.nrc.cadc.reg.Standards;
import ca.nrc.cadc.util.StringUtil;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.security.auth.Subject;
import net.canfar.storage.PathUtils;
import net.canfar.storage.web.*;
import net.canfar.storage.web.config.StorageConfiguration;
import net.canfar.storage.web.config.VOSpaceServiceConfig;
import net.canfar.storage.web.config.VOSpaceServiceConfigManager;
import net.canfar.storage.web.restlet.JSONRepresentation;

import javax.security.auth.Subject;

import org.apache.commons.fileupload.FileItemIterator;
import org.apache.commons.fileupload.FileItemStream;
import org.apache.commons.fileupload.FileUploadException;
Expand Down Expand Up @@ -112,19 +121,6 @@
import org.restlet.resource.Put;
import org.restlet.resource.ResourceException;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;


public class FileItemServerResource extends StorageItemServerResource {

private static final Logger LOGGER = Logger.getLogger(FileItemServerResource.class);
Expand All @@ -135,18 +131,17 @@ public class FileItemServerResource extends StorageItemServerResource {
private final UploadVerifier uploadVerifier;
private final FileValidator fileValidator;

private final static AuthMethod[] PROTOCOL_AUTH_METHODS = new AuthMethod[] {
AuthMethod.ANON,
AuthMethod.CERT,
AuthMethod.COOKIE
};


FileItemServerResource(StorageConfiguration storageConfiguration,
VOSpaceServiceConfigManager voSpaceServiceConfigManager,
StorageItemFactory storageItemFactory, VOSpaceClient voSpaceClient,
VOSpaceServiceConfig serviceConfig, UploadVerifier uploadVerifier,
FileValidator fileValidator) {
private static final AuthMethod[] PROTOCOL_AUTH_METHODS =
new AuthMethod[] {AuthMethod.ANON, AuthMethod.CERT, AuthMethod.COOKIE};

FileItemServerResource(
StorageConfiguration storageConfiguration,
VOSpaceServiceConfigManager voSpaceServiceConfigManager,
StorageItemFactory storageItemFactory,
VOSpaceClient voSpaceClient,
VOSpaceServiceConfig serviceConfig,
UploadVerifier uploadVerifier,
FileValidator fileValidator) {
super(storageConfiguration, voSpaceServiceConfigManager, storageItemFactory, voSpaceClient, serviceConfig);
this.uploadVerifier = uploadVerifier;
this.fileValidator = fileValidator;
Expand Down Expand Up @@ -185,32 +180,25 @@ void download(final DataNode dataNode) throws Exception {

/**
* Check both the new prototype and old Files service lookup endpoints.
* @param serviceURI The URI that identifies the Service to use.
* @param authMethod The AuthMethod interface to pick out.
* @return URL, never null.
*
* @param serviceURI The URI that identifies the Service to use.
* @param authMethod The AuthMethod interface to pick out.
* @return URL, never null.
* @throws IllegalStateException if no URL can be found.
*/
private URL lookupDownloadEndpoint(final URI serviceURI, final AuthMethod authMethod) {
final URI[] downloadEndpointStandards = new URI[] {
Standards.VOSPACE_FILES,
Standards.VOSPACE_FILES_20
};
final URI[] downloadEndpointStandards = new URI[] {Standards.VOSPACE_FILES, Standards.VOSPACE_FILES_20};

for (final URI uri : downloadEndpointStandards) {
final URL serviceURL = lookupDownloadEndpoint(serviceURI, uri, authMethod);
final URL serviceURL = lookupEndpoint(serviceURI, uri, authMethod);
if (serviceURL != null) {
return serviceURL;
}
}

throw new IllegalStateException("Incomplete configuration in the registry. No endpoint for "
+ serviceURI + " could be found from ("
+ Arrays.toString(downloadEndpointStandards) + ")");
}

private URL lookupDownloadEndpoint(final URI serviceURI, final URI capabilityStandardURI,
final AuthMethod authMethod) {
return getRegistryClient().getServiceURL(serviceURI, capabilityStandardURI, authMethod);
+ serviceURI + " could be found from ("
+ Arrays.toString(downloadEndpointStandards) + ")");
}

String toEndpoint(final URI downloadURI) {
Expand Down Expand Up @@ -240,8 +228,7 @@ public void accept(final Representation payload) throws Exception {

if (!fileItemIterator.hasNext()) {
// Some problem occurs, sent back a simple line of text.
uploadError(Status.CLIENT_ERROR_BAD_REQUEST,
"Unable to upload corrupted or incompatible data.");
uploadError(Status.CLIENT_ERROR_BAD_REQUEST, "Unable to upload corrupted or incompatible data.");
} else {
upload(fileItemIterator);
}
Expand Down Expand Up @@ -302,23 +289,22 @@ Path upload(final FileItemStream fileItemStream) throws Exception {

return PathUtils.toPath(dataNode);
} else {
throw new ResourceException(new IllegalArgumentException(
String.format("Invalid file name: %s -- File name must match %s.", filename,
fileValidator.getRule())));
throw new ResourceException(new IllegalArgumentException(String.format(
"Invalid file name: %s -- File name must match %s.", filename, fileValidator.getRule())));
}
}

/**
* Do the secure upload.
*
* @param inputStream The InputStream to pull from.
* @param dataNode The DataNode to upload to.
* @param contentType The file content type.
* @param dataNode The DataNode to upload to.
* @param contentType The file content type.
*/
protected void upload(final InputStream inputStream, final DataNode dataNode, final String contentType)
throws Exception {
final UploadOutputStreamWrapper outputStreamWrapper = new UploadOutputStreamWrapperImpl(inputStream,
BUFFER_SIZE);
final UploadOutputStreamWrapper outputStreamWrapper =
new UploadOutputStreamWrapperImpl(inputStream, BUFFER_SIZE);

try {
// Due to a bug in VOSpace that returns a 400 while checking
Expand All @@ -332,7 +318,7 @@ protected void upload(final InputStream inputStream, final DataNode dataNode, fi
if (cause instanceof IllegalStateException) {
final Throwable illegalStateCause = cause.getCause();
if ((illegalStateCause instanceof NodeNotFoundException)
|| (illegalStateCause instanceof ResourceNotFoundException)) {
|| (illegalStateCause instanceof ResourceNotFoundException)) {
createNode(dataNode);
} else {
throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST, e.getCause());
Expand Down Expand Up @@ -365,22 +351,24 @@ protected void upload(final InputStream inputStream, final DataNode dataNode, fi
}

/**
* Abstract away the Transfer stuff. It's cumbersome.
* Abstract away the Transfer stuff. It's cumbersome.
*
* @param outputStreamWrapper The OutputStream wrapper.
* @param dataNode The node to upload.
* @param contentType The file content type.
* @param dataNode The node to upload.
* @param contentType The file content type.
* @throws Exception To capture transfer and upload failures.
*/
void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode dataNode, final String contentType)
throws Exception {
final VOSURI dataNodeVOSURI = toURI(dataNode);

final List<Protocol> protocols = Arrays.stream(FileItemServerResource.PROTOCOL_AUTH_METHODS).map(authMethod -> {
final Protocol httpsAuth = new Protocol(VOS.PROTOCOL_HTTPS_PUT);
httpsAuth.setSecurityMethod(Standards.getSecurityMethod(authMethod));
return httpsAuth;
}).collect(Collectors.toList());
final List<Protocol> protocols = Arrays.stream(FileItemServerResource.PROTOCOL_AUTH_METHODS)
.map(authMethod -> {
final Protocol httpsAuth = new Protocol(VOS.PROTOCOL_HTTPS_PUT);
httpsAuth.setSecurityMethod(Standards.getSecurityMethod(authMethod));
return httpsAuth;
})
.collect(Collectors.toList());

final Transfer transfer = new Transfer(dataNodeVOSURI.getURI(), Direction.pushToVoSpace);
transfer.setView(new View(VOS.VIEW_DEFAULT));
Expand All @@ -396,8 +384,9 @@ void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode
VOSClientUtil.checkTransferFailure(ct);

if (ct.getHttpTransferDetails().getDigest() != null) {
uploadVerifier.verifyMD5(outputStreamWrapper.getCalculatedMD5(),
ct.getHttpTransferDetails().getDigest().getSchemeSpecificPart());
uploadVerifier.verifyMD5(
outputStreamWrapper.getCalculatedMD5(),
ct.getHttpTransferDetails().getDigest().getSchemeSpecificPart());
}

uploadSuccess();
Expand All @@ -406,7 +395,7 @@ void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode
/**
* Parse the representation into a Map for easier access to Form elements.
*
* @return Map of field names to File Items, or empty Map. Never null.
* @return Map of field names to File Items, or empty Map. Never null.
*/
private ServletFileUpload parseRepresentation() {
// 1/ Create a factory for disk-based file items
Expand All @@ -427,26 +416,21 @@ private ServletFileUpload createFileUpload(final DiskFileItemFactory factory) {
return new ServletFileUpload(factory);
}


private void uploadError(final Status status, final String message) {
writeResponse(status,
new JSONRepresentation() {
@Override
public void write(final JSONWriter jsonWriter)
throws JSONException {
jsonWriter.object().key("error").value(message).endObject();
}
});
writeResponse(status, new JSONRepresentation() {
@Override
public void write(final JSONWriter jsonWriter) throws JSONException {
jsonWriter.object().key("error").value(message).endObject();
}
});
}

private void uploadSuccess() {
writeResponse(Status.SUCCESS_CREATED,
new JSONRepresentation() {
@Override
public void write(final JSONWriter jsonWriter)
throws JSONException {
jsonWriter.object().key("code").value(0).endObject();
}
});
writeResponse(Status.SUCCESS_CREATED, new JSONRepresentation() {
@Override
public void write(final JSONWriter jsonWriter) throws JSONException {
jsonWriter.object().key("code").value(0).endObject();
}
});
}
}
Loading

0 comments on commit 9bfeda1

Please sign in to comment.