Skip to content

Commit

Permalink
Fix abstract store and implement jcloud rollback
Browse files Browse the repository at this point in the history
  • Loading branch information
@tylerjmchugh
tylerjmchugh committed Jan 2, 2025
1 parent e73fb98 commit c0b921b
Show file tree
Hide file tree
Showing 2 changed files with 98 additions and 49 deletions.
91 changes: 46 additions & 45 deletions core/src/main/java/org/fao/geonet/api/records/attachments/AbstractStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@

import jeeves.server.context.ServiceContext;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.input.BoundedInputStream;
import org.fao.geonet.ApplicationContextHolder;
import org.fao.geonet.api.exception.GeonetMaxUploadSizeExceededException;
import org.fao.geonet.api.exception.NotAllowedException;
Expand All @@ -46,6 +45,7 @@
import org.springframework.web.multipart.MultipartFile;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
Expand Down Expand Up @@ -164,46 +164,6 @@ protected int canDownload(ServiceContext context, String metadataUuid, MetadataR
return metadataId;
}

protected String getFilenameFromHeader(final URL fileUrl) {
HttpURLConnection connection = null;
try {
connection = (HttpURLConnection) fileUrl.openConnection();
connection.setRequestMethod("HEAD");
connection.connect();
String contentDisposition = connection.getHeaderField("Content-Disposition");

if (contentDisposition != null && contentDisposition.contains("filename=")) {
String filename = contentDisposition.split("filename=")[1].replace("\"", "").trim();
return filename.isEmpty() ? null : filename;
}
return null;
} catch (Exception e) {
log.error("Error retrieving resource filename from header", e);
return null;
} finally {
if (connection != null) {
connection.disconnect();
}
}
}

protected long getContentLengthFromHeader(final URL fileUrl) {
HttpURLConnection connection = null;
try {
connection = (HttpURLConnection) fileUrl.openConnection();
connection.setRequestMethod("HEAD");
connection.connect();
return connection.getContentLengthLong();
} catch (Exception e) {
log.error("Error retrieving resource content length from header", e);
return -1;
} finally {
if (connection != null) {
connection.disconnect();
}
}
}

protected String getFilenameFromUrl(final URL fileUrl) {
String fileName = FilenameUtils.getName(fileUrl.getPath());
if (fileName.contains("?")) {
Expand All @@ -212,6 +172,30 @@ protected String getFilenameFromUrl(final URL fileUrl) {
return fileName;
}

/**
* Attempts to extract the filename from the Content-Disposition header.
*
* Example header:
* Content-Disposition: attachment; filename="myfile.txt"
*
* @param contentDisposition The Content-Disposition header value
* @return The filename if present, otherwise null
*/
private String extractFilenameFromContentDisposition(String contentDisposition) {
if (contentDisposition == null) {
return null;
}
for (String token : contentDisposition.split(";")) {
token = token.trim();
if (token.toLowerCase().startsWith("filename=")) {
return token.substring("filename=".length())
.replace("\"", "") // Remove surrounding quotes if any
.trim();
}
}
return null;
}

@Override
public final MetadataResource putResource(final ServiceContext context, final String metadataUuid, final MultipartFile file,
final MetadataResourceVisibility visibility) throws Exception {
Expand Down Expand Up @@ -250,12 +234,27 @@ public final MetadataResource putResource(ServiceContext context, String metadat
@Override
public final MetadataResource putResource(ServiceContext context, String metadataUuid, URL fileUrl,
MetadataResourceVisibility visibility, Boolean approved) throws Exception {
String filename = getFilenameFromHeader(fileUrl);
if (filename == null) {

// Open a connection to the URL
HttpURLConnection connection = (HttpURLConnection) fileUrl.openConnection();
connection.setInstanceFollowRedirects(true);
connection.setRequestMethod("GET");

// Check if the response code is OK
int responseCode = connection.getResponseCode();
if (responseCode != HttpURLConnection.HTTP_OK) {
throw new IOException("Unexpected response code: " + responseCode);
}

// Extract filename from Content-Disposition header if present otherwise use the filename from the URL
String contentDisposition = connection.getHeaderField("Content-Disposition");
String filename = extractFilenameFromContentDisposition(contentDisposition);
if (filename.isEmpty()) {
filename = getFilenameFromUrl(fileUrl);
}

long contentLength = getContentLengthFromHeader(fileUrl);
// Check if the content length is within the allowed limit
long contentLength = connection.getContentLengthLong();
if (contentLength > maxUploadSize) {
throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException")
.withMessageKey("exception.maxUploadSizeExceeded",
Expand All @@ -265,7 +264,9 @@ public final MetadataResource putResource(ServiceContext context, String metadat
FileUtil.humanizeFileSize(maxUploadSize)});
}

try (InputStream is = new LimitedInputStream(fileUrl.openStream(), maxUploadSize+1)) {
// Put the resource but limit the input stream to the max upload size plus one byte
// so we can check if the file is larger than the allowed size
try (InputStream is = new LimitedInputStream(connection.getInputStream(), maxUploadSize+1)) {
return putResource(context, metadataUuid, filename, is, null, visibility, approved);
}
}
Expand Down
56 changes: 52 additions & 4 deletions datastorages/jcloud/src/main/java/org/fao/geonet/api/records/attachments/JCloudStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
import org.fao.geonet.lib.Lib;
import org.fao.geonet.resources.JCloudConfiguration;
import org.fao.geonet.util.FileUtil;
import org.fao.geonet.util.LimitedInputStream;
import org.fao.geonet.util.LimitedIntputStream;
import org.fao.geonet.utils.IO;
import org.fao.geonet.utils.Log;
import org.jclouds.blobstore.ContainerNotFoundException;
Expand Down Expand Up @@ -280,12 +280,16 @@ protected MetadataResource putResource(final ServiceContext context, final Strin
try {
Map<String, String> properties = null;
boolean isNewResource = true;
String backupKey = key + ".backup";

try {
StorageMetadata storageMetadata = jCloudConfiguration.getClient().getBlobStore().blobMetadata(jCloudConfiguration.getContainerName(), key);
if (storageMetadata != null) {
isNewResource = false;

// Copy the existing blob to a backup location.
jCloudConfiguration.getClient().getBlobStore().copyBlob(jCloudConfiguration.getContainerName(), key, jCloudConfiguration.getContainerName(), backupKey, CopyOptions.NONE);

// Copy existing properties
properties = new HashMap<>(storageMetadata.getUserMetadata());
}
Expand All @@ -312,24 +316,68 @@ protected MetadataResource putResource(final ServiceContext context, final Strin
String.format("Put(2) blob '%s' with version label '%s'.", key, properties.get(jCloudConfiguration.getExternalResourceManagementVersionPropertyName())));

// Upload the Blob in multiple chunks to supports large files.
jCloudConfiguration.getClient().getBlobStore().putBlob(jCloudConfiguration.getContainerName(), blob, multipart());
try {
jCloudConfiguration.getClient().getBlobStore().putBlob(jCloudConfiguration.getContainerName(), blob, multipart());
} catch (Exception e) {
// If the exception was caused by the size limit being exceeded then rollback the resource
if (isUploadSizeExceeded(is)) {
rollbackPutResource(key, backupKey, isNewResource);
throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException")
.withMessageKey("exception.maxUploadSizeExceeded",
new String[]{FileUtil.humanizeFileSize(maxUploadSize)})
.withDescriptionKey("exception.maxUploadSizeExceededUnknownSize.description",
new String[]{FileUtil.humanizeFileSize(maxUploadSize)});
} else {
throw e;
}
}

Blob blobResults = jCloudConfiguration.getClient().getBlobStore().getBlob(jCloudConfiguration.getContainerName(), key);

if (is instanceof LimitedInputStream && ((LimitedInputStream) is).isLimitExceeded()) {
delResource(context, metadataUuid, visibility, filename, approved);
// Rollback the upload if the size was exceeded
if (isUploadSizeExceeded(is)) {
rollbackPutResource(key, backupKey, isNewResource);
throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException")
.withMessageKey("exception.maxUploadSizeExceeded",
new String[]{FileUtil.humanizeFileSize(maxUploadSize)})
.withDescriptionKey("exception.maxUploadSizeExceededUnknownSize.description",
new String[]{FileUtil.humanizeFileSize(maxUploadSize)});
}

if (!isNewResource) {
// Remove the backup if the new resource was successfully uploaded.
jCloudConfiguration.getClient().getBlobStore().removeBlob(jCloudConfiguration.getContainerName(), backupKey);
}

return createResourceDescription(context, metadataUuid, visibility, filename, blobResults.getMetadata(), metadataId, approved);
} finally {
locks.remove(key);
}
}
}

protected void rollbackPutResource(final String key, final String backupKey, boolean isNewResource) {
// If the resource is not new then restore the backup
// otherwise remove the new resource.
if (!isNewResource) {
// Check if the backup exists before restoring it.
StorageMetadata storageMetadata = jCloudConfiguration.getClient().getBlobStore().blobMetadata(jCloudConfiguration.getContainerName(), backupKey);
if (storageMetadata == null) {
Log.warning(Geonet.RESOURCES,
String.format("Backup resource '%s' not found for rollback. Resource '%s' will not be restored.", backupKey, key));
return;
}
jCloudConfiguration.getClient().getBlobStore().copyBlob(jCloudConfiguration.getContainerName(), backupKey, jCloudConfiguration.getContainerName(), key, CopyOptions.NONE);
jCloudConfiguration.getClient().getBlobStore().removeBlob(jCloudConfiguration.getContainerName(), backupKey);
} else {
jCloudConfiguration.getClient().getBlobStore().removeBlob(jCloudConfiguration.getContainerName(), key);
}
}

protected boolean isUploadSizeExceeded(InputStream is) {
return is instanceof LimitedInputStream && ((LimitedInputStream) is).isLimitReached();
}

protected void setProperties(Map<String, String> properties, String metadataUuid, Date changeDate, Map<String, String> additionalProperties) {

// Add additional properties if exists.
Expand Down

0 comments on commit c0b921b

Please sign in to comment.