From c0b921b419fb0865d28b0c6acda04c4890a409f3 Mon Sep 17 00:00:00 2001 From: tylerjmchugh Date: Thu, 2 Jan 2025 10:18:02 -0500 Subject: [PATCH] Fix abstract store and implement jcloud rollback --- .../records/attachments/AbstractStore.java | 91 ++++++++++--------- .../api/records/attachments/JCloudStore.java | 56 +++++++++++- 2 files changed, 98 insertions(+), 49 deletions(-) diff --git a/core/src/main/java/org/fao/geonet/api/records/attachments/AbstractStore.java b/core/src/main/java/org/fao/geonet/api/records/attachments/AbstractStore.java index 2d0eb2157f9..cb6f53248c6 100644 --- a/core/src/main/java/org/fao/geonet/api/records/attachments/AbstractStore.java +++ b/core/src/main/java/org/fao/geonet/api/records/attachments/AbstractStore.java @@ -26,7 +26,6 @@ import jeeves.server.context.ServiceContext; import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.input.BoundedInputStream; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.api.exception.GeonetMaxUploadSizeExceededException; import org.fao.geonet.api.exception.NotAllowedException; @@ -46,6 +45,7 @@ import org.springframework.web.multipart.MultipartFile; import java.io.BufferedInputStream; +import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; @@ -164,46 +164,6 @@ protected int canDownload(ServiceContext context, String metadataUuid, MetadataR return metadataId; } - protected String getFilenameFromHeader(final URL fileUrl) { - HttpURLConnection connection = null; - try { - connection = (HttpURLConnection) fileUrl.openConnection(); - connection.setRequestMethod("HEAD"); - connection.connect(); - String contentDisposition = connection.getHeaderField("Content-Disposition"); - - if (contentDisposition != null && contentDisposition.contains("filename=")) { - String filename = contentDisposition.split("filename=")[1].replace("\"", "").trim(); - return filename.isEmpty() ? null : filename; - } - return null; - } catch (Exception e) { - log.error("Error retrieving resource filename from header", e); - return null; - } finally { - if (connection != null) { - connection.disconnect(); - } - } - } - - protected long getContentLengthFromHeader(final URL fileUrl) { - HttpURLConnection connection = null; - try { - connection = (HttpURLConnection) fileUrl.openConnection(); - connection.setRequestMethod("HEAD"); - connection.connect(); - return connection.getContentLengthLong(); - } catch (Exception e) { - log.error("Error retrieving resource content length from header", e); - return -1; - } finally { - if (connection != null) { - connection.disconnect(); - } - } - } - protected String getFilenameFromUrl(final URL fileUrl) { String fileName = FilenameUtils.getName(fileUrl.getPath()); if (fileName.contains("?")) { @@ -212,6 +172,30 @@ protected String getFilenameFromUrl(final URL fileUrl) { return fileName; } + /** + * Attempts to extract the filename from the Content-Disposition header. + * + * Example header: + * Content-Disposition: attachment; filename="myfile.txt" + * + * @param contentDisposition The Content-Disposition header value + * @return The filename if present, otherwise null + */ + private String extractFilenameFromContentDisposition(String contentDisposition) { + if (contentDisposition == null) { + return null; + } + for (String token : contentDisposition.split(";")) { + token = token.trim(); + if (token.toLowerCase().startsWith("filename=")) { + return token.substring("filename=".length()) + .replace("\"", "") // Remove surrounding quotes if any + .trim(); + } + } + return null; + } + @Override public final MetadataResource putResource(final ServiceContext context, final String metadataUuid, final MultipartFile file, final MetadataResourceVisibility visibility) throws Exception { @@ -250,12 +234,27 @@ public final MetadataResource putResource(ServiceContext context, String metadat @Override public final MetadataResource putResource(ServiceContext context, String metadataUuid, URL fileUrl, MetadataResourceVisibility visibility, Boolean approved) throws Exception { - String filename = getFilenameFromHeader(fileUrl); - if (filename == null) { + + // Open a connection to the URL + HttpURLConnection connection = (HttpURLConnection) fileUrl.openConnection(); + connection.setInstanceFollowRedirects(true); + connection.setRequestMethod("GET"); + + // Check if the response code is OK + int responseCode = connection.getResponseCode(); + if (responseCode != HttpURLConnection.HTTP_OK) { + throw new IOException("Unexpected response code: " + responseCode); + } + + // Extract filename from Content-Disposition header if present otherwise use the filename from the URL + String contentDisposition = connection.getHeaderField("Content-Disposition"); + String filename = extractFilenameFromContentDisposition(contentDisposition); + if (filename.isEmpty()) { filename = getFilenameFromUrl(fileUrl); } - long contentLength = getContentLengthFromHeader(fileUrl); + // Check if the content length is within the allowed limit + long contentLength = connection.getContentLengthLong(); if (contentLength > maxUploadSize) { throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException") .withMessageKey("exception.maxUploadSizeExceeded", @@ -265,7 +264,9 @@ public final MetadataResource putResource(ServiceContext context, String metadat FileUtil.humanizeFileSize(maxUploadSize)}); } - try (InputStream is = new LimitedInputStream(fileUrl.openStream(), maxUploadSize+1)) { + // Put the resource but limit the input stream to the max upload size plus one byte + // so we can check if the file is larger than the allowed size + try (InputStream is = new LimitedInputStream(connection.getInputStream(), maxUploadSize+1)) { return putResource(context, metadataUuid, filename, is, null, visibility, approved); } } diff --git a/datastorages/jcloud/src/main/java/org/fao/geonet/api/records/attachments/JCloudStore.java b/datastorages/jcloud/src/main/java/org/fao/geonet/api/records/attachments/JCloudStore.java index 07102063616..5d495c7127c 100644 --- a/datastorages/jcloud/src/main/java/org/fao/geonet/api/records/attachments/JCloudStore.java +++ b/datastorages/jcloud/src/main/java/org/fao/geonet/api/records/attachments/JCloudStore.java @@ -43,7 +43,7 @@ import org.fao.geonet.lib.Lib; import org.fao.geonet.resources.JCloudConfiguration; import org.fao.geonet.util.FileUtil; -import org.fao.geonet.util.LimitedInputStream; +import org.fao.geonet.util.LimitedIntputStream; import org.fao.geonet.utils.IO; import org.fao.geonet.utils.Log; import org.jclouds.blobstore.ContainerNotFoundException; @@ -280,12 +280,16 @@ protected MetadataResource putResource(final ServiceContext context, final Strin try { Map properties = null; boolean isNewResource = true; + String backupKey = key + ".backup"; try { StorageMetadata storageMetadata = jCloudConfiguration.getClient().getBlobStore().blobMetadata(jCloudConfiguration.getContainerName(), key); if (storageMetadata != null) { isNewResource = false; + // Copy the existing blob to a backup location. + jCloudConfiguration.getClient().getBlobStore().copyBlob(jCloudConfiguration.getContainerName(), key, jCloudConfiguration.getContainerName(), backupKey, CopyOptions.NONE); + // Copy existing properties properties = new HashMap<>(storageMetadata.getUserMetadata()); } @@ -312,17 +316,39 @@ protected MetadataResource putResource(final ServiceContext context, final Strin String.format("Put(2) blob '%s' with version label '%s'.", key, properties.get(jCloudConfiguration.getExternalResourceManagementVersionPropertyName()))); // Upload the Blob in multiple chunks to supports large files. - jCloudConfiguration.getClient().getBlobStore().putBlob(jCloudConfiguration.getContainerName(), blob, multipart()); + try { + jCloudConfiguration.getClient().getBlobStore().putBlob(jCloudConfiguration.getContainerName(), blob, multipart()); + } catch (Exception e) { + // If the exception was caused by the size limit being exceeded then rollback the resource + if (isUploadSizeExceeded(is)) { + rollbackPutResource(key, backupKey, isNewResource); + throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException") + .withMessageKey("exception.maxUploadSizeExceeded", + new String[]{FileUtil.humanizeFileSize(maxUploadSize)}) + .withDescriptionKey("exception.maxUploadSizeExceededUnknownSize.description", + new String[]{FileUtil.humanizeFileSize(maxUploadSize)}); + } else { + throw e; + } + } + Blob blobResults = jCloudConfiguration.getClient().getBlobStore().getBlob(jCloudConfiguration.getContainerName(), key); - if (is instanceof LimitedInputStream && ((LimitedInputStream) is).isLimitExceeded()) { - delResource(context, metadataUuid, visibility, filename, approved); + // Rollback the upload if the size was exceeded + if (isUploadSizeExceeded(is)) { + rollbackPutResource(key, backupKey, isNewResource); throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException") .withMessageKey("exception.maxUploadSizeExceeded", new String[]{FileUtil.humanizeFileSize(maxUploadSize)}) .withDescriptionKey("exception.maxUploadSizeExceededUnknownSize.description", new String[]{FileUtil.humanizeFileSize(maxUploadSize)}); } + + if (!isNewResource) { + // Remove the backup if the new resource was successfully uploaded. + jCloudConfiguration.getClient().getBlobStore().removeBlob(jCloudConfiguration.getContainerName(), backupKey); + } + return createResourceDescription(context, metadataUuid, visibility, filename, blobResults.getMetadata(), metadataId, approved); } finally { locks.remove(key); @@ -330,6 +356,28 @@ protected MetadataResource putResource(final ServiceContext context, final Strin } } + protected void rollbackPutResource(final String key, final String backupKey, boolean isNewResource) { + // If the resource is not new then restore the backup + // otherwise remove the new resource. + if (!isNewResource) { + // Check if the backup exists before restoring it. + StorageMetadata storageMetadata = jCloudConfiguration.getClient().getBlobStore().blobMetadata(jCloudConfiguration.getContainerName(), backupKey); + if (storageMetadata == null) { + Log.warning(Geonet.RESOURCES, + String.format("Backup resource '%s' not found for rollback. Resource '%s' will not be restored.", backupKey, key)); + return; + } + jCloudConfiguration.getClient().getBlobStore().copyBlob(jCloudConfiguration.getContainerName(), backupKey, jCloudConfiguration.getContainerName(), key, CopyOptions.NONE); + jCloudConfiguration.getClient().getBlobStore().removeBlob(jCloudConfiguration.getContainerName(), backupKey); + } else { + jCloudConfiguration.getClient().getBlobStore().removeBlob(jCloudConfiguration.getContainerName(), key); + } + } + + protected boolean isUploadSizeExceeded(InputStream is) { + return is instanceof LimitedInputStream && ((LimitedInputStream) is).isLimitReached(); + } + protected void setProperties(Map properties, String metadataUuid, Date changeDate, Map additionalProperties) { // Add additional properties if exists.