From c5193fa0120f055c8c282209010b6ec51da8aa8c Mon Sep 17 00:00:00 2001 From: tylerjmchugh Date: Mon, 30 Dec 2024 14:49:44 -0500 Subject: [PATCH] Stream file instead of using temp file --- .../records/attachments/AbstractStore.java | 23 +--- .../records/attachments/FilesystemStore.java | 11 ++ .../fao/geonet/util/LimitedInputStream.java | 102 ++++++++++++++++++ .../org/fao/geonet/api/Messages.properties | 1 + .../fao/geonet/api/Messages_fre.properties | 1 + .../org/fao/geonet/api/Messages.properties | 1 + .../fao/geonet/api/Messages_fre.properties | 1 + 7 files changed, 120 insertions(+), 20 deletions(-) create mode 100644 core/src/main/java/org/fao/geonet/util/LimitedInputStream.java diff --git a/core/src/main/java/org/fao/geonet/api/records/attachments/AbstractStore.java b/core/src/main/java/org/fao/geonet/api/records/attachments/AbstractStore.java index fba41dad8b0..2d0eb2157f9 100644 --- a/core/src/main/java/org/fao/geonet/api/records/attachments/AbstractStore.java +++ b/core/src/main/java/org/fao/geonet/api/records/attachments/AbstractStore.java @@ -38,6 +38,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.util.FileUtil; +import org.fao.geonet.util.LimitedInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -45,13 +46,11 @@ import org.springframework.web.multipart.MultipartFile; import java.io.BufferedInputStream; -import java.io.FileInputStream; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Base64; import java.util.List; @@ -65,7 +64,7 @@ public abstract class AbstractStore implements Store { private static final Logger log = LoggerFactory.getLogger(AbstractStore.class); @Value("${api.params.maxUploadSize}") - private int maxUploadSize; + protected int maxUploadSize; @Override public final List getResources(final ServiceContext context, final String metadataUuid, final Sort sort, @@ -266,24 +265,8 @@ public final MetadataResource putResource(ServiceContext context, String metadat FileUtil.humanizeFileSize(maxUploadSize)}); } - Path tempFilePath = Files.createTempFile("uploaded_resource", null); - try (BoundedInputStream boundedInputStream = new BoundedInputStream(fileUrl.openStream(), maxUploadSize+1)) { - Files.copy(boundedInputStream, tempFilePath, StandardCopyOption.REPLACE_EXISTING); - } - - if (Files.size(tempFilePath) > maxUploadSize) { - Files.deleteIfExists(tempFilePath); - throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException") - .withMessageKey("exception.maxUploadSizeExceeded", - new String[]{FileUtil.humanizeFileSize(maxUploadSize)}) - .withDescriptionKey("exception.maxUploadSizeExceededUnknownSize.description", - new String[]{FileUtil.humanizeFileSize(maxUploadSize)}); - } - - try (InputStream is = new FileInputStream(tempFilePath.toFile())) { + try (InputStream is = new LimitedInputStream(fileUrl.openStream(), maxUploadSize+1)) { return putResource(context, metadataUuid, filename, is, null, visibility, approved); - } finally { - Files.deleteIfExists(tempFilePath); } } diff --git a/core/src/main/java/org/fao/geonet/api/records/attachments/FilesystemStore.java b/core/src/main/java/org/fao/geonet/api/records/attachments/FilesystemStore.java index 469bfc296ea..1dcb33efc92 100644 --- a/core/src/main/java/org/fao/geonet/api/records/attachments/FilesystemStore.java +++ b/core/src/main/java/org/fao/geonet/api/records/attachments/FilesystemStore.java @@ -26,6 +26,7 @@ package org.fao.geonet.api.records.attachments; import jeeves.server.context.ServiceContext; +import org.fao.geonet.api.exception.GeonetMaxUploadSizeExceededException; import org.fao.geonet.api.exception.ResourceAlreadyExistException; import org.fao.geonet.api.exception.ResourceNotFoundException; import org.fao.geonet.constants.Geonet; @@ -35,6 +36,8 @@ import org.fao.geonet.kernel.GeonetworkDataDirectory; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.lib.Lib; +import org.fao.geonet.util.FileUtil; +import org.fao.geonet.util.LimitedInputStream; import org.fao.geonet.utils.IO; import org.fao.geonet.utils.Log; import org.springframework.beans.factory.annotation.Autowired; @@ -203,6 +206,14 @@ public MetadataResource putResource(final ServiceContext context, final String m checkResourceId(filename); Path filePath = getPath(context, metadataId, visibility, filename, approved); Files.copy(is, filePath, StandardCopyOption.REPLACE_EXISTING); + if (is instanceof LimitedInputStream && ((LimitedInputStream) is).isLimitReached()) { + Files.deleteIfExists(filePath); + throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException") + .withMessageKey("exception.maxUploadSizeExceeded", + new String[]{FileUtil.humanizeFileSize(maxUploadSize)}) + .withDescriptionKey("exception.maxUploadSizeExceededUnknownSize.description", + new String[]{FileUtil.humanizeFileSize(maxUploadSize)}); + } if (changeDate != null) { IO.touch(filePath, FileTime.from(changeDate.getTime(), TimeUnit.MILLISECONDS)); } diff --git a/core/src/main/java/org/fao/geonet/util/LimitedInputStream.java b/core/src/main/java/org/fao/geonet/util/LimitedInputStream.java new file mode 100644 index 00000000000..56a65c07797 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/util/LimitedInputStream.java @@ -0,0 +1,102 @@ +package org.fao.geonet.util; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class LimitedInputStream extends InputStream { + private final InputStream in; + private final long max; + private long pos; + private long mark; + private boolean propagateClose; + + public LimitedInputStream(InputStream in, long size) { + this.pos = 0L; + this.mark = -1L; + this.propagateClose = true; + this.max = size; + this.in = in; + } + + public LimitedInputStream(InputStream in) { + this(in, -1L); + } + + public int read() throws IOException { + if (this.max >= 0L && this.pos >= this.max) { + return -1; + } else { + int result = this.in.read(); + ++this.pos; + return result; + } + } + + public int read(byte[] b) throws IOException { + return this.read(b, 0, b.length); + } + + public int read(byte[] b, int off, int len) throws IOException { + if (this.max >= 0L && this.pos >= this.max) { + return -1; + } else { + long maxRead = this.max >= 0L ? Math.min((long)len, this.max - this.pos) : (long)len; + int bytesRead = this.in.read(b, off, (int)maxRead); + if (bytesRead == -1) { + return -1; + } else { + this.pos += (long)bytesRead; + return bytesRead; + } + } + } + + public long skip(long n) throws IOException { + long toSkip = this.max >= 0L ? Math.min(n, this.max - this.pos) : n; + long skippedBytes = this.in.skip(toSkip); + this.pos += skippedBytes; + return skippedBytes; + } + + public int available() throws IOException { + return this.max >= 0L && this.pos >= this.max ? 0 : this.in.available(); + } + + public boolean isLimitReached() { + return this.pos >= this.max; + } + + public String toString() { + return this.in.toString(); + } + + public void close() throws IOException { + if (this.propagateClose) { + this.in.close(); + } + + } + + public synchronized void reset() throws IOException { + this.in.reset(); + this.pos = this.mark; + } + + public synchronized void mark(int readlimit) { + this.in.mark(readlimit); + this.mark = this.pos; + } + + public boolean markSupported() { + return this.in.markSupported(); + } + + public boolean isPropagateClose() { + return this.propagateClose; + } + + public void setPropagateClose(boolean propagateClose) { + this.propagateClose = propagateClose; + } +} diff --git a/core/src/test/resources/org/fao/geonet/api/Messages.properties b/core/src/test/resources/org/fao/geonet/api/Messages.properties index 2657aa3088a..a2825472f84 100644 --- a/core/src/test/resources/org/fao/geonet/api/Messages.properties +++ b/core/src/test/resources/org/fao/geonet/api/Messages.properties @@ -178,6 +178,7 @@ api.exception.unsatisfiedRequestParameter=Unsatisfied request parameter api.exception.unsatisfiedRequestParameter.description=Unsatisfied request parameter. exception.maxUploadSizeExceeded=Maximum upload size of {0} exceeded. exception.maxUploadSizeExceeded.description=The request was rejected because its size ({0}) exceeds the configured maximum ({1}). +exception.maxUploadSizeExceededUnknownSize.description=The request was rejected because its size exceeds the configured maximum ({0}). exception.resourceNotFound.metadata=Metadata not found exception.resourceNotFound.metadata.description=Metadata with UUID ''{0}'' not found. exception.resourceNotFound.resource=Metadata resource ''{0}'' not found diff --git a/core/src/test/resources/org/fao/geonet/api/Messages_fre.properties b/core/src/test/resources/org/fao/geonet/api/Messages_fre.properties index af2b4d80fd1..4fcae3fb969 100644 --- a/core/src/test/resources/org/fao/geonet/api/Messages_fre.properties +++ b/core/src/test/resources/org/fao/geonet/api/Messages_fre.properties @@ -172,6 +172,7 @@ api.exception.unsatisfiedRequestParameter=Param\u00E8tre de demande non satisfai api.exception.unsatisfiedRequestParameter.description=Param\u00E8tre de demande non satisfait. exception.maxUploadSizeExceeded=La taille maximale du t\u00E9l\u00E9chargement de {0} a \u00E9t\u00E9 exc\u00E9d\u00E9e. exception.maxUploadSizeExceeded.description=La demande a \u00E9t\u00E9 refus\u00E9e car sa taille ({0}) exc\u00E8de le maximum configur\u00E9 ({1}). +exception.maxUploadSizeExceededUnknownSize.description=La demande a \u00E9t\u00E9 refus\u00E9e car sa taille exc\u00E8de le maximum configur\u00E9 ({0}). exception.resourceNotFound.metadata=Fiches introuvables exception.resourceNotFound.metadata.description=La fiche ''{0}'' est introuvable. exception.resourceNotFound.resource=Ressource ''{0}'' introuvable diff --git a/web/src/main/webapp/WEB-INF/classes/org/fao/geonet/api/Messages.properties b/web/src/main/webapp/WEB-INF/classes/org/fao/geonet/api/Messages.properties index 3d33a2fc02b..0d5044c4a65 100644 --- a/web/src/main/webapp/WEB-INF/classes/org/fao/geonet/api/Messages.properties +++ b/web/src/main/webapp/WEB-INF/classes/org/fao/geonet/api/Messages.properties @@ -183,6 +183,7 @@ api.exception.unsatisfiedRequestParameter=Unsatisfied request parameter api.exception.unsatisfiedRequestParameter.description=Unsatisfied request parameter. exception.maxUploadSizeExceeded=Maximum upload size of {0} exceeded. exception.maxUploadSizeExceeded.description=The request was rejected because its size ({0}) exceeds the configured maximum ({1}). +exception.maxUploadSizeExceededUnknownSize.description=The request was rejected because its size exceeds the configured maximum ({0}). exception.resourceNotFound.metadata=Metadata not found exception.resourceNotFound.metadata.description=Metadata with UUID ''{0}'' not found. exception.resourceNotFound.resource=Metadata resource ''{0}'' not found diff --git a/web/src/main/webapp/WEB-INF/classes/org/fao/geonet/api/Messages_fre.properties b/web/src/main/webapp/WEB-INF/classes/org/fao/geonet/api/Messages_fre.properties index 7e0bc9048ea..0ab9496aa5b 100644 --- a/web/src/main/webapp/WEB-INF/classes/org/fao/geonet/api/Messages_fre.properties +++ b/web/src/main/webapp/WEB-INF/classes/org/fao/geonet/api/Messages_fre.properties @@ -177,6 +177,7 @@ api.exception.unsatisfiedRequestParameter=Param\u00E8tre de demande non satisfai api.exception.unsatisfiedRequestParameter.description=Param\u00E8tre de demande non satisfait. exception.maxUploadSizeExceeded=La taille maximale du t\u00E9l\u00E9chargement de {0} a \u00E9t\u00E9 exc\u00E9d\u00E9e. exception.maxUploadSizeExceeded.description=La demande a \u00E9t\u00E9 refus\u00E9e car sa taille ({0}) exc\u00E8de le maximum configur\u00E9 ({1}). +exception.maxUploadSizeExceededUnknownSize.description=La demande a \u00E9t\u00E9 refus\u00E9e car sa taille exc\u00E8de le maximum configur\u00E9 ({0}). exception.resourceNotFound.metadata=Fiches introuvables exception.resourceNotFound.metadata.description=La fiche ''{0}'' est introuvable. exception.resourceNotFound.resource=Ressource ''{0}'' introuvable