diff --git a/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java b/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java index 182d9eb0db6..48a33984fa0 100644 --- a/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java +++ b/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java @@ -22,18 +22,20 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.json.JSONArray; +import org.labkey.api.action.ApiUsageException; import org.labkey.api.assay.plate.AssayPlateMetadataService; import org.labkey.api.assay.sample.AssaySampleLookupContext; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.collections.Sets; -import org.labkey.api.data.AssayResultsFileConverter; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ConvertHelper; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.DbScope; +import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.data.ForeignKey; import org.labkey.api.data.ImportAliasable; import org.labkey.api.data.MvUtil; @@ -326,8 +328,6 @@ else if (mvIndicatorColumns.contains(column.name)) else column.errorValues = ERROR_VALUE; - if (run != null && column.clazz == File.class) - column.converter = new AssayResultsFileConverter(run); } return loader; @@ -917,9 +917,9 @@ else if (o instanceof MvFieldWrapper mvWrapper) valueMissing = false; } - // If the column is a file link or attachment, resolve the value to a File object + // If the column is a file link, resolve the value to a File object String uri = pd.getType().getTypeURI(); - if (uri.equals(PropertyType.FILE_LINK.getTypeUri()) || uri.equals(PropertyType.ATTACHMENT.getTypeUri())) + if (uri.equals(PropertyType.FILE_LINK.getTypeUri())) { if ("".equals(o)) { @@ -932,16 +932,16 @@ else if (o instanceof MvFieldWrapper mvWrapper) // File column values are stored as the absolute resolved path try { - File resolvedFile = AssayUploadFileResolver.resolve(o, container, pd); + File resolvedFile = ExpDataFileConverter.convert(o); if (resolvedFile != null) { o = resolvedFile; map.put(pd.getName(), o); } } - catch (ValidationException e) + catch (ConvertHelper.FileConversionException e) { - throw new RuntimeValidationException(e); + throw new ApiUsageException(e); } } } diff --git a/api/src/org/labkey/api/assay/AssayRunUploadContext.java b/api/src/org/labkey/api/assay/AssayRunUploadContext.java index c64fd23022d..6d44da7a716 100644 --- a/api/src/org/labkey/api/assay/AssayRunUploadContext.java +++ b/api/src/org/labkey/api/assay/AssayRunUploadContext.java @@ -22,6 +22,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.collections.CollectionUtils; import org.labkey.api.data.Container; +import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.dataiterator.DataIteratorBuilder; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.api.ExpProtocol; @@ -104,7 +105,7 @@ default DataIteratorBuilder getRawData() /** * Map of inputs to roles that will be attached to the assay run. - * The map key will be converted into an ExpData object using {@link org.labkey.api.data.ExpDataFileConverter} + * The map key will be converted into an ExpData object using {@link ExpDataFileConverter} * The map value is the role of the file. * Each input file will be attached as an input ExpData to the imported assay run. * NOTE: These files will not be parsed or imported by the assay's DataHandler -- use {@link #getUploadedData()} instead. @@ -336,7 +337,7 @@ public final FACTORY setRunProperties(Map rawProperties) /** * Map of inputs to roles that will be attached to the assay run. - * The map key will be converted into an ExpData object using {@link org.labkey.api.data.ExpDataFileConverter} + * The map key will be converted into an ExpData object using {@link ExpDataFileConverter} * The map value is the role of the file. * Each input file will be attached as an input ExpData to the imported assay run. * NOTE: These files will not be parsed or imported by the assay's DataHandler -- use {@link #getUploadedData()} instead. diff --git a/api/src/org/labkey/api/assay/AssayUploadFileResolver.java b/api/src/org/labkey/api/assay/AssayUploadFileResolver.java deleted file mode 100644 index 55daa9fb9e1..00000000000 --- a/api/src/org/labkey/api/assay/AssayUploadFileResolver.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2017-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.assay; - -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.Container; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.files.FileContentService; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.query.ValidationException; -import org.labkey.api.util.URIUtil; - -import java.io.File; -import java.nio.file.Path; - -/** - * Created by klum on 6/2/2017. - */ -public class AssayUploadFileResolver -{ - /** - * Resolves files for attachment or file property columns - * - * @param o the value to resolve - * @param property the DomainProperty of the destination column - */ - @Nullable - public static File resolve(Object o, Container container, DomainProperty property) throws ValidationException - { - if (o == null) - return null; - - String uri = property.getType().getTypeURI(); - if (uri.equals(PropertyType.FILE_LINK.getTypeUri()) || uri.equals(PropertyType.ATTACHMENT.getTypeUri())) - { - File fileToResolve = null; - - if (o instanceof File) - { - fileToResolve = (File) o; - } - else if (o instanceof String) - { - // Issue 36502: Do not resolve blank string as a file name - String s = StringUtils.trimToNull((String)o); - if (s == null) - return null; - - fileToResolve = new File(s); - } - - if (fileToResolve != null) - { - // For security reasons, make sure the user hasn't tried to reference a file that's not under - // the pipeline root or @assayfiles root. Otherwise, they could get access to any file on the server - - Path assayFilesRoot = FileContentService.get().getFileRootPath(container, FileContentService.ContentType.assayfiles); - if (assayFilesRoot != null && URIUtil.isDescendant(assayFilesRoot.toUri(), fileToResolve.toURI())) - { - return null; // return null since we do not need to convert the file path - } - - PipeRoot root = PipelineService.get().findPipelineRoot(container); - if (root == null) - { - throw new ValidationException("Pipeline root not available in container " + container); - } - - if (!root.isUnderRoot(fileToResolve)) - { - File resolved = root.resolvePath(fileToResolve.toString()); - if (resolved == null) - throw new ValidationException("Cannot reference file " + fileToResolve + " from container " + container); - - return resolved; - } - } - } - return null; - } -} diff --git a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java index b73de6969a9..1427a0ae910 100644 --- a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java +++ b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java @@ -23,6 +23,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.json.JSONArray; +import org.labkey.api.action.ApiUsageException; import org.labkey.api.assay.actions.AssayRunUploadForm; import org.labkey.api.assay.pipeline.AssayRunAsyncContext; import org.labkey.api.assay.pipeline.AssayUploadPipelineJob; @@ -86,10 +87,12 @@ import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.api.writer.ContainerUser; import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.FileFilter; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -408,6 +411,13 @@ public ExpExperiment saveExperimentRun( AssayResultsFileWriter resultsFileWriter = new AssayResultsFileWriter(context.getProtocol(), run, null); resultsFileWriter.savePostedFiles(context); + Path assayResultsRunDir = AssayResultsFileWriter.getAssayFilesDirectoryPath(run); + if (null != assayResultsRunDir && !FileUtil.hasCloudScheme(assayResultsRunDir)) + { + FileLike assayResultFileRoot = FileSystemLike.wrapFile(assayResultsRunDir); + if (assayResultFileRoot != null) + QueryService.get().setEnvironment(QueryService.Environment.ASSAYFILESPATH, assayResultFileRoot); + } importResultData(context, run, inputDatas, outputDatas, info, xarContext, transformResult, insertedDatas); Integer reRunId = context.getReRunId(); @@ -471,7 +481,7 @@ public ExpExperiment saveExperimentRun( return batch; } - catch (ExperimentException | IOException e) + catch (ExperimentException | IOException | ConvertHelper.FileConversionException e) { // clean up the run results file dir here if it was created, for non-async imports AssayResultsFileWriter resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), run, null); @@ -481,6 +491,8 @@ public ExpExperiment saveExperimentRun( if (e instanceof ExperimentException) throw (ExperimentException)e; + else if (e instanceof ConvertHelper.FileConversionException) + throw new ApiUsageException(e.getMessage(), e); else throw new ExperimentException(e); } @@ -725,8 +737,6 @@ protected void addInputDatas( // Resolve submitted values into ExpData objects protected void addDatas(Container c, @NotNull Map resolved, @NotNull Map unresolved, @Nullable Logger log) throws ValidationException { - ExpDataFileConverter expDataFileConverter = new ExpDataFileConverter(); - for (Map.Entry entry : unresolved.entrySet()) { Object o = entry.getKey(); @@ -738,7 +748,7 @@ protected void addDatas(Container c, @NotNull Map resolved, @No } else { - File file = (File) expDataFileConverter.convert(File.class, o); + File file = ExpDataFileConverter.convert(o); if (file != null) { ExpData data = ExperimentService.get().getExpDataByURL(file, c); @@ -1104,10 +1114,11 @@ protected void savePropertyObject(ExpObject object, Container container, Map getPropertyMapFromRequest(List { - renderPopup(renderHelper, url, fileIconUrl, displayName, filename, thumbnail, isImage, out); + renderPopup(renderHelper, url, fileIconUrl, displayName, fileValue, thumbnail, isImage, out); return ret; } ).appendTo(out); } else { - renderPopup(renderHelper, url, fileIconUrl, displayName, filename, thumbnail, isImage, out); + renderPopup(renderHelper, url, fileIconUrl, displayName, fileValue, thumbnail, isImage, out); } } else diff --git a/api/src/org/labkey/api/data/AbstractTableInfo.java b/api/src/org/labkey/api/data/AbstractTableInfo.java index 16b03db90a6..7b70be7085b 100644 --- a/api/src/org/labkey/api/data/AbstractTableInfo.java +++ b/api/src/org/labkey/api/data/AbstractTableInfo.java @@ -104,6 +104,7 @@ import static java.util.Collections.unmodifiableCollection; import static org.apache.poi.util.StringUtil.isNotBlank; +import static org.labkey.api.data.AbstractFileDisplayColumn.UNAVAILABLE_FILE_SUFFIX; abstract public class AbstractTableInfo implements TableInfo, AuditConfigurable, MemTrackable { @@ -2094,7 +2095,7 @@ public List> getValidatedImportTemplates(ViewContext ctx) else { boolean fileExist = FileLinkDisplayColumn.filePathExist(template.second, ctx.getContainer(), ctx.getUser()); - templates.add(new Pair<>(template.first, template.second + (fileExist ? "" : " (unavailable)"))); + templates.add(new Pair<>(template.first, template.second + (fileExist ? "" : UNAVAILABLE_FILE_SUFFIX))); } } return templates; diff --git a/api/src/org/labkey/api/data/AssayResultsFileConverter.java b/api/src/org/labkey/api/data/AssayResultsFileConverter.java deleted file mode 100644 index 659da4c6f8f..00000000000 --- a/api/src/org/labkey/api/data/AssayResultsFileConverter.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.labkey.api.data; - -import org.labkey.api.assay.AssayResultsFileWriter; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.URIUtil; - -import java.io.File; - -public class AssayResultsFileConverter extends ExpDataFileConverter -{ - ExpRun _run; - - public AssayResultsFileConverter(ExpRun run) - { - _run = run; - } - - @Override - public Object convert(Class type, Object value) - { - Object convertedValue = super.convert(type, value); - - // if value was null or the ExpDataFileConverter was unable to convert it, return null - if (convertedValue == null) - return null; - - // if we have a run, first try to resolve the file relative to the run's results directory - if (_run != null && value instanceof String) - { - File runRoot = AssayResultsFileWriter.getAssayFilesDirectoryPath(_run).toFile(); - if (runRoot.exists()) - { - String valueStr = value.toString(); - - for (int i = 0; i < 5; i++) // try up to 5 times to find a case-sensitive match - { - String resultsFileName = FileUtil.getAppendedFileName(valueStr, i); - File resultsFile = FileUtil.appendName(runRoot, resultsFileName); - if (!resultsFile.exists() || !URIUtil.isDescendant(runRoot.toURI(), resultsFile.toURI())) - break; - - if (isCaseSensitiveFileNameMatch(resultsFileName, resultsFile)) - return resultsFile; - } - } - } - - return convertedValue; - } - - // if two files were uploaded with the same name but different casing, then the file system will uniquify the names - // when saved (i.e. test.txt and Test-1.txt) on a case-sensitive file system, so we have to check here to see if - // the resolved results file name matches the canonical file name - private boolean isCaseSensitiveFileNameMatch(String value, File resultsFile) - { - String caseSensitivePath = FileUtil.getAbsoluteCaseSensitiveFile(resultsFile).getAbsolutePath(); - String caseSensitiveName = AssayResultsFileWriter.getFileNameWithoutPath(caseSensitivePath); - return value.equals(caseSensitiveName); - } -} diff --git a/api/src/org/labkey/api/data/ConvertHelper.java b/api/src/org/labkey/api/data/ConvertHelper.java index e8ae8d0094b..48684c6e3cc 100644 --- a/api/src/org/labkey/api/data/ConvertHelper.java +++ b/api/src/org/labkey/api/data/ConvertHelper.java @@ -61,6 +61,7 @@ import org.labkey.api.util.GUID; import org.labkey.api.util.ReturnURLString; import org.labkey.api.util.SimpleTime; +import org.labkey.api.util.SkipMothershipLogging; import org.labkey.api.util.StringExpression; import org.labkey.api.util.StringExpressionFactory; import org.labkey.api.util.TimeOnlyDate; @@ -178,7 +179,7 @@ protected void register() _register(new SimpleTimeConverter(), SimpleTime.class); _register(new ShowRowsConverter(), ShowRows.class); _register(new UserConverter(), User.class); - _register(new ExpDataFileConverter(), File.class); + _register(new NoOpConverter(), File.class); // let data iterator handle conversion _register(new FacetingBehaviorTypeConverter(), FacetingBehaviorType.class); _register(new DefaultScaleConverter(), DefaultScaleType.class); _register(new SchemaKey.Converter(), SchemaKey.class); @@ -450,6 +451,13 @@ public ContainerConversionException(String msg) } } + public static class FileConversionException extends ConversionException implements SkipMothershipLogging + { + public FileConversionException(String msg) + { + super(msg); + } + } public static class ContainerConverter implements Converter { @@ -878,6 +886,15 @@ public Object convert(Class type, Object value) } } + public static class NoOpConverter implements Converter + { + @Override + public Object convert(Class type, Object value) + { + return value; + } + } + public static class FacetingBehaviorTypeConverter implements Converter { @Override diff --git a/api/src/org/labkey/api/data/ExpDataFileConverter.java b/api/src/org/labkey/api/data/ExpDataFileConverter.java index 610c52b1806..70af860cf31 100644 --- a/api/src/org/labkey/api/data/ExpDataFileConverter.java +++ b/api/src/org/labkey/api/data/ExpDataFileConverter.java @@ -23,6 +23,7 @@ import org.json.JSONObject; import org.labkey.api.assay.AbstractAssayProvider; import org.labkey.api.assay.AssayDataType; +import org.labkey.api.assay.AssayResultsFileWriter; import org.labkey.api.exp.api.DataType; import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.api.ExpDataClass; @@ -42,12 +43,14 @@ import org.labkey.api.view.NotFoundException; import org.labkey.api.webdav.WebdavResource; import org.labkey.api.webdav.WebdavService; +import org.labkey.vfs.FileLike; import java.io.File; import java.net.URI; import java.net.URISyntaxException; import java.util.Collection; import java.util.Collections; +import java.util.List; import static org.labkey.api.dataiterator.SimpleTranslator.getFileRootSubstitutedFilePath; @@ -55,9 +58,9 @@ * User: jeckels * Date: Sep 30, 2011 */ -public class ExpDataFileConverter implements Converter +public class ExpDataFileConverter { - private static final Converter FILE_CONVERTER = new FileConverter(); + public static final Converter FILE_CONVERTER = new FileConverter(); public static ExpData resolveExpData(JSONObject dataObject, @NotNull Container container, @NotNull User user, @NotNull Collection knownTypes) { @@ -206,61 +209,124 @@ private static DataType getDataType(@NotNull File file, @NotNull Collection fileRootTypes = List.of(FileContentService.ContentType.files, FileContentService.ContentType.pipeline, FileContentService.ContentType.assayfiles); + for (FileContentService.ContentType fileRootType : fileRootTypes) + { + File fileRoot = fileContent.getFileRoot(container, fileRootType); + if (fileRoot != null) { - return f; + if (f.isFile() && URIUtil.isDescendant(fileRoot.toURI(), f.toURI())) + return f; + + if (!f.isAbsolute()) + { + try + { + File fileWithRoot = FileUtil.appendPath(fileRoot, Path.parse(f.getPath())); + if (fileWithRoot.isFile() && URIUtil.isDescendant(fileRoot.toURI(), fileWithRoot.toURI())) + return fileWithRoot; + } + catch (IllegalArgumentException ignore) + { + } + } } } } + + if (value instanceof String) + throw new ConvertHelper.FileConversionException("Invalid file path: " + value); + else + throw new ConvertHelper.FileConversionException("Invalid file path: " + f.getPath()); } + return null; } - private File convertToFile(Object value, @NotNull Container container, @NotNull User user, @Nullable String fileRootPath) + public static File convertToFile(Object value, @NotNull Container container, @NotNull User user, @Nullable String fileRootPath, @Nullable FileLike assayResultFileRoot) { if (value instanceof File f) { @@ -270,7 +336,7 @@ private File convertToFile(Object value, @NotNull Container container, @NotNull if (value instanceof JSONObject json) { // Assume the same structure as the saveBatch and getBatch APIs work with - ExpData data = resolveExpData(json, container, user, Collections.emptyList()); + ExpData data = ExpDataFileConverter.resolveExpData(json, container, user, Collections.emptyList()); if (data != null && data.getFile() != null) { return data.getFile(); @@ -301,6 +367,29 @@ private File convertToFile(Object value, @NotNull Container container, @NotNull String webdav = value.toString(); if (null != StringUtils.trimToNull(webdav)) { + if (assayResultFileRoot != null) + { + try + { + for (int i = 0; i < 5; i++) // try up to 5 times to find a case-sensitive match + { + String resultsFileName = FileUtil.getAppendedFileName(webdav, i); + FileLike assayResultFile = assayResultFileRoot.resolveChild(resultsFileName); + + if (!assayResultFile.isFile()) + break; + + File resultsFile = FileUtil.toFileForRead(assayResultFile); + if (isCaseSensitiveFileNameMatch(resultsFileName, resultsFile)) + return resultsFile; + } + } + catch (Exception ignore) + { + } + + } + webdav = getFileRootSubstitutedFilePath(webdav, fileRootPath); Path path = Path.decode(FileUtil.encodeForURL(webdav, true /*Issue 51207*/).replace(AppProps.getInstance().getBaseServerUrl() + AppProps.getInstance().getContextPath(), "")); WebdavResource resource = WebdavService.get().getResolver().lookup(path); @@ -341,7 +430,26 @@ private File convertToFile(Object value, @NotNull Container container, @NotNull return result; } } - // Otherwise, treat it as a plain path + + String filePath = value.toString(); + if (filePath.startsWith("file:")) + { + URI uri = FileUtil.createUri(filePath); + if (FileUtil.FILE_SCHEME.equals(uri.getScheme())) + return new File(uri); + } + + // Otherwise, treat it as a plain path (processed by getFileRootSubstitutedFilePath) return FILE_CONVERTER.convert(File.class, webdav); } + + // if two files were uploaded with the same name but different casing, then the file system will uniquify the names + // when saved (i.e. test.txt and Test-1.txt) on a case-sensitive file system, so we have to check here to see if + // the resolved results file name matches the canonical file name + private static boolean isCaseSensitiveFileNameMatch(String value, File resultsFile) + { + String caseSensitivePath = FileUtil.getAbsoluteCaseSensitiveFile(resultsFile).getAbsolutePath(); + String caseSensitiveName = AssayResultsFileWriter.getFileNameWithoutPath(caseSensitivePath); + return value.equals(caseSensitiveName); + } } diff --git a/api/src/org/labkey/api/data/URLDisplayColumn.java b/api/src/org/labkey/api/data/URLDisplayColumn.java index 327e33d8bdf..55bb96d703d 100644 --- a/api/src/org/labkey/api/data/URLDisplayColumn.java +++ b/api/src/org/labkey/api/data/URLDisplayColumn.java @@ -115,7 +115,7 @@ protected InputStream getFileContents(RenderContext ctx, Object value) } @Override - protected void renderIconAndFilename(RenderContext ctx, HtmlWriter out, String filename, boolean link, boolean thumbnail) + protected void renderIconAndFilename(RenderContext ctx, HtmlWriter out, String fileValue, boolean link, boolean thumbnail) { Object value = getValue(ctx); String url = renderURL(ctx); @@ -125,11 +125,11 @@ protected void renderIconAndFilename(RenderContext ctx, HtmlWriter out, String f if (value != null && (url != null || imageUrl != null || popupImageUrl != null)) { // custom image URLs through the column metadata - super.renderIconAndFilename(ctx, out, filename, imageUrl, popupImageUrl, false, thumbnail); + super.renderIconAndFilename(ctx, out, fileValue, imageUrl, popupImageUrl, false, thumbnail); } else { - super.renderIconAndFilename(ctx, out, filename, link, thumbnail); + super.renderIconAndFilename(ctx, out, fileValue, link, thumbnail); } } diff --git a/api/src/org/labkey/api/dataiterator/SimpleTranslator.java b/api/src/org/labkey/api/dataiterator/SimpleTranslator.java index 1341dd81e11..c26be4f509c 100644 --- a/api/src/org/labkey/api/dataiterator/SimpleTranslator.java +++ b/api/src/org/labkey/api/dataiterator/SimpleTranslator.java @@ -42,6 +42,7 @@ import org.labkey.api.data.DbSequence; import org.labkey.api.data.DbSequenceManager; import org.labkey.api.data.EnumTableInfo; +import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.data.ForeignKey; import org.labkey.api.data.JdbcType; import org.labkey.api.data.LookupResolutionType; @@ -1863,7 +1864,7 @@ else if (file instanceof FileLike fl) } } else if (value instanceof String filePath) - return getFileRootSubstitutedFilePath(filePath, _fileRootPath); + return ExpDataFileConverter.convert(filePath); return value; } } diff --git a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java index bbb54df8096..872719b2918 100644 --- a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java +++ b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java @@ -40,8 +40,10 @@ import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConvertHelper; import org.labkey.api.data.DbScope; import org.labkey.api.data.DbSequenceManager; +import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.data.ImportAliasable; import org.labkey.api.data.MultiValuedForeignKey; import org.labkey.api.data.PropertyStorageSpec; @@ -548,7 +550,15 @@ protected DataIteratorBuilder _toDataIteratorBuilder(String debugName, List> convertedRows = new ArrayList<>(); + for (int i = 0; i < rows.size(); i++) + { + Map row = rows.get(i); + row = coerceTypes(row); + convertedRows.add(row); + } + return MapDataIterator.of(colNames, convertedRows, debugName); } @@ -706,7 +716,14 @@ protected Map coerceTypes(Map row) { try { - value = ConvertUtils.convert(value.toString(), col.getJavaObjectClass()); + if (PropertyType.FILE_LINK.equals(col.getPropertyType()) && value instanceof String strVal) + value = ExpDataFileConverter.convert(strVal); + else + value = ConvertUtils.convert(value.toString(), col.getJavaObjectClass()); + } + catch (ConvertHelper.FileConversionException e) + { + throw e; } catch (ConversionException e) { diff --git a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java index ce2de00a17f..4d4c06109a5 100644 --- a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java +++ b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java @@ -26,6 +26,7 @@ import org.labkey.api.collections.CaseInsensitiveMapWrapper; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; +import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.data.JdbcType; import org.labkey.api.data.MvUtil; import org.labkey.api.data.Parameter; @@ -841,10 +842,15 @@ protected Object convertColumnValue(ColumnInfo col, Object value, User user, Con case DATE, TIME, TIMESTAMP: return value instanceof Date ? value : ConvertUtils.convert(value.toString(), Date.class); default: - if (PropertyType.FILE_LINK == col.getPropertyType() && (value instanceof MultipartFile || value instanceof AttachmentFile)) + if (PropertyType.FILE_LINK == col.getPropertyType()) { - FileLike fl = (FileLike)_fileColumnValueMapping.saveFileColumnValue(user, c, fileLinkDirPath, col.getName(), value); - value = fl.toNioPathForRead().toString(); + if ((value instanceof MultipartFile || value instanceof AttachmentFile)) + { + FileLike fl = (FileLike)_fileColumnValueMapping.saveFileColumnValue(user, c, fileLinkDirPath, col.getName(), value); + value = fl.toNioPathForRead().toString(); + } + + return ExpDataFileConverter.convert(value); } return ConvertUtils.convert(value.toString(), col.getJdbcType().getJavaClass()); } diff --git a/api/src/org/labkey/api/query/QueryService.java b/api/src/org/labkey/api/query/QueryService.java index efc0abd2594..0e6a8d6301e 100644 --- a/api/src/org/labkey/api/query/QueryService.java +++ b/api/src/org/labkey/api/query/QueryService.java @@ -319,7 +319,8 @@ enum Environment USER(JdbcType.OTHER), CONTAINER(JdbcType.OTHER), ACTION(JdbcType.OTHER), - FILEROOTPATH(JdbcType.VARCHAR); + FILEROOTPATH(JdbcType.VARCHAR), + ASSAYFILESPATH(JdbcType.OTHER); public final JdbcType type; diff --git a/api/src/org/labkey/api/reader/DataLoader.java b/api/src/org/labkey/api/reader/DataLoader.java index ea020b9ec6a..2250ca156d4 100644 --- a/api/src/org/labkey/api/reader/DataLoader.java +++ b/api/src/org/labkey/api/reader/DataLoader.java @@ -31,6 +31,7 @@ import org.labkey.api.data.BaseColumnInfo; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; +import org.labkey.api.data.ConvertHelper; import org.labkey.api.data.ImportAliasable; import org.labkey.api.data.JdbcType; import org.labkey.api.data.MvUtil; @@ -43,6 +44,7 @@ import org.labkey.api.dataiterator.ScrollableDataIterator; import org.labkey.api.exp.MvColumn; import org.labkey.api.exp.MvFieldWrapper; +import org.labkey.api.exp.PropertyType; import org.labkey.api.iterator.CloseableFilteredIterator; import org.labkey.api.iterator.CloseableIterator; import org.labkey.api.query.BatchValidationException; @@ -327,7 +329,6 @@ private void inferColumnInfo(@NotNull Map renamedColumns) throws for (int f = 0; f < nCols; f++) { List classesToTest = new ArrayList<>(Arrays.asList(CONVERT_CLASSES)); - Class knownColumnClass = null; int classIndex = -1; //NOTE: this means we have a header row @@ -337,27 +338,17 @@ private void inferColumnInfo(@NotNull Map renamedColumns) throws { String name = lineFields[0][f]; name = StringUtilsLabKey.sanitizeSeparatorsAndTrim(name); - if (_columnInfoMap.containsKey(name)) - { - //preferentially use this class if it matches - knownColumnClass = _columnInfoMap.get(name).getJavaClass(); - classesToTest.add(0, knownColumnClass); - } - else if (renamedColumns.containsKey(name) && _columnInfoMap.containsKey(renamedColumns.get(name))) + + ColumnInfo knownColumn = getKnownColumn(name, renamedColumns); + + if (knownColumn != null) { - knownColumnClass = _columnInfoMap.get(renamedColumns.get(name)).getJavaClass(); + Class knownColumnClass = knownColumn.getJavaClass(); classesToTest.add(0, knownColumnClass); } } } - // Issue 49830: if we know the column class is File based on the columnInfoMap, use it instead of trying to infer the class based on the data - if (File.class.equals(knownColumnClass)) - { - colDescs[f].clazz = knownColumnClass; - continue; - } - for (int line = inferStartLine; line < numLines; line++) { if (f >= lineFields[line].length) @@ -473,6 +464,16 @@ else if (renamedColumns.containsKey(name) && _columnInfoMap.containsKey(renamedC _columns = colDescs; } + private ColumnInfo getKnownColumn(String name, @NotNull Map renamedColumns) + { + ColumnInfo knownColumn = null; + if (_columnInfoMap.containsKey(name)) + knownColumn = _columnInfoMap.get(name); + else if (renamedColumns.containsKey(name) && _columnInfoMap.containsKey(renamedColumns.get(name))) + knownColumn = _columnInfoMap.get(renamedColumns.get(name)); + return knownColumn; + } + protected String getDefaultColumnName(int col) { return "column" + col; diff --git a/api/src/org/labkey/api/study/assay/FileLinkDisplayColumn.java b/api/src/org/labkey/api/study/assay/FileLinkDisplayColumn.java index 3880c267301..7e23d8cb357 100644 --- a/api/src/org/labkey/api/study/assay/FileLinkDisplayColumn.java +++ b/api/src/org/labkey/api/study/assay/FileLinkDisplayColumn.java @@ -20,11 +20,11 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.admin.CoreUrls; -import org.labkey.api.assay.AssayFileWriter; import org.labkey.api.attachments.Attachment; import org.labkey.api.data.AbstractFileDisplayColumn; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; import org.labkey.api.data.DisplayColumn; import org.labkey.api.data.RemappingDisplayColumnFactory; import org.labkey.api.data.RenderContext; @@ -57,6 +57,7 @@ import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -246,12 +247,6 @@ public void addQueryFieldKeys(Set keys) keys.add(_objectURIFieldKey); } - @Override - protected String getFileName(RenderContext ctx, Object value) - { - return getFileName(ctx, value, false); - } - public static boolean filePathExist(String path, Container container, User user) { String davPath = path; @@ -292,7 +287,7 @@ public static boolean filePathExist(String path, Container container, User user) } @Override - protected String getFileName(RenderContext ctx, Object value, boolean isDisplay) + protected String getFileName(RenderContext ctx, Object value) { String result = value == null ? null : StringUtils.trimToNull(value.toString()); if (result != null) @@ -314,19 +309,44 @@ protected String getFileName(RenderContext ctx, Object value, boolean isDisplay) f = FileUtil.getAbsoluteCaseSensitiveFile(new File(result)); NetworkDrive.ensureDrive(f.getPath()); List fileRootTypes = List.of(FileContentService.ContentType.files, FileContentService.ContentType.pipeline, FileContentService.ContentType.assayfiles); - for (FileContentService.ContentType fileRootType : fileRootTypes) + boolean valid = false; + List containers = new ArrayList<>(); + containers.add(_container); + // Not ideal, but needed in case data is queried from cross folder context + if (ctx.get("folder") != null || ctx.get("container") != null) + { + Object folderObj = ctx.get("folder"); + if (folderObj == null) + folderObj = ctx.get("container"); + if (folderObj instanceof String containerId) + { + Container dataContainer = ContainerManager.getForId(containerId); + if (dataContainer != null && !dataContainer.equals(_container)) + containers.add(dataContainer); + } + } + for (Container container : containers) { - result = relativize(f, FileContentService.get().getFileRoot(_container, fileRootType)); - if (result != null) + if (valid) break; + + for (FileContentService.ContentType fileRootType : fileRootTypes) + { + result = relativize(f, FileContentService.get().getFileRoot(container, fileRootType)); + if (result != null) + { + valid = true; + break; + } + } } if (result == null) { result = f.getName(); } - if (isDisplay && !f.exists() && !result.endsWith("(unavailable)")) - result += " (unavailable)"; + if ((!valid || !f.exists()) && !result.endsWith(UNAVAILABLE_FILE_SUFFIX)) + result += UNAVAILABLE_FILE_SUFFIX; } return result; } @@ -364,48 +384,78 @@ protected InputStream getFileContents(RenderContext ctx, Object ignore) throws F } @Override - protected void renderIconAndFilename(RenderContext ctx, HtmlWriter out, String filename, boolean link, boolean thumbnail) + protected void renderIconAndFilename( + RenderContext ctx, + HtmlWriter out, + String fileValue /*Could be raw path value, or processed filename by `getFileName`*/, + boolean link, + boolean thumbnail) { Object value = getValue(ctx); - String s = value == null ? null : StringUtils.trimToNull(value.toString()); - if (s != null) + String strValue = value == null ? null : StringUtils.trimToNull(value.toString()); + if (strValue != null && !fileValue.endsWith(UNAVAILABLE_FILE_SUFFIX)) { File f; - if (s.startsWith("file:")) - f = new File(URI.create(s)); + if (strValue.startsWith("file:")) + f = new File(URI.create(strValue)); else - f = new File(s); + f = new File(strValue); if (!f.exists()) { - String fullPath = PipelineService.get().findPipelineRoot(_container).getRootPath().getAbsolutePath() + File.separator + AssayFileWriter.DIR_NAME + File.separator + value; - f = new File(fullPath); + // try all file root + List fileRootTypes = List.of(FileContentService.ContentType.files, FileContentService.ContentType.pipeline, FileContentService.ContentType.assayfiles); + for (FileContentService.ContentType fileRootType : fileRootTypes) + { + String fullPath = FileContentService.get().getFileRoot(_container, fileRootType).getAbsolutePath()+ File.separator + value; + f = new File(fullPath); + if (f.exists()) + break; + } } // It's probably a file, so check that first if (f.isFile()) { - super.renderIconAndFilename(ctx, out, filename, link, thumbnail); + super.renderIconAndFilename(ctx, out, strValue, link, thumbnail); } else if (f.isDirectory()) { - super.renderIconAndFilename(ctx, out, filename, Attachment.getFileIcon(".folder"), null, link, false); + super.renderIconAndFilename(ctx, out, strValue, Attachment.getFileIcon(".folder"), null, link, false); } else { // It's not on the file system anymore, so don't offer a link and tell the user it's unavailable - super.renderIconAndFilename(ctx, out, filename, Attachment.getFileIcon(filename), null, false, false); + super.renderIconAndFilename(ctx, out, strValue, Attachment.getFileIcon(fileValue), null, false, false); } } else { - super.renderIconAndFilename(ctx, out, filename, link, thumbnail); + super.renderIconAndFilename(ctx, out, fileValue, link, thumbnail); } } @Override public Object getDisplayValue(RenderContext ctx) { - return getFileName(ctx, super.getDisplayValue(ctx), true); + return getFileName(ctx, super.getDisplayValue(ctx)); + } + + @Override + public Object getJsonValue(RenderContext ctx) + { + return getDisplayValue(ctx); } + + @Override + public boolean isFilterable() + { + return false; + } + @Override + public boolean isSortable() + { + return false; + } + } diff --git a/core/src/org/labkey/core/query/UserAvatarDisplayColumnFactory.java b/core/src/org/labkey/core/query/UserAvatarDisplayColumnFactory.java index fdae14dc26a..96f51fa6eff 100644 --- a/core/src/org/labkey/core/query/UserAvatarDisplayColumnFactory.java +++ b/core/src/org/labkey/core/query/UserAvatarDisplayColumnFactory.java @@ -74,7 +74,7 @@ public void renderDetailsCellContents(RenderContext ctx, HtmlWriter out) } @Override - protected void renderIconAndFilename(RenderContext ctx, HtmlWriter out, String filename, @Nullable String fileIconUrl, @Nullable String popupIconUrl, boolean link, boolean thumbnail) + protected void renderIconAndFilename(RenderContext ctx, HtmlWriter out, String fileValue, @Nullable String fileIconUrl, @Nullable String popupIconUrl, boolean link, boolean thumbnail) { renderDetailsCellContents(ctx, out); } diff --git a/experiment/src/client/test/integration/MoveSamplesAction.ispec.ts b/experiment/src/client/test/integration/MoveSamplesAction.ispec.ts index 9a84b590b82..7a2ee57bc98 100644 --- a/experiment/src/client/test/integration/MoveSamplesAction.ispec.ts +++ b/experiment/src/client/test/integration/MoveSamplesAction.ispec.ts @@ -712,7 +712,7 @@ describe('Move Samples', () => { const sampleData = await _getSampleData(sampleRowId1, subfolder1Options, SAMPLE_TYPE_NAME_2, "RowId," + FILE_FIELD_1_NAME); expect(sampleData.length).toBe(1); - expect(getSlashedPath(sampleData[0][FILE_FIELD_1_NAME]).endsWith(subfolder1Options.containerPath + "/@files/sampletype/fileA.txt")).toBe(true); + expect(getSlashedPath(sampleData[0][FILE_FIELD_1_NAME]).endsWith("sampletype/fileA.txt")).toBe(true); await verifyDetailedAuditLogs(topFolderOptions, subfolder1Options, [sampleRowId1], undefined, userComment, [{ @@ -745,8 +745,8 @@ describe('Move Samples', () => { expect(updateCounts.sampleFiles).toBe(2); const sampleData = await _getSampleData(sampleRowId1, topFolderOptions, SAMPLE_TYPE_NAME_2, "RowId," + FILE_FIELD_1_NAME + "," + FILE_FIELD_2_NAME); expect(sampleData.length).toBe(1); - expect(getSlashedPath(sampleData[0][FILE_FIELD_1_NAME]).endsWith(topFolderOptions.containerPath + "/@files/sampletype/fileB.txt")).toBe(true); - expect(getSlashedPath(sampleData[0][FILE_FIELD_2_NAME]).endsWith(topFolderOptions.containerPath + "/@files/sampletype/fileC.txt")).toBe(true); + expect(getSlashedPath(sampleData[0][FILE_FIELD_1_NAME]).endsWith("sampletype/fileB.txt")).toBe(true); + expect(getSlashedPath(sampleData[0][FILE_FIELD_2_NAME]).endsWith("sampletype/fileC.txt")).toBe(true); await verifyDetailedAuditLogs(subfolder1Options, topFolderOptions, [sampleRowId1], undefined, undefined, [{ oldValue: getSlashedPath(subfolder1Options.containerPath + "/@files/sampletype/fileB.txt"), @@ -783,8 +783,8 @@ describe('Move Samples', () => { expect(updateCounts.sampleFiles).toBe(2); const sampleData = await _getSampleData(sampleRowId1, topFolderOptions, SAMPLE_TYPE_NAME_2, "RowId," + FILE_FIELD_1_NAME + "," + FILE_FIELD_2_NAME); expect(sampleData.length).toBe(1); - expect(getSlashedPath(sampleData[0][FILE_FIELD_1_NAME]).endsWith(topFolderOptions.containerPath + "/@files/sampletype/fileD-1.txt")).toBe(true); - expect(getSlashedPath(sampleData[0][FILE_FIELD_2_NAME]).endsWith(topFolderOptions.containerPath + "/@files/sampletype/fileE.txt")).toBe(true); + expect(getSlashedPath(sampleData[0][FILE_FIELD_1_NAME]).endsWith("sampletype/fileD-1.txt")).toBe(true); + expect(getSlashedPath(sampleData[0][FILE_FIELD_2_NAME]).endsWith("sampletype/fileE.txt")).toBe(true); await verifyDetailedAuditLogs(subfolder1Options, topFolderOptions, [sampleRowId1], undefined, undefined, [{ oldValue: getSlashedPath(subfolder1Options.containerPath + "/@files/sampletype/fileD.txt"), @@ -815,7 +815,7 @@ describe('Move Samples', () => { expect(updateCounts.sampleFiles).toBe(1); const sampleData = await _getSampleData(sampleRowId1, subfolder2Options, SAMPLE_TYPE_NAME_2, "RowId," + FILE_FIELD_1_NAME); expect(sampleData.length).toBe(1); - expect(getSlashedPath(sampleData[0][FILE_FIELD_1_NAME]).endsWith(subfolder2Options.containerPath + "/@files/sampletype/fileF.txt")).toBe(true); + expect(getSlashedPath(sampleData[0][FILE_FIELD_1_NAME]).endsWith("sampletype/fileF.txt")).toBe(true); await verifyDetailedAuditLogs(subfolder1Options, subfolder2Options, [sampleRowId1], undefined, undefined, [{ oldValue: getSlashedPath(subfolder1Options.containerPath + "/@files/sampletype/fileF.txt"), diff --git a/experiment/src/org/labkey/experiment/DefaultCustomPropertyRenderer.java b/experiment/src/org/labkey/experiment/DefaultCustomPropertyRenderer.java index 0186d9b1e72..51c60e04d86 100644 --- a/experiment/src/org/labkey/experiment/DefaultCustomPropertyRenderer.java +++ b/experiment/src/org/labkey/experiment/DefaultCustomPropertyRenderer.java @@ -32,6 +32,8 @@ import java.util.Date; import java.util.List; +import static org.labkey.api.data.AbstractFileDisplayColumn.UNAVAILABLE_FILE_SUFFIX; + /** * Responsible for showing custom field values (like assay run properties or sample type columns) in experiment module detail pages. * User: jeckels @@ -57,7 +59,7 @@ public String getValue(ObjectProperty prop, List siblingProperti } if (o == null) { - o = f.toString(); + o = f.getName() + UNAVAILABLE_FILE_SUFFIX; } } diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 8dc01ff28f1..ae4b4e5bcc5 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -32,6 +32,7 @@ import org.labkey.api.data.ContainerManager; import org.labkey.api.data.CounterDefinition; import org.labkey.api.data.DbScope; +import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.data.NameGenerator; import org.labkey.api.data.RemapCache; import org.labkey.api.data.SimpleFilter; @@ -2099,6 +2100,9 @@ public static class FileLinkDataIterator extends WrapperDataIterator value = null; } } + + if (value instanceof String filePath) + return ExpDataFileConverter.convert(filePath); return value; }; } @@ -2475,6 +2479,7 @@ private int _importSplitFile(TypeData typeData, File splitFile, Container dataCo configureLoader(loader, dataTable, null, true, aliasNames); if (loader instanceof TabLoader tabLoader) tabLoader.setIncludeComments(true); // don't skip lines that starts with "#" (if the original file is Excel) + QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, dataContainer); return updateService.loadRows(_user, dataContainer, loader, _context, null); } catch (SQLException | IOException e) diff --git a/experiment/src/org/labkey/experiment/api/ExpRunTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpRunTableImpl.java index c2794b0db88..c2574d49933 100644 --- a/experiment/src/org/labkey/experiment/api/ExpRunTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpRunTableImpl.java @@ -32,6 +32,7 @@ import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.DataColumn; +import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.data.ForeignKey; import org.labkey.api.data.JdbcType; import org.labkey.api.data.MultiValuedForeignKey; @@ -1031,9 +1032,11 @@ else if (Column.WorkflowTask.toString().equalsIgnoreCase(columnName)) { PropertyDescriptor propertyDescriptor = col.getPropertyDescriptor(); Object oldValue = run.getProperty(propertyDescriptor); - if (propertyDescriptor.getPropertyType() == PropertyType.FILE_LINK && (value instanceof MultipartFile || value instanceof SpringAttachmentFile)) + if (propertyDescriptor.getPropertyType() == PropertyType.FILE_LINK) { - value = saveFile(user, container, col.getName(), value, AssayFileWriter.DIR_NAME); + if (value instanceof MultipartFile || value instanceof SpringAttachmentFile) + value = saveFile(user, container, col.getName(), value, AssayFileWriter.DIR_NAME); + value = ExpDataFileConverter.convert(value); } ForeignKey fk = col.getFk(); diff --git a/experiment/src/org/labkey/experiment/controllers/property/PropertyController.java b/experiment/src/org/labkey/experiment/controllers/property/PropertyController.java index d318dd83d83..6b5b3638c84 100644 --- a/experiment/src/org/labkey/experiment/controllers/property/PropertyController.java +++ b/experiment/src/org/labkey/experiment/controllers/property/PropertyController.java @@ -46,6 +46,7 @@ import org.labkey.api.data.ContainerService; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.data.NameExpressionValidationResult; import org.labkey.api.data.SimpleFilter; import org.labkey.api.defaults.DefaultValueService; @@ -1187,7 +1188,7 @@ protected ObjectMapper createResponseObjectMapper() public Object execute(InferDomainForm form, BindException errors) throws Exception { Map fileMap = getFileMap(); - File file = form.getFile() != null ? (File) ConvertUtils.convert(form.getFile().toString(), File.class) : null; + File file = form.getFile() != null ? ExpDataFileConverter.convert(form.getFile()) : null; FileType guessFormat = form.isGuessFormatAsTSV() ? TabLoader.TSV_FILE_TYPE : null; DataLoader loader = null; diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index d8249bebac9..e7881e47afc 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -4662,6 +4662,7 @@ protected JSONObject executeJson(JSONObject json, CommandType commandType, boole } } + QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, container); List> responseRows = commandType.saveRows(qus, rowsToProcess, getUser(), container, configParameters, extraContext); if (auditEvent != null) diff --git a/study/test/src/org/labkey/test/tests/study/StudyDatasetFileFieldTest.java b/study/test/src/org/labkey/test/tests/study/StudyDatasetFileFieldTest.java index f7272198103..3bd79fbf2e7 100644 --- a/study/test/src/org/labkey/test/tests/study/StudyDatasetFileFieldTest.java +++ b/study/test/src/org/labkey/test/tests/study/StudyDatasetFileFieldTest.java @@ -16,12 +16,19 @@ import org.labkey.test.components.domain.DomainFormPanel; import org.labkey.test.components.ext4.Checkbox; import org.labkey.test.pages.DatasetInsertPage; +import org.labkey.test.pages.ImportDataPage; +import org.labkey.test.pages.ViewDatasetDataPage; import org.labkey.test.pages.study.DatasetDesignerPage; import org.labkey.test.params.FieldDefinition; import org.labkey.test.util.AuditLogHelper; +import org.labkey.test.util.ApiPermissionsHelper; import org.labkey.test.util.DataRegionTable; import org.labkey.test.util.DomainUtils; import org.labkey.test.util.TestDataGenerator; +import org.labkey.test.util.FileBrowserHelper; +import org.labkey.test.util.PasswordUtil; +import org.labkey.test.util.data.TestDataUtils; +import org.openqa.selenium.NoSuchElementException; import java.io.File; import java.io.IOException; @@ -89,6 +96,9 @@ protected void doCleanup(boolean afterTest) throws TestTimeoutException @Test public void testFileField() throws IOException, CommandException { + new ApiPermissionsHelper(this) + .setSiteRoleUserPermissions(PasswordUtil.getUsername(), "See Absolute File Paths"); + String datasetName = "Dataset-1"; File inputFile = TestFileUtils.getSampleData("fileTypes/sample.txt"); //arbitrary file goToProjectHome(); @@ -120,6 +130,8 @@ public void testFileField() throws IOException, CommandException File downloadedFile = doAndWaitForDownload(() -> waitAndClick(WAIT_FOR_JAVASCRIPT, Locator.tagWithAttribute("a", "title", "Download attached file"), 0)); checker().verifyTrue("Incorrect file name ", FileUtils.contentEquals(downloadedFile, inputFile)); + FileBrowserHelper.FileDetailInfo fileInfoOriginalFile = _fileBrowserHelper.getFileDetailInfo(getProjectName(), "sample.txt"); + goToFolderManagement().goToExportTab(); new Checkbox(Locator.tagWithText("label", "Files").precedingSibling("input").findElement(getDriver())).check(); File exportedFolderFile = doAndWaitForDownload(()->findButton("Export").click()); @@ -173,6 +185,77 @@ public void testFileField() throws IOException, CommandException setFormElement(Locator.name("quf_" + INT_FIELD), "2"); setFormElement(Locator.name("quf_" + FILE_FIELD_1), updateFile.toString()); clickButton("Submit"); + + FileBrowserHelper.FileDetailInfo fileInfoImportedFile = _fileBrowserHelper.getFileDetailInfo(IMPORT_PROJECT, "sample.txt"); + + // error case: import, update, merge with invalid file path + ViewDatasetDataPage datasetDataPage = new ViewDatasetDataPage(getDriver()); + ImportDataPage importDataPage = datasetDataPage.importBulkData(); + importDataPage.setCopyPasteMerge(false, false); + importFilePathError("badNew", "101", fileInfoOriginalFile.absoluteFilePath()); + importFilePathError("badNew", "101", fileInfoOriginalFile.dataFileUrl()); + importFilePathError("badNew", "101", fileInfoOriginalFile.webDavUrl()); + importFilePathError("badNew", "101", "bad.txt"); + importFilePathError("badNew", "101", fileInfoOriginalFile.absoluteFilePath().replace("sample.txt", "")); + importFilePathError("badNew", "101", "."); + importFilePathError("badNew", "101", "../.."); + importDataPage.setCopyPasteMerge(true, true); + importFilePathError("badNew", "101", fileInfoOriginalFile.absoluteFilePath()); + importFilePathError("1", "2", fileInfoOriginalFile.dataFileUrl()); + importFilePathError("badNew", "101", fileInfoOriginalFile.webDavUrl()); + importFilePathError("1", "2", "bad.txt"); + importFilePathError("badNew", "101", fileInfoOriginalFile.absoluteFilePath().replace("sample.txt", "")); + importFilePathError("1", "2", "."); + importFilePathError("badNew", "101", "../.."); + importDataPage.setCopyPasteMerge(false, true); + importFilePathError("1", "2", fileInfoOriginalFile.absoluteFilePath()); + importFilePathError("1", "2", fileInfoOriginalFile.dataFileUrl()); + importFilePathError("1", "2", fileInfoOriginalFile.webDavUrl()); + importFilePathError("1", "2", "bad.txt"); + importFilePathError("1", "2", fileInfoOriginalFile.absoluteFilePath().replace("sample.txt", "")); + importFilePathError("1", "2", "."); + importFilePathError("1", "2", "../.."); + // happy case, import/update/merge with valid file path + importDataPage.setCopyPasteMerge(false, false); + String header = "ParticipantId\tSequenceNum\tfileField\n"; + String data = "2\t3\t" + fileInfoImportedFile.absoluteFilePath() + "\n" + + "3\t4\t" + fileInfoImportedFile.dataFileUrl() + "\n" + + "4\t5\t" + fileInfoImportedFile.webDavUrl() + "\n" + + "5\t6\t" + fileInfoImportedFile.webDavUrlRelative() + "\n"; + setFormElement(Locator.name("text"), header + data); + new ImportDataPage(getDriver()).submit(); + datasetDataPage = new ViewDatasetDataPage(getDriver()); + importDataPage = datasetDataPage.importBulkData(); + importDataPage.setCopyPasteMerge(false, true); + data = "2\t3\t" + fileInfoImportedFile.dataFileUrl() + "\n" + + "3\t4\t" + fileInfoImportedFile.webDavUrl() + "\n" + + "4\t5\t" + fileInfoImportedFile.webDavUrlRelative() + "\n" + + "5\t6\t" + fileInfoImportedFile.absoluteFilePath() + "\n"; + setFormElement(Locator.name("text"), header + data); + new ImportDataPage(getDriver()).submit(); + datasetDataPage = new ViewDatasetDataPage(getDriver()); + importDataPage = datasetDataPage.importBulkData(); + importDataPage.setCopyPasteMerge(true, true); + data += "6\t7\t" + fileInfoImportedFile.webDavUrlRelative() + "\n"; + setFormElement(Locator.name("text"), header + data); + new ImportDataPage(getDriver()).submit(); + } + + private void importFilePathError(String participantId, String sequenceNum, String filePath) + { + String pasteData = TestDataUtils.tsvStringFromRowMapsEscapeBackslash(List.of( + Map.of("ParticipantId", participantId, "SequenceNum", sequenceNum, FILE_FIELD_1, filePath)), + List.of("ParticipantId", "SequenceNum", FILE_FIELD_1), true); + setFormElement(Locator.name("text"), pasteData); + new ImportDataPage(getDriver()).submitExpectingError(); + try + { + waitForElementToBeVisible(Locator.xpath("//div[contains(@class, 'labkey-error')][contains(text(),'Invalid file path: " + filePath + "')]")); + } + catch(NoSuchElementException nse) + { + checker().fatal().error("Invalid file path error not present."); + } } @Test