File path related issue bundle

api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java

@@ -22,18 +22,20 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.json.JSONArray;
+import org.labkey.api.action.ApiUsageException;
 import org.labkey.api.assay.plate.AssayPlateMetadataService;
 import org.labkey.api.assay.sample.AssaySampleLookupContext;
 import org.labkey.api.collections.CaseInsensitiveHashMap;
 import org.labkey.api.collections.CaseInsensitiveHashSet;
 import org.labkey.api.collections.Sets;
-import org.labkey.api.data.AssayResultsFileConverter;
 import org.labkey.api.data.ColumnInfo;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerFilter;
+import org.labkey.api.data.ConvertHelper;
 import org.labkey.api.data.DbSchema;
 import org.labkey.api.data.DbSchemaType;
 import org.labkey.api.data.DbScope;
+import org.labkey.api.data.ExpDataFileConverter;
 import org.labkey.api.data.ForeignKey;
 import org.labkey.api.data.ImportAliasable;
 import org.labkey.api.data.MvUtil;
@@ -326,8 +328,6 @@ else if (mvIndicatorColumns.contains(column.name))
                 else
                     column.errorValues = ERROR_VALUE;
 
-                if (run != null && column.clazz == File.class)
-                    column.converter = new AssayResultsFileConverter(run);
             }
             return loader;
 
@@ -917,9 +917,9 @@ else if (o instanceof MvFieldWrapper mvWrapper)
                         valueMissing = false;
                     }
 
-                    // If the column is a file link or attachment, resolve the value to a File object
+                    // If the column is a file link, resolve the value to a File object
                     String uri = pd.getType().getTypeURI();
-                    if (uri.equals(PropertyType.FILE_LINK.getTypeUri()) || uri.equals(PropertyType.ATTACHMENT.getTypeUri()))
+                    if (uri.equals(PropertyType.FILE_LINK.getTypeUri()))
                     {
                         if ("".equals(o))
                         {
@@ -932,16 +932,16 @@ else if (o instanceof MvFieldWrapper mvWrapper)
                             // File column values are stored as the absolute resolved path
                             try
                             {
-                                File resolvedFile = AssayUploadFileResolver.resolve(o, container, pd);
+                                File resolvedFile = ExpDataFileConverter.convert(o);
                                 if (resolvedFile != null)
                                 {
                                     o = resolvedFile;
                                     map.put(pd.getName(), o);
                                 }
                             }
-                            catch (ValidationException e)
+                            catch (ConvertHelper.FileConversionException e)
                             {
-                                throw new RuntimeValidationException(e);
+                                throw new ApiUsageException(e);
                             }
                         }
                     }

api/src/org/labkey/api/assay/AssayRunUploadContext.java

@@ -22,6 +22,7 @@
 import org.jetbrains.annotations.Nullable;
 import org.labkey.api.collections.CollectionUtils;
 import org.labkey.api.data.Container;
+import org.labkey.api.data.ExpDataFileConverter;
 import org.labkey.api.dataiterator.DataIteratorBuilder;
 import org.labkey.api.exp.ExperimentException;
 import org.labkey.api.exp.api.ExpProtocol;
@@ -104,7 +105,7 @@ default DataIteratorBuilder getRawData()
 
     /**
      * Map of inputs to roles that will be attached to the assay run.
-     * The map key will be converted into an ExpData object using {@link org.labkey.api.data.ExpDataFileConverter}
+     * The map key will be converted into an ExpData object using {@link ExpDataFileConverter}
      * The map value is the role of the file.
      * Each input file will be attached as an input ExpData to the imported assay run.
      * NOTE: These files will not be parsed or imported by the assay's DataHandler -- use {@link #getUploadedData()} instead.
@@ -336,7 +337,7 @@ public final FACTORY setRunProperties(Map<String, Object> rawProperties)
 
         /**
          * Map of inputs to roles that will be attached to the assay run.
-         * The map key will be converted into an ExpData object using {@link org.labkey.api.data.ExpDataFileConverter}
+         * The map key will be converted into an ExpData object using {@link ExpDataFileConverter}
          * The map value is the role of the file.
          * Each input file will be attached as an input ExpData to the imported assay run.
          * NOTE: These files will not be parsed or imported by the assay's DataHandler -- use {@link #getUploadedData()} instead.

api/src/org/labkey/api/assay/DefaultAssayRunCreator.java

@@ -23,6 +23,7 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.json.JSONArray;
+import org.labkey.api.action.ApiUsageException;
 import org.labkey.api.assay.actions.AssayRunUploadForm;
 import org.labkey.api.assay.pipeline.AssayRunAsyncContext;
 import org.labkey.api.assay.pipeline.AssayUploadPipelineJob;
@@ -86,10 +87,12 @@
 import org.labkey.api.view.ViewBackgroundInfo;
 import org.labkey.api.writer.ContainerUser;
 import org.labkey.vfs.FileLike;
+import org.labkey.vfs.FileSystemLike;
 
 import java.io.File;
 import java.io.FileFilter;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -408,6 +411,13 @@ public ExpExperiment saveExperimentRun(
             AssayResultsFileWriter resultsFileWriter = new AssayResultsFileWriter(context.getProtocol(), run, null);
             resultsFileWriter.savePostedFiles(context);
 
+            Path assayResultsRunDir = AssayResultsFileWriter.getAssayFilesDirectoryPath(run);
+            if (null != assayResultsRunDir && !FileUtil.hasCloudScheme(assayResultsRunDir))
+            {
+                FileLike assayResultFileRoot = FileSystemLike.wrapFile(assayResultsRunDir);
+                if (assayResultFileRoot != null)
+                    QueryService.get().setEnvironment(QueryService.Environment.ASSAYFILESPATH, assayResultFileRoot);
+            }
             importResultData(context, run, inputDatas, outputDatas, info, xarContext, transformResult, insertedDatas);
 
             Integer reRunId = context.getReRunId();
@@ -471,7 +481,7 @@ public ExpExperiment saveExperimentRun(
 
             return batch;
         }
-        catch (ExperimentException | IOException e)
+        catch (ExperimentException | IOException | ConvertHelper.FileConversionException e)
         {
             // clean up the run results file dir here if it was created, for non-async imports
             AssayResultsFileWriter<?> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), run, null);
@@ -481,6 +491,8 @@ public ExpExperiment saveExperimentRun(
 
             if (e instanceof ExperimentException)
                 throw (ExperimentException)e;
+            else if (e instanceof ConvertHelper.FileConversionException)
+                throw new ApiUsageException(e.getMessage(), e);
             else
                 throw new ExperimentException(e);
         }
@@ -725,8 +737,6 @@ protected void addInputDatas(
     // Resolve submitted values into ExpData objects
     protected void addDatas(Container c, @NotNull Map<ExpData, String> resolved, @NotNull Map<?, String> unresolved, @Nullable Logger log) throws ValidationException
     {
-        ExpDataFileConverter expDataFileConverter = new ExpDataFileConverter();
-
         for (Map.Entry<?, String> entry : unresolved.entrySet())
         {
             Object o = entry.getKey();
@@ -738,7 +748,7 @@ protected void addDatas(Container c, @NotNull Map<ExpData, String> resolved, @No
             }
             else
             {
-                File file = (File) expDataFileConverter.convert(File.class, o);
+                File file = ExpDataFileConverter.convert(o);
                 if (file != null)
                 {
                     ExpData data = ExperimentService.get().getExpDataByURL(file, c);
@@ -1104,10 +1114,11 @@ protected void savePropertyObject(ExpObject object, Container container, Map<Dom
                 String value = entry.getValue();
 
                 // resolve any file links for batch or run properties
-                File resolvedFile = AssayUploadFileResolver.resolve(value, container, entry.getKey());
-                if (resolvedFile != null)
+                if (pd.getType().getTypeURI().equals(PropertyType.FILE_LINK.getTypeUri()))
                 {
-                    value = resolvedFile.getAbsolutePath();
+                    File resolvedFile = ExpDataFileConverter.convert(value);
+                    if (resolvedFile != null)
+                        value = resolvedFile.getAbsolutePath();
                 }
 
                 // Treat the empty string as a null in the database, which is our normal behavior when receiving data
@@ -1201,7 +1212,11 @@ else if (!missing)
         {
             try
             {
-                Object o = ConvertUtils.convert(value, type);
+                Object o;
+                if (type == File.class)
+                    o = ExpDataFileConverter.convert(value);
+                else
+                    o = ConvertUtils.convert(value, type);
                 ValidatorContext validatorContext = new ValidatorContext(context.getContainer(), context.getUser());
                 for (ColumnValidator validator : validators)
                 {

api/src/org/labkey/api/assay/actions/AssayRunUploadForm.java

@@ -165,7 +165,12 @@ protected Map<DomainProperty, String> getPropertyMapFromRequest(List<? extends D
             }
 
             if (additionalFiles.containsKey(pd))
-                properties.put(pd, additionalFiles.get(pd).toNioPathForRead().toString());
+            {
+                if (additionalFiles.get(pd).equals(BLANK_FILE)) // file has been removed
+                    properties.put(pd, null);
+                else
+                    properties.put(pd, additionalFiles.get(pd).toNioPathForRead().toString());
+            }
             else
                 properties.put(pd, value);
         }

api/src/org/labkey/api/data/AbstractFileDisplayColumn.java

@@ -55,6 +55,8 @@ public abstract class AbstractFileDisplayColumn extends DataColumn
     protected String _thumbnailWidth;
     protected String _popupWidth;
 
+    public static final String UNAVAILABLE_FILE_SUFFIX = " (unavailable)";
+
     public AbstractFileDisplayColumn(ColumnInfo col)
     {
         super(col);
@@ -75,16 +77,11 @@ public void renderGridCellContents(RenderContext ctx, HtmlWriter out)
     /** @return the short name of the file (not including full path) */
     protected abstract String getFileName(RenderContext ctx, Object value);
 
-    protected String getFileName(RenderContext ctx, Object value, boolean isDisplay)
-    {
-        return getFileName(ctx, value);
-    }
-
     protected abstract InputStream getFileContents(RenderContext ctx, Object value) throws FileNotFoundException;
 
-    protected void renderIconAndFilename(RenderContext ctx, HtmlWriter out, String filename, boolean link, boolean thumbnail)
+    protected void renderIconAndFilename(RenderContext ctx, HtmlWriter out, String fileValue, boolean link, boolean thumbnail)
     {
-        renderIconAndFilename(ctx, out, filename, null, null, link, thumbnail);
+        renderIconAndFilename(ctx, out, fileValue, null, null, link, thumbnail);
     }
 
     protected boolean isImage(String filename)
@@ -95,34 +92,35 @@ protected boolean isImage(String filename)
                 || filename.toLowerCase().endsWith(".gif");
     }
 
-    protected void renderIconAndFilename(RenderContext ctx, HtmlWriter out, String filename, @Nullable String fileIconUrl, @Nullable String popupIconUrl, boolean link, boolean thumbnail)
+    protected void renderIconAndFilename(RenderContext ctx, HtmlWriter out, String fileValue, @Nullable String fileIconUrl, @Nullable String popupIconUrl, boolean link, boolean thumbnail)
     {
-        if (null != filename && !StringUtils.isEmpty(filename))
+        if (null != fileValue && !StringUtils.isEmpty(fileValue))
         {
             // equivalent of DisplayColumn.renderURL.
             // Don't want to call renderUrl (DataColumn.renderUrl) to skip unnecessary displayValue check
             StringExpression s = compileExpression(ctx.getViewContext());
-            String url = null == s ? null : s.eval(ctx);
-            String displayName = getFileName(ctx, filename, true);
-            boolean isImage = isImage(filename);
-            FileImageRenderHelper renderHelper = createRenderHelper(ctx, url, filename, displayName, fileIconUrl, popupIconUrl, thumbnail, isImage);
+            String displayName = getFileName(ctx, fileValue);
+            boolean unavailable = displayName.endsWith(UNAVAILABLE_FILE_SUFFIX);
+            String url = null == s || unavailable ? null : s.eval(ctx);
+            boolean isImage = isImage(fileValue);
+            FileImageRenderHelper renderHelper = createRenderHelper(ctx, url, fileValue, displayName, fileIconUrl, popupIconUrl, thumbnail, isImage);
 
 
             if (link && null != url)
             {
                 A(
                     at(title, "Download attached file")
-                    .at(getLinkTarget() != null && MimeMap.DEFAULT.canInlineFor(filename), target, getLinkTarget())
+                    .at(getLinkTarget() != null && MimeMap.DEFAULT.canInlineFor(fileValue), target, getLinkTarget())
                     .at(href, url),
                     (Renderable) ret -> {
-                        renderPopup(renderHelper, url, fileIconUrl, displayName, filename, thumbnail, isImage, out);
+                        renderPopup(renderHelper, url, fileIconUrl, displayName, fileValue, thumbnail, isImage, out);
                         return ret;
                     }
                 ).appendTo(out);
             }
             else
             {
-                renderPopup(renderHelper, url, fileIconUrl, displayName, filename, thumbnail, isImage, out);
+                renderPopup(renderHelper, url, fileIconUrl, displayName, fileValue, thumbnail, isImage, out);
             }
         }
         else

api/src/org/labkey/api/data/AbstractTableInfo.java

@@ -104,6 +104,7 @@
 
 import static java.util.Collections.unmodifiableCollection;
 import static org.apache.poi.util.StringUtil.isNotBlank;
+import static org.labkey.api.data.AbstractFileDisplayColumn.UNAVAILABLE_FILE_SUFFIX;
 
 abstract public class AbstractTableInfo implements TableInfo, AuditConfigurable, MemTrackable
 {
@@ -2094,7 +2095,7 @@ public List<Pair<String, String>> getValidatedImportTemplates(ViewContext ctx)
             else
             {
                 boolean fileExist = FileLinkDisplayColumn.filePathExist(template.second, ctx.getContainer(), ctx.getUser());
-                templates.add(new Pair<>(template.first, template.second + (fileExist ? "" : " (unavailable)")));
+                templates.add(new Pair<>(template.first, template.second + (fileExist ? "" : UNAVAILABLE_FILE_SUFFIX)));
             }
         }
         return templates;