From 143274aef9828b4da1eb42fd002ef46924437d0f Mon Sep 17 00:00:00 2001 From: roby Date: Tue, 14 Oct 2025 12:36:44 -0600 Subject: [PATCH] IRSA-7208: Support additional MOCs - includes reponse to feedback More work; - fixes Firefly-1870 - fixes Firefly-1863 - fixes IRSA-7339 - fixes Firefly-1881 - fixes test failing --- config/app.config | 4 + src/firefly/config/app.prop | 2 + .../caltech/ipac/firefly/data/FileInfo.java | 29 +- .../ipac/firefly/data/HttpResultInfo.java | 21 +- .../ipac/firefly/data/ServerParams.java | 1 + .../server/catquery/CatSummaryQuery.java | 21 +- .../firefly/server/db/DuckDbReadable.java | 6 +- .../ipac/firefly/server/util/QueryUtil.java | 1 - .../server/visualize/hips/HiPSMasterList.java | 37 +- .../hips/HiPSMasterListSourceType.java | 1 + .../visualize/hips/IrsaHiPSListSource.java | 54 +++ .../imageretrieve/URIFileRetriever.java | 4 +- .../util/download/ConcurrentDownload.java | 75 +++- .../ipac/util/download/Downloader.java | 244 ++++------- .../caltech/ipac/util/download/GcsRef.java | 49 +-- .../ipac/util/download/S3Download.java | 9 +- .../edu/caltech/ipac/util/download/S3Ref.java | 42 +- .../ipac/util/download/URLDownload.java | 401 ++++++++++-------- .../caltech/ipac/util/download/UriRef.java | 10 +- .../visualize/net/SloanDssImageGetter.java | 5 +- src/firefly/js/FFEntryPoint.js | 1 + src/firefly/js/data/ServerParams.js | 1 + .../js/drawingLayers/ExtractLineTool.js | 15 +- src/firefly/js/tables/TableUtil.js | 7 +- src/firefly/js/ui/HiPSImageSelect.jsx | 3 + src/firefly/js/ui/TargetPanel.jsx | 3 +- src/firefly/js/visualize/HiPSListUtil.js | 5 +- src/firefly/js/visualize/PlotViewUtil.js | 2 +- src/firefly/js/visualize/task/PlotHipsTask.js | 4 +- .../ipac/util/download/UriRefTest.java | 2 +- 30 files changed, 591 insertions(+), 468 deletions(-) diff --git a/config/app.config b/config/app.config index 3500c8ea5b..00dcbee396 100644 --- a/config/app.config +++ b/config/app.config @@ -40,6 +40,7 @@ GatorHost = "https://irsa.ipac.caltech.edu" irsa.base.url = "https://irsa.ipac.caltech.edu" irsa.gator.dd.hostname = "https://irsa.ipac.caltech.edu" irsa.gator.hostname = "https://irsa.ipac.caltech.edu" +irsa.moc.list.path = "/irsadata/hips/irsa_moc.list" //MOST (moving object - precovery - search) host most.host = "https://irsasearchops.ipac.caltech.edu/cgi-bin/MOST/nph-most" // IRSA Periodogram API @@ -63,15 +64,18 @@ __$tap.maxrec.hardlimit = 10000000 environments { local { BuildType = "Development" + irsa.moc.list.path= "https://bacchus1.ipac.caltech.edu/data/hips/irsa_moc.list" ehcache.multicast.port = 4015 visualize.fits.Security= false ehcache.multicast.ttl = 0 } dev { BuildType = "Development" + irsa.moc.list.path= "https://bacchus1.ipac.caltech.edu/data/hips/irsa_moc.list" ehcache.multicast.port = "5015" lsst.dataAccess.uri = "http://localhost:8661/db/v0/query?" lsst.dataAccess.db = "smm_bremerton" + irsa.base.url= "https://irsadev.ipac.caltech.edu" } test { BuildType = "Beta" diff --git a/src/firefly/config/app.prop b/src/firefly/config/app.prop index 961217e85f..1611b6e676 100644 --- a/src/firefly/config/app.prop +++ b/src/firefly/config/app.prop @@ -2,8 +2,10 @@ # this path does not need to include the upload or cache directory visualize.fits.search.path=@visualize.fits.search.path@:@alerts.dir@:@work.directory@/@app-name@:@wise.filesystem_basepath@:@planck.filesystem_basepath@:@ptf.filesystem_basepath@:@ztf.filesystem_basepath@:@lcogt.filesystem_basepath@:@lsst.filesystem_basepath@ +irsa.base.url=@irsa.base.url@ irsa.gator.hostname=@irsa.gator.hostname@ irsa.gator.dd.hostname=@irsa.gator.dd.hostname@ +irsa.moc.list.path=@irsa.moc.list.path@ wise.ibe.host=@wise.ibe.host@ twomass.ibe.host=@twomass.ibe.host@ ztf.ibe.host=@ztf.ibe.host@ diff --git a/src/firefly/java/edu/caltech/ipac/firefly/data/FileInfo.java b/src/firefly/java/edu/caltech/ipac/firefly/data/FileInfo.java index b44b10aea6..77e77cbf63 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/data/FileInfo.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/data/FileInfo.java @@ -15,7 +15,12 @@ import java.util.List; import java.util.Map; -import static edu.caltech.ipac.firefly.data.HttpResultInfo.*; +import static edu.caltech.ipac.firefly.data.HttpResultInfo.CONTENT_TYPE; +import static edu.caltech.ipac.firefly.data.HttpResultInfo.EXTERNAL_NAME; +import static edu.caltech.ipac.firefly.data.HttpResultInfo.RESPONSE_CODE; +import static edu.caltech.ipac.firefly.data.HttpResultInfo.RESPONSE_CODE_MSG; +import static edu.caltech.ipac.firefly.data.HttpResultInfo.SIZE_IN_BYTES; +import static edu.caltech.ipac.firefly.data.HttpResultInfo.SUFFIX; public class FileInfo implements HasAccessInfo, Serializable, CacheKey { @@ -103,6 +108,7 @@ public static FileInfo blankFilePlaceholder() { public void putAttribute(String key, String value) { attributes.put(key,value); } public String getAttribute(String key) { + if (attributes.containsKey(key)) return attributes.get(key); var matchingKey= attributes.keySet().stream().filter(k -> k.equalsIgnoreCase(key)).findFirst().orElse(null); if (matchingKey!=null) return attributes.get(matchingKey); return null; @@ -173,9 +179,7 @@ public void setSuffix(String filename) { @Override public boolean equals(Object o) { - if (o instanceof FileInfo) { - FileInfo fi = (FileInfo) o; - // external name is unique + if (o instanceof FileInfo fi) { return getInternalFilename().equals(fi.getInternalFilename()); } else { return false; @@ -205,6 +209,9 @@ public boolean hasFileNameResolver() { public String resolveFileName(String name) { return resolver.getResolvedName(name); } + public long getContentLength() { return StringUtils.getLong(getAttribute("Content-Length"),0); } + public String getContentEncoding() { return getAttribute("Content-Encoding"); } + public String getContentDisposition() { return getAttribute("Content-Disposition"); } public FileInfo copy() { FileInfo fi= new FileInfo(); @@ -222,5 +229,19 @@ public FileInfo copy() { public interface FileNameResolver { String getResolvedName(String input); } + + public boolean isReservedKey(String k) { + return (k.equals(RESPONSE_CODE) || + k.equals(RESPONSE_CODE_MSG) || + k.equals(EXTERNAL_NAME) || + k.equals(CONTENT_TYPE) || + k.equals(SUFFIX) || + k.equals(INTERNAL_NAME) || + k.equals(FILE_DOWNLOADED) || + k.equals(DESC) || + k.equals(BLANK) || + k.equals(SIZE_IN_BYTES) || + k.equals(HAS_ACCESS)); + } } diff --git a/src/firefly/java/edu/caltech/ipac/firefly/data/HttpResultInfo.java b/src/firefly/java/edu/caltech/ipac/firefly/data/HttpResultInfo.java index 13cbdeb77f..b18d0e9054 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/data/HttpResultInfo.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/data/HttpResultInfo.java @@ -9,6 +9,7 @@ import edu.caltech.ipac.util.StringUtils; import edu.caltech.ipac.util.download.ResponseMessage; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -26,6 +27,7 @@ public class HttpResultInfo { public static final String SUFFIX= "suffix"; private final byte[] result; + private Map sendHeaders= null; private final Map attributes= new HashMap<>(7); @@ -40,20 +42,25 @@ public HttpResultInfo(byte[] result,int responseCode, String msg, String content putAttribute(RESPONSE_CODE_MSG, msg!=null ? msg : ResponseMessage.getHttpResponseMessage(responseCode)); putAttribute(EXTERNAL_NAME,suggestedFileName); if (contentType!=null) putAttribute(CONTENT_TYPE, contentType); - if (result!=null) putAttribute(SIZE_IN_BYTES, result.length+""); } public void putAttribute(String key, String value) { attributes.put(key,value); } public String getAttribute(String key) { + if (attributes.containsKey(key)) return attributes.get(key); var matchingKey= attributes.keySet().stream().filter(k -> k.equalsIgnoreCase(key)).findFirst().orElse(null); if (matchingKey!=null) return attributes.get(matchingKey); return null; } + public void setSendHeaders(Map sendHeaders) { this.sendHeaders=sendHeaders; } + public Map getSendHeaders() { + return this.sendHeaders!=null ? this.sendHeaders : Collections.emptyMap(); + } + public Map getAttributes() { return attributes; } + public int getResponseCode() { return StringUtils.getInt(attributes.get(RESPONSE_CODE), 200); } public boolean isOK() { return getResponseCode()==200; } public String getResponseCodeMsg() { return attributes.get(RESPONSE_CODE_MSG); } - public long getSizeInBytes() { return StringUtils.getInt(getAttribute(SIZE_IN_BYTES),0); } public String getContentType() { return getAttribute(CONTENT_TYPE); } public String getExternalName() { return getAttribute(EXTERNAL_NAME); } public String getLocation() { return getAttribute(LOCATION); } @@ -66,4 +73,14 @@ public String getAttribute(String key) { public long getContentLength() { return StringUtils.getLong(getAttribute("Content-Length"),0); } public String getContentEncoding() { return getAttribute("Content-Encoding"); } public String getContentDisposition() { return getAttribute("Content-Disposition"); } + + public boolean isReservedKey(String k) { + return (k.equals(RESPONSE_CODE) || + k.equals(RESPONSE_CODE_MSG) || + k.equals(REDIRECTED) || + k.equals(LOCATION) || + k.equals(EXTERNAL_NAME) || + k.equals(CONTENT_TYPE)|| + k.equals(SUFFIX)); + } } diff --git a/src/firefly/java/edu/caltech/ipac/firefly/data/ServerParams.java b/src/firefly/java/edu/caltech/ipac/firefly/data/ServerParams.java index c8a11fbdce..9da9406cb3 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/data/ServerParams.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/data/ServerParams.java @@ -104,6 +104,7 @@ public class ServerParams { public static final String HIPS_LIST_SOURCE_NAME= "hipsListSourceName"; public static final String ENSURE_SOURCE= "ensureSource"; public static final String ADHOC_SOURCE = "adhocSource"; + public static final String ADHOC_MOC_INCLUDE= "AdhocMOCInclude"; public static final String HIPS_DATATYPES = "hipsDataTypes"; public static final String HIPS_MERGE_PRIORITY = "mergedListPriority"; public static final String HIPS_TABLE_TYPE= "hipsTableType"; diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/catquery/CatSummaryQuery.java b/src/firefly/java/edu/caltech/ipac/firefly/server/catquery/CatSummaryQuery.java index 0e20769b92..6352b1e51b 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/catquery/CatSummaryQuery.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/catquery/CatSummaryQuery.java @@ -3,29 +3,26 @@ */ package edu.caltech.ipac.firefly.server.catquery; -import edu.caltech.ipac.table.io.IpacTableWriter; import edu.caltech.ipac.firefly.data.TableServerRequest; import edu.caltech.ipac.firefly.server.query.DataAccessException; import edu.caltech.ipac.firefly.server.query.IpacTablePartProcessor; import edu.caltech.ipac.firefly.server.query.ParamDoc; import edu.caltech.ipac.firefly.server.query.SearchManager; import edu.caltech.ipac.firefly.server.query.SearchProcessorImpl; -import edu.caltech.ipac.table.DataGroupPart; -import edu.caltech.ipac.util.AppProperties; import edu.caltech.ipac.table.DataGroup; +import edu.caltech.ipac.table.DataGroupPart; import edu.caltech.ipac.table.DataObject; import edu.caltech.ipac.table.DataType; +import edu.caltech.ipac.table.io.IpacTableWriter; +import edu.caltech.ipac.util.AppProperties; import edu.caltech.ipac.util.StringUtils; import edu.caltech.ipac.util.download.FailedRequestException; import edu.caltech.ipac.util.download.URLDownload; -import java.io.BufferedInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; import java.io.File; import java.io.IOException; import java.net.URL; -import java.net.URLConnection; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.regex.Matcher; @@ -70,14 +67,8 @@ protected File loadDataFile(TableServerRequest req) throws IOException, DataAcce try { - URLConnection conn = URLDownload.makeConnection(url, null, null); - conn.addRequestProperty("accept", "*/*"); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataInputStream in= new DataInputStream(new BufferedInputStream( - conn.getInputStream())); - URLDownload.logHeader(conn); - URLDownload.netCopy(in, baos, conn, 0, null); - String jsonContent = baos.toString(); + var headers= Collections.singletonMap("accept", "*.*"); + String jsonContent = URLDownload.getDataFromURL(url,null,headers).getResultAsString(); HashMap keyValues = parse(jsonContent); DataType setType = new DataType("set", String.class); setType.setWidth(15); diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/db/DuckDbReadable.java b/src/firefly/java/edu/caltech/ipac/firefly/server/db/DuckDbReadable.java index 30cb6ad08d..027f98386a 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/db/DuckDbReadable.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/db/DuckDbReadable.java @@ -215,8 +215,7 @@ public void export(TableServerRequest treq, OutputStream out) throws DataAccessE File tmp = File.createTempFile("duck-", "."+getName(), QueryUtil.getTempDir(treq)); execUpdate(exportSql.formatted(selectSql, tmp.getAbsolutePath(), voTable.toString(StandardCharsets.UTF_8).replaceAll("'", "''"))); if (tmp.canRead()) { - Downloader downloader = new Downloader(new DataInputStream(new FileInputStream(tmp)), out, tmp.length()); - downloader.download(); + Downloader.download(new DataInputStream(new FileInputStream(tmp)), out); tmp.delete(); } } catch (Exception e) { @@ -244,8 +243,7 @@ public void export(TableServerRequest treq, OutputStream out) throws DataAccessE File tmp = File.createTempFile("export-", "."+getName(), ServerContext.getTempWorkDir()); execUpdate("COPY (%s) TO '%s' (HEADER, DELIMITER '%c')".formatted(sql, tmp.getAbsolutePath(), getDelimiter())); if (tmp.canRead()) { - Downloader downloader = new Downloader(new DataInputStream(new FileInputStream(tmp)), out, tmp.length()); - downloader.download(); + Downloader.download(new DataInputStream(new FileInputStream(tmp)), out); tmp.delete(); } } catch (Exception e) { diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java b/src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java index 0e5fd69703..1c42558064 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java @@ -193,7 +193,6 @@ public static File resolveFileFromSource(String source, boolean checkForUpdates, FileUtil.writeStringToFile(outFile, "workaround"); outFile.setLastModified(res.lastModified()); ops.setOnlyIfModified(false); - ops.setExpectStaticFile(true); FileInfo finfo = RetrieveUtil.download(uri, outFile, inputs.getCookies(), inputs.getHeaders(), ops); if (finfo.getResponseCode() != HttpURLConnection.HTTP_NOT_MODIFIED) { checkForFailures(finfo); diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterList.java b/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterList.java index 64d4ef310d..db73203f06 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterList.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterList.java @@ -39,13 +39,15 @@ * @author Cindy Wang */ @SearchProcessorImpl(id = "HiPSSearch", params = - {@ParamDoc(name = ServerParams.HIPS_DATATYPES, desc = "types of HiPS data to search"), - @ParamDoc(name = ServerParams.HIPS_SOURCES, desc = "HiPS sources"), - @ParamDoc(name = HIPS_LIST_SOURCE, desc = "HiPS list source url"), - @ParamDoc(name = ENSURE_SOURCE, desc = "if true then only ensure the source and an empty table is return"), - @ParamDoc(name = ServerParams.ADHOC_SOURCE, desc = "a comma list of IVOA ids to make the source, the HIPS_SOURCE parameter much include adhoc"), - @ParamDoc(name = ServerParams.SORT_ORDER, desc = "HiPS order, source based"), - @ParamDoc(name = ServerParams.HIPS_TABLE_TYPE, desc = "hips or moc, default hips") + { + @ParamDoc(name = ServerParams.HIPS_DATATYPES, desc = "types of HiPS data to search"), + @ParamDoc(name = ServerParams.HIPS_SOURCES, desc = "HiPS sources"), + @ParamDoc(name = HIPS_LIST_SOURCE, desc = "HiPS list source url"), + @ParamDoc(name = ENSURE_SOURCE, desc = "if true then only ensure the source and an empty table is return"), + @ParamDoc(name = ServerParams.ADHOC_SOURCE, desc = "a comma list of IVOA ids to make the source, the HIPS_SOURCE parameter much include adhoc"), + @ParamDoc(name = ServerParams.ADHOC_MOC_INCLUDE, desc = "always use the additional mocs from these sources in adhoc"), + @ParamDoc(name = ServerParams.SORT_ORDER, desc = "HiPS order, source based"), + @ParamDoc(name = ServerParams.HIPS_TABLE_TYPE, desc = "hips or moc, default hips") }) public class HiPSMasterList extends EmbeddedDbProcessor { public final static String INFO_ICON_STUB = ""; @@ -113,10 +115,15 @@ public DataGroup fetchDataGroup(TableServerRequest request) throws DataAccessExc String[] prioritySources = (hipsMergePriority != null) ? hipsMergePriority.split(",") : null; List allSourceData = new ArrayList<>(); List adhocSources= Collections.emptyList(); + List adhocMocInclude= Collections.emptyList(); + boolean usingAdhoc= false; if (workingSources!=null && Arrays.asList(workingSources).contains("adhoc") && adhocSrcParam!=null) { - adhocSources= Arrays.asList(adhocSrcParam.split(",")); + adhocSources= new ArrayList<>(Arrays.asList(adhocSrcParam.split(","))); workingSources= new String[] {ServerParams.ALL}; + usingAdhoc= true; + var v= request.getParam(ServerParams.ADHOC_MOC_INCLUDE); + if (v!=null) adhocMocInclude= Arrays.asList(v.split(",")); } if (workingSources == null || workingSources.length == 0 || @@ -146,7 +153,21 @@ public DataGroup fetchDataGroup(TableServerRequest request) throws DataAccessExc if (hipsL != null) { allSourceData.addAll(hipsL); } + if (!hipsTable) { + List mocL = hipsls.getAdditionalMOCS(source); + if (mocL != null) { + if (usingAdhoc && adhocMocInclude.contains(source)) { + var moreMocs= mocL + .stream() + .map( v -> v.getMapInfo().get(PARAMS.IVOID.getKey())) + .toList(); + adhocSources.addAll(moreMocs); + } + allSourceData.addAll(mocL); + } + } } + } if (allSourceData.isEmpty()) { diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterListSourceType.java b/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterListSourceType.java index b5c9a1044f..4c29b8c1e8 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterListSourceType.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterListSourceType.java @@ -11,5 +11,6 @@ public interface HiPSMasterListSourceType { List getHiPSListData(String[] dataTypes, String source); + default List getAdditionalMOCS(String source) {return null;} String getUrl(); } diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/IrsaHiPSListSource.java b/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/IrsaHiPSListSource.java index 605689951f..1a3a055c23 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/IrsaHiPSListSource.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/IrsaHiPSListSource.java @@ -1,16 +1,31 @@ package edu.caltech.ipac.firefly.server.visualize.hips; +import edu.caltech.ipac.firefly.data.FileInfo; import edu.caltech.ipac.firefly.data.ServerParams; +import edu.caltech.ipac.firefly.server.servlets.HiPSRetrieve; +import edu.caltech.ipac.firefly.server.util.Logger; +import edu.caltech.ipac.table.DataGroup; +import edu.caltech.ipac.table.DataObject; +import edu.caltech.ipac.table.TableUtil; import edu.caltech.ipac.util.AppProperties; +import java.net.URI; +import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static edu.caltech.ipac.firefly.server.visualize.hips.HiPSMasterListEntry.PARAMS; /** * Created by cwang on 2/27/18. */ public class IrsaHiPSListSource implements HiPSMasterListSourceType { + + private static final String irsaBaseUrl= AppProperties.getProperty("irsa.base.url","https://irsa.ipac.caltech.edu"); + private static final String irsaAddMocPath = AppProperties.getProperty("irsa.moc.list.path","/irsadata/hips/irsa_moc.list"); + static final Logger.LoggerImpl log = Logger.getLogger(); + private static final String irsaHipsListUrl = AppProperties.getProperty( "irsa.hips.masterUrl", "https://irsa.ipac.caltech.edu/data/hips/list"); @@ -26,5 +41,44 @@ public List getHiPSListData(String[] dataTypes, String sour } } + public List getAdditionalMOCS(String source) { + try { + URL url= new URI(getMocUrl()).toURL(); + FileInfo listFileInfo = HiPSRetrieve.retrieveHiPSData(url.toString(), source, false); + if (listFileInfo.getResponseCode()!=200 && listFileInfo.getResponseCode()!=304) return null; + DataGroup dg= TableUtil.readAnyFormat(listFileInfo.getFile()); + var retList = new ArrayList(); + + for(var i=0; i cookie newHeaders.put("If-Modified-Since", outfile.lastModified()+""); } - HttpResultInfo result= URLDownload.getHeader(url,cookies,newHeaders,6); + HttpResultInfo result= URLDownload.getHeader(url,cookies,newHeaders,8); var status= result.getResponseCode(); if (status>=400 && status!=HttpURLConnection.HTTP_BAD_METHOD) { + logHeadFail(url,result); return new FileInfo(result.getResponseCode(), result.getResponseCodeMsg()); } if (status == HttpURLConnection.HTTP_NOT_MODIFIED) return notModified(url,outfile,result); @@ -65,7 +66,7 @@ public static FileInfo getData(URL url, File outfile, Map cookie else { return URLDownload.getDataToFile(finalUrl,outfile,cookies,requestHeaders,ops); } - } catch (MalformedURLException | URISyntaxException e) { + } catch (MalformedURLException | URISyntaxException | IllegalArgumentException e) { throw new FailedRequestException("redirect URL could not be parsed"); } } @@ -74,10 +75,6 @@ public static FileInfo getData(URL url, File outfile, Map cookie private static FileInfo doMultiThreadedDownload(URL url, File outfile, Map cookies, Map requestHeaders, URLDownload.Options ops, HttpResultInfo headResult) throws FailedRequestException{ - - - - StopWatch.Tracker tracker = new StopWatch.Tracker("Multi-threaded Download", null); tracker.starts(); var len= headResult.getContentLength(); @@ -90,11 +87,10 @@ private static FileInfo doMultiThreadedDownload(URL url, File outfile, Map makeExecutorService()); - File outDir= outfile.getParentFile(); try (var outRaf = new RandomAccessFile(outfile, "rw") ) { outRaf.setLength(len); } catch (IOException e) { - throw new FailedRequestException("could not creaste output file"); + throw new FailedRequestException("could not create output file"); } try (var outRaf = new RandomAccessFile(outfile, "rw") ) { @@ -110,7 +106,7 @@ private static FileInfo doMultiThreadedDownload(URL url, File outfile, Map pd.getStatus()==HttpURLConnection.HTTP_PARTIAL)) { - callListener(dl, len, len); + outRaf.close(); //closing explicitly here - I want the tme to reflect any flushing callListener(dl, len, 0); tracker.stops(); double dlSeconds = tracker.getElapsedTime(StopWatch.Unit.SECONDS); - outRaf.close(); - logSuccess(headResult,outfile,url,dlSeconds,pdList.size()); + logSuccess(headResult,outfile,url,dlSeconds,pdList.size(),headResult); return new FileInfo(outfile, headResult.getExternalName(), 200, headResult.getContentType()); } } catch (IOException e) { @@ -185,7 +180,7 @@ private static boolean partialDownloadQualified(HttpResultInfo result) { private static long transferredBytes(ArrayList pdList) { var total=0L; - for (var pd : pdList) {total+= pd.getTransferredBytes(); } + for (var pd : pdList) {total+= pd.getTransferredBytes();} return total; } @@ -215,16 +210,53 @@ private static void logNotModified(File outfile, URL url) { log.info(send+file); } - private static void logSuccess(HttpResultInfo r, File outfile, URL url, double dSeconds, int parts) { + private static void logSuccess(HttpResultInfo r, File outfile, URL url, double dSeconds, int parts, HttpResultInfo headerR) { String formatedSize= FileUtil.getSizeAsString(r.getContentLength()); - String send= String.format( "Download (%.1f sec, %s, %d parts): %s\n", dSeconds, formatedSize, parts, url.toString() ); - String stat= String.format( - " length: %d, contentType: %s, encoding: %s, disposition %s\n", - r.getContentLength(), r.getContentType(), r.getContentEncoding(), r.getContentDisposition() ); - String file= " File: "+ outfile.toPath(); - log.info(send+stat+file); + String lastMod= r.getAttribute("Last-Modified")!=null ? ", Last-Modified: " +r.getAttribute("Last-Modified") : ""; + log.info( + String.format( "DOWNLOAD (%.1f sec, %s, %d parts): Content-Type: %s, Content-Length: %s%s", + dSeconds, formatedSize, parts, r.getContentType(), r.getContentLength(), lastMod), + "url: "+ url.toString(), + "file: "+ outfile.toPath(), + "headers sent: " + sendHeadersToStr(headerR), + "more headers: "+otherHeadersToStr(r) + ); } + private static String otherHeadersToStr(HttpResultInfo r) { + StringBuilder out= new StringBuilder(); + int cnt=0; + for(var a : r.getAttributes().entrySet()) { + var k = a.getKey();; + if (!r.isReservedKey(k) && !k.equalsIgnoreCase("") && + !k.equalsIgnoreCase("content-type") && !k.equalsIgnoreCase("content-length") && + !k.equalsIgnoreCase("Last-Modified") ) { + if (cnt>0) out.append(", "); + out.append(String.format("%s: %s", k, a.getValue())); + cnt++; + } + } + return out.toString(); + } + + private static String sendHeadersToStr(HttpResultInfo r) { + if (r.getSendHeaders()==null) return ""; + StringBuilder out= new StringBuilder(); + int cnt=0; + for(var a : r.getSendHeaders().entrySet()) { + var k = a.getKey(); + if (cnt>0) out.append(", "); + out.append(String.format("%s: %s", k, a.getValue())); + cnt++; + } + return out.toString(); + } + + private static void logHeadFail(URL url, HttpResultInfo r) { + String send= String.format( "FAIL: Concurrent Download Header (%d, %s): %s", + r.getResponseCode(), r.getResponseCodeMsg(), url); + log.info(send,"headers sent: " + sendHeadersToStr(r)); + } private static void logFail(Exception e, File outfile, URL url, int statusCode, String statusMessage, double seconds) { if (e!=null) log.error(e); @@ -295,7 +327,6 @@ public PartialDownload(int index, MappedByteBuffer outBuf, URL url, Map out.write(buff, 0, bytesRead)); + } } - public static void doDownload(DataInputStream in, - File outfile, - long contentLength, - long maxSize, - DownloadListener listener) throws IOException, FailedRequestException { - var downloader = new Downloader(in, outfile, contentLength); - downloader.setDownloadListener(listener); - downloader.setMaxDownloadSize(maxSize); - downloader.download(); - } - public static void doDownload(DataInputStream in, - ByteBuffer mappedOutBuf, - long contentLength, - long maxSize, - DownloadListener listener) throws IOException, FailedRequestException { - var downloader = new Downloader(in, (OutputStream) null, contentLength); - downloader.setDownloadListener(listener); - downloader.setMaxDownloadSize(maxSize); - downloader.mappedOutBuf= mappedOutBuf; - downloader.download(); + public static void download(DataInputStream in, + File outfile, + long contentLength, + long maxSize, + DownloadListener listener) throws IOException, FailedRequestException { + try (var fc = FileChannel.open(outfile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { + var downloader = new Downloader(in, contentLength, listener, maxSize); + downloader.processDownload( + (buff, bytesRead) -> { + var ignore= fc.write(ByteBuffer.wrap(buff, 0, bytesRead)); + }); + } } - public void setMaxDownloadSize(long maxDownloadSize) { _maxDownloadSize= maxDownloadSize; } - - public void download() throws IOException, FailedRequestException { + public static void download(DataInputStream in, + ByteBuffer outByteBuffer, + long contentLength, + long maxSize, + DownloadListener listener) throws IOException, FailedRequestException { + var downloader = new Downloader(in, contentLength, listener, maxSize); + downloader.processDownload((buff, bytesRead) -> outByteBuffer.put(ByteBuffer.wrap(buff, 0, bytesRead))); + } - FileChannel fc = outfile!= null - ? FileChannel.open(outfile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE) - : null; + private void processDownload(Writer writer) throws IOException, FailedRequestException { - long total = _downloadSize; - String messStr; String outStr; Date startDate = null; - TimeStats timeStats = null; + TimeStats timeStats; long totalRead = 0; - if (total > 0) { - messStr = " out of " + FileUtil.getSizeAsString(total); - } else { - messStr = ""; - } - try { - int read; + var messStr = (contentLength > 0) ? " out of " + FileUtil.getSizeAsString(contentLength) : ""; + try (in) { + int byteRead; byte[] buffer = new byte[BUFFER_SIZE]; - long sTime= System.currentTimeMillis(); - while ((read = _in.read(buffer)) != -1) { - totalRead += read; + long sTime = System.currentTimeMillis(); + while ((byteRead = in.read(buffer)) != -1) { + totalRead += byteRead; if (System.currentTimeMillis() - sTime > 750) { sTime = System.currentTimeMillis(); if (startDate == null) startDate = new Date(); - if (total > 0) { - timeStats = computeTimeStats(startDate, totalRead, total); + if (contentLength > 0) { + timeStats = computeTimeStats(startDate, totalRead, contentLength); outStr = FileUtil.getSizeAsString(totalRead) + messStr + " - " + timeStats.remainingStr; } else { outStr = (totalRead / 1024) + messStr; - timeStats = new TimeStats(); - } - if (_maxDownloadSize>0 && totalRead>_maxDownloadSize) { - throw new FailedRequestException( - "File too big to download, Exceeds maximum size of: "+ FileUtil.getSizeAsString(_maxDownloadSize), - "URL does not have a content length header but the " + - "downloaded data exceeded the max size of " +_maxDownloadSize); - } - fireDownloadListeners(totalRead, total, timeStats, outStr); - } - if (_out!=null) _out.write(buffer, 0, read); - if (fc!=null) { - ByteBuffer fcBuff = ByteBuffer.wrap(buffer,0,read); - while (fcBuff.hasRemaining()) { - int ignore= fc.write(fcBuff); - } - } - if (mappedOutBuf!=null) { - ByteBuffer b = ByteBuffer.wrap(buffer,0,read); - while (b.hasRemaining()) { - mappedOutBuf.put(b); + timeStats = null; } + checkSize(totalRead); + fireDownloadListeners(totalRead, contentLength, timeStats, outStr); } + writer.write(buffer, byteRead); } - } catch (EOFException e) { if (totalRead == 0) { - throw new IOException("No data was downloaded",e); + throw new IOException("No data was downloaded", e); } - } finally { - FileUtil.silentClose(_out); - if (fc!=null) fc.close(); } - _out = null; - _in = null; } -//===================================================================== -//----------- add / remove listener methods ----------- -//===================================================================== + private void checkSize(long totalRead) throws FailedRequestException { + if (maxDownloadSize > 0 && totalRead > maxDownloadSize) { + throw new FailedRequestException( + "File too big to download, Exceeds maximum size of: " + FileUtil.getSizeAsString(maxDownloadSize), + "URL does not have a content length header but the " + + "downloaded data exceeded the max size of " + maxDownloadSize); + } + } - public void setDownloadListener(DownloadListener l) { - this.downloadListener= l; + private void fireDownloadListeners(long current, long max, TimeStats timeStats, String mess) { + if (downloadListener == null) return; + DownloadEvent ev = timeStats == null + ? new DownloadEvent(this, current, max, 0, 0, "", "", mess) + : new DownloadEvent(this, current, max, timeStats.elapseSec, timeStats.remainSec, + timeStats.elapseStr, timeStats.remainingStr, mess); + downloadListener.dataDownloading(ev); } + //====================================================================== //------------------ Private / Protected Methods ----------------------- //====================================================================== - private TimeStats computeTimeStats(Date startDate, long cnt, long totalSize) { - TimeStats timeStats = new TimeStats(); + private static TimeStats computeTimeStats(Date startDate, long cnt, long totalSize) { Date now = new Date(); long elapseTime = now.getTime() - startDate.getTime(); long projectedTime = (elapseTime * totalSize) / cnt; double percentLeft = 1.0F - ((double) cnt / (double) totalSize); long remainingTime = (long) (projectedTime * percentLeft + 1000L); - timeStats.elapseSec = elapseTime / 1000; - timeStats.remainSec = remainingTime / 1000; - timeStats.remainingStr = millsecToFormatStr(remainingTime, true); - timeStats.elapseStr = millsecToFormatStr(elapseTime); - - return timeStats; + return new TimeStats(timeeFormatedStr(remainingTime, true), millToSecStr(elapseTime), + remainingTime / 1000, elapseTime / 1000); } - public static String millsecToFormatStr(long milliSec, - boolean userFriendly) { + public static String timeeFormatedStr(long milliSec, boolean userFriendly) { String retval; if (userFriendly) { - long sec= milliSec / 1000; + long sec = milliSec / 1000; if (sec < 3300) { - if (sec <=5) retval= "Less than 5 sec"; - else if (sec <=30) retval= "Less than 30 sec"; - else if (sec <=45) retval= "Less than a minute"; - else if (sec < 75) retval= "About a minute"; - else retval= "About " + sec/60 + " minutes"; - } - else { - float hour= sec / 3600F; + if (sec <= 5) retval = "Less than 5 sec"; + else if (sec <= 30) retval = "Less than 30 sec"; + else if (sec <= 45) retval = "Less than a minute"; + else if (sec < 75) retval = "About a minute"; + else retval = "About " + sec / 60 + " minutes"; + } else { + float hour = sec / 3600F; if (hour < 1.2F && hour > .8F) { - retval= "About an hour"; - } - else { - retval= millsecToFormatStr(milliSec); + retval = "About an hour"; + } else { + retval = millToSecStr(milliSec); } } - } - else { - retval= millsecToFormatStr(milliSec); + } else { + retval = millToSecStr(milliSec); } return retval; } - public static String millsecToFormatStr(long milliSec) { + public static String millToSecStr(long milliSec) { String minStr, secStr; - long inSec= milliSec / 1000; - long hours= inSec/3600; - long mins= (inSec - (hours*3600)) / 60; - minStr= (mins < 10) ? "0" + mins : mins + ""; - long secs= inSec - ((hours*3600) + (mins*60)); - secStr= (secs < 10) ? "0" + secs : secs + ""; + long inSec = milliSec / 1000; + long hours = inSec / 3600; + long mins = (inSec - (hours * 3600)) / 60; + minStr = (mins < 10) ? "0" + mins : mins + ""; + long secs = inSec - ((hours * 3600) + (mins * 60)); + secStr = (secs < 10) ? "0" + secs : secs + ""; return hours + ":" + minStr + ":" + secStr; } - private void fireDownloadListeners(long current, long max, TimeStats timeStats, String mess) { - if (downloadListener==null) return; - DownloadEvent ev; - if (timeStats != null) { - ev = new DownloadEvent(this, current, max, - timeStats.elapseSec, - timeStats.remainSec, - timeStats.elapseStr, - timeStats.remainingStr, - mess); - } else { - ev = new DownloadEvent(this, current, max, 0, 0, "", "", mess); - } - downloadListener.dataDownloading(ev); - } //====================================================================== //------------------ Private Inners classes ---------------------------- //====================================================================== - private static class TimeStats { - String remainingStr = ""; - String elapseStr = ""; - long remainSec = 0; - long elapseSec = 0; - } - + private record TimeStats(String remainingStr, String elapseStr, long remainSec, long elapseSec) { } + private interface Writer { void write(byte[] buffer, int bytesRead) throws IOException; } } diff --git a/src/firefly/java/edu/caltech/ipac/util/download/GcsRef.java b/src/firefly/java/edu/caltech/ipac/util/download/GcsRef.java index 0b9908feb0..bbe43ec7f8 100644 --- a/src/firefly/java/edu/caltech/ipac/util/download/GcsRef.java +++ b/src/firefly/java/edu/caltech/ipac/util/download/GcsRef.java @@ -6,9 +6,9 @@ */ -import java.net.MalformedURLException; +import edu.caltech.ipac.firefly.core.Util; + import java.net.URI; -import java.net.URISyntaxException; import java.net.URL; import java.util.Objects; @@ -35,11 +35,9 @@ public String toUrlStr() { } public URL toUrl() { - try { - return sourceUrl!=null ? sourceUrl : new URI(toUrlStr()).toURL(); - } catch (MalformedURLException | URISyntaxException e) { - return null; - } + return sourceUrl!=null + ? sourceUrl + : Util.Try.it(() -> new URI(toUrlStr()).toURL()).getOrElse((URL)null); } public static GcsRef makeFromUri(Object uri) { @@ -56,27 +54,24 @@ public static GcsRef makeFromUri(Object uri) { private static GcsRef makeFromString(String s) { if (s.length() < 10) return null; if (s.toLowerCase().startsWith("http") && s.toLowerCase().contains(GOOGLE)) { - try { - URL url = new URI(s).toURL(); - var path = url.getPath(); - if (path.length() < 2) return null; - var cleanPath = path.substring(1); - if (cleanPath.length() < 2) return null; - var host = url.getHost().toLowerCase(); - if (GOOGLE.equals(host)) { - var sAry = cleanPath.split("/",2); - if (sAry.length != 2) return null; - return new GcsRef(null, sAry[0], sAry[1], url); - } - else if (host.endsWith(GOOGLE)) { - var sAry = cleanPath.split("\\.",2); - if (sAry.length != 2) return null; - return new GcsRef(null, sAry[0], cleanPath, url); - } - return null; - } catch (MalformedURLException | URISyntaxException e) { - return null; + URL url= Util.Try.it(() -> new URI(s).toURL()).getOrElse((URL)null); + if (url == null) return null; + var path = url.getPath(); + if (path.length() < 2) return null; + var cleanPath = path.substring(1); + if (cleanPath.length() < 2) return null; + var host = url.getHost().toLowerCase(); + if (GOOGLE.equals(host)) { + var sAry = cleanPath.split("/",2); + if (sAry.length != 2) return null; + return new GcsRef(null, sAry[0], sAry[1], url); } + else if (host.endsWith(GOOGLE)) { + var sAry = cleanPath.split("\\.",2); + if (sAry.length != 2) return null; + return new GcsRef(null, sAry[0], cleanPath, url); + } + return null; } return null; } diff --git a/src/firefly/java/edu/caltech/ipac/util/download/S3Download.java b/src/firefly/java/edu/caltech/ipac/util/download/S3Download.java index 0ad8dfbf7a..7f38981877 100644 --- a/src/firefly/java/edu/caltech/ipac/util/download/S3Download.java +++ b/src/firefly/java/edu/caltech/ipac/util/download/S3Download.java @@ -24,19 +24,14 @@ import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest; import software.amazon.awssdk.transfer.s3.model.FileDownload; -import java.io.BufferedOutputStream; import java.io.DataInputStream; import java.io.File; -import java.io.FileOutputStream; -import java.io.OutputStream; import java.net.HttpURLConnection; import java.time.Instant; import java.util.Collections; import java.util.List; import java.util.Map; -import static edu.caltech.ipac.firefly.server.network.HttpServices.BUFFER_SIZE; - public class S3Download { /* @@ -227,8 +222,8 @@ private static FileInfo NOT_USED_getPublicDataNonThreaded( S3Ref ref, try (ResponseInputStream s3Object = client.getObject(getObjectRequest)) { var hInfo= getResponseInfo(s3Object.response()); extName= URLDownload.getSuggestedFileName(hInfo.contentDisposition()); - OutputStream out= new BufferedOutputStream(new FileOutputStream(outfile), BUFFER_SIZE); - URLDownload.netCopy(new DataInputStream(s3Object),out,hInfo.contentLength,0, options.dl()); + Downloader.download(new DataInputStream(s3Object), + outfile, hInfo.contentLength, options.maxFileSize(), options.dl()); return new FileInfo(outfile, extName, 200, ResponseMessage.getHttpResponseMessage(200), hInfo.contentType()); } catch (S3Exception e) { int status= e.statusCode(); diff --git a/src/firefly/java/edu/caltech/ipac/util/download/S3Ref.java b/src/firefly/java/edu/caltech/ipac/util/download/S3Ref.java index dd03642f42..df8b4c315e 100644 --- a/src/firefly/java/edu/caltech/ipac/util/download/S3Ref.java +++ b/src/firefly/java/edu/caltech/ipac/util/download/S3Ref.java @@ -6,11 +6,10 @@ */ +import edu.caltech.ipac.firefly.core.Util; import edu.caltech.ipac.util.StringUtils; -import java.net.MalformedURLException; import java.net.URI; -import java.net.URISyntaxException; import java.net.URL; import java.util.Objects; @@ -68,28 +67,25 @@ private static S3Ref makeFromString(String s) { String bucket = path.substring(0, idx); return new S3Ref(null, bucket, key); } else if (s.toLowerCase().startsWith("https")) { - try { - URL url = new URI(s).toURL(); - var path = url.getPath(); - if (path.length() < 2) return null; - if (!StringUtils.isEmpty(url.getQuery())) return null; - var host = url.getHost().toLowerCase(); - if (!host.endsWith(AMAZON) || !host.contains("s3.")) return null; - var cleanPath = path.substring(1); - var workingHost = host.substring(0, host.length() - amazonLen - 1); + URL url= Util.Try.it(() -> new URI(s).toURL()).getOrElse((URL)null); + if (url == null) return null; + var path = url.getPath(); + if (path.length() < 2) return null; + if (!StringUtils.isEmpty(url.getQuery())) return null; + var host = url.getHost().toLowerCase(); + if (!host.endsWith(AMAZON) || !host.contains("s3.")) return null; + var cleanPath = path.substring(1); + var workingHost = host.substring(0, host.length() - amazonLen - 1); - if (workingHost.startsWith("s3.")) { // s3 path style - var region = workingHost.substring(3); - var sAry = cleanPath.split("/",2); - if (sAry.length != 2) return null; - return new S3Ref(region, sAry[0], sAry[1]); - } else { // s3 host style - var sAry = workingHost.split("\\.s3\\."); - if (sAry.length != 2) return null; - return new S3Ref(sAry[1], sAry[0], cleanPath); - } - } catch (MalformedURLException | URISyntaxException e) { - return null; + if (workingHost.startsWith("s3.")) { // s3 path style + var region = workingHost.substring(3); + var sAry = cleanPath.split("/",2); + if (sAry.length != 2) return null; + return new S3Ref(region, sAry[0], sAry[1]); + } else { // s3 host style + var sAry = workingHost.split("\\.s3\\."); + if (sAry.length != 2) return null; + return new S3Ref(sAry[1], sAry[0], cleanPath); } } return null; diff --git a/src/firefly/java/edu/caltech/ipac/util/download/URLDownload.java b/src/firefly/java/edu/caltech/ipac/util/download/URLDownload.java index 1283b2cf3d..a6f2e971e1 100644 --- a/src/firefly/java/edu/caltech/ipac/util/download/URLDownload.java +++ b/src/firefly/java/edu/caltech/ipac/util/download/URLDownload.java @@ -3,16 +3,17 @@ */ package edu.caltech.ipac.util.download; +import edu.caltech.ipac.firefly.core.Util; import edu.caltech.ipac.firefly.data.FileInfo; import edu.caltech.ipac.firefly.data.HttpResultInfo; import edu.caltech.ipac.firefly.server.RequestOwner; import edu.caltech.ipac.firefly.server.network.HttpServiceInput; import edu.caltech.ipac.firefly.server.util.Logger; +import edu.caltech.ipac.firefly.server.util.StopWatch; import edu.caltech.ipac.firefly.server.util.VersionUtil; import edu.caltech.ipac.util.Base64; import edu.caltech.ipac.util.FileUtil; import edu.caltech.ipac.util.StringUtils; -import edu.caltech.ipac.util.UTCTimeUtil; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; @@ -33,7 +34,6 @@ import java.net.MalformedURLException; import java.net.SocketTimeoutException; import java.net.URI; -import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; import java.net.UnknownHostException; @@ -115,6 +115,25 @@ public static String getSuggestedFileName(URLConnection conn) { return getSuggestedFileName(disposition); } + private static int codeFromException(Exception e) { + return switch (e) { + case SSLException ignored -> 495; + case SocketTimeoutException ignored -> 408; + case UnknownHostException ignored -> 404; + default -> 500; + }; + } + + private static HttpResultInfo exceptionToResponse(Exception e, Map sendHeaders) { + var r= new HttpResultInfo(codeFromException(e),ResponseMessage.getNetworkCallFailureMessage(e)); + r.setSendHeaders(sendHeaders); + return r; + } + + private static FileInfo exceptionToFileInfo(Exception e) { + return new FileInfo(codeFromException(e),ResponseMessage.getNetworkCallFailureMessage(e)); + } + public static String getSuggestedFileName(String disposition) { if (disposition == null) return null; String[] strs = disposition.split(";"); @@ -133,6 +152,19 @@ public static String getFileNameFromUrl(URL url) { return sanitizeFilename(suggestedFileName); } + public static Map buildReqHeaders(URL url, Map requestHeaders, Options ops) { + if (requestHeaders== null) requestHeaders= Collections.emptyMap(); + Map h = new HashMap<>(requestHeaders); + if (ops==null || ops.useCredentials) { + var inputs= new HttpServiceInput(url.toString()); + var credentials= inputs.getHeaders(); + if (credentials!=null && !credentials.isEmpty()) { + if (!credentials.keySet().stream().allMatch(h::containsKey)) h.putAll(credentials); + } + } + return h; + } + public static String sanitizeFilename(String fName) { if (StringUtils.isEmpty(fName)) return ""; //trim leading/trailing whitespace and quotes @@ -144,11 +176,44 @@ public static String sanitizeFilename(String fName) { return fName; } + private static String sendHeadersToCompactStr(Map> sendHeaders) { + + if (sendHeaders.isEmpty()) return ""; + var outStr= new StringBuilder(); + for(Map.Entry> se: sendHeaders.entrySet()) { + if (!outStr.isEmpty()) outStr.append(", "); + StringBuilder workBuff = new StringBuilder(100); + var key= (se.getKey() == null) ? "" : se.getKey(); + workBuff.append(key); + workBuff.append(": "); + if (key.equalsIgnoreCase("cookie")) { + try { + List cValList= se.getValue(); + int lenTotal= cValList.stream().map( (s) -> s==null ? 0 : s.length()).reduce(0, Integer::sum); + String names= cValList.stream() + .reduce("", (all,t) -> all+ Arrays.stream(t.split(";")) + .map( (s) -> s.split("=")[0]) + .reduce("", (allV,tv) -> allV+tv+",")); + workBuff.append("0) workBuff.append(" length: ").append(lenTotal); + workBuff.append(">"); + } + catch (Exception e) { + workBuff.append(""); + } + } + else { + workBuff.append(sanitizeHeader(se.getKey(), String.valueOf(se.getValue()))); + } + outStr.append(workBuff.toString()); + } + return outStr.toString(); + } - private static int getResponseCode(URLConnection conn) { + private static int getResponseCode(HttpURLConnection conn) { if (conn==null) return -1; try { - return (conn instanceof HttpURLConnection) ? ((HttpURLConnection)conn).getResponseCode() : 200; + return conn.getResponseCode(); } catch (SocketTimeoutException e) { return 408; } catch (UnknownHostException e) { @@ -159,11 +224,7 @@ private static int getResponseCode(URLConnection conn) { } private static URL makeURL(String urlStr) { - try { - return new URI(urlStr).toURL(); - } catch (MalformedURLException | URISyntaxException e) { - return null; - } + return Util.Try.it(() -> new URI(urlStr).toURL()).getOrElse((URL)null); } private static URL urlFromLocation(HttpURLConnection conn) { return makeURL(conn.getHeaderField("Location")); } @@ -183,19 +244,19 @@ public static URLConnection makeConnection(URL url, Map requestHeaders) throws IOException { try { URLConnection conn = url.openConnection(); - if (conn instanceof HttpURLConnection) { - conn.setRequestProperty("User-Agent", VersionUtil.getUserAgentString()); - conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); + if (conn instanceof HttpURLConnection httpConn) { + httpConn.setRequestProperty("User-Agent", VersionUtil.getUserAgentString()); + httpConn.setRequestProperty("Accept-Encoding", "gzip, deflate"); if (cookies != null) { var filteredCookies= new HashMap<>(cookies); filteredCookies.remove("JSESSIONID"); filteredCookies.remove(RequestOwner.USER_KEY); - if (!filteredCookies.isEmpty()) addCookiesToConnection(conn, filteredCookies); + if (!filteredCookies.isEmpty()) addCookiesToConnection(httpConn, filteredCookies); } String[] userInfo = getUserInfo(url); if (userInfo != null) { String authStringEnc = Base64.encode(userInfo[0] + ":" + userInfo[1]); - conn.setRequestProperty("Authorization", "Basic " + authStringEnc); + httpConn.setRequestProperty("Authorization", "Basic " + authStringEnc); } } if (requestHeaders != null && !requestHeaders.isEmpty()) { @@ -213,11 +274,15 @@ public static URLConnection makeConnection(URL url, public static HttpURLConnection makeURLConnection(URL url, Map cookies, Map requestHeaders) throws IOException { - return (HttpURLConnection)makeConnection(url, cookies, requestHeaders); + + var conn= makeConnection(url, cookies, requestHeaders); + if (conn==null) throw new IOException("HTTP connection not be created"); + if (conn instanceof HttpURLConnection httpConn) return httpConn; + throw new IOException("HTTP connection not be created, could not be cast to HttpURLConnection"); } - private static void addCookiesToConnection(URLConnection conn, Map cookies) { - if (!(conn instanceof HttpURLConnection) || cookies == null) return; + private static void addCookiesToConnection(HttpURLConnection conn, Map cookies) { + if (cookies == null) return; StringBuilder sb = new StringBuilder(200); for (Map.Entry entry : cookies.entrySet()) { if (!sb.isEmpty()) sb.append("; "); @@ -251,43 +316,42 @@ public static HttpResultInfo getDataFromURL(URL url, Map requestHeaders, ByteBuffer outByteBuffer, Options ops) throws FailedRequestException { - URLConnection conn= null; + HttpURLConnection conn= null; try { - Map h= new HashMap<>(); - if (requestHeaders!=null) h.putAll(requestHeaders); - if (ops.useCredentials) { - var inputs= new HttpServiceInput(url.toString()); - var credentials= inputs.getHeaders(); - if (credentials!=null && !credentials.isEmpty()) { - if (!credentials.keySet().stream().allMatch(h::containsKey)) h.putAll(credentials); - } - } - conn= makeConnection(url,cookies,h); + StopWatch.Tracker tracker = new StopWatch.Tracker("Download", null); + tracker.starts(); + var h= buildReqHeaders(url, requestHeaders, ops); + conn= makeURLConnection(url,cookies,h); Map> reqProp= conn.getRequestProperties(); pushPostData(conn, postData); byte[] results = null; + + DataInputStream in= makeAnyInStream(conn, false); + long conLen= conn.getContentLength(); if (outByteBuffer!=null) { - netCopy(makeAnyInStream(conn, false), outByteBuffer, conn.getContentLength(), ops); + Downloader.download(in, outByteBuffer, conLen, ops.maxFileSize, ops.dl); } else { ByteArrayOutputStream out = new ByteArrayOutputStream(4096); - netCopy(makeAnyInStream(conn, false), out, conn, 0, ops.dl); + Downloader.download(in, out, conLen, 0, ops.dl); results = out.toByteArray(); } - if (!ops.logErrorsOnly) logHeader(url.toString(), postData, conn, reqProp); - - Set>> hSet = getResponseCode(conn)==-1 ? null : conn.getHeaderFields().entrySet(); + var responseCode= getResponseCode(conn); + Set>> hSet = responseCode==-1 ? Collections.emptySet() : conn.getHeaderFields().entrySet(); var result= new HttpResultInfo(results,getResponseCode(conn),null, conn.getContentType(), getSuggestedFileName(conn)); - if (hSet!=null) { - for (Map.Entry> e : hSet) { - result.putAttribute(e.getKey()!=null ? e.getKey() : "",combineValues(e.getValue())); - } + result.setSendHeaders(h); + for (Map.Entry> e : hSet) { + result.putAttribute(e.getKey()!=null ? e.getKey() : "",combineValues(e.getValue())); } - if (!ops.logErrorsOnly) logCompletedDownload(conn.getURL(), result.getContentLength()); + tracker.stops(); + double dlSeconds = tracker.getElapsedTime(StopWatch.Unit.SECONDS); + if (responseCode>300) logHeader(url.toString(), postData, conn, reqProp); + if (!ops.logErrorsOnly) logSuccess(result,url,dlSeconds,reqProp, postData); return result; - } catch (SSLException e) { - return new HttpResultInfo(null,495,ResponseMessage.getNetworkCallFailureMessage(e), null,null); + } catch (SSLException | SocketTimeoutException | UnknownHostException e) { + logError(url, postData, e); + return exceptionToResponse(e,requestHeaders); } catch (IOException e) { logError(url, postData, e); throw new FailedRequestException(ResponseMessage.getNetworkCallFailureMessage(e), e, getResponseCode(conn)); @@ -298,15 +362,13 @@ public static HttpResultInfo getHeader(URL url, Map cookies, Map requestHeaders, int timeoutInSec ) throws FailedRequestException { + Map h= null; try { - HttpURLConnection conn= makeURLConnection(url,cookies,requestHeaders); - return getHeaderFromConnection(conn,timeoutInSec,MAX_REDIRECT,cookies,requestHeaders); - } catch (SSLException e) { - return new HttpResultInfo(null,495,ResponseMessage.getNetworkCallFailureMessage(e), null,null); - } catch (SocketTimeoutException e) { - return new HttpResultInfo(null,408,ResponseMessage.getNetworkCallFailureMessage(e), null,null); - } catch (UnknownHostException e) { - return new HttpResultInfo(null,404,ResponseMessage.getNetworkCallFailureMessage(e), null,null); + h= buildReqHeaders(url,requestHeaders,null); + HttpURLConnection conn= makeURLConnection(url,cookies,h); + return getHeaderFromConnection(conn,timeoutInSec,MAX_REDIRECT,cookies,h); + } catch (SSLException | SocketTimeoutException | UnknownHostException e) { + return exceptionToResponse(e,h); } catch (IOException e) { logError(url, null, e); throw new FailedRequestException(ResponseMessage.getNetworkCallFailureMessage(e), e, -1); @@ -321,12 +383,15 @@ private static HttpResultInfo getHeaderFromConnection(HttpURLConnection conn, try { if (timeoutInSec>0) { conn.setConnectTimeout(timeoutInSec * 1000); - conn.setReadTimeout(timeoutInSec * 1000); + conn.setReadTimeout(timeoutInSec * 1000 + 3000); } conn.setRequestMethod("HEAD"); + + conn.connect(); Set>> hSet = getResponseCode(conn)==-1 ? Collections.emptySet() : conn.getHeaderFields().entrySet(); HttpResultInfo result= new HttpResultInfo(null,getResponseCode(conn),null, conn.getContentType(), getSuggestedFileName(conn)); + result.setSendHeaders(requestHeaders); for (var e : hSet) { result.putAttribute(e.getKey()!=null ? e.getKey() : "",combineValues(e.getValue())); @@ -346,12 +411,9 @@ private static HttpResultInfo getHeaderFromConnection(HttpURLConnection conn, "Response Code: " + responseCode, responseCode); } return result; - } catch (SSLException e) { - return new HttpResultInfo(495, ResponseMessage.getNetworkCallFailureMessage(e)); - } catch (SocketTimeoutException e) { - return new HttpResultInfo(408, ResponseMessage.getNetworkCallFailureMessage(e)); - } catch (UnknownHostException e) { - return new HttpResultInfo(404, ResponseMessage.getNetworkCallFailureMessage(e)); + } catch (SSLException | SocketTimeoutException | UnknownHostException e) { + logError(conn.getURL(), null , e); + return exceptionToResponse(e,requestHeaders); } catch (IOException e) { logError(conn.getURL(), null, e); throw new FailedRequestException(ResponseMessage.getNetworkCallFailureMessage(e), e, getResponseCode(conn)); @@ -377,7 +439,7 @@ public static FileInfo getDataToFileUsingPost(URL url, Map postDa int timeoutInSec) throws FailedRequestException { try { Options ops= new Options(true, true, 0L, false, false, timeoutInSec, dl, false, false); - return getDataToFile(makeConnection(url, cookies, requestHeader), outfile, ops, postData,0); + return getDataToFile(makeURLConnection(url, cookies, requestHeader), outfile, ops, postData,0); } catch (IOException e) { logError(url, postData, e); throw new FailedRequestException(ResponseMessage.getNetworkCallFailureMessage(e), e); @@ -426,16 +488,8 @@ public static FileInfo getDataToFile(URL url, Map requestHeaders, Options ops) throws FailedRequestException { try { - Map h= new HashMap<>(); - if (requestHeaders!=null) h.putAll(requestHeaders); - if (ops.useCredentials) { - var inputs= new HttpServiceInput(url.toString()); - var credentials= inputs.getHeaders(); - if (credentials!=null && !credentials.isEmpty()) { - if (!credentials.keySet().stream().allMatch(h::containsKey)) h.putAll(credentials); - } - } - return getDataToFile(makeConnection(url, cookies, h), outfile, ops, null, ops.allowRedirect?MAX_REDIRECT:0); + var h= buildReqHeaders(url,requestHeaders,ops); + return getDataToFile(makeURLConnection(url, cookies, h), outfile, ops, null, ops.allowRedirect?MAX_REDIRECT:0); } catch (IOException e) { throw new FailedRequestException(ResponseMessage.getNetworkCallFailureMessage(e), e); } @@ -452,13 +506,15 @@ public static FileInfo getDataToFile(URL url, * @return an array of FileInfo objects * @throws FailedRequestException Any Network Error with simple message, cause will probably be IOException */ - public static FileInfo getDataToFile(URLConnection conn, + public static FileInfo getDataToFile(HttpURLConnection conn, File outfile, Options ops, Map postData, int redirectCnt) throws FailedRequestException { try { + StopWatch.Tracker tracker = new StopWatch.Tracker("Download", null); + tracker.starts(); String originalUrl= conn.getURL().toString(); FileInfo outFileData; Map> reqProp = conn.getRequestProperties(); @@ -469,18 +525,14 @@ public static FileInfo getDataToFile(URLConnection conn, conn.setConnectTimeout(ops.timeoutInSec * 1000);//Sets a specified timeout value, in milliseconds conn.setReadTimeout(ops.timeoutInSec * 1000); } - if (conn instanceof HttpURLConnection) { - pushPostData(conn, postData); - sendHeaders = conn.getRequestProperties(); - if (ops.onlyIfModified) { - outFileData = checkAlreadyDownloaded(conn, outfile); - if (outFileData != null) return outFileData; - if (getResponseCode(conn) == 408) { - throw new FailedRequestException("Timeout", "Timeout", 408); - } + pushPostData(conn, postData); + sendHeaders = conn.getRequestProperties(); + if (ops.onlyIfModified) { + outFileData = checkAlreadyDownloaded(conn, outfile); + if (outFileData != null) return outFileData; + if (getResponseCode(conn) == 408) { + throw new FailedRequestException("Timeout", "Timeout", 408); } - } else if (postData != null) { - doPostDataException(conn, postData); } } catch (IllegalStateException e) { // if I get this exception then the connection was already open and I can't set any more headers @@ -490,19 +542,19 @@ public static FileInfo getDataToFile(URLConnection conn, //---From here on the server should be responding //------ conn.connect(); - if (!ops.logErrorsOnly) logHeader(originalUrl, postData, conn, sendHeaders); validFileSize(conn, ops.maxFileSize); -// netCopy(makeAnyInStream(conn, ops.uncompress), makeOutStream(outfile), conn, ops.maxFileSize, ops.dl); -// Downloader.download(makeAnyInStream(conn, ops.uncompress), outfile, conn.getContentLength(), ops.maxFileSize, ops.dl); - netCopy(makeAnyInStream(conn, ops.uncompress), makeOutStream(outfile), conn, ops.maxFileSize, ops.dl); -// netCopy(makeAnyInStream(conn, ops.uncompress), outfile, conn.getContentLength(), ops); + Downloader.download(makeAnyInStream(conn, ops.uncompress), outfile, conn.getContentLength(), ops.maxFileSize, ops.dl); long elapse = System.currentTimeMillis() - start; int responseCode = getResponseCode(conn); outFileData = new FileInfo(outfile, getSuggestedFileName(conn), responseCode, ResponseMessage.getHttpResponseMessage(responseCode), conn.getContentType()); + Set>> hSet = responseCode==-1 ? Collections.emptySet() : conn.getHeaderFields().entrySet(); + for (Map.Entry> e : hSet) { + outFileData.putAttribute(e.getKey()!=null ? e.getKey() : "",combineValues(e.getValue())); + } if (conn.getContentEncoding() != null) outFileData.putAttribute("content-encoding", conn.getContentEncoding()); - if (!ops.logErrorsOnly || responseCode>=300) logDownload(outFileData, conn.getURL().toString(), elapse); +// if (responseCode>=300) logDownload(outFileData, conn.getURL().toString(), elapse); if (responseCode >= 300 && responseCode < 400) { if (redirectCnt > 0 && Arrays.asList(301,302,303,307,308).contains(responseCode)) { @@ -512,13 +564,13 @@ public static FileInfo getDataToFile(URLConnection conn, throw new FailedRequestException(ResponseMessage.getHttpResponseMessage(responseCode), "Response Code: " + responseCode, responseCode, outFileData); } + tracker.stops(); + double dlSeconds = tracker.getElapsedTime(StopWatch.Unit.SECONDS); + if (responseCode>300) logHeader(originalUrl, postData, conn, sendHeaders); + if (!ops.logErrorsOnly && responseCode<300) logSuccess(outFileData,outfile,conn.getURL(),dlSeconds,sendHeaders); return outFileData; - } catch (SSLException e) { - return new FileInfo(495, ResponseMessage.getNetworkCallFailureMessage(e)); - } catch (SocketTimeoutException e) { - return new FileInfo(408, ResponseMessage.getNetworkCallFailureMessage(e)); - } catch (UnknownHostException e) { - return new FileInfo(404, ResponseMessage.getNetworkCallFailureMessage(e)); + } catch (SSLException | SocketTimeoutException | UnknownHostException e) { + return exceptionToFileInfo(e); } catch (IOException e) { logError(conn.getURL(), null, e); throw new FailedRequestException(ResponseMessage.getNetworkCallFailureMessage(e),e, getResponseCode(conn)); @@ -526,15 +578,15 @@ public static FileInfo getDataToFile(URLConnection conn, } - private static FileInfo redirect(URLConnection conn, - File outfile, - Map> reqProp, - Options ops, - int redirectCnt) throws FailedRequestException, IOException { + private static FileInfo redirect(HttpURLConnection conn, + File outfile, + Map> reqProp, + Options ops, + int redirectCnt) throws FailedRequestException, IOException { var ignore= outfile.delete(); String urlStr= conn.getHeaderField("Location"); - HttpURLConnection newConn= (HttpURLConnection)makeConnection(makeURL(urlStr), null, null); + HttpURLConnection newConn= makeURLConnection(makeURL(urlStr), null, null); for(Map.Entry> entry : reqProp.entrySet()) { for(String s : entry.getValue()) newConn.setRequestProperty(entry.getKey(), s); } @@ -557,10 +609,10 @@ private static String postDataToString(Map postData) { return sBuff.toString(); } - private static void pushPostData(URLConnection conn, Map postData) throws IOException { - if (!(conn instanceof HttpURLConnection) || postData==null) return; + private static void pushPostData(HttpURLConnection conn, Map postData) throws IOException { + if (postData==null) return; String postStr= postDataToString(postData); - ((HttpURLConnection)conn).setRequestMethod("POST"); + conn.setRequestMethod("POST"); if (conn.getRequestProperty("Content-Type")==null) { conn.setRequestProperty( "Content-Type", "application/x-www-form-urlencoded" ); } @@ -574,7 +626,7 @@ private static void pushPostData(URLConnection conn, Map postData } - private static void validFileSize(URLConnection conn, long maxFileSize) throws FailedRequestException { + private static void validFileSize(HttpURLConnection conn, long maxFileSize) throws FailedRequestException { long contLen = conn.getContentLength(); if (maxFileSize > 0 && contLen > 0 && contLen > maxFileSize) { throw new FailedRequestException( @@ -596,7 +648,7 @@ private static void validFileSize(URLConnection conn, long maxFileSize) throws F * @return the FileInfo if the file exist and is not out of date, otherwise null * @throws IOException if something goes wrong */ - private static FileInfo checkAlreadyDownloaded(URLConnection urlConn, File outfile) throws IOException { + private static FileInfo checkAlreadyDownloaded(HttpURLConnection urlConn, File outfile) throws IOException { FileInfo retval = null; try { if (outfile != null && outfile.canRead() && outfile.length() > 0) { @@ -616,53 +668,6 @@ private static FileInfo checkAlreadyDownloaded(URLConnection urlConn, File outfi return retval; } - public static void netCopy(DataInputStream in, - OutputStream out, - long contentLength, - long maxSize, - DownloadListener dl) throws FailedRequestException, IOException { - try { - Downloader downloader = new Downloader(in, out, contentLength); - downloader.setMaxDownloadSize(maxSize); - downloader.setDownloadListener(dl); - downloader.download(); - } finally { - FileUtil.silentClose(in); - FileUtil.silentClose(out); - } - - } - - public static void netCopy(DataInputStream in, File outfile, long contentLength, Options ops) - throws FailedRequestException, IOException { - //todo -don't use! this one does not work yet - Downloader.doDownload(in, outfile, contentLength, ops.maxFileSize, ops.dl); - } - - public static void netCopy(DataInputStream in, ByteBuffer outBuff, long contentLength, Options ops) - throws FailedRequestException, IOException { - Downloader.doDownload(in, outBuff, contentLength, ops.maxFileSize, ops.dl); - } - - - public static void netCopy(DataInputStream in, - OutputStream out, - URLConnection conn, - long maxSize, - DownloadListener dl) throws FailedRequestException, IOException { - netCopy(in,out,conn.getContentLength(),maxSize,dl); - } - - - private static void logDownload(FileInfo retFile, String urlStr, long elapse) { - if (retFile == null) return; - String timeStr = (elapse>0) ? ", time: "+UTCTimeUtil.getHMSFromMills(elapse) : ""; - List outList = new ArrayList<>(2); - outList.add(String.format("Download Complete: %s : %d bytes%s", - retFile.getFile().getName(), retFile.getFile().length(), timeStr)); - outList.add(urlStr); - _log.info(outList.toArray(new String[0])); - } private static void logError(URL url, Map postData, Exception e) { List strList = new ArrayList<>(6); @@ -681,16 +686,13 @@ private static void logError(URL url, Map postData, Exception e) _log.warn(strList.toArray(new String[0])); } - public static void logHeader(URLConnection conn) { logHeader(null,null, conn, null); } - - public static void logHeader(String originalUrl, URLConnection conn) { logHeader(originalUrl,null, conn, null); } - - private static void logHeader(String originalUrl, Map postData, URLConnection conn, Map> sendHeaders) { + private static void logHeader(String originalUrl, Map postData, HttpURLConnection conn, Map> sendHeaders) { StringBuilder workBuff; try { String verb= ""; - if (conn instanceof HttpURLConnection) verb= ((HttpURLConnection)conn).getRequestMethod(); - Set>> hSet = getResponseCode(conn)==-1 ? null : conn.getHeaderFields().entrySet(); + verb= conn.getRequestMethod(); + Set>> hSet= Collections.emptySet(); + hSet = getResponseCode(conn)==-1 ? null : conn.getHeaderFields().entrySet(); List outStr= new ArrayList<>(40); String key; if (conn.getURL() != null) { @@ -732,12 +734,7 @@ private static void logHeader(String originalUrl, Map postData, if (postData != null) { outStr.add(StringUtils.pad(20,"Post Data ") + ": " + postDataToString(postData)); } - if (conn instanceof HttpURLConnection) { - outStr.add("----------Received Headers, response status code: " + getResponseCode(conn)); - } - else { - outStr.add("----------Received Headers"); - } + outStr.add("----------Received Headers, response status code: " + getResponseCode(conn)); if (hSet!=null) { List values; for (Map.Entry> e : hSet) { @@ -766,14 +763,66 @@ private static void logHeader(String originalUrl, Map postData, } - private static void logCompletedDownload(URL url, long size) { - _log.info(String.format("Download Complete- %d bytes", size), url != null ? url.toString() : null); - } - - private static void doPostDataException(URLConnection conn, Map postData) throws FailedRequestException { - FailedRequestException fe = new FailedRequestException("Can only do post with http(s): " + conn.getURL().toString()); - logError(conn.getURL(), postData, fe); - throw fe; + private static void logSuccess(FileInfo fileInfo, File outfile, URL url, double dSeconds, Map> sendHeaders) { + String formatedSize= FileUtil.getSizeAsString(fileInfo.getSizeInBytes()); + String lastMod= fileInfo.getAttribute("Last-Modified")!=null ? ", Last-Modified: " +fileInfo.getAttribute("Last-Modified") : ""; + _log.info( + String.format( "DOWNLOAD (%.1f sec, %s, response: %d): Content-Type: %s, Content-Length: %s%s", + dSeconds, formatedSize, fileInfo.getResponseCode(), fileInfo.getContentType(), fileInfo.getSizeInBytes(), lastMod), + "url: "+ url.toString(), + "file: "+ outfile.toPath(), + "send headers: "+sendHeadersToCompactStr(sendHeaders), + "more response headers: "+otherHeadersToStr(fileInfo) + ); + } + + private static void logSuccess(HttpResultInfo r, URL url, double dSeconds, Map> sendHeaders, Map postData) { + String formatedSize= FileUtil.getSizeAsString(r.getContentLength()); + String lastMod= r.getAttribute("Last-Modified")!=null ? ", Last-Modified: " +r.getAttribute("Last-Modified") : ""; + String postStr= postData.isEmpty() ? "" : "\n Post Data :" + postDataToString(postData); + String send= "send headers: "+sendHeadersToCompactStr(sendHeaders) + postStr; + + _log.info( + String.format( "DOWNLOAD to memory (%.1f sec, %s, response: %d): Content-Type: %s, Content-Length: %s%s", + dSeconds, formatedSize, r.getResponseCode(), r.getContentType(), r.getContentLength(), lastMod), + "url: "+ url.toString(), + send, + "more response headers: "+otherHeadersToStr(r) + ); + } + + + private static String otherHeadersToStr(FileInfo r) { + StringBuilder out = new StringBuilder(); + int cnt = 0; + for (var a : r.getAttributeMap().entrySet()) { + var k = a.getKey(); + ; + if (!r.isReservedKey(k) && !k.equalsIgnoreCase("") && + !k.equalsIgnoreCase("content-type") && !k.equalsIgnoreCase("content-length") && + !k.equalsIgnoreCase("Last-Modified")) { + if (cnt > 0) out.append(", "); + out.append(String.format("%s: %s", k, a.getValue())); + cnt++; + } + } + return out.toString(); + } + + private static String otherHeadersToStr(HttpResultInfo r) { + StringBuilder out= new StringBuilder(); + int cnt=0; + for(var a : r.getAttributes().entrySet()) { + var k = a.getKey();; + if (r.isReservedKey(k) && !k.equalsIgnoreCase("") && + !k.equalsIgnoreCase("content-type") && !k.equalsIgnoreCase("content-length") && + !k.equalsIgnoreCase("Last-Modified") ) { + if (cnt>0) out.append(", "); + out.append(String.format("%s: %s", k, a.getValue())); + cnt++; + } + } + return out.toString(); } //====================================================================== @@ -789,14 +838,14 @@ private static DataInputStream makeGZipInStream(URLConnection conn) throws IOExc return new DataInputStream(new GZIPInputStream(conn.getInputStream(), BUFFER_SIZE)); } - private static DataInputStream makeAnyInStream(URLConnection conn, boolean uncompress) throws IOException { + private static DataInputStream makeAnyInStream(HttpURLConnection conn, boolean uncompress) throws IOException { String contentType = conn.getContentType(); if (conn.getContentEncoding() != null) return makeEncodedInStream(conn); else if (uncompress && contentType != null && contentType.toLowerCase().endsWith("gzip")) return makeGZipInStream(conn); else return makeDataInStream(conn); } - private static DataInputStream makeEncodedInStream(URLConnection conn) throws IOException { + private static DataInputStream makeEncodedInStream(HttpURLConnection conn) throws IOException { String encodeType = conn.getContentEncoding(); if (encodeType == null) return null; if (encodeType.toLowerCase().endsWith("gzip")) { @@ -809,10 +858,8 @@ private static DataInputStream makeEncodedInStream(URLConnection conn) throws IO } } - private static DataInputStream makeDataInStream(URLConnection conn) throws IOException { - if (conn instanceof HttpURLConnection && getResponseCode(conn)==-1) { -// throw new IOException("Http Response Code is -1, invalid http protocol, " + -// "probably no status line in response headers"); + private static DataInputStream makeDataInStream(HttpURLConnection conn) throws IOException { + if (getResponseCode(conn)==-1) { _log.warn("Http Response Code is -1, invalid http protocol, " + "probably no status line in response headers- trying anyway"); return new DataInputStream(makeInStream(conn)); @@ -822,8 +869,7 @@ private static DataInputStream makeDataInStream(URLConnection conn) throws IOExc return new DataInputStream(makeInStream(conn)); } catch (IOException e) { - if (!(conn instanceof HttpURLConnection)) throw e; - return new DataInputStream(makeErrStream((HttpURLConnection) conn)); + return new DataInputStream(makeErrStream(conn)); } } } @@ -907,6 +953,7 @@ public static Options modifiedAndTimeoutOp(boolean onlyIfModified, int timeoutIn public DownloadListener dl() { return dl; } public boolean onlyIfModified() { return onlyIfModified; } public boolean expectStaticFile() { return expectStaticFile; } + public long maxFileSize() { return maxFileSize; } } } \ No newline at end of file diff --git a/src/firefly/java/edu/caltech/ipac/util/download/UriRef.java b/src/firefly/java/edu/caltech/ipac/util/download/UriRef.java index 121d4aefe5..9f7290634d 100644 --- a/src/firefly/java/edu/caltech/ipac/util/download/UriRef.java +++ b/src/firefly/java/edu/caltech/ipac/util/download/UriRef.java @@ -1,8 +1,8 @@ package edu.caltech.ipac.util.download; -import java.net.MalformedURLException; +import edu.caltech.ipac.firefly.core.Util; + import java.net.URI; -import java.net.URISyntaxException; import java.net.URL; import java.util.Objects; @@ -93,11 +93,7 @@ static private UriRef makeString(String uriStr) { private static URL makeURLFromStr(String urlStr) { if (urlStr == null) return null; - try { - return new URI(urlStr.trim()).toURL(); - } catch (URISyntaxException | MalformedURLException ignore) { - return null; - } + return Util.Try.it(() -> new URI(urlStr.trim()).toURL()).getOrElse((URL)null); } } diff --git a/src/firefly/java/edu/caltech/ipac/visualize/net/SloanDssImageGetter.java b/src/firefly/java/edu/caltech/ipac/visualize/net/SloanDssImageGetter.java index 0f114c7d03..f19f9bcbf1 100644 --- a/src/firefly/java/edu/caltech/ipac/visualize/net/SloanDssImageGetter.java +++ b/src/firefly/java/edu/caltech/ipac/visualize/net/SloanDssImageGetter.java @@ -8,8 +8,8 @@ import edu.caltech.ipac.table.io.VoTableReader; import edu.caltech.ipac.util.AppProperties; import edu.caltech.ipac.util.FileUtil; -import edu.caltech.ipac.util.download.FileCacheHelper; import edu.caltech.ipac.util.download.FailedRequestException; +import edu.caltech.ipac.util.download.FileCacheHelper; import edu.caltech.ipac.util.download.URLDownload; import edu.caltech.ipac.visualize.plot.WorldPt; @@ -17,7 +17,6 @@ import java.io.IOException; import java.net.SocketTimeoutException; import java.net.URI; -import java.net.URISyntaxException; import java.net.URL; public class SloanDssImageGetter { @@ -45,7 +44,7 @@ public static FileInfo get(SloanDssImageParams params, File outFile) throws Fail } catch (SocketTimeoutException timeOutE) { if (outFile.canWrite()) outFile.delete(); throw timeOutE; - } catch (URISyntaxException me) { + } catch (Exception me) { throw new FailedRequestException( "Invalid URL", "Details in exception", me ); } } diff --git a/src/firefly/js/FFEntryPoint.js b/src/firefly/js/FFEntryPoint.js index 68cb3ffc57..e5b5acfd8d 100644 --- a/src/firefly/js/FFEntryPoint.js +++ b/src/firefly/js/FFEntryPoint.js @@ -51,6 +51,7 @@ const defOptions = { sources: getDefaultMOCList(), label: 'Featured MOC ' }, + adhocMocIncludeAdditionSources: 'irsa', mergedListPriority: 'Irsa' }, coverage : { } diff --git a/src/firefly/js/data/ServerParams.js b/src/firefly/js/data/ServerParams.js index fb4793fe50..7f2e454064 100644 --- a/src/firefly/js/data/ServerParams.js +++ b/src/firefly/js/data/ServerParams.js @@ -95,6 +95,7 @@ export const ServerParams = { HIPS_LIST_SOURCE_NAME: 'hipsListSourceName', ENSURE_SOURCE: 'ensureSource', ADHOC_SOURCE : 'adhocSource', + ADHOC_MOC_INCLUDE: 'AdhocMOCInclude', HIPS_DATATYPES: 'hipsDataTypes', HIPS_DEFSOURCES: 'defHipsSources', HIPS_TABLE_TYPE: 'hipsTableType', diff --git a/src/firefly/js/drawingLayers/ExtractLineTool.js b/src/firefly/js/drawingLayers/ExtractLineTool.js index 4ff542296d..70e45f5f38 100644 --- a/src/firefly/js/drawingLayers/ExtractLineTool.js +++ b/src/firefly/js/drawingLayers/ExtractLineTool.js @@ -247,7 +247,7 @@ function getMode(plot) { function start(drawLayer, action) { const {imagePt,plotId,shiftDown}= action.payload; if (drawLayer.selectionType===LINE_SELECTION || drawLayer.selectionType===COLUMN_SELECTION) { - const newDl= {...drawLayer, ...setupSelect(imagePt,drawLayer)}; + const newDl= {...drawLayer, ...setupSelect(plotId,imagePt,drawLayer)}; return drag(newDl, action); } const plot= primePlot(visRoot(),plotId); @@ -256,7 +256,7 @@ function start(drawLayer, action) { const cc= CsysConverter.make(plot); let retObj= {}; if (mode==='select' || shiftDown) { - retObj= setupSelect(imagePt,drawLayer); + retObj= setupSelect(plotId,imagePt,drawLayer); } else if (mode==='edit') { const ptAry= getPtAry(plot); @@ -281,7 +281,7 @@ function start(drawLayer, action) { return Object.assign(retObj, makeBaseReturnObj(retObj.firstPt, retObj.currentPt, drawAry)); } else { - retObj= setupSelect(imagePt,drawLayer) ; + retObj= setupSelect(plotId,imagePt,drawLayer) ; } } if (retObj.startSelectCnt>2) { @@ -332,9 +332,12 @@ function end(action) { } -function setupSelect(imagePt, drawLayer) { - return {firstPt: imagePt, currentPt: imagePt, activePt:undefined, - moveHead: true, helpWarning:false, startSelectCnt:drawLayer.startSelectCnt+1}; +function setupSelect(plotId, imagePt, drawLayer) { + return { + firstPt: imagePt, currentPt: imagePt, activePt:undefined, selectPlotId: plotId, + moveHead: true, helpWarning:false, + startSelectCnt: drawLayer.selectPlotId===plotId ? drawLayer.startSelectCnt+1 : 0 + }; } function findClosestPtIdx(ptAry, pt) { diff --git a/src/firefly/js/tables/TableUtil.js b/src/firefly/js/tables/TableUtil.js index b8ef2c1fac..8fcd3ad3f6 100644 --- a/src/firefly/js/tables/TableUtil.js +++ b/src/firefly/js/tables/TableUtil.js @@ -1020,8 +1020,11 @@ export function isSubHighlightRow(tableOrId, rowIdx, hlRowIdx) { const highlightedRow= hlRowIdx ?? tableModel.highlightedRow; const makeCellKey= (row) => colNameAry.map((cname) => getCellValue(tableModel, row, cname)).join('|'); - - return makeCellKey(highlightedRow) === makeCellKey(rowIdx); + + const hKey= makeCellKey(highlightedRow); + const rKey= makeCellKey(rowIdx); + if (!hKey && !rKey) return false; + return hKey===rKey; } export function hasSubHighlightRows(tableOrId) { diff --git a/src/firefly/js/ui/HiPSImageSelect.jsx b/src/firefly/js/ui/HiPSImageSelect.jsx index d77ed6f1cd..5444e1e5de 100644 --- a/src/firefly/js/ui/HiPSImageSelect.jsx +++ b/src/firefly/js/ui/HiPSImageSelect.jsx @@ -114,6 +114,7 @@ export function showHiPSSurveysPopup(pv, moc= false) { ivoid: getIvoaId(), title: getTitle(), hipsUrl: rootUrl, + mocFile: rootUrl.toLowerCase().endsWith('fits') ? '' : undefined, plot: primePlot(pv), visible: true } ) : @@ -229,10 +230,12 @@ function HiPSSurveyTable({groupKey, variant, extraHiPSListName , moc=false}) { useEffect ( () => { if (!getTblById(activeHipsTblId)) { const mocSources= getAppOptions().hips?.adhocMocSource?.sources ?? getDefaultMOCList(); + const adhocMocIncludeAdditionSources= getAppOptions().hips?.adhocMocIncludeAdditionSources ?? ''; const req = makeHiPSRequest( moc===true? 'moc' : 'hips', finalSources, moc ? mocSources : undefined, + moc ? adhocMocIncludeAdditionSources : undefined, activeHipsTblId); req && dispatchTableFetch(req); diff --git a/src/firefly/js/ui/TargetPanel.jsx b/src/firefly/js/ui/TargetPanel.jsx index 2b2bb32f8d..e9fb7c9fa1 100644 --- a/src/firefly/js/ui/TargetPanel.jsx +++ b/src/firefly/js/ui/TargetPanel.jsx @@ -247,7 +247,8 @@ function makePayloadAndUpdateActive(displayValue, parseResults, resolvePromise, function replaceValue(v,defaultToActiveTarget, computedState) { if (!defaultToActiveTarget) return v; if ((computedState.displayValue || computedState.message) && !computedState.valid) return ''; - if (v?.trim() && v===computedState.value && computedState.valid && isValidPoint(parseWorldPt(v))) return v; + if (isString(v)) v= v.trim(); + if (v && v===computedState.value && computedState.valid && isValidPoint(parseWorldPt(v))) return v; return getActiveTarget()?.worldPt?.toString() ?? v; } diff --git a/src/firefly/js/visualize/HiPSListUtil.js b/src/firefly/js/visualize/HiPSListUtil.js index 66cb999f83..60af079ec5 100644 --- a/src/firefly/js/visualize/HiPSListUtil.js +++ b/src/firefly/js/visualize/HiPSListUtil.js @@ -79,7 +79,9 @@ export function getHiPSMergePriority() { } -export function makeHiPSRequest(tableType, sources=getHiPSSources(), mocSources, tbl_id, +export function makeHiPSRequest(tableType, + sources=getHiPSSources(), mocSources, + adhocMocIncludeAdditionSources, tbl_id, types=HiPSData) { const sourceMergePriority = getHiPSMergePriority(); const sp = sourceMergePriority?.join?.(',') || sourceMergePriority; @@ -91,6 +93,7 @@ export function makeHiPSRequest(tableType, sources=getHiPSSources(), mocSources, [ServerParams.HIPS_MERGE_PRIORITY]: sp, }; if (mocSources) params[ServerParams.ADHOC_SOURCE]= mocSources.join(','); + if (adhocMocIncludeAdditionSources) params[ServerParams.ADHOC_MOC_INCLUDE]= adhocMocIncludeAdditionSources; return makeTblRequest('HiPSSearch', 'HiPS Maps', params, { diff --git a/src/firefly/js/visualize/PlotViewUtil.js b/src/firefly/js/visualize/PlotViewUtil.js index da2f113713..d9ce881cf7 100644 --- a/src/firefly/js/visualize/PlotViewUtil.js +++ b/src/firefly/js/visualize/PlotViewUtil.js @@ -990,7 +990,7 @@ export const getPtWavelength= (plot, pt, cubeIdx, band) => getPtSpectralCoords(p export function getPtSpectralCoords(plot, pt, cubeIdx, band= Band.NO_BAND) { if (!plot?.wlDataAry?.[band.value] || !hasWLInfo(plot)) return [0,0]; const ipt= CCUtil.getImageCoords(plot,pt); - if (!ipt) return [0,0]; + if (!ipt && plot?.wlData?.hasPixelLevelCoordInfo) return [0,0]; return getWavelength(ipt, cubeIdx, plot.wlDataAry[band.value]) ?? [0,0]; } diff --git a/src/firefly/js/visualize/task/PlotHipsTask.js b/src/firefly/js/visualize/task/PlotHipsTask.js index 37aea9f36f..83157b2a39 100644 --- a/src/firefly/js/visualize/task/PlotHipsTask.js +++ b/src/firefly/js/visualize/task/PlotHipsTask.js @@ -299,7 +299,9 @@ export function createHiPSMocLayerFromPreloadedTable({tbl_id,title, fitsPath, mo export async function createHiPSMocLayer({ivoid, title, hipsUrl, plot, visible=false, mocFile = 'Moc.fits', color, mocGroupDefColorId}) { - const mocUrl = (mocFile && isString(mocFile)) ? hipsUrl.endsWith('/') ? hipsUrl + mocFile : hipsUrl+'/'+mocFile : hipsUrl; + const mocUrl = (mocFile && isString(mocFile)) + ? hipsUrl.endsWith('/') ? hipsUrl + mocFile : hipsUrl+'/'+mocFile + : hipsUrl; const tbl_id = makeMocTableId(ivoid); const dls = getDrawLayersByType(getDlAry(), HiPSMOC.TYPE_ID); diff --git a/src/firefly/test/edu/caltech/ipac/util/download/UriRefTest.java b/src/firefly/test/edu/caltech/ipac/util/download/UriRefTest.java index 9435d0eab6..29f9aa267d 100644 --- a/src/firefly/test/edu/caltech/ipac/util/download/UriRefTest.java +++ b/src/firefly/test/edu/caltech/ipac/util/download/UriRefTest.java @@ -39,7 +39,7 @@ public void testSimpleCases(){ var srRefwithQ= UriRef.make("https://s3.us-west-2.amazonaws.com/amzn-s3-demo-bucket1/puppy.jpg?a=2"); assertEquals(UriRef.ResourceType.OnPrimUrl, srRefwithQ.getType()); - } catch (URISyntaxException | MalformedURLException e) { + } catch (URISyntaxException | MalformedURLException | IllegalArgumentException e) { throw new RuntimeException(e); }