Caltech-IPAC · robyww · Oct 17, 2025 · Oct 14, 2025
diff --git a/config/app.config b/config/app.config
@@ -40,6 +40,7 @@ GatorHost     = "https://irsa.ipac.caltech.edu"
 irsa.base.url           = "https://irsa.ipac.caltech.edu"
 irsa.gator.dd.hostname  = "https://irsa.ipac.caltech.edu"
 irsa.gator.hostname     = "https://irsa.ipac.caltech.edu"
+irsa.moc.list.path     = "/irsadata/hips/irsa_moc.list"
 //MOST (moving object - precovery - search) host
 most.host = "https://irsasearchops.ipac.caltech.edu/cgi-bin/MOST/nph-most"
 // IRSA Periodogram API
@@ -63,15 +64,18 @@ __$tap.maxrec.hardlimit = 10000000
 environments {
     local {
         BuildType = "Development"
+        irsa.moc.list.path= "https://bacchus1.ipac.caltech.edu/data/hips/irsa_moc.list"
         ehcache.multicast.port = 4015
         visualize.fits.Security= false
         ehcache.multicast.ttl = 0
     }
     dev {
         BuildType = "Development"
+        irsa.moc.list.path= "https://bacchus1.ipac.caltech.edu/data/hips/irsa_moc.list"
         ehcache.multicast.port = "5015"
         lsst.dataAccess.uri = "http://localhost:8661/db/v0/query?"
         lsst.dataAccess.db = "smm_bremerton"
+        irsa.base.url= "https://irsadev.ipac.caltech.edu"
     }
     test {
         BuildType = "Beta"

diff --git a/src/firefly/config/app.prop b/src/firefly/config/app.prop
@@ -2,8 +2,10 @@
 # this path does not need to include the upload or cache directory
 [email protected]@:@alerts.dir@:@work.directory@/@app-name@:@wise.filesystem_basepath@:@planck.filesystem_basepath@:@ptf.filesystem_basepath@:@ztf.filesystem_basepath@:@lcogt.filesystem_basepath@:@lsst.filesystem_basepath@
 
+[email protected]@
 [email protected]@
 [email protected]@
+[email protected]@
 [email protected]@
 [email protected]@
 [email protected]@

diff --git a/src/firefly/java/edu/caltech/ipac/firefly/data/FileInfo.java b/src/firefly/java/edu/caltech/ipac/firefly/data/FileInfo.java
@@ -15,7 +15,12 @@
 import java.util.List;
 import java.util.Map;
 
-import static edu.caltech.ipac.firefly.data.HttpResultInfo.*;
+import static edu.caltech.ipac.firefly.data.HttpResultInfo.CONTENT_TYPE;
+import static edu.caltech.ipac.firefly.data.HttpResultInfo.EXTERNAL_NAME;
+import static edu.caltech.ipac.firefly.data.HttpResultInfo.RESPONSE_CODE;
+import static edu.caltech.ipac.firefly.data.HttpResultInfo.RESPONSE_CODE_MSG;
+import static edu.caltech.ipac.firefly.data.HttpResultInfo.SIZE_IN_BYTES;
+import static edu.caltech.ipac.firefly.data.HttpResultInfo.SUFFIX;
 
 
 public class FileInfo implements HasAccessInfo, Serializable, CacheKey {
@@ -103,6 +108,7 @@ public static FileInfo blankFilePlaceholder() {
 
     public void putAttribute(String key, String value) { attributes.put(key,value); }
     public String getAttribute(String key) {
+        if (attributes.containsKey(key)) return attributes.get(key);
         var matchingKey= attributes.keySet().stream().filter(k -> k.equalsIgnoreCase(key)).findFirst().orElse(null);
         if (matchingKey!=null) return attributes.get(matchingKey);
         return null;
@@ -173,9 +179,7 @@ public void setSuffix(String filename) {
 
     @Override
     public boolean equals(Object o) {
-        if (o instanceof FileInfo) {
-            FileInfo fi = (FileInfo) o;
-            // external name is unique
+        if (o instanceof FileInfo fi) {
             return getInternalFilename().equals(fi.getInternalFilename());
         } else {
             return false;
@@ -205,6 +209,9 @@ public boolean hasFileNameResolver() {
 
     public String resolveFileName(String name) { return resolver.getResolvedName(name); }
 
+    public long getContentLength() { return StringUtils.getLong(getAttribute("Content-Length"),0); }
+    public String getContentEncoding() { return getAttribute("Content-Encoding"); }
+    public String getContentDisposition() { return getAttribute("Content-Disposition"); }
 
     public FileInfo copy() {
         FileInfo fi= new FileInfo();
@@ -222,5 +229,19 @@ public FileInfo copy() {
     public interface FileNameResolver {
         String getResolvedName(String input);
     }
+
+    public boolean isReservedKey(String k) {
+        return (k.equals(RESPONSE_CODE) ||
+                k.equals(RESPONSE_CODE_MSG) ||
+                k.equals(EXTERNAL_NAME) ||
+                k.equals(CONTENT_TYPE) ||
+                k.equals(SUFFIX) ||
+                k.equals(INTERNAL_NAME) ||
+                k.equals(FILE_DOWNLOADED) ||
+                k.equals(DESC) ||
+                k.equals(BLANK) ||
+                k.equals(SIZE_IN_BYTES) ||
+                k.equals(HAS_ACCESS));
+    }
 }
 
diff --git a/src/firefly/java/edu/caltech/ipac/firefly/data/HttpResultInfo.java b/src/firefly/java/edu/caltech/ipac/firefly/data/HttpResultInfo.java
@@ -9,6 +9,7 @@
 import edu.caltech.ipac.util.StringUtils;
 import edu.caltech.ipac.util.download.ResponseMessage;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -26,6 +27,7 @@ public class HttpResultInfo {
     public static final String SUFFIX= "suffix";
 
     private final byte[] result;
+    private Map<String, String> sendHeaders= null;
 
     private final Map<String,String> attributes= new HashMap<>(7);
 
@@ -40,20 +42,25 @@ public HttpResultInfo(byte[] result,int responseCode, String msg, String content
         putAttribute(RESPONSE_CODE_MSG, msg!=null ? msg : ResponseMessage.getHttpResponseMessage(responseCode));
         putAttribute(EXTERNAL_NAME,suggestedFileName);
         if (contentType!=null) putAttribute(CONTENT_TYPE, contentType);
-        if (result!=null) putAttribute(SIZE_IN_BYTES, result.length+"");
     }
 
     public void putAttribute(String key, String value) { attributes.put(key,value); }
     public String getAttribute(String key) {
+        if (attributes.containsKey(key)) return attributes.get(key);
         var matchingKey= attributes.keySet().stream().filter(k -> k.equalsIgnoreCase(key)).findFirst().orElse(null);
         if (matchingKey!=null) return attributes.get(matchingKey);
         return null;
     }
 
+    public void setSendHeaders(Map<String,String> sendHeaders) { this.sendHeaders=sendHeaders; }
+    public Map<String, String> getSendHeaders() {
+        return this.sendHeaders!=null ? this.sendHeaders : Collections.emptyMap();
+    }
+    public Map<String,String> getAttributes() { return attributes; }
+
     public int getResponseCode() { return StringUtils.getInt(attributes.get(RESPONSE_CODE), 200); }
     public boolean isOK() { return getResponseCode()==200; }
     public String getResponseCodeMsg() { return attributes.get(RESPONSE_CODE_MSG); }
-    public long getSizeInBytes() { return StringUtils.getInt(getAttribute(SIZE_IN_BYTES),0); }
     public String getContentType() { return getAttribute(CONTENT_TYPE); }
     public String getExternalName() { return getAttribute(EXTERNAL_NAME); }
     public String getLocation() { return getAttribute(LOCATION); }
@@ -66,4 +73,14 @@ public String getAttribute(String key) {
     public long getContentLength() { return StringUtils.getLong(getAttribute("Content-Length"),0); }
     public String getContentEncoding() { return getAttribute("Content-Encoding"); }
     public String getContentDisposition() { return getAttribute("Content-Disposition"); }
+
+    public boolean isReservedKey(String k) {
+        return (k.equals(RESPONSE_CODE) ||
+                k.equals(RESPONSE_CODE_MSG) ||
+                k.equals(REDIRECTED) ||
+                k.equals(LOCATION) ||
+                k.equals(EXTERNAL_NAME) ||
+                k.equals(CONTENT_TYPE)||
+                k.equals(SUFFIX));
+    }
 }
diff --git a/src/firefly/java/edu/caltech/ipac/firefly/data/ServerParams.java b/src/firefly/java/edu/caltech/ipac/firefly/data/ServerParams.java
@@ -104,6 +104,7 @@ public class ServerParams {
     public static final String HIPS_LIST_SOURCE_NAME= "hipsListSourceName";
     public static final String ENSURE_SOURCE= "ensureSource";
     public static final String ADHOC_SOURCE = "adhocSource";
+    public static final String ADHOC_MOC_INCLUDE= "AdhocMOCInclude";
     public static final String HIPS_DATATYPES = "hipsDataTypes";
     public static final String HIPS_MERGE_PRIORITY = "mergedListPriority";
     public static final String HIPS_TABLE_TYPE= "hipsTableType";

diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/catquery/CatSummaryQuery.java b/src/firefly/java/edu/caltech/ipac/firefly/server/catquery/CatSummaryQuery.java
@@ -3,29 +3,26 @@
  */
 package edu.caltech.ipac.firefly.server.catquery;
 
-import edu.caltech.ipac.table.io.IpacTableWriter;
 import edu.caltech.ipac.firefly.data.TableServerRequest;
 import edu.caltech.ipac.firefly.server.query.DataAccessException;
 import edu.caltech.ipac.firefly.server.query.IpacTablePartProcessor;
 import edu.caltech.ipac.firefly.server.query.ParamDoc;
 import edu.caltech.ipac.firefly.server.query.SearchManager;
 import edu.caltech.ipac.firefly.server.query.SearchProcessorImpl;
-import edu.caltech.ipac.table.DataGroupPart;
-import edu.caltech.ipac.util.AppProperties;
 import edu.caltech.ipac.table.DataGroup;
+import edu.caltech.ipac.table.DataGroupPart;
 import edu.caltech.ipac.table.DataObject;
 import edu.caltech.ipac.table.DataType;
+import edu.caltech.ipac.table.io.IpacTableWriter;
+import edu.caltech.ipac.util.AppProperties;
 import edu.caltech.ipac.util.StringUtils;
 import edu.caltech.ipac.util.download.FailedRequestException;
 import edu.caltech.ipac.util.download.URLDownload;
 
-import java.io.BufferedInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.net.URL;
-import java.net.URLConnection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.regex.Matcher;
@@ -70,14 +67,8 @@ protected File loadDataFile(TableServerRequest req) throws IOException, DataAcce
 
         try {
 
-            URLConnection conn = URLDownload.makeConnection(url, null, null);
-            conn.addRequestProperty("accept", "*/*");
-            ByteArrayOutputStream baos = new  ByteArrayOutputStream();
-            DataInputStream in= new DataInputStream(new BufferedInputStream(
-                        conn.getInputStream()));
-            URLDownload.logHeader(conn);
-            URLDownload.netCopy(in, baos, conn, 0, null);
-            String jsonContent = baos.toString();
+            var headers= Collections.singletonMap("accept", "*.*");
+            String jsonContent  = URLDownload.getDataFromURL(url,null,headers).getResultAsString();
             HashMap<String, Integer> keyValues = parse(jsonContent);
             DataType setType = new DataType("set", String.class);
             setType.setWidth(15);

diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/db/DuckDbReadable.java b/src/firefly/java/edu/caltech/ipac/firefly/server/db/DuckDbReadable.java
@@ -215,8 +215,7 @@ public void export(TableServerRequest treq, OutputStream out) throws DataAccessE
                 File tmp = File.createTempFile("duck-", "."+getName(), QueryUtil.getTempDir(treq));
                 execUpdate(exportSql.formatted(selectSql, tmp.getAbsolutePath(), voTable.toString(StandardCharsets.UTF_8).replaceAll("'", "''")));
                 if (tmp.canRead()) {
-                    Downloader downloader = new Downloader(new DataInputStream(new FileInputStream(tmp)), out, tmp.length());
-                    downloader.download();
+                    Downloader.download(new DataInputStream(new FileInputStream(tmp)), out);
                     tmp.delete();
                 }
             } catch (Exception e) {
@@ -244,8 +243,7 @@ public void export(TableServerRequest treq, OutputStream out) throws DataAccessE
                 File tmp = File.createTempFile("export-", "."+getName(), ServerContext.getTempWorkDir());
                 execUpdate("COPY (%s) TO '%s' (HEADER, DELIMITER '%c')".formatted(sql, tmp.getAbsolutePath(), getDelimiter()));
                 if (tmp.canRead()) {
-                    Downloader downloader = new Downloader(new DataInputStream(new FileInputStream(tmp)), out, tmp.length());
-                    downloader.download();
+                    Downloader.download(new DataInputStream(new FileInputStream(tmp)), out);
                     tmp.delete();
                 }
             } catch (Exception e) {

diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java b/src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java
@@ -193,7 +193,6 @@ public static File resolveFileFromSource(String source, boolean checkForUpdates,
                     FileUtil.writeStringToFile(outFile, "workaround");
                     outFile.setLastModified(res.lastModified());
                     ops.setOnlyIfModified(false);
-                    ops.setExpectStaticFile(true);
                     FileInfo finfo = RetrieveUtil.download(uri, outFile, inputs.getCookies(), inputs.getHeaders(), ops);
                     if (finfo.getResponseCode() != HttpURLConnection.HTTP_NOT_MODIFIED) {
                         checkForFailures(finfo);

diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterList.java b/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterList.java
@@ -39,13 +39,15 @@
  * @author Cindy Wang
  */
 @SearchProcessorImpl(id = "HiPSSearch", params =
-        {@ParamDoc(name = ServerParams.HIPS_DATATYPES, desc = "types of HiPS data to search"),
-         @ParamDoc(name = ServerParams.HIPS_SOURCES, desc = "HiPS sources"),
-         @ParamDoc(name = HIPS_LIST_SOURCE, desc = "HiPS list source url"),
-         @ParamDoc(name = ENSURE_SOURCE, desc = "if true then only ensure the source and an empty table is return"),
-         @ParamDoc(name = ServerParams.ADHOC_SOURCE, desc = "a comma list of IVOA ids to make the source, the HIPS_SOURCE parameter much include adhoc"),
-         @ParamDoc(name = ServerParams.SORT_ORDER, desc = "HiPS order, source based"),
-         @ParamDoc(name = ServerParams.HIPS_TABLE_TYPE, desc = "hips or moc, default hips")
+        {
+                @ParamDoc(name = ServerParams.HIPS_DATATYPES, desc = "types of HiPS data to search"),
+                @ParamDoc(name = ServerParams.HIPS_SOURCES, desc = "HiPS sources"),
+                @ParamDoc(name = HIPS_LIST_SOURCE, desc = "HiPS list source url"),
+                @ParamDoc(name = ENSURE_SOURCE, desc = "if true then only ensure the source and an empty table is return"),
+                @ParamDoc(name = ServerParams.ADHOC_SOURCE, desc = "a comma list of IVOA ids to make the source, the HIPS_SOURCE parameter much include adhoc"),
+                @ParamDoc(name = ServerParams.ADHOC_MOC_INCLUDE, desc = "always use the additional mocs from these sources in adhoc"),
+                @ParamDoc(name = ServerParams.SORT_ORDER, desc = "HiPS order, source based"),
+                @ParamDoc(name = ServerParams.HIPS_TABLE_TYPE, desc = "hips or moc, default hips")
         })
 public class HiPSMasterList extends EmbeddedDbProcessor {
     public final static String INFO_ICON_STUB = "<img data-src='info'/>";
@@ -113,10 +115,15 @@ public DataGroup fetchDataGroup(TableServerRequest request) throws DataAccessExc
         String[] prioritySources = (hipsMergePriority != null) ? hipsMergePriority.split(",") : null;
         List<HiPSMasterListEntry> allSourceData = new ArrayList<>();
         List<String> adhocSources= Collections.emptyList();
+        List<String> adhocMocInclude= Collections.emptyList();
+        boolean usingAdhoc= false;
 
         if (workingSources!=null && Arrays.asList(workingSources).contains("adhoc") && adhocSrcParam!=null) {
-            adhocSources= Arrays.asList(adhocSrcParam.split(","));
+            adhocSources= new ArrayList<>(Arrays.asList(adhocSrcParam.split(",")));
             workingSources= new String[] {ServerParams.ALL};
+            usingAdhoc= true;
+            var v= request.getParam(ServerParams.ADHOC_MOC_INCLUDE);
+            if (v!=null) adhocMocInclude= Arrays.asList(v.split(","));
         }
 
         if (workingSources == null || workingSources.length == 0 ||
@@ -146,7 +153,21 @@ public DataGroup fetchDataGroup(TableServerRequest request) throws DataAccessExc
                     if (hipsL != null) {
                         allSourceData.addAll(hipsL);
                     }
+                    if (!hipsTable) {
+                        List<HiPSMasterListEntry> mocL = hipsls.getAdditionalMOCS(source);
+                        if (mocL != null) {
+                            if (usingAdhoc && adhocMocInclude.contains(source)) {
+                                var moreMocs= mocL
+                                        .stream()
+                                        .map( v -> v.getMapInfo().get(PARAMS.IVOID.getKey()))
+                                        .toList();
+                                adhocSources.addAll(moreMocs);
+                            }
+                            allSourceData.addAll(mocL);
+                        }
+                    }
                 }
+
             }
 
             if (allSourceData.isEmpty()) {

diff --git a/...firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterListSourceType.java b/...firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterListSourceType.java
@@ -11,5 +11,6 @@
 public interface HiPSMasterListSourceType {
 
     List<HiPSMasterListEntry> getHiPSListData(String[] dataTypes, String source);
+    default List<HiPSMasterListEntry> getAdditionalMOCS(String source) {return null;}
     String getUrl();
 }
diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/IrsaHiPSListSource.java b/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/IrsaHiPSListSource.java
@@ -1,16 +1,31 @@
 package edu.caltech.ipac.firefly.server.visualize.hips;
 
+import edu.caltech.ipac.firefly.data.FileInfo;
 import edu.caltech.ipac.firefly.data.ServerParams;
+import edu.caltech.ipac.firefly.server.servlets.HiPSRetrieve;
+import edu.caltech.ipac.firefly.server.util.Logger;
+import edu.caltech.ipac.table.DataGroup;
+import edu.caltech.ipac.table.DataObject;
+import edu.caltech.ipac.table.TableUtil;
 import edu.caltech.ipac.util.AppProperties;
 
+import java.net.URI;
+import java.net.URL;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import static edu.caltech.ipac.firefly.server.visualize.hips.HiPSMasterListEntry.PARAMS;
 
 /**
  * Created by cwang on 2/27/18.
  */
 public class IrsaHiPSListSource implements HiPSMasterListSourceType {
+
+    private static final String irsaBaseUrl= AppProperties.getProperty("irsa.base.url","https://irsa.ipac.caltech.edu");
+    private static final String irsaAddMocPath = AppProperties.getProperty("irsa.moc.list.path","/irsadata/hips/irsa_moc.list");
+    static final Logger.LoggerImpl log = Logger.getLogger();
+
     private static final String irsaHipsListUrl = AppProperties.getProperty(
             "irsa.hips.masterUrl",
             "https://irsa.ipac.caltech.edu/data/hips/list");
@@ -26,5 +41,44 @@ public List<HiPSMasterListEntry> getHiPSListData(String[] dataTypes, String sour
         }
     }
 
+    public List<HiPSMasterListEntry> getAdditionalMOCS(String source) {
+        try {
+            URL url= new URI(getMocUrl()).toURL();
+            FileInfo listFileInfo = HiPSRetrieve.retrieveHiPSData(url.toString(), source, false);
+            if (listFileInfo.getResponseCode()!=200 && listFileInfo.getResponseCode()!=304) return null;
+            DataGroup dg= TableUtil.readAnyFormat(listFileInfo.getFile());
+            var retList = new ArrayList<HiPSMasterListEntry>();
+
+            for(var i=0; i<dg.size();i++) {
+                DataObject row= dg.get(i);
+                HiPSMasterListEntry entry = new HiPSMasterListEntry();
+                setV(entry,row, PARAMS.FRACTION, "coverage");
+                setV(entry,row, PARAMS.WAVELENGTH, "waveband");
+                setV(entry,row, PARAMS.TITLE, "title");
+                setV(entry,row, PARAMS.URL, "moc_url");
+                setV(entry,row, PARAMS.IVOID, "moc_url");
+                setV(entry,row, PARAMS.ORDER, "order");
+                setV(entry,row, PARAMS.RELEASEDATE, "release");
+                setV(entry,row, PARAMS.PROPERTIES, "info_url");
+                entry.set(PARAMS.SOURCE.getKey(),source);
+
+                retList.add(entry);
+            }
+            return retList;
+        } catch (Exception e) {
+            log.error(e, "cannot find additional MOCS for " + source);
+            return null;
+        }
+    }
+
+    public void setV(HiPSMasterListEntry entry, DataObject row, HiPSMasterListEntry.PARAMS key, String col) {
+        Object obj= row.getDataElement(col);
+        if (obj!=null) entry.set(key.getKey(), obj.toString());
+    }
+
     public String getUrl() { return irsaHipsListUrl; }
+    public String getMocUrl() {
+        if (irsaAddMocPath.startsWith("https")) return irsaAddMocPath;
+        else return irsaBaseUrl+irsaAddMocPath;
+    }
 }