diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index df30fefd..2bdf0e2d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,7 +7,7 @@ on: types: [run-all-tool-tests-command] env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_23.0 + GALAXY_BRANCH: release_23.1 MAX_CHUNKS: 40 jobs: setup: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 93fe3a86..c73784f7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,7 +17,7 @@ on: - '*' env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_23.0 + GALAXY_BRANCH: release_23.1 MAX_CHUNKS: 4 MAX_FILE_SIZE: 1M concurrency: diff --git a/tools/archives/pyvo_integration/.shed.yml b/tools/archives/pyvo_integration/.shed.yml new file mode 100644 index 00000000..0a91019f --- /dev/null +++ b/tools/archives/pyvo_integration/.shed.yml @@ -0,0 +1,23 @@ +name: astronomicalarchivestool +owner: volodymyrss +type: unrestricted +categories: +- Data Source +description: Astronomical archives tools contains tools for querying and fetching resources from astronomical archives into Galaxy +long_description: | + This set of tool uses VO protocols (like TAP) through python libraries like pyvo, in order to enable the + user to interact with astronomical archives which have registered themselves (in the IVOA registry for instance). + + They offer the possibility to search and preview those archives content as well as download resources made available + by them. + +homepage_url: + +remote_repository_url: https://github.com/esg-epfl-apc/tools-astro/tree/main/tools/ +auto_tool_repositories: + name_template: "{{ tool_id }}" + description_template: "{{ tool_name }}" +suite: + name: "suite_astronomicalarchivestools" + description: "A suite of Galaxy tools to interact with astronomical archives" + type: repository_suite_definition \ No newline at end of file diff --git a/tools/archives/pyvo_integration/astronomical_archives.py b/tools/archives/pyvo_integration/astronomical_archives.py new file mode 100644 index 00000000..9997d59d --- /dev/null +++ b/tools/archives/pyvo_integration/astronomical_archives.py @@ -0,0 +1,1153 @@ +import json +import os +import sys +import urllib +from urllib import request + +import pyvo +from pyvo import DALAccessError, DALQueryError, DALServiceError +from pyvo import registry + +MAX_ALLOWED_ENTRIES = 100 +MAX_REGISTRIES_TO_SEARCH = 100 + + +class Service: + # https://pyvo.readthedocs.io/en/latest/api/pyvo.registry.Servicetype.html + + services = { + 'TAP': 'tap', + 'SIA': 'sia', + 'SIA2': 'sia2', + 'SPECTRUM': 'spectrum', + 'SCS': 'scs', + 'LINE': 'line' + } + + supported_services = { + 'TAP': 'tap' + } + + def __init__(self): + pass + + @staticmethod + def is_service_supported(service_type) -> bool: + is_supported = True + + if service_type not in Service.services.keys(): + is_supported = False + elif service_type not in Service.supported_services.keys(): + is_supported = False + + return is_supported + + +class Waveband: + # https://pyvo.readthedocs.io/en/latest/api/pyvo.registry.Waveband.html + # https://www.ivoa.net/rdf/messenger/2020-08-26/messenger.html + + wavebands = { + 'Extreme UV': 'EUV', + 'Gamma ray': 'Gamma-ray', + 'Infrared': 'Infrared', + 'Millimeter': 'Millimeter', + 'Neutrino': 'Neutrino', + 'Optical': 'Optical', + 'Photon': 'Photon', + 'Radio': 'Radio', + 'Ultra violet': 'UV', + 'X-ray': 'X-ray' + } + + def __init__(self): + pass + + @staticmethod + def is_waveband_supported(waveband) -> bool: + is_supported = True + + if waveband not in Waveband.wavebands.keys(): + is_supported = False + + return is_supported + + +class TapArchive: + # https://www.ivoa.net/documents/ObsCore/20170509/REC-ObsCore-v1.1-20170509 + + service_type = Service.services['TAP'] + + def __init__(self, + id=1, + title="Unknown title", + name="Unknown name", + access_url=""): + + self.id = id, + self.title = title, + self.name = name, + self.access_url = access_url + self.initialized = False + self.archive_service = None + self.tables = None + + def get_resources(self, + query, + number_of_results, + url_field='access_url'): + + resource_list_hydrated = [] + + error_message = None + + if self.initialized: + + try: + raw_resource_list = self.archive_service.search(query) + + for i, resource in enumerate(raw_resource_list): + if i < number_of_results: + resource_list_hydrated.append( + self._get_resource_object(resource)) + else: + break + + except DALQueryError: + if self.has_obscore_table(): + error_message = "Error in query -> " + query + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_DOWNLOAD, + error_message) + else: + error_message = "No obscore table in the archive" + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_DOWNLOAD, + error_message) + + except DALServiceError: + error_message = "Error communicating with the service" + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_DOWNLOAD, + error_message) + + except Exception: + error_message = "Unknow error while querying the service" + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_DOWNLOAD, + error_message) + + return resource_list_hydrated, error_message + + def _get_resource_object(self, resource): + resource_hydrated = {} + + for key, value in resource.items(): + resource_hydrated[key] = value + + return resource_hydrated + + def initialize(self): + error_message = None + + try: + self._get_service() + + if self.archive_service: + self._set_archive_tables() + self.initialized = True + + except DALAccessError: + error_message = \ + "A connection to the service could not be established" + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_ARCHIVE_CONNECTION, + error_message) + + except Exception: + error_message = "Unknow error while initializing TAP service" + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_ARCHIVE_CONNECTION, + error_message) + + return self.initialized, error_message + + def _get_service(self): + if self.access_url: + self.archive_service = pyvo.dal.TAPService(self.access_url) + + def _set_archive_tables(self): + + self.tables = [] + + for table in self.archive_service.tables: + archive_table = { + 'name': table.name, + 'type': table.type, + 'fields': None + } + + fields = [] + + for table_field in table.columns: + field = { + 'name': table_field.name, + 'description': table_field.description, + 'unit': table_field.unit, + 'datatype': table_field.datatype.content + } + + fields.append(field) + + archive_table['fields'] = fields + + self.tables.append(archive_table) + + def _is_query_valid(self, query) -> bool: + is_valid = True + + attribute_from = 'from' + attribute_where = 'where' + + idx_from = query.index(attribute_from) + idx_where = query.index(attribute_where) + + table_name = '' + + for idx in range(idx_from + len('from') + 1, idx_where): + table_name = table_name + query[idx] + + if not next( + (item for item in self.tables if + item["name"] == table_name), + False): + + is_valid = False + + return is_valid + + def has_obscore_table(self) -> bool: + has_obscore_table = self._has_table("ivoa.obscore") + + return has_obscore_table + + def _has_table(self, table_name) -> bool: + _has_table = False + + _has_table = next( + (item for item in self.tables if item["name"] == table_name), + False) + + return _has_table + + def get_archive_name(self, archive_type): + try: + if archive_type == 'registry': + name = str(self.title).strip("',()") + else: + name = self.access_url + except Exception: + name = 'Unknown archive title' + + return name + + +class RegistrySearchParameters: + + def __init__(self, keyword=None, waveband=None, service_type=None): + self.keyword = keyword + self.waveband = waveband + self.service_type = service_type + + def get_parameters(self): + + parameters = { + 'keywords': '', + 'waveband': '', + 'service_type': '' + } + + if self.keyword: + parameters['keywords'] = self.keyword + + if Waveband.is_waveband_supported(self.waveband): + parameters['waveband'] = \ + Waveband.wavebands[self.waveband] + + if Service.is_service_supported(self.service_type): + parameters['service_type'] = \ + Service.services[self.service_type] + else: + parameters['service_type'] = Service.services['TAP'] + + return parameters + + +class Registry: + + def __init__(self): + pass + + @staticmethod + def search_registries(rsp: RegistrySearchParameters, + number_of_registries): + + parameters = rsp.get_parameters() + + keywords = parameters['keywords'] + waveband = parameters['waveband'] + service_type = parameters['service_type'] + + if not waveband: + registry_list = registry.search( + keywords=keywords, + servicetype=service_type) + else: + registry_list = registry.search( + keywords=keywords, + waveband=waveband, + servicetype=service_type) + + if registry_list: + registry_list = Registry._get_registries_from_list( + registry_list, + number_of_registries) + + return registry_list + + @staticmethod + def _get_registries_from_list(registry_list, number_of_registries): + + archive_list = [] + + for i, ivoa_registry in enumerate(registry_list): + if i < number_of_registries: + archive = TapArchive(ivoa_registry.standard_id, + ivoa_registry.res_title, + ivoa_registry.short_name, + ivoa_registry.access_url) + + archive_list.append(archive) + + return archive_list + + +class TapQuery: + + def __init__(self, query): + self.raw_query = query + + def get_query(self): + return urllib.parse.unquote(self.raw_query).replace("+", " ") + + +class BaseADQLQuery: + + def __init__(self): + pass + + def _get_order_by_clause(self, order_type): + order_by_clause = 'ORDER BY ' + order_type + + return order_by_clause + + def _get_where_clause(self, parameters): + where_clause = '' + is_first_statement = True + + for key, value in parameters.items(): + + if value != '': + statement = str(key) + ' = ' + '\'' + str(value) + '\' ' + + if is_first_statement: + is_first_statement = False + where_clause += 'WHERE ' + else: + statement = 'AND ' + statement + + where_clause += statement + + return where_clause + + +class ToolRunner: + + def __init__(self, + run_parameters, + output, + output_csv, + output_html, + output_basic_html, + output_error): + + self._raw_parameters_path = run_parameters + self._json_parameters = json.load(open(run_parameters, "r")) + self._archive_type = '' + self._query_type = '' + self._archives = [] + self._adql_query = '' + self._services_access_url = '' + self._url_field = 'access_url' + self._number_of_files = '' + self._is_initialised = False + + self._csv_file = False + self._image_file = False + self._html_file = False + self._basic_html_file = False + + self._output = output + self._output_csv = output_csv + self._output_html = output_html + self._output_basic_html = output_basic_html + self._output_error = output_error + + self._set_run_main_parameters() + self._is_initialised, error_message = self._set_archive() + + if self._is_initialised and error_message is None: + self._set_query() + self._set_output() + + def _set_run_main_parameters(self): + + qs = "query_section" + qsl = "query_selection" + + self._archive_type = \ + self._json_parameters['archive_selection']['archive_type'] + self._query_type = \ + self._json_parameters[qs][qsl]['query_type'] + + def _set_archive(self): + + error_message = None + + if self._archive_type == 'archive': + self._service_access_url =\ + self._json_parameters['archive_selection']['archive'] + + self._archives.append( + TapArchive(access_url=self._service_access_url)) + + else: + keyword = \ + self._json_parameters['archive_selection']['keyword'] + waveband = \ + self._json_parameters['archive_selection']['wavebands'] + service_type = \ + self._json_parameters['archive_selection']['service_type'] + + rsp = RegistrySearchParameters( + keyword=keyword, + waveband=waveband, + service_type=service_type) + + archive_list = Registry.search_registries( + rsp, + MAX_REGISTRIES_TO_SEARCH) + + if len(archive_list) >= 1: + self._archives = archive_list + else: + error_message = "no archive matching search parameters" + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_ARCHIVE_CONNECTION, + error_message) + + if error_message is None: + + self._archives[:] = \ + [archive for archive in self._archives if + archive.initialize()[0]] + + if len(self._archives) >= 1: + return True, None + else: + return False, \ + "no archive matching search" \ + " parameters could be initialized" + + else: + return False, error_message + + def _set_query(self): + + qs = 'query_section' + qsl = 'query_selection' + + if self._query_type == 'obscore_query': + + dataproduct_type = \ + self._json_parameters[qs][qsl]['dataproduct_type'] + obs_collection = \ + self._json_parameters[qs][qsl]['obs_collection'] + obs_title = \ + self._json_parameters[qs][qsl]['obs_title'] + obs_id = \ + self._json_parameters[qs][qsl]['obs_id'] + facility_name = \ + self._json_parameters[qs][qsl]['facility_name'] + instrument_name = \ + self._json_parameters[qs][qsl]['instrument_name'] + em_min = \ + self._json_parameters[qs][qsl]['em_min'] + em_max = \ + self._json_parameters[qs][qsl]['em_max'] + target_name = \ + self._json_parameters[qs][qsl]['target_name'] + obs_publisher_id = \ + self._json_parameters[qs][qsl]['obs_publisher_id'] + s_fov = \ + self._json_parameters[qs][qsl]['s_fov'] + calibration_level = \ + self._json_parameters[qs][qsl]['calibration_level'] + t_min = \ + self._json_parameters[qs][qsl]['t_min'] + t_max = \ + self._json_parameters[qs][qsl]['t_max'] + order_by = \ + self._json_parameters[qs][qsl]['order_by'] + + obscore_query_object = ADQLObscoreQuery(dataproduct_type, + obs_collection, + obs_title, + obs_id, + facility_name, + instrument_name, + em_min, + em_max, + target_name, + obs_publisher_id, + s_fov, + calibration_level, + t_min, + t_max, + order_by) + + self._adql_query = obscore_query_object.get_query() + + elif self._query_type == 'raw_query': + + wc = 'where_clause' + + tap_table = \ + self._json_parameters[qs][qsl]['table'] + + where_field = \ + self._json_parameters[qs][qsl][wc]['where_field'] + where_condition = \ + self._json_parameters[qs][qsl][wc]['where_condition'] + + self._url_field = \ + self._json_parameters[qs][qsl]['url_field'] + + self._adql_query = \ + ADQLTapQuery().get_query( + tap_table, + where_field, + where_condition) + else: + self._adql_query = ADQLObscoreQuery.base_query + + def _set_output(self): + self._number_of_files = \ + int( + self._json_parameters['output_section']['number_of_files'] + ) + + if self._number_of_files < 1: + self._number_of_files = 1 + elif self._number_of_files > 100: + self._number_of_files = MAX_ALLOWED_ENTRIES + + output_selection = \ + self._json_parameters['output_section']['output_selection'] + + if output_selection is not None: + if 'c' in output_selection: + self._csv_file = True + if 'i' in output_selection: + self._image_file = True + if 'h' in output_selection: + self._html_file = True + if 'b' in output_selection: + self._basic_html_file = True + + def _validate_json_parameters(self, json_parameters): + self._json_parameters = json.load(open(json_parameters, "r")) + + def run(self): + if self._is_initialised: + error_message = None + file_url = [] + + archive_name = self._archives[0].get_archive_name( + self._archive_type) + + for archive in self._archives: + _file_url, error_message = archive.get_resources( + self._adql_query, + self._number_of_files, + self._url_field) + + file_url.extend(_file_url) + + if len(file_url) >= int(self._number_of_files): + file_url = file_url[:int(self._number_of_files)] + break + + if file_url: + + if self._csv_file: + FileHandler.write_urls_to_output( + file_url, + self._output_csv, + self._url_field) + + if self._image_file: + + try: + fits_file = FileHandler.download_file_from_url( + file_url[0][self._url_field]) + + FileHandler.write_file_to_output( + fits_file, + self._output, "wb") + + log_message = "from url " +\ + file_url[0][self._url_field] + + Logger.create_action_log( + Logger.ACTION_SUCCESS, + Logger.ACTION_TYPE_DOWNLOAD, + log_message) + + except Exception: + error_message = "from url " + \ + file_url[0][self._url_field] + + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_DOWNLOAD, + error_message) + + for i, url in enumerate(file_url[1:], start=1): + try: + fits_file = \ + FileHandler.download_file_from_url( + url[self._url_field]) + + FileHandler.write_file_to_subdir( + fits_file, + FileHandler.get_file_name_from_url( + url[self._url_field])) + + log_message = "from url " + \ + url[self._url_field] + + Logger.create_action_log( + Logger.ACTION_SUCCESS, + Logger.ACTION_TYPE_DOWNLOAD, + log_message) + + except Exception: + error_message = "from url " + \ + url[self._url_field] + + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_DOWNLOAD, + error_message) + + if self._html_file: + html_file = OutputHandler.generate_html_output( + file_url, + archive_name, + self._adql_query) + + FileHandler.write_file_to_output(html_file, + self._output_html) + + if self._basic_html_file: + html_file = \ + OutputHandler.generate_basic_html_output( + file_url, + archive_name, + self._adql_query) + + FileHandler.write_file_to_output( + html_file, + self._output_basic_html) + + summary_file = Logger.create_log_file(archive_name, + self._adql_query) + summary_file += "\n Tool run executed with success" + + FileHandler.write_file_to_output(summary_file, + self._output_error) + + else: + + summary_file = Logger.create_log_file(archive_name, + self._adql_query) + + if error_message is None: + summary_file += \ + "\n No resources matching parameters found" + else: + summary_file += error_message + + FileHandler.write_file_to_output(summary_file, + self._output_error) + + else: + summary_file = Logger.create_log_file("Archive", + self._adql_query) + + summary_file += "Unable to initialize archive" + + FileHandler.write_file_to_output(summary_file, + self._output_error) + + +class ADQLObscoreQuery(BaseADQLQuery): + order_by_field = { + 'size': 'access_estsize', + 'collection': 'obs_collection', + 'object': 'target_name' + } + + base_query = 'SELECT TOP ' + \ + str(MAX_ALLOWED_ENTRIES) + \ + ' * FROM ivoa.obscore ' + + def __init__(self, + dataproduct_type, + obs_collection, + obs_title, + obs_id, + facility_name, + instrument_name, + em_min, + em_max, + target_name, + obs_publisher_id, + s_fov, + calibration_level, + t_min, + t_max, + order_by): + + super().__init__() + + if calibration_level == 'none': + calibration_level = '' + + if order_by == 'none': + order_by = '' + + if t_min == 'None' or t_min is None: + t_min = '' + + if t_max == 'None' or t_max is None: + t_max = '' + + if em_min == 'None' or em_min is None: + em_min = '' + + if em_max == 'None' or em_max is None: + em_max = '' + + if dataproduct_type == 'none' or dataproduct_type is None: + dataproduct_type = '' + + self.parameters = { + 'dataproduct_type': dataproduct_type, + 'obs_collection': obs_collection, + 'obs_title': obs_title, + 'obs_id': obs_id, + 'facility_name': facility_name, + 'instrument_name': instrument_name, + 'em_min': em_min, + 'em_max': em_max, + 'target_name': target_name, + 'obs_publisher_id': obs_publisher_id, + 's_fov': s_fov, + 'calibration_level': calibration_level, + 't_min': t_min, + 't_max': t_max + } + + self.order_by = order_by + + def get_query(self): + return ADQLObscoreQuery.base_query + \ + self.get_where_statement() + \ + self.get_order_by_statement() + + def get_order_by_statement(self): + if self.order_by != '': + return self._get_order_by_clause(self.order_by) + else: + return '' + + def _get_order_by_clause(self, order_type): + + obscore_order_type = ADQLObscoreQuery.order_by_field[order_type] + + return super()._get_order_by_clause(obscore_order_type) + + def get_where_statement(self): + return self._get_where_clause(self.parameters) + + def _get_where_clause(self, parameters): + return super()._get_where_clause(parameters) + + +class ADQLTapQuery(BaseADQLQuery): + base_query = 'SELECT TOP '+str(MAX_ALLOWED_ENTRIES)+' * FROM ' + + def __init__(self): + super().__init__() + + def get_order_by_clause(self, order_type): + return super()._get_order_by_clause(order_type) + + def get_query(self, table, where_field, where_condition): + if where_field != '' and where_condition != '': + return ADQLTapQuery.base_query + \ + str(table) + \ + ' WHERE ' + \ + str(where_field) + ' = ' + '\'' + \ + str(where_condition) + '\'' + else: + return ADQLTapQuery.base_query + str(table) + + +class HTMLReport: + _html_report_base_header = '' + _html_report_base_body = '' + _html_report_base_footer = '' + _html_report_base_script = '' + + def __init__(self): + pass + + +class OutputHandler: + + def __init__(self): + pass + + @staticmethod + def generate_html_output(urls_data, archive_name, adql_query): + return OutputHandler.html_header + \ + OutputHandler.generate_html_content( + urls_data, + archive_name, + adql_query, + div_attr='class="title"', + table_attr='class="fl-table"') + + @staticmethod + def generate_basic_html_output(urls_data, + archive_name, + adql_query, ): + return OutputHandler.generate_html_content(urls_data, + archive_name, + adql_query) + + @staticmethod + def generate_html_content(urls_data, archive_name, adql_query, + div_attr="", table_attr="border='1'"): + html_file = \ + f""" +
+

Resources Preview archive: + + {archive_name} + +

+ ADQL query : {adql_query} +
""" + + html_file += f'' + + for key in Utils.collect_resource_keys(urls_data): + html_file += '' + + html_file += '' + + for resource in urls_data: + html_file += '' + + for key, value in resource.items(): + html_file += f'' + + html_file += '' + html_file += '' + + html_file += '
' + str(key) + '
{value}' + for preview_key in \ + ['preview', 'preview_url', 'postcard_url']: + if preview_key in resource: + html_file += ( + '
Preview' + f'' + '
' + ) + html_file += '
' + return html_file + + html_header = """ """ + + +class FileHandler: + + def __init__(self): + pass + + @staticmethod + def download_file_from_url(file_url): + with request.urlopen(file_url) as response: + fits_file = response.read() + + return fits_file + + @staticmethod + def write_file_to_output(file, output, write_type="w"): + with open(output, write_type) as file_output: + file_output.write(file) + + @staticmethod + def write_urls_to_output(urls: [], output, access_url="access_url"): + with open(output, "w") as file_output: + for url in urls: + try: + file_output.write(url[access_url] + ',') + except Exception: + error_message = "url field not found for url" + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_WRITE_URL, + error_message) + + @staticmethod + def write_file_to_subdir(file, index): + dir = os.getcwd() + + dir += '/fits' + + upload_dir = os.path.join(dir, str(index) + '.fits') + + with open(upload_dir, "wb") as file_output: + file_output.write(file) + + @staticmethod + def get_file_name_from_url(url, index=None): + url_parts = url.split('/') + + file_name = '' + + try: + if (url_parts[-1]) != '': + file_name = url_parts[-1] + elif len(url_parts) > 1: + file_name = url_parts[-2] + except Exception: + file_name = 'archive file ' + + return file_name + + +class Utils: + + def __init__(self): + pass + + @staticmethod + def collect_resource_keys(urls_data: list) -> list: + """ + Collect all the keys from the resources, + keeping the order in the order of key appearance in the resources + """ + + resource_keys = [] + for resource in urls_data: + for key in resource.keys(): + if key not in resource_keys: + resource_keys.append(key) + return resource_keys + + +class Logger: + _logs = [] + + ACTION_SUCCESS = 1 + ACTION_ERROR = 2 + + ACTION_TYPE = 1 + INFO_TYPE = 2 + + ACTION_TYPE_DOWNLOAD = 1 + ACTION_TYPE_ARCHIVE_CONNECTION = 2 + ACTION_TYPE_WRITE_URL = 3 + ACTION_TYPE_WRITE_FILE = 4 + + def __init__(self): + pass + + @staticmethod + def create_action_log(outcome, action, message) -> bool: + + is_log_created = False + log = "" + + if action == Logger.ACTION_TYPE_DOWNLOAD: + if outcome == Logger.ACTION_SUCCESS: + log += "Success downloading file : " + message + else: + log += "Error downloading file : " + message + + is_log_created = True + elif action == Logger.ACTION_TYPE_ARCHIVE_CONNECTION: + if outcome == Logger.ACTION_SUCCESS: + log += "Success connecting to archive : " + message + else: + log += "Error connecting to archive : " + message + + is_log_created = True + elif action == Logger.ACTION_TYPE_WRITE_URL: + if outcome == Logger.ACTION_SUCCESS: + log += "Success writing url to file : " + message + else: + log += "Error writing to file : " + message + + is_log_created = True + + if is_log_created: + Logger._insert_log(Logger.ACTION_TYPE, log) + + return is_log_created + + @staticmethod + def create_info_log(message): + pass + + @staticmethod + def _insert_log(type, log): + Logger._logs.append(log) + + @staticmethod + def create_log_file(archive_name, query): + log_file = "" + + log_file += "Run summary for archive : " + archive_name + "\n" + log_file += "With query : " + query + "\n" + + for log in Logger._logs: + log_file += log + "\n" + + return log_file + + +if __name__ == "__main__": + output = sys.argv[1] + output_csv = sys.argv[2] + output_html = sys.argv[3] + output_basic_html = sys.argv[4] + output_error = sys.argv[5] + + inputs = sys.argv[6] + + tool_runner = ToolRunner(inputs, + output, + output_csv, + output_html, + output_basic_html, + output_error) + + tool_runner.run() diff --git a/tools/archives/pyvo_integration/astronomical_archives.xml b/tools/archives/pyvo_integration/astronomical_archives.xml new file mode 100644 index 00000000..9be9b468 --- /dev/null +++ b/tools/archives/pyvo_integration/astronomical_archives.xml @@ -0,0 +1,977 @@ + + queries astronomical archives through Virtual Observatory protocols + + operation_0224 + + + astropy + pyvo + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + +
+
+ + + + + + + +
+ + + + + + 'i' in output_section['output_selection'] + output_section['output_selection'] is not None + + + 'c' in output_section['output_selection'] + output_section['output_selection'] is not None + + + 'h' in output_section['output_selection'] + output_section['output_selection'] is not None + + + 'b' in output_section['output_selection'] + output_section['output_selection'] is not None + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +.. class:: infomark + +The help section contains general tips on how to use the tool as well as a list of some of the archives available through the tool with a description of their content + +.. class:: infomark + +For the non basic html report to be rendered correctly the tool needs to be allowed to render html in : Admin -> Tool Management -> Manage Allowlist + +**What it does** + +This tool lets you explore and query different archives available in the IVOA registry of registries. + +You can choose to download a list of files url matching your search parameters or directly download the files alongside the manifest containing the urls you can also choose to preview the matching urls values in an html report + +The html report will list all the fields of the queried table and let you preview the different files when a preview is available + +You have the choice between letting the tool select the archive matching your search criteria or you can directly choose the archive you want to query + +**How to use it** + +OUTPUT SELECTION + +The file type and number of files selection let you choose if you want to download the chosen files alongside the url manifest or if you want to preview the urls in an html report and how many of them will be listed and downloaded + +ARCHIVE SELECTION + +The archive selection lets you decide what archive will be queried : + +If you want the tool to select an archive for you : select "Query IVOA Registry" and enter a specific keyword describing the kind of archive you are looking for (for instance "hess" or "apertif") +The tool will choose the first archive matching your criterias + +If you already know what archive you want to query : select "Query specific IVOA archive" and choose the corresponding archive + +QUERY SELECTION + +The query selection lets you decide what kind of ADQL query will be run against the chosen archive + +The two first choices ("No specific query" and "IVOA obscore table") assume that an obscore table (as specified in the IVOA guidelines) is present, if the service has not implemented an obscore table you will need to run a custom query against a specific exposed table + +If you don't want to run any specific query choose "No specific query", this choice will run a basic "SELECT" query and return the results + +If you want to run an ADQL query against the obscore table of the archive choose "IVOA obscore table", that will let you specify the value of some of the most used mandatory obscore field + +If the archive does not have an obscore table or if you want to run a very specific query choose "Raw ADQL query builder" for that to work you will need to know the name of the table you want to query and the field containing the access urls if you want to download the files + +**Example** + +Browsing a specific archive to find and download a specific file (LoLSS collection from Astron archive) : + +Choose "Download urls preview (html format)" from "File type selection" and put 10 in the number of files you want to preview + +In Archive Selection choose "Query specific IVOA archive" and then select "The VO @ Astron" + +In Query Selection choose "IVOA obscore query builder" and put "LoLSS" in the collection field + +Once run the tool should have created an html report containing a number of files from the LoLSS collection, explore the report by clicking the "Visualize" icon and copy the "obs_title" of one of the files + +Repeat the previous steps except that this time you will choose "Download urls + files" with 1 file to download and enter the obs_title in the obs_title field + +The tool should now be downloading this specific file alongside a csv file containing the download url + +**The tool doesn't find any files** + +There are several reason for the tool to output a file explaining that no files matching your criterias were found: + + +Everything ran fine but no files were matching your criterias + +The archive does not have an obscore table (if you ran an obscore query) + +The files access urls are not working properly + +The table or field you choose in your raw ADQL query doesn't exist or does not contain a download url + +The selected archive is not available + + +In most cases the best course of action is to try to download the urls only and if some are returned to preview them and to manually try their download urls + +Another possibility is to manually review the TAP schema of the archive to craft a custom ADQL query and preview the results + +----- + +**References** + +Obscore Table IVOA specifications: https://www.ivoa.net/documents/ObsCore/20160224/WD-ObsCore-v1.1-20160224.pdf + +Obscore Table dataproduct_type values specifications: https://www.ivoa.net/rdf/product-type/2023-06-26/product-type.html + +ADQL specifications: https://www.ivoa.net/documents/ADQL/20180112/PR-ADQL-2.1-20180112.html + +IVOA VO protocol summary : https://ivoa.net/deployers/intro_to_vo_concepts.html + +----- + +**List of archives description** + +----- + +**AIP GAVO TAP** http://gavo.aip.de/tap AIP DaCHS TAP service + +----- + +The AIP DaCHS's TAP end point. The Table Access +Protocol (TAP) lets you execute queries against our database tables, +inspect various metadata, and upload your own data. It is thus the +VO's premier way to access public data holdings. + +Tables exposed through this endpoint include: dens256, fof, redshifts from the bolshoi schema, columns, services, tables from the glots schema, fof, fofmtree, fofparticles, particles85, redshifts, treesnapnums from the mdr1 schema, rave_dr4 from the ravedr4 schema, alt_identifier, authorities, capability, g_num_stat, interface, intf_param, registries, relationship, res_date, res_detail, res_role, res_schema, res_subject, res_table, resource, stc_spatial, stc_spectral, stc_temporal, subject_uat, table_column, tap_table, validation from the rr schema, columns, groups, key_columns, keys, schemas, tables from the tap_schema schema. + +----- + +**STScI CAOMTAP** https://vao.stsci.edu/CAOMTAP/TapService.aspx MAST STScI CAOM and ObsCore TAP service + +----- + +The MAST Archive at STScI TAP end point for observational data, saved in the Common Archive Data Model format and made available through the ObsCore limited view. + +The Table Access Protocol (TAP) lets you execute queries against our database tables, and inspect various metadata. Upload is not currently supported. + +Missions and projects with data available through the CAOMTAP service include: +BEFS, EUVE, FUSE, GALEX, HLA, HST, HUT, IUE, JWST, K2, KEPLER, PS1 (PanSTARRS 1) Data Release 2, SPITZER_SHA, SWIFT, TESS, TUES, WUPPE. + +High Level Science Products (HLSPs) available include: +3D-DASH, 3DHST, CDIPS, CLASSY, DIAMANTE, ELEANOR, EVEREST, GSFC-ELEANOR-LITE, HALO, HFF-DEEPSPACE, hlsp_3cr, hlsp_acsggct, hlsp_andromeda, hlsp_angrrr, hlsp_angst, hlsp_appp, hlsp_borg, hlsp_candels, hlsp_carina, hlsp_clash, hlsp_coma, hlsp_cosmos, hlsp_frontier, hlsp_gems, hlsp_ghosts, hlsp_goods, hlsp_goodsnic, hlsp_heritage, hlsp_hippies, hlsp_hlastarclusters, hlsp_hudf09, hlsp_hudf12, hlsp_ison, hlsp_legus, hlsp_m82, hlsp_m83mos, hlsp_orion, hlsp_phat, hlsp_sgal, hlsp_sm4ero, hlsp_stages, hlsp_starcat, hlsp_udf, hlsp_uvudf, hlsp_wfc3ers, hlsp_wisp, hlsp_xdf, IRIS, K2SC, K2SFF, K2VARCAT, KBONUS-APEXBA, KEPSEISMIC, PATHOS, PSFK2, QLP, TASOC, TESS-SPOC, ULLYSES + +Tables exposed through this endpoint include the ObsCore (version 1.1) view of our CAOM (version 2.4) database, as well as all other CAOM tables + +----- + +**HSC TAP** https://mast.stsci.edu/vo-tap/api/v0.1/hsc/ MAST STScI Hubble Source Catalog Version 3 (HSCv3) TAP service + +----- + +The MAST Archive at STScI TAP end point for the Hubble Source Catalog, version 3.1. The Hubble Source Catalog (HSC) is designed to optimize science from the Hubble Space Telescope by combining the tens of thousands of visit-based source lists in the Hubble Legacy Archive (HLA) into a single master catalog. The Hubble Source Catalog version 3.1 also provides proper motions for over 400,00 stars in the augmented Sagittarius Window Eclipsing Extrasolar Planet Search (SWEEPS) HST field, and the Hubble Catalog of Variables (HCV). + +The Table Access Protocol (TAP) lets you execute queries against our database tables, and inspect various metadata. Upload is not currently supported. + +----- + +**HSCv3 TAP** https://mast.stsci.edu/vo-tap/api/v0.1/hsc/ MAST STScI Hubble Source Catalog Version 3 (HSCv3) TAP service + +----- + +The MAST Archive at STScI TAP end point for the Hubble Source Catalog, version 3.1. The Hubble Source Catalog (HSC) is designed to optimize science from the Hubble Space Telescope by combining the tens of thousands of visit-based source lists in the Hubble Legacy Archive (HLA) into a single master catalog. The Hubble Source Catalog version 3.1 also provides proper motions for over 400,00 stars in the augmented Sagittarius Window Eclipsing Extrasolar Planet Search (SWEEPS) HST field, and the Hubble Catalog of Variables (HCV). + +The Table Access Protocol (TAP) lets you execute queries against our database tables, and inspect various metadata. Upload is not currently supported. + +----- + +**MissionMAST TAP** https://vao.stsci.edu/MissionMAST/TapService.aspx MAST STScI MissionMAST TAP service for HST Science Data + +----- + +The Table Access Protocol (TAP) lets you execute queries against our database tables, and inspect various metadata. Upload is not currently supported. + +----- + +**PS1DR2 TAP** https://vao.stsci.edu/PS1DR2/TapService.aspx MAST STScI PanSTARRS 1 DR2 TAP service + +----- + +The Panoramic Survey Telescope and Rapid Response System (Pan-STARRS or PS1) is a wide-field imaging facility developed at the University of Hawaii's Institute for Astronomy for a variety of scientific studies from the nearby to the very distant Universe. The PS1 catalog includes measurements in five filters (grizy) covering 30,000 square degrees of the sky north of declination -30 degrees, with typically ~12 epochs for each filter. This interface allows searches for the mean measurements and the deeper stacked measurements from images combining all the epochs. The DR2 release also includes the detection catalog containing all the multi-epoch observations. + +The Table Access Protocol (TAP) lets you execute queries against our database tables, and inspect various metadata. Upload is not currently supported. + +----- + +**STScI RegTAP** https://vao.stsci.edu/RegTAP/TapService.aspx MAST STScI Registry TAP service + +----- + +The MAST Archive at STScI TAP end point for registry metadata. +The Table Access Protocol (TAP) lets you execute queries against our database tables, inspect various metadata, and upload your own data. +Tables exposed through this endpoint include are only for the Registry Relational database + +----- + +**HLSP CLASSY TAP** https://vao.stsci.edu/HLSP_CLASSY/TapService.aspx MAST STScI CLASSY HLSP TAP service + +----- + +The COS Legacy Archive Spectroscopic SurveY (CLASSY): A UV Treasury of Star-Forming Galaxies is a High Level Science Product hosted at the Mikulski Archive for Space Telescopes. + +The Table Access Protocol (TAP) lets you execute queries against our database tables, and inspect various metadata. Upload is not currently supported. + +----- + +**HLSP ULLYSES TAP** https://vao.stsci.edu/HLSP_ULLYSES/TapService.aspx MAST STScI ULLYSES HLSP TAP service + +----- + +The Hubble Space Telescope’s (HST) Ultraviolet Legacy Library of Young Stars as Essential Standards (ULLYSES) program is an ultraviolet spectroscopic library of young high- and low-mass stars in the local universe. + +The Table Access Protocol (TAP) lets you execute queries against our database tables, and inspect various metadata. Upload is not currently supported. + +----- + +**TIC TAP** https://mast.stsci.edu/vo-tap/api/v0.1/tic MAST STScI TESS Input Catalog TAP service + +----- + +The MAST Archive at STScI TAP end point for the TESS Input Catalog.

The TIC is used to help identify two-minute cadence target selection for the TESS mission, and to calculate physical and observational properties of planet candidates. It is for use by both the TESS science team and the public, and it is periodically updated – the current version is TIC-8. TIC-8 uses the GAIA DR2 catalog as a base and merges a large number of other photometric catalogs, including 2MASS, UCAC4, APASS, SDSS, WISE, etc. There are roughly 1.5 billion stellar and extended sources in TIC-8, containing compiled magnitudes including B, V, u, g, r, i, z, J, H, K, W1-W4, and G. +The TIC can be directly accessed through the Mikulski Archive for Space Telescopes (MAST), using either queries or bulk download. + +The Table Access Protocol (TAP) lets you execute queries against our database tables, and inspect various metadata. Upload is not currently supported. + +----- + +**UCL Astro TAP** http://astroweb.projects.phys.ucl.ac.uk:8000/tap UCL DaCHS server TAP service + +----- + +The UCL DaCHS server's TAP end point. The Table Access +Protocol (TAP) lets you execute queries against our database tables, +inspect various metadata, and upload your own data. It is thus the +VO's premier way to access public data holdings. + +Tables exposed through this endpoint include: columns, groups, key_columns, keys, schemas, tables from the tap_schema schema, epn_core from the jasmin schema, emptyobscore, obscore from the ivoa schema, epn_core from the mdisc schema. + +----- + +**ASTRON VO TAP** https://vo.astron.nl/tap The VO @ ASTRON TAP service + +----- + +The The VO @ ASTRON's TAP end point. The Table Access +Protocol (TAP) lets you execute queries against our database tables, +inspect various metadata, and upload your own data. It is thus the +VO's premier way to access public data holdings. + +Tables exposed through this endpoint include: main from the lbcs schema, main, msssvf_img_main from the mvf schema, frb_det, frb_obs from the arts_dr1 schema, img_main, main from the lofartier1 schema, mosaic, source_catalog from the lolss schema, main from the svc schema, img_main, main from the tgssadr schema, main, mom0 from the sauron schema, images, source_catalog from the apertif_dr_bootes schema, columns, groups, key_columns, keys, schemas, tables from the tap_schema schema, hetdex_images, main, main_merged from the hetdex schema, img_main from the msss schema, obscore from the ivoa schema, beam_cubes, calibrated_visibilities, continuum_images, flux_cal_visibilities, pol_cal_visibilities, pol_cubes, raw_visibilities, spectral_cubes from the apertif_dr1 schema, main_gausses, main_sources, mosaics from the lotss_dr2 schema. + +----- + +**AIASCR TAP** http://vos2.asu.cas.cz/tap AIASCR VO Services TAP service + +----- + +The AIASCR VO Services's TAP end point. The Table Access +Protocol (TAP) lets you execute queries against our database tables, +inspect various metadata, and upload your own data. It is thus the +VO's premier way to access public data holdings. + +Tables exposed through this endpoint include: objcat, objects_observed, objobs_complete, objobs_lightcurves, observation_info from the bextract schema, dim_exposure, dim_observation_objcat, exposure, objcat, objobs_complete, objobs_lightcurves, observation, observation_objcat from the bextract_jan16 schema, reduced_images from the dk154_reduced schema, exposure, objcat, objobs_complete, objobs_lightcurves, observation from the bextract_jan15 schema, columns, groups, key_columns, keys, schemas, tables from the tap_schema schema, dim_exposure, dim_observation_objcat, exposure, objcat, objobs_complete, objobs_lightcurves, objobs_lightcurves_ssap, observation, observation_objcat from the bextract_apr18 schema, data from the ccd700 schema, dim_exposure, dim_observation_objcat, exposure, objcat, objobs_complete, objobs_lightcurves, observation, observation_objcat from the bextract_jan15_v2 schema, emptyobscore, obscore from the ivoa schema, data from the heros schema, reduced_images from the dk154_reduced_jan18 schema, dim_exposure, dim_observation_objcat, exposure, objcat, objobs_complete, objobs_lightcurves, observation, observation_objcat from the bextract_jul15 schema, dim_exposure, dim_observation_objcat, exposure, objcat, objobs_complete, objobs_lightcurves, objobs_lightcurves_ssap, observation, observation_objcat from the bextract_jul16 schema, data from the lamost_dr2 schema, data from the lamost_dr5 schema, data from the lamost_dr3 schema, data from the lamost schema, data from the lamost_dr1 schema, dim_exposure, dim_observation_objcat, exposure, objcat, objobs_complete, objobs_lightcurves, objobs_lightcurves_ssap, observation, observation_objcat from the bextract_jan18 schema, dim_exposure, dim_observation_objcat, exposure, objcat, objobs_complete, objobs_lightcurves, objobs_lightcurves_ssap, observation, observation_objcat from the bextract_dec17 schema, raw_images from the dk154_rawdata schema. + +----- + +**CSIRO ATOA TAP** https://atoavo.atnf.csiro.au/tap CSIRO Australia Telescope Online Archive TAP Service + +----- + +Table Access Protocol service for accessing Australia Telescope Online Archive + +----- + +**CSIRO ASKAP TAP** https://casda.csiro.au/casda_vo_tools/tap CSIRO ASKAP Science Data Archive TAP Service + +----- + +Table Access Protocol service for accessing catalogues from ASKAP radio astronomy observations + +----- + +**CSIRO Pulsar TAP** https://data.csiro.au/psrdavo/tap CSIRO Parkes Pulsar Data Archive + +----- + +Repository of pulsar observations made at the Parkes, Australia radio telescope + +----- + +**BIRA-IASB TAP** http://vespa-ae.oma.be/tap VESPA PA team server TAP service + +----- + +The VESPA PA team server's TAP end point. The Table Access +Protocol (TAP) lets you execute queries against our database tables, +inspect various metadata, and upload your own data. It is thus the +VO's premier way to access public data holdings. + +Tables exposed through this endpoint include: epn_core from the gem_mars schema, obscore from the ivoa schema, epn_core from the nomad schema, epn_core from the soir schema, columns, groups, key_columns, keys, schemas, tables from the tap_schema schema. + +----- + +**ArVO Byu TAP** http://arvo-registry.sci.am/tap ArVO Byurakan TAP service + +----- + +The ArVO Byurakan's TAP end point. The Table Access +Protocol (TAP) lets you execute queries against our database tables, +inspect various metadata, and upload your own data. It is thus the +VO's premier way to access public data holdings. + +Tables exposed through this endpoint include: columns, groups, key_columns, keys, schemas, tables from the tap_schema schema, main from the dfbsplates schema, spectra, ssa from the dfbsspec schema, obscore from the ivoa schema. + +----- + +**CADC TAP** https://ws.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/argus CADC Archive Query Service + +----- + +Query service for observation metadata for all CADC archival data holdings. This service + provides Table Access Protocol (TAP) API access to observation metadata conforming to the + Common Archive Observation Model (CAOM) and the IVOA ObsCore data model. + + The current dynamic collections (new data arriving) include: CFHT, CFHTMEGAPIPE, DAO, DRAO, GEMINI, HST, + JCMT, NEOSSAT, OMM, TESS, VLASS. + + The current static collections include: APASS, BLAST, CFHTTERAPIX, CFHTWIRWOLF, + CGPS, DAOPLATES, FUSE, HSTHLA, IRIS, MACHO, MOST, NGVS, UKIRT, VGPS. + + The curent external collections (metadata only, data served by original data centre) include: + CHANDRA, NOAO, SDSS, SUBARU, XMM. + +----- + +**YouCat** https://ws-cadc.canfar.net/youcat CANFAR User Catalogue Query Service + +----- + +YouCat is a query service for catalogues hosted by CANFAR. This service + provides Table Access Protocol (TAP) API access to catalogues created by + project teams. + + YouCat also implements a prototype VOSI-tables extension that allows users + to create, update table metadata add table content (rows), create indices, + and delete tables in the database. Users can also control access to their + own tables (public to allow anonymous querying, protected so only members + of specified groups can query, and private where only the owner can query). + + Users who want to create tables must request an allocation (schema) by email + to support@canfar.net. + + Currently available catalogues include: VLASS, CFHTLS, KiDS, PAndAS, RCSLens. + +----- + +**AMDA** http://cdpp-epntap.irap.omp.eu/__system__/tap/run/tap Planetary and heliophysics plasma data at CDPP/AMDA + +----- + +Planetary and heliophysics plasma data at CDPP/AMDA + +----- + +**ILLU67P** http://cdpp-epntap.irap.omp.eu/__system__/tap/run/tap Illumination maps of 67P + +----- + +Illumination by the Sun of each face of the comet 67P/Churyumov-Gerasimenko based on the shape model + CSHP_DV_130_01_LORES_OBJ.OBJ. The service provides the cosine between the normal of each face (in the same order as the faces defined in the shape model) and the Sun direction; both + numerical values and images of the illumination are available. Each map is defined for a given position of the Sun + in the frame of 67P (67P/C-G_CK). Longitude 0 is at the center of each map. The code is developed by A. Beth, + Imperial College London, UK and the service is provided by CDPP (http://cdpp.eu). Acknowlegment: The illumination models + have been developed at the Department of Physics at Imperial College London (UK) under the financial support of STFC + grant of UK ST/N000692/1 and ESA contract 4000119035/16/ES/JD (Rosetta RPC-PIU). We would also like to warmly + thank Bernhard Geiger (ESA) for his support in validating the 2D-illumination maps. + +----- + +**PSWS Transplanet** http://cdpp-epntap.irap.omp.eu/__system__/tap/run/tap Magnetosphere Ionosphere coupling simulation runs + +----- + +A Transplanet model of magnetosphere-ionosphere coupling at Earth, Mars, and Jupiter + +----- + +**SIMBAD TAP** http://simbad.cds.unistra.fr/simbad/sim-tap SIMBAD TAP query engine + +----- + +This service provides TAP access to a simplified view of the SIMBAD database + +----- + +**VizieR ObsTAP** cdsarc.cds.unistra.fr/saadavizier.tap/tap TAP VizieR query engine + +----- + +This service provides ObsTAP access to VizieR spectra, time-series and images. + +----- + +**TAPVizieR** tapvizier.cds.unistra.fr/TAPVizieR/tap TAP VizieR query engine + +----- + +This service provides TAP access to a simplified view of the VizieR database + +----- + +**J-PLUS-DR1** https://archive.cefca.es/catalogues/jplus-dr1/tap_async.html J-PLUS DR1 Catalogue (July, 2018) + +----- + +J-PLUS DR1 Catalogue (July, 2018) is based on scientific images in 12 filters collected from November 2015 to January 2018 covering a total area of ~1020 square degrees. The Javalambre Photometric Local Universe Survey (J-PLUS) is an ongoing 12-band photometric optical survey, observing thousands of square degrees of the Northern Hemisphere from the dedicated JAST80 telescope at the Observatorio Astrofísico de Javalambre (OAJ, Teruel, Spain) . Please include the following in any published material that makes use of this data: "Based on observations made with the JAST80 telescope for the J-PLUS project at the Observatorio Astrofísico de Javalambre, in Teruel, owned, managed and operated by the Centro de Estudios de Física del Cosmos de Aragón." + +----- + +**J-PLUS-DR2** https://archive.cefca.es/catalogues/jplus-dr2/tap_async.html J-PLUS DR2 Catalogue (July, 2020) + +----- + +J-PLUS DR2 Catalogue (July, 2020) is based on scientific images in 12 filters collected from November 2015 to February 2020 covering a total area of ~2000 square degrees. The Javalambre Photometric Local Universe Survey (J-PLUS) is an ongoing 12-band photometric optical survey, observing thousands of square degrees of the Northern Hemisphere from the dedicated JAST80 telescope at the Observatorio Astrofísico de Javalambre (OAJ, Teruel, Spain) . Please include the following in any published material that makes use of this data: "Based on observations made with the JAST80 telescope for the J-PLUS project at the Observatorio Astrofísico de Javalambre, in Teruel, owned, managed and operated by the Centro de Estudios de Física del Cosmos de Aragón." + +----- + +**J-PLUS-DR3** https://archive.cefca.es/catalogues/jplus-dr3/tap_async.html J-PLUS DR3 Catalogue (July, 2022) + +----- + +J-PLUS DR3 Catalogue (July, 2022) is based on scientific images in 12 filters collected from November 2015 to February 2022 covering a total area of ~3000 square degrees. The Javalambre Photometric Local Universe Survey (J-PLUS) is an ongoing 12-band photometric optical survey, observing thousands of square degrees of the Northern Hemisphere from the dedicated JAST80 telescope at the Observatorio Astrofísico de Javalambre (OAJ, Teruel, Spain) . Please include the following in any published material that makes use of this data: "Based on observations made with the JAST80 telescope for the J-PLUS project at the Observatorio Astrofísico de Javalambre, in Teruel, owned, managed and operated by the Centro de Estudios de Física del Cosmos de Aragón." + +----- + +**MINJPASPDR201912** https://archive.cefca.es/catalogues/minijpas-pdr201912/tap_async.html MINIJ-PAS PDR201912 Catalogue (December, 2019) + +----- + +MINIJ-PAS PDR201912 Catalogue (December, 2019) is based on scientific images in 60 filters covering a total area of ~1 square degree. MiniJ-PAS is a 60-band photometric optical survey based on images collected by the JST250 telescope and the Pathfinder instrument at the Observatorio Astrofísico de Javalambre (OAJ, Teruel, Spain) . Please include the following in any published material that makes use of this data: "Based on observations made with the JST250 telescope and PathFinder camera for Mini J-PAS project at the Observatorio Astrofísico de Javalambre, in Teruel, owned, managed and operated by the Centro de Estudios de Física del Cosmos de Aragón." + +----- + +**ChiVO TAP** https://vo.chivo.cl/tap ChiVO TAP service + +----- + +The ChiVO's TAP end point. The Table Access +Protocol (TAP) lets you execute queries against our database tables, +inspect various metadata, and upload your own data. It is thus the +VO's premier way to access public data holdings. + +Tables exposed through this endpoint include: alma_fits from the alma_fits schema, m83_obs from the m83_test1 schema, m83_obs from the m83_data schema, hd163296band6 from the hd163296band6 schema, columns, groups, key_columns, keys, schemas, tables from the tap_schema schema, main from the arihip schema, main from the tgas schema, dr1 from the gaia schema, data from the openngc schema, emptyobscore, obscore from the ivoa schema. + +----- + +**CDA** https://cda.harvard.edu/cxctap Chandra X-ray Observatory Data Archive + +----- + +The Chandra X-ray Observatory is the U.S. follow-on to the Einstein +Observatory. Chandra was formerly known as AXAF, the Advanced X-ray +Astrophysics Facility, but renamed by NASA in December, 1998. +Originally three instruments and a high-resolution mirror carried in +one spacecraft, the project was reworked in 1992 and 1993. The Chandra +spacecraft carries a high resolution mirror, two imaging detectors, +and two sets of transmission gratings. Important Chandra features are: +an order of magnitude improvement in spatial resolution, good +sensitivity from 0.1 to 10 keV, and the capability for high spectral +resolution observations over most of this range. + +----- + +**CSC** http://cda.cfa.harvard.edu/csctap Chandra Source Catalog + +----- + +The Chandra X-ray Observatory is the U.S. follow-on to the Einstein +Observatory and one of NASA"s Great Observatories. +Chandra was formerly known as AXAF, the Advanced X-ray +Astrophysics Facility, but renamed by NASA in December, 1998. +Originally three instruments and a high-resolution mirror carried in +one spacecraft, the project was reworked in 1992 and 1993. The Chandra +spacecraft carries a high resolution mirror, two imaging detectors, +and two sets of transmission gratings. Important Chandra features are: +an order of magnitude improvement in spatial resolution, good +sensitivity from 0.1 to 10 keV, and the capability for high spectral +resolution observations over most of this range. + +The Chandra Source Catalog (CSC) includes information about X-ray +sources detected in observations obtained using the Chandra X-ray Observatory. +Release 2.0 of the catalog includes 317,167 point, compact, and extended +sources detected in ACIS and HRC-I imaging observations released +publicly prior to the end of 2014. + +Observed source positions and multi-band count rates are reported, as +well as numerous derived spatial, photometric, spectral, and temporal +calibrated source properties that may be compared with data obtained +by other telescopes. Each record includes the best estimates of the +properties of a source based on data extracted from all observations +in which the source was detected. + +The Chandra Source Catalog is extracted from the CXC"s Chandra Data +Archive (CDA). The CXC should be acknowledged as the source of Chandra data. + +For detailed information on the Chandra Observatory and datasets see: + +http://cxc.harvard.edu/ for general Chandra information; + +http://cxc.harvard.edu/cda/ for the Chandra Data Archive; + +http://cxc.harvard.edu/csc/ for Chandra Source Catalog information. + +----- + +**CSCR1** http://cda.cfa.harvard.edu/csc1tap Chandra Source Catalog Release 1 + +----- + +The Chandra X-ray Observatory is the U.S. follow-on to the Einstein +Observatory and one of NASA"s Great Observatories. +Chandra was formerly known as AXAF, the Advanced X-ray +Astrophysics Facility, but renamed by NASA in December, 1998. +Originally three instruments and a high-resolution mirror carried in +one spacecraft, the project was reworked in 1992 and 1993. The Chandra +spacecraft carries a high resolution mirror, two imaging detectors, +and two sets of transmission gratings. Important Chandra features are: +an order of magnitude improvement in spatial resolution, good +sensitivity from 0.1 to 10 keV, and the capability for high spectral +resolution observations over most of this range. + +The Chandra Source Catalog (CSC) includes information about X-ray +sources detected in observations obtained using the Chandra X-ray +Observatory. Release 1.1 of the catalog includes about 138,000 point +and compact sources with observed spatial extents less than ~30 arcsec +detected in a subset of ACIS and HRC-I imaging observations released +publicly prior to the end of 2009. + +Observed source positions and multi-band count rates are reported, as +well as numerous derived spatial, photometric, spectral, and temporal +calibrated source properties that may be compared with data obtained +by other telescopes. Each record includes the best estimates of the +properties of a source based on data extracted from all observations +in which the source was detected. + +The Chandra Source Catalog is extracted from the CXC"s Chandra Data +Archive (CDA). The CXC should be acknowledged as the source of Chandra data. + +For detailed information on the Chandra Observatory and datasets see: + +http://cxc.harvard.edu/ for general Chandra information; + +http://cxc.harvard.edu/cda/ for the Chandra Data Archive; + +http://cxc.harvard.edu/csc/ for Chandra Source Catalog information. + +----- + +**CSCR2** http://cda.cfa.harvard.edu/csc2tap Chandra Source Catalog Release 2 + +----- + +The Chandra X-ray Observatory is the U.S. follow-on to the Einstein +Observatory and one of NASA"s Great Observatories. +Chandra was formerly known as AXAF, the Advanced X-ray +Astrophysics Facility, but renamed by NASA in December, 1998. +Originally three instruments and a high-resolution mirror carried in +one spacecraft, the project was reworked in 1992 and 1993. The Chandra +spacecraft carries a high resolution mirror, two imaging detectors, +and two sets of transmission gratings. Important Chandra features are: +an order of magnitude improvement in spatial resolution, good +sensitivity from 0.1 to 10 keV, and the capability for high spectral +resolution observations over most of this range. + +The Chandra Source Catalog (CSC) includes information about X-ray +sources detected in observations obtained using the Chandra X-ray +Observatory. Release 2.0 of the catalog includes 317,167 point, +compact, and extended sources detected in ACIS and HRC-I imaging +observations released publicly prior to the end of 2014. + +Observed source positions and multi-band count rates are reported, as +well as numerous derived spatial, photometric, spectral, and temporal +calibrated source properties that may be compared with data obtained +by other telescopes. Each record includes the best estimates of the +properties of a source based on data extracted from all observations +in which the source was detected. + +The Chandra Source Catalog is extracted from the CXC"s Chandra Data +Archive (CDA). The CXC should be acknowledged as the source of Chandra data. + +For detailed information on the Chandra Observatory and datasets see: + +http://cxc.harvard.edu/ for general Chandra information; + +http://cxc.harvard.edu/cda/ for the Chandra Data Archive; + +http://cxc.harvard.edu/csc/ for Chandra Source Catalog information. + +----- + +**Hubble/TAP** https://hst.esac.esa.int/tap-server/tap European HST Archive TAP + +----- + +This service provides access to observations and catalogs from the ESA Hubble Space Observatory mission hosted at the ESAC Science Data Centre + +----- + +**ESASky** https://sky.esa.int/esasky-tap/tap ESASky TAP + +----- + +ESASky is a science-driven discovery portal providing full access to the entire sky as observed with all ESA Space astronomy missions. This service provides access to catalogues, observations, and spectra hosted at the ESAC Science Data Centre. + +----- + +**ESASky Legacy** https://esaskylegacy.esac.esa.int/esasky-legacy-sl-tap/tap ESASky Legacy TAP + +----- + +ESASky legacy is a TAP service to provide the community with complete, self-standing catalogues and data collections from ESA legacy astronomy missions, ensuring long-term preservation. This service provides a unified access to data products and their successors that can be exposed by ESASky. + +----- + +**GAIA** https://gea.esac.esa.int/tap-server/tap Gaia TAP + +----- + +This service provides access to catalogues generated by the ESA Gaia mission hosted at the ESAC Science Data Centre + +----- + +**HREDA** https://hreda.esac.esa.int/hreda-sl-tap/tap HREDA TAP + +----- + +This service provides access to investigation for the HREDA archive hosted at the ESAC Science Data Centre + +----- + +**Herschel** http://archives.esac.esa.int/hsa/whsa-tap-server/tap Herschel TAP + +----- + +This service provides access to observations and catalogs from the ESA Herschel Space Observatory mission hosted at the ESAC Science Data Centre + +----- + +**ISLA** https://isla.esac.esa.int/tap/tap Integral Science Archive TAP + +----- + +ObsLocTAP and Integral observation resources + +----- + +**ISO TAP** http://nida.esac.esa.int/nida-sl-tap/tap Infrared Space Observatory (ISO) TAP + +----- + +This service provides access to tables and catalogues of the ISO ESA mission. + The Infrared Space Observatory (ISO) was the world's first true orbiting infrared observatory. Equipped with four highly-sophisticated and versatile scientific instruments, it was launched by Ariane in November 1995 and provided astronomers world-wide with a facility of unprecedented sensitivity and capabilities for a detailed exploration of the Universe at infrared wavelengths. The two spectrometers (SWS and LWS), a camera (ISOCAM) and an imaging photo-polarimeter (ISOPHOT) jointly covered wavelengths from 2.5 to around 240 microns with spatial resolutions ranging from 1.5 arcseconds (at the shortest wavelengths) to 90 arcseconds (at the longer wavelengths). Its 60 cm diameter telescope was cooled by superfluid liquid helium to temperatures of 2-4 K. + +----- + +**JWST** https://jwst.esac.esa.int/server/tap European JWST Archive TAP + +----- + +This service provides access to observations from the European JWST Science Archive hosted at the ESAC Science Data Centre. + +----- + +**PSA** https://archives.esac.esa.int/psa-tap/tap ESA PSA EPNTAP + +----- + +This service provides access to observations and catalogs from the ESA Planetary Science Archive hosted at the ESAC Science Data Centre + +----- + +**ESAVO TAP** https://registry.euro-vo.org/regtap/tap EURO-VO Registry TAP + +----- + +This is the TAP service for EuroVO Registry. + +----- + +**XMM** http://nxsa.esac.esa.int/tap-server/tap XMM-Newton data and catalogues + +----- + +This service provides access to XMM-Newton data and catalogues generated by the ESA XMM mission hosted at the ESAC Science Data Centre + +----- + +**FAI NVO DC TAP** http://vo.fai.kz/tap FAI archives TAP service + +----- + +The FAI archives's TAP end point. The Table Access +Protocol (TAP) lets you execute queries against our database tables, +inspect various metadata, and upload your own data. It is thus the +VO's premier way to access public data holdings. + +Tables exposed through this endpoint include: main from the fai50mak schema, main from the fai_agn schema, main from the fai_schmidt_lc schema, main from the grb_photometry schema, obscore from the ivoa schema, columns, groups, key_columns, keys, schemas, tables from the tap_schema schema. + +----- + +**PRSFUB TAP** http://dachs.planet.fu-berlin.de/tap Planetary web services @ FUB TAP service + +----- + +The Planetary web services @ FUB's TAP end point. The Table Access +Protocol (TAP) lets you execute queries against our database tables, +inspect various metadata, and upload your own data. It is thus the +VO's premier way to access public data holdings. + +Tables exposed through this endpoint include: columns, groups, key_columns, keys, schemas, tables from the tap_schema schema, epn_core from the hrsc3nd schema, emptyobscore, obscore from the ivoa schema. + + + + + 10.3847/1538-4357/ac7c74 + + + 10.3847/1538-3881/aabc4f + + + 10.5479/ADS/bib/2008ivoa.spec.1030O + + + 10.1051/0004-6361/201322068 + + + @ARTICLE{ivoa_arch, + author = {Patrick Dowler and Janet Evans and Christophe Arviset and Severin Gaudet and Technical Coordination Group}, + title = {IVOA Architecture}, + url = {https://www.ivoa.net/documents/IVOAArchitecture/20211101/index.html}, + year = {2021}, + } + + + diff --git a/tools/archives/pyvo_integration/test-data/astronomical_archives_gen.loc b/tools/archives/pyvo_integration/test-data/astronomical_archives_gen.loc new file mode 100644 index 00000000..11c4bc6a --- /dev/null +++ b/tools/archives/pyvo_integration/test-data/astronomical_archives_gen.loc @@ -0,0 +1,140 @@ +# Table used for listing astronomical archives TAP service urls +# +0 AIP DaCHS TAP service http://gavo.aip.de/tap +1 MAST STScI CAOM and ObsCore TAP service https://vao.stsci.edu/CAOMTAP/TapService.aspx +2 MAST STScI Hubble Source Catalog Version 3 (HSCv3) TAP service https://vao.stsci.edu/hsctap/TapService.aspx +3 MAST STScI Hubble Source Catalog Version 2 (HSCv2) TAP service http://vao.stsci.edu/hscv2tap/TapService.aspx +4 MAST STScI Hubble Source Catalog Version 3 (HSCv3) TAP service http://vao.stsci.edu/hsctap/TapService.aspx +5 MAST STScI MissionMAST TAP service for HST Science Data https://vao.stsci.edu/MissionMAST/TapService.aspx +6 MAST STScI PanSTARRS 1 DR2 TAP service https://vao.stsci.edu/PS1DR2/TapService.aspx +7 MAST STScI Registry TAP service https://vao.stsci.edu/RegTAP/TapService.aspx +8 MAST STScI CLASSY HLSP TAP service https://vao.stsci.edu/HLSP_CLASSY/TapService.aspx +9 MAST STScI ULLYSES HLSP TAP service https://vao.stsci.edu/HLSP_ULLYSES/TapService.aspx +10 UCL DaCHS server TAP service http://astroweb.projects.phys.ucl.ac.uk:8000/tap +11 The VO @ ASTRON TAP service https://vo.astron.nl/tap +12 AIASCR VO Services TAP service http://vos2.asu.cas.cz/tap +13 CSIRO Australia Telescope Online Archive TAP Service https://atoavo.atnf.csiro.au/tap +14 CSIRO ASKAP Science Data Archive TAP Service https://casda.csiro.au/casda_vo_tools/tap +15 CSIRO Parkes Pulsar Data Archive https://data.csiro.au/psrdavo/tap +16 VESPA PA team server TAP service http://vespa-ae.oma.be/tap +17 ArVO Byurakan TAP service http://arvo-registry.sci.am/tap +18 CADC Archive Query Service https://ws.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/argus +19 CANFAR User Catalogue Query Service https://ws-cadc.canfar.net/youcat +20 Planetary and heliophysics plasma data at CDPP/AMDA http://cdpp-epntap.irap.omp.eu/__system__/tap/run/tap +21 Illumination maps of 67P http://cdpp-epntap.irap.omp.eu/__system__/tap/run/tap +22 Magnetosphere Ionosphere coupling simulation runs http://cdpp-epntap.irap.omp.eu/__system__/tap/run/tap +23 SIMBAD TAP query engine http://simbad.cds.unistra.fr/simbad/sim-tap +24 TAP VizieR query engine http://cdsarc.cds.unistra.fr/saadavizier.tap/tap +25 TAP VizieR query engine http://tapvizier.cds.unistra.fr/TAPVizieR/tap +26 J-PLUS DR1 Catalogue (July, 2018) https://archive.cefca.es/catalogues/jplus-dr1/tap_async.html +27 J-PLUS DR2 Catalogue (July, 2020) https://archive.cefca.es/catalogues/jplus-dr2/tap_async.html +28 J-PLUS DR3 Catalogue (July, 2022) https://archive.cefca.es/catalogues/jplus-dr3/tap_async.html +29 MINIJ-PAS PDR201912 Catalogue (December, 2019) https://archive.cefca.es/catalogues/minijpas-pdr201912/tap_async.html +30 ChiVO TAP service https://vo.chivo.cl/tap +31 Chandra X-ray Observatory Data Archive https://cda.harvard.edu/cxctap +32 Chandra Source Catalog http://cda.cfa.harvard.edu/csctap +33 Chandra Source Catalog Release 1 http://cda.cfa.harvard.edu/csc1tap +34 Chandra Source Catalog Release 2 http://cda.cfa.harvard.edu/csc2tap +35 European HST Archive TAP https://hst.esac.esa.int/tap-server/tap +36 ESASky TAP https://sky.esa.int/esasky-tap/tap +37 ESASky Legacy TAP https://esaskylegacy.esac.esa.int/esasky-legacy-sl-tap/tap +38 Gaia TAP https://gea.esac.esa.int/tap-server/tap +39 HREDA TAP https://hreda.esac.esa.int/hreda-sl-tap/tap +40 Herschel TAP http://archives.esac.esa.int/hsa/whsa-tap-server/tap +41 Integral Science Archive TAP https://isla.esac.esa.int/tap/tap +42 Infrared Space Observatory (ISO) TAP http://nida.esac.esa.int/nida-sl-tap/tap +43 European JWST Archive TAP https://jwst.esac.esa.int/server/tap +44 ESA PSA EPNTAP https://archives.esac.esa.int/psa-tap/tap +45 EURO-VO Registry TAP https://registry.euro-vo.org/regtap/tap +46 XMM-Newton data and catalogues http://nxsa.esac.esa.int/tap-server/tap +47 ESO TAP_CAT: a TAP service to query the astronomical catalogs generated by ESO observers, including the catalogs of the ESO Public Surveys. http://archive.eso.org/tap_cat +48 ESO TAP_OBS: a TAP service to browse and access raw and reduced data, and to query the ambient measurements, of the La Silla Paranal Observatory. http://archive.eso.org/tap_obs +49 FAI archives TAP service http://vo.fai.kz/tap +50 Planetary web services @ FUB TAP service http://dachs.planet.fu-berlin.de/tap +51 Gaia@AIP TAP Service https://gaia.aip.de/tap +52 GEOPS/IPSL DaCHS Server TAP service https://dachs-vo.ipsl.fr/tap +53 Laurino et al 2011 Catalog of WGE photometric redshifts for SDSS candidate qsos and galaxies http://ia2-tap.oats.inaf.it:8080/wgetap +54 IAPS TAP SERVICE http://vo-node1.iaps.inaf.it/tap +55 IAPS TAP SERVICES TAP service http://vo-node1.iaps.inaf.it/tap +56 INAF IA2 Project Resources TAP Service http://archives.ia2.inaf.it/vo/tap/projects +57 IAP - DaCHS Services TAP service http://dorotka.ufa.cas.cz/tap +58 IDOC: Integrated Data and Operating Center TAP service http://idoc-dachs.ias.u-psud.fr/tap +59 IRSA Table Access Protocol (TAP) Service https://irsa.ipac.caltech.edu/TAP +60 JacobsUni EPN TAP service https://epn1.epn-vespa.jacobs-university.de/tap +61 Official ALMA Table Access (EA server at NAOJ) https://almascience.nao.ac.jp/tap +62 Official ALMA Table Access (EU server at ESO) https://almascience.eso.org/tap +63 Official ALMA Table Access (NA server at NRAO) https://almascience.nrao.edu/tap +64 EVN Data Archive TAP service http://evn-vo.jive.eu/tap +65 Catalog Quasars and Active Galactic Nuclei http://jvo.nao.ac.jp/skynode/do/tap/agn +66 JVO ALMA VO Service http://jvo.nao.ac.jp/skynode/do/tap/alma +67 AKARI Far-infrared All-Sky Survey Maps http://jvo.nao.ac.jp/skynode/do/tap/akari +68 AKARI Asteroid Flux Catalogue Version 1 http://jvo.nao.ac.jp/skynode/do/tap/akari/catalog/asteroid_flux +69 AKARI/IRC NIR Low-resolution Spectral Catalogue of Diffuse Sky Patches http://jvo.nao.ac.jp/skynode/do/tap/akari/ivoa/obscore +70 AKARI/IRC NIR Spectral Atlas of Galactic Planetary Nebulae http://jvo.nao.ac.jp/skynode/do/tap/akari/ivoa/obscore +71 AKARI/IRC MIR-S slit-less 9um point source catalogue http://jvo.nao.ac.jp/skynode/do/tap/akari/catalog/slitless_mir_psc +72 AKARI/IRC MIR-S slit-less spectroscopic catalogue http://jvo.nao.ac.jp/skynode/do/tap/akari/ivoa/obscore +73 HALCA VSOP (the VLBI Space Observatory Programme) Correlated Data http://jvo.nao.ac.jp/skynode/do/tap/halca +74 The VSOP (the VLBI Space Observatory Programme) 5 GHz AGN (Active Galactic Nuclei) Survey Program Analysis Data http://jvo.nao.ac.jp/skynode/do/tap/halca_agn +75 Hitomi Master Catalog http://jvo.nao.ac.jp/skynode/do/tap/hitomi +76 Comprehensive Catalogue of Kiso UV-X Galaxies (KUG2000) (Nakajima+ 2010) http://jvo.nao.ac.jp/skynode/do/tap/kug2000 +77 Nobeyama Radio Telescope FITS Archive http://jvo.nao.ac.jp/skynode/do/tap/nobeyama/ +78 SAGA -Stellar Abundances for Galactic Archeology Database- http://jvo.nao.ac.jp/skynode/do/tap/saga +79 Subaru HDS Spectrum data service http://jvo.nao.ac.jp/skynode/do/tap/hds +80 Subaru MOIRCS data service http://jvo.nao.ac.jp/skynode/do/tap/moircs +81 Subaru Suprime-Cam data service http://jvo.nao.ac.jp/skynode/do/tap/spcam +82 Subaru/XMM-Newton Deep Survey v1.0 http://jvo.nao.ac.jp/skynode/do/tap/sxds_v1 +83 KM3NeT Open Data TAP service http://vo.km3net.de/tap +84 KOA Table Access Protocol (TAP) Service https://koa.ipac.caltech.edu/TAP +85 TAP server for Konkoly TAP service http://nadir.konkoly.hu:80/tap +86 KSB-ROB TAP service http://vo-tap.oma.be/tap +87 CeSAM Virtual Observatory Server TAP service https://vo.lam.fr/tap +88 LATMOS DaCHS server TAP service http://vo.projet.latmos.ipsl.fr/tap +89 KIS Science Data Centre TAP service http://dachs.sdc.leibniz-kis.de/tap +90 DaCHS Server for LMD TAP service http://vo.lmd.jussieu.fr/tap +91 MUSE-Wide survey TAP Service https://musewide.aip.de/tap +92 HEASARC Xamin Catalog Interface https://heasarc.gsfc.nasa.gov/xamin/vo/tap +93 MACHO TAP https://machotap.asvo.nci.org.au/ncitap/tap +94 SkyMapper TAP https://api.skymapper.nci.org.au/public/tap +95 WiggleZ Final Data Release TAP https://wiggleztap.asvo.nci.org.au/ncitap/tap +96 NED TAP VO Service https://ned.ipac.caltech.edu/tap/ +97 OCA Data Center TAP service https://dachs.oca.eu/tap +98 GAVO Data Center TAP service http://dc.zah.uni-heidelberg.de/tap +99 OSUG Virtual Observatory TAP service http://osug-vo.osug.fr:8080/tap +100 CLIMSO coronagraphs at pic du midi de Bigorre http://climso.irap.omp.eu/__system__/tap/run/tap +101 Synthetic Spectra Convolution Service https://pollux.oreme.org/vo/datalink/speconvol +102 PADC TAP Server on voparis-tap-astro.obspm.fr TAP service http://voparis-tap-astro.obspm.fr/tap +103 PADC TAP Server on voparis-tap-he.obspm.fr TAP service http://voparis-tap-he.obspm.fr/tap +104 PADC TAP Server on voparis-tap-helio.obspm.fr TAP service http://voparis-tap-helio.obspm.fr/tap +105 PADC TAP Server on voparis-tap-maser.obspm.fr TAP service http://voparis-tap-maser.obspm.fr/tap +106 PADC TAP Server on voparis-tap-planeto.obspm.fr TAP service http://voparis-tap-planeto.obspm.fr/tap +107 PDS-PPI VO server TAP service https://vo-pds-ppi.igpp.ucla.edu/tap +108 Southern Photometric Local Universe Survey https://splus.cloud/public-TAP/tap +109 OCA Data Center TAP service https://dachs.oca.eu/tap +110 PADC TAP Server on voparis-rr.obspm.fr TAP service http://voparis-rr.obspm.fr:80/tap +111 PVOL EPN-TAP TAP service http://pvol2.ehu.eus:8083/tap +112 Cross section values TAP service http://spectrum.iaa.es/tap +113 VO HPSL datacenter TAP service http://vespa.cbk.waw.pl/tap +114 SSDC TAP Service https://tools.ssdc.asi.it/TAPSSDC-1.1 +115 ARI's Gaia TAP Service https://gaia.ari.uni-heidelberg.de/tap +116 Gaia catalog release 2 https://gaia.obspm.fr/tap-server/tap +117 H.E.S.S. DL3 public test data release 1 http://voparis-tap-astro.obspm.fr/__system__/tap/run/tap +118 Nancay Decameter Array observation database http://vogate.obs-nancay.fr/__system__/tap/run/tap +119 VOXastro Data Center TAP service http://rcsed-vo.sai.msu.ru/tap +120 6dF Galaxy Survey Data Release 2 http://wfaudata.roe.ac.uk/6df-dsa/TAP +121 6dF Galaxy Survey Data Release 3 http://wfaudata.roe.ac.uk/6dfdr3-dsa/TAP +122 ATLAS DR1 - VST ATLAS Survey http://wfaudata.roe.ac.uk/atlasDR1-dsa/TAP +123 GALEX Release 6 http://wfaudata.roe.ac.uk/galexgr6-dsa/TAP +124 OSA VST ATLAS Survey http://tap.roe.ac.uk/osa +125 Personal SuperCOSMOS Science Archive (SSA) http://wfaudata.roe.ac.uk/pssa-dsa/TAP +126 SuperCOSMOS Science Archive (SSA) http://wfaudata.roe.ac.uk/ssa-dsa/TAP +127 SuperCOSMOS Science Archive (SSA) http://tap.roe.ac.uk/ssa +128 2MASS Photometric Redshift catalogue (2MPZ) http://wfaudata.roe.ac.uk/twompz-dsa/TAP +129 UHS DR2 - UKIRT Hemisphere Survey Data Release 2 http://wfaudata.roe.ac.uk/uhsDR2/TAP +130 VHS DR4 - VISTA Hemisphere Survey Data Release 4 http://wfaudata.roe.ac.uk/vhsDR4-dsa/TAP +131 VSA - VISTA Science Archive http://tap.roe.ac.uk/vsa +132 WSA - WFCAM Science Archive http://tap.roe.ac.uk/wsa +133 XMM-Newton Serendipitous Source Catalogue (2XMM) http://wfaudata.roe.ac.uk/xmm-dsa/TAP +134 APPLAUSE - Archives of Photographic PLates for Astronomical USE TAP Service https://www.plate-archive.org/tap +135 TAP interface of the 3XMM-DR5 XMM-Newton Catalogue http://xcatdb.unistra.fr/3xmmdr5/tap +136 TAP interface of the 3XMM-dr7 XMM-Newton Catalogue http://xcatdb.unistra.fr/3xmmdr7/tap +137 TAP interface of the 4XMM dr13 XMM-Newton Catalogue https://xcatdb.unistra.fr/xtapdb diff --git a/tools/archives/pyvo_integration/tool-data/astronomical_archives_gen.loc b/tools/archives/pyvo_integration/tool-data/astronomical_archives_gen.loc new file mode 100644 index 00000000..11c4bc6a --- /dev/null +++ b/tools/archives/pyvo_integration/tool-data/astronomical_archives_gen.loc @@ -0,0 +1,140 @@ +# Table used for listing astronomical archives TAP service urls +# +0 AIP DaCHS TAP service http://gavo.aip.de/tap +1 MAST STScI CAOM and ObsCore TAP service https://vao.stsci.edu/CAOMTAP/TapService.aspx +2 MAST STScI Hubble Source Catalog Version 3 (HSCv3) TAP service https://vao.stsci.edu/hsctap/TapService.aspx +3 MAST STScI Hubble Source Catalog Version 2 (HSCv2) TAP service http://vao.stsci.edu/hscv2tap/TapService.aspx +4 MAST STScI Hubble Source Catalog Version 3 (HSCv3) TAP service http://vao.stsci.edu/hsctap/TapService.aspx +5 MAST STScI MissionMAST TAP service for HST Science Data https://vao.stsci.edu/MissionMAST/TapService.aspx +6 MAST STScI PanSTARRS 1 DR2 TAP service https://vao.stsci.edu/PS1DR2/TapService.aspx +7 MAST STScI Registry TAP service https://vao.stsci.edu/RegTAP/TapService.aspx +8 MAST STScI CLASSY HLSP TAP service https://vao.stsci.edu/HLSP_CLASSY/TapService.aspx +9 MAST STScI ULLYSES HLSP TAP service https://vao.stsci.edu/HLSP_ULLYSES/TapService.aspx +10 UCL DaCHS server TAP service http://astroweb.projects.phys.ucl.ac.uk:8000/tap +11 The VO @ ASTRON TAP service https://vo.astron.nl/tap +12 AIASCR VO Services TAP service http://vos2.asu.cas.cz/tap +13 CSIRO Australia Telescope Online Archive TAP Service https://atoavo.atnf.csiro.au/tap +14 CSIRO ASKAP Science Data Archive TAP Service https://casda.csiro.au/casda_vo_tools/tap +15 CSIRO Parkes Pulsar Data Archive https://data.csiro.au/psrdavo/tap +16 VESPA PA team server TAP service http://vespa-ae.oma.be/tap +17 ArVO Byurakan TAP service http://arvo-registry.sci.am/tap +18 CADC Archive Query Service https://ws.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/argus +19 CANFAR User Catalogue Query Service https://ws-cadc.canfar.net/youcat +20 Planetary and heliophysics plasma data at CDPP/AMDA http://cdpp-epntap.irap.omp.eu/__system__/tap/run/tap +21 Illumination maps of 67P http://cdpp-epntap.irap.omp.eu/__system__/tap/run/tap +22 Magnetosphere Ionosphere coupling simulation runs http://cdpp-epntap.irap.omp.eu/__system__/tap/run/tap +23 SIMBAD TAP query engine http://simbad.cds.unistra.fr/simbad/sim-tap +24 TAP VizieR query engine http://cdsarc.cds.unistra.fr/saadavizier.tap/tap +25 TAP VizieR query engine http://tapvizier.cds.unistra.fr/TAPVizieR/tap +26 J-PLUS DR1 Catalogue (July, 2018) https://archive.cefca.es/catalogues/jplus-dr1/tap_async.html +27 J-PLUS DR2 Catalogue (July, 2020) https://archive.cefca.es/catalogues/jplus-dr2/tap_async.html +28 J-PLUS DR3 Catalogue (July, 2022) https://archive.cefca.es/catalogues/jplus-dr3/tap_async.html +29 MINIJ-PAS PDR201912 Catalogue (December, 2019) https://archive.cefca.es/catalogues/minijpas-pdr201912/tap_async.html +30 ChiVO TAP service https://vo.chivo.cl/tap +31 Chandra X-ray Observatory Data Archive https://cda.harvard.edu/cxctap +32 Chandra Source Catalog http://cda.cfa.harvard.edu/csctap +33 Chandra Source Catalog Release 1 http://cda.cfa.harvard.edu/csc1tap +34 Chandra Source Catalog Release 2 http://cda.cfa.harvard.edu/csc2tap +35 European HST Archive TAP https://hst.esac.esa.int/tap-server/tap +36 ESASky TAP https://sky.esa.int/esasky-tap/tap +37 ESASky Legacy TAP https://esaskylegacy.esac.esa.int/esasky-legacy-sl-tap/tap +38 Gaia TAP https://gea.esac.esa.int/tap-server/tap +39 HREDA TAP https://hreda.esac.esa.int/hreda-sl-tap/tap +40 Herschel TAP http://archives.esac.esa.int/hsa/whsa-tap-server/tap +41 Integral Science Archive TAP https://isla.esac.esa.int/tap/tap +42 Infrared Space Observatory (ISO) TAP http://nida.esac.esa.int/nida-sl-tap/tap +43 European JWST Archive TAP https://jwst.esac.esa.int/server/tap +44 ESA PSA EPNTAP https://archives.esac.esa.int/psa-tap/tap +45 EURO-VO Registry TAP https://registry.euro-vo.org/regtap/tap +46 XMM-Newton data and catalogues http://nxsa.esac.esa.int/tap-server/tap +47 ESO TAP_CAT: a TAP service to query the astronomical catalogs generated by ESO observers, including the catalogs of the ESO Public Surveys. http://archive.eso.org/tap_cat +48 ESO TAP_OBS: a TAP service to browse and access raw and reduced data, and to query the ambient measurements, of the La Silla Paranal Observatory. http://archive.eso.org/tap_obs +49 FAI archives TAP service http://vo.fai.kz/tap +50 Planetary web services @ FUB TAP service http://dachs.planet.fu-berlin.de/tap +51 Gaia@AIP TAP Service https://gaia.aip.de/tap +52 GEOPS/IPSL DaCHS Server TAP service https://dachs-vo.ipsl.fr/tap +53 Laurino et al 2011 Catalog of WGE photometric redshifts for SDSS candidate qsos and galaxies http://ia2-tap.oats.inaf.it:8080/wgetap +54 IAPS TAP SERVICE http://vo-node1.iaps.inaf.it/tap +55 IAPS TAP SERVICES TAP service http://vo-node1.iaps.inaf.it/tap +56 INAF IA2 Project Resources TAP Service http://archives.ia2.inaf.it/vo/tap/projects +57 IAP - DaCHS Services TAP service http://dorotka.ufa.cas.cz/tap +58 IDOC: Integrated Data and Operating Center TAP service http://idoc-dachs.ias.u-psud.fr/tap +59 IRSA Table Access Protocol (TAP) Service https://irsa.ipac.caltech.edu/TAP +60 JacobsUni EPN TAP service https://epn1.epn-vespa.jacobs-university.de/tap +61 Official ALMA Table Access (EA server at NAOJ) https://almascience.nao.ac.jp/tap +62 Official ALMA Table Access (EU server at ESO) https://almascience.eso.org/tap +63 Official ALMA Table Access (NA server at NRAO) https://almascience.nrao.edu/tap +64 EVN Data Archive TAP service http://evn-vo.jive.eu/tap +65 Catalog Quasars and Active Galactic Nuclei http://jvo.nao.ac.jp/skynode/do/tap/agn +66 JVO ALMA VO Service http://jvo.nao.ac.jp/skynode/do/tap/alma +67 AKARI Far-infrared All-Sky Survey Maps http://jvo.nao.ac.jp/skynode/do/tap/akari +68 AKARI Asteroid Flux Catalogue Version 1 http://jvo.nao.ac.jp/skynode/do/tap/akari/catalog/asteroid_flux +69 AKARI/IRC NIR Low-resolution Spectral Catalogue of Diffuse Sky Patches http://jvo.nao.ac.jp/skynode/do/tap/akari/ivoa/obscore +70 AKARI/IRC NIR Spectral Atlas of Galactic Planetary Nebulae http://jvo.nao.ac.jp/skynode/do/tap/akari/ivoa/obscore +71 AKARI/IRC MIR-S slit-less 9um point source catalogue http://jvo.nao.ac.jp/skynode/do/tap/akari/catalog/slitless_mir_psc +72 AKARI/IRC MIR-S slit-less spectroscopic catalogue http://jvo.nao.ac.jp/skynode/do/tap/akari/ivoa/obscore +73 HALCA VSOP (the VLBI Space Observatory Programme) Correlated Data http://jvo.nao.ac.jp/skynode/do/tap/halca +74 The VSOP (the VLBI Space Observatory Programme) 5 GHz AGN (Active Galactic Nuclei) Survey Program Analysis Data http://jvo.nao.ac.jp/skynode/do/tap/halca_agn +75 Hitomi Master Catalog http://jvo.nao.ac.jp/skynode/do/tap/hitomi +76 Comprehensive Catalogue of Kiso UV-X Galaxies (KUG2000) (Nakajima+ 2010) http://jvo.nao.ac.jp/skynode/do/tap/kug2000 +77 Nobeyama Radio Telescope FITS Archive http://jvo.nao.ac.jp/skynode/do/tap/nobeyama/ +78 SAGA -Stellar Abundances for Galactic Archeology Database- http://jvo.nao.ac.jp/skynode/do/tap/saga +79 Subaru HDS Spectrum data service http://jvo.nao.ac.jp/skynode/do/tap/hds +80 Subaru MOIRCS data service http://jvo.nao.ac.jp/skynode/do/tap/moircs +81 Subaru Suprime-Cam data service http://jvo.nao.ac.jp/skynode/do/tap/spcam +82 Subaru/XMM-Newton Deep Survey v1.0 http://jvo.nao.ac.jp/skynode/do/tap/sxds_v1 +83 KM3NeT Open Data TAP service http://vo.km3net.de/tap +84 KOA Table Access Protocol (TAP) Service https://koa.ipac.caltech.edu/TAP +85 TAP server for Konkoly TAP service http://nadir.konkoly.hu:80/tap +86 KSB-ROB TAP service http://vo-tap.oma.be/tap +87 CeSAM Virtual Observatory Server TAP service https://vo.lam.fr/tap +88 LATMOS DaCHS server TAP service http://vo.projet.latmos.ipsl.fr/tap +89 KIS Science Data Centre TAP service http://dachs.sdc.leibniz-kis.de/tap +90 DaCHS Server for LMD TAP service http://vo.lmd.jussieu.fr/tap +91 MUSE-Wide survey TAP Service https://musewide.aip.de/tap +92 HEASARC Xamin Catalog Interface https://heasarc.gsfc.nasa.gov/xamin/vo/tap +93 MACHO TAP https://machotap.asvo.nci.org.au/ncitap/tap +94 SkyMapper TAP https://api.skymapper.nci.org.au/public/tap +95 WiggleZ Final Data Release TAP https://wiggleztap.asvo.nci.org.au/ncitap/tap +96 NED TAP VO Service https://ned.ipac.caltech.edu/tap/ +97 OCA Data Center TAP service https://dachs.oca.eu/tap +98 GAVO Data Center TAP service http://dc.zah.uni-heidelberg.de/tap +99 OSUG Virtual Observatory TAP service http://osug-vo.osug.fr:8080/tap +100 CLIMSO coronagraphs at pic du midi de Bigorre http://climso.irap.omp.eu/__system__/tap/run/tap +101 Synthetic Spectra Convolution Service https://pollux.oreme.org/vo/datalink/speconvol +102 PADC TAP Server on voparis-tap-astro.obspm.fr TAP service http://voparis-tap-astro.obspm.fr/tap +103 PADC TAP Server on voparis-tap-he.obspm.fr TAP service http://voparis-tap-he.obspm.fr/tap +104 PADC TAP Server on voparis-tap-helio.obspm.fr TAP service http://voparis-tap-helio.obspm.fr/tap +105 PADC TAP Server on voparis-tap-maser.obspm.fr TAP service http://voparis-tap-maser.obspm.fr/tap +106 PADC TAP Server on voparis-tap-planeto.obspm.fr TAP service http://voparis-tap-planeto.obspm.fr/tap +107 PDS-PPI VO server TAP service https://vo-pds-ppi.igpp.ucla.edu/tap +108 Southern Photometric Local Universe Survey https://splus.cloud/public-TAP/tap +109 OCA Data Center TAP service https://dachs.oca.eu/tap +110 PADC TAP Server on voparis-rr.obspm.fr TAP service http://voparis-rr.obspm.fr:80/tap +111 PVOL EPN-TAP TAP service http://pvol2.ehu.eus:8083/tap +112 Cross section values TAP service http://spectrum.iaa.es/tap +113 VO HPSL datacenter TAP service http://vespa.cbk.waw.pl/tap +114 SSDC TAP Service https://tools.ssdc.asi.it/TAPSSDC-1.1 +115 ARI's Gaia TAP Service https://gaia.ari.uni-heidelberg.de/tap +116 Gaia catalog release 2 https://gaia.obspm.fr/tap-server/tap +117 H.E.S.S. DL3 public test data release 1 http://voparis-tap-astro.obspm.fr/__system__/tap/run/tap +118 Nancay Decameter Array observation database http://vogate.obs-nancay.fr/__system__/tap/run/tap +119 VOXastro Data Center TAP service http://rcsed-vo.sai.msu.ru/tap +120 6dF Galaxy Survey Data Release 2 http://wfaudata.roe.ac.uk/6df-dsa/TAP +121 6dF Galaxy Survey Data Release 3 http://wfaudata.roe.ac.uk/6dfdr3-dsa/TAP +122 ATLAS DR1 - VST ATLAS Survey http://wfaudata.roe.ac.uk/atlasDR1-dsa/TAP +123 GALEX Release 6 http://wfaudata.roe.ac.uk/galexgr6-dsa/TAP +124 OSA VST ATLAS Survey http://tap.roe.ac.uk/osa +125 Personal SuperCOSMOS Science Archive (SSA) http://wfaudata.roe.ac.uk/pssa-dsa/TAP +126 SuperCOSMOS Science Archive (SSA) http://wfaudata.roe.ac.uk/ssa-dsa/TAP +127 SuperCOSMOS Science Archive (SSA) http://tap.roe.ac.uk/ssa +128 2MASS Photometric Redshift catalogue (2MPZ) http://wfaudata.roe.ac.uk/twompz-dsa/TAP +129 UHS DR2 - UKIRT Hemisphere Survey Data Release 2 http://wfaudata.roe.ac.uk/uhsDR2/TAP +130 VHS DR4 - VISTA Hemisphere Survey Data Release 4 http://wfaudata.roe.ac.uk/vhsDR4-dsa/TAP +131 VSA - VISTA Science Archive http://tap.roe.ac.uk/vsa +132 WSA - WFCAM Science Archive http://tap.roe.ac.uk/wsa +133 XMM-Newton Serendipitous Source Catalogue (2XMM) http://wfaudata.roe.ac.uk/xmm-dsa/TAP +134 APPLAUSE - Archives of Photographic PLates for Astronomical USE TAP Service https://www.plate-archive.org/tap +135 TAP interface of the 3XMM-DR5 XMM-Newton Catalogue http://xcatdb.unistra.fr/3xmmdr5/tap +136 TAP interface of the 3XMM-dr7 XMM-Newton Catalogue http://xcatdb.unistra.fr/3xmmdr7/tap +137 TAP interface of the 4XMM dr13 XMM-Newton Catalogue https://xcatdb.unistra.fr/xtapdb diff --git a/tools/archives/pyvo_integration/tool_data_table_conf.xml.sample b/tools/archives/pyvo_integration/tool_data_table_conf.xml.sample new file mode 100644 index 00000000..cb8ac2e2 --- /dev/null +++ b/tools/archives/pyvo_integration/tool_data_table_conf.xml.sample @@ -0,0 +1,7 @@ + + + + id, name, value + +
+ \ No newline at end of file