Skip to content

Commit

Permalink
use pds api instead of older method
Browse files Browse the repository at this point in the history 
Change the validate.properties to use the registry-api. Adjusted code to get all data then walk through the results as lists and maps of the JSON response object.
  • Loading branch information
Al Niessner authored and Al Niessner committed Aug 30, 2023
1 parent 5425b07 commit 0dbd9bc
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 81 deletions.
142 changes: 64 additions & 78 deletions src/main/java/gov/nasa/pds/validate/ValidateLauncher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
Expand All @@ -52,6 +54,7 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.TimeZone;
import java.util.stream.Collectors;
import java.util.stream.Stream;
Expand All @@ -73,18 +76,13 @@
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.Priority;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.ls.LSInput;
import org.xml.sax.InputSource;
import org.xml.sax.SAXParseException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
Expand Down Expand Up @@ -505,49 +503,43 @@ else if (Flag.MODEL.getShortName().equals(o.getOpt())) {
}
}

@SuppressWarnings("unchecked")
private void getLatestJsonContext() {

String url = ToolInfo.getSearchURL();
List<ValidationProblem> pList = new ArrayList<>();
ObjectMapper mapper = new ObjectMapper();
String base = ToolInfo.getSearchURL();
String endpoint = ToolInfo.getEndpoint();
String query = ToolInfo.getQuery();

SolrClient client = new HttpSolrClient.Builder(url).build();
SolrQuery solrQuery = new SolrQuery(query);
solrQuery.setRequestHandler("/" + endpoint);
solrQuery.setStart(0);
solrQuery.setParam("fl",
"identifier, " + "version_id, " + "data_product_type, " + "target_name, "
+ "instrument_name, " + "instrument_host_name, " + "resource_name, "
+ "investigation_name, " + "target_type, " + "instrument_type, "
+ "instrument_host_type, " + "resource_type, " + "investigation_type, "
+ "facility_name, facility_type, airborne_name, airborne_type");

QueryResponse resp;
List<ValidationProblem> pList = new ArrayList<>();
URL url = null;
try {
resp = client.query(solrQuery);
SolrDocumentList res = resp.getResults();
solrQuery.setRows((int) res.getNumFound());
resp = client.query(solrQuery);
res = resp.getResults();
parseJsonObjectWriteTofile(res);

client.close();
int total = 0;
List<Map<String,Object>> contexts = new ArrayList<Map<String,Object>>();
do {
url = new URL(base + "/" + endpoint + "?" + URLEncoder.encode(query, StandardCharsets.UTF_8) + "&start=" + contexts.size());
Scanner reader = new Scanner(url.openStream());
StringBuffer buffer = new StringBuffer();
while (reader.hasNext()) {
buffer.append(reader.next());
}
Map<String, Object> response = mapper.readValue(buffer.toString(), HashMap.class);
total = (Integer)((Map<String,Object>)response.get("summary")).get("hits");
contexts.addAll((List<Map<String,Object>>)response.get("data"));
} while (contexts.size() < total);
parseJsonObjectWriteTofile(contexts);
ValidationProblem p1 =
new ValidationProblem(new ProblemDefinition(ExceptionType.INFO, ProblemType.GENERAL_INFO,
"Successfully updated registered context products config file. "), new URL(url));
"Successfully updated registered context products config file. "), url);
pList.add(p1);
ValidationProblem p2 =
new ValidationProblem(new ProblemDefinition(ExceptionType.INFO, ProblemType.GENERAL_INFO,
res.size() + " registered context products found."), new URL(url));
contexts.size() + " registered context products found."), url);
pList.add(p2);

} catch (SolrServerException | IOException ex) {
} catch (IOException ex) {
try {
ValidationProblem p = new ValidationProblem(new ProblemDefinition(ExceptionType.ERROR,
ProblemType.INTERNAL_ERROR,
"Error connecting to Registry to update registered context products config file. Verify internet connection and try again."),
new URL(url));
url);
report.record(new URI(
System.getProperty("resources.home") + File.separator + ToolInfo.getOutputFileName()),
p);
Expand All @@ -568,16 +560,20 @@ private void getLatestJsonContext() {
}
}

private void parseJsonObjectWriteTofile(SolrDocumentList docs) {
private void parseJsonObjectWriteTofile(List<Map<String,Object>> documents) {
final List<String> empty = Arrays.asList("N/A");
final List<String> fieldNames = Arrays.asList(
"pds:Airborne.pds",
"pds:Facility.pds",
"pds:Instrument.pds",
"pds:Instrument_Host.pds",
"pds:Investigation.pds",
"pds:Resource.pds",
"pds:Target.pds");
// backup old file
try {
copyFile(registeredProductsFile, new File(System.getProperty("resources.home")
+ File.separator + ToolInfo.getOutputFileName() + ".backup"));
// System.out.println("back up " +
// System.getProperty("resources.home") + "/" +
// ToolInfo.getOutputFileName()
// + " to " + System.getProperty("resources.home") + "/" +
// ToolInfo.getOutputFileName() + ".backup");
} catch (IOException e) {
e.printStackTrace();
}
Expand All @@ -586,46 +582,38 @@ private void parseJsonObjectWriteTofile(SolrDocumentList docs) {
try {
jsonWriter = new JsonWriter(new FileWriter(
System.getProperty("resources.home") + File.separator + ToolInfo.getOutputFileName()));

jsonWriter.setIndent(" ");
jsonWriter.beginObject(); // start Product_Context
jsonWriter.name("Product_Context");
jsonWriter.beginArray();
for (SolrDocument document : docs) {
String id = (String) document.getFirstValue("identifier");
String ver = (String) document.getFirstValue("version_id");
String data_type = (String) document.getFirstValue("data_product_type");
List<Object> names =
(ArrayList<Object>) document.getFieldValues(data_type.toLowerCase() + "_name");
List<Object> types =
(ArrayList<Object>) document.getFieldValues(data_type.toLowerCase() + "_type");

jsonWriter.beginObject(); // start a product

jsonWriter.name("name");
jsonWriter.beginArray();
if (names == null) {
jsonWriter.value("N/A");
} else {
for (Object n : names) {
jsonWriter.value((String) n);
}
}
jsonWriter.endArray();

jsonWriter.name("type");
jsonWriter.beginArray();
if (types == null) {
jsonWriter.value("N/A");
} else {
for (Object t : types) {
jsonWriter.value((String) t);
for (Map<String,Object> document : documents) {
@SuppressWarnings("unchecked")
Map<String,Object> properties = (Map<String,Object>)document.get("properties");
@SuppressWarnings("unchecked")
String lidvid = ((List<String>)properties.get("lidvid")).get(0);
for (String fieldName : fieldNames) {
if (properties.containsKey (fieldName + ":name") || properties.containsKey (fieldName + ":type")) {
@SuppressWarnings("unchecked")
List<Object> names = (List<Object>)properties.getOrDefault(fieldName + ":name", empty);
@SuppressWarnings("unchecked")
List<Object> types = (List<Object>)properties.getOrDefault(fieldName + ":type", empty);
jsonWriter.beginObject(); // start a product
jsonWriter.name("name");
jsonWriter.beginArray();
for (Object n : names) {
jsonWriter.value((String) n);
}
jsonWriter.endArray();
jsonWriter.name("type");
jsonWriter.beginArray();
for (Object t : types) {
jsonWriter.value((String) t);
}
jsonWriter.endArray();
jsonWriter.name("lidvid").value(lidvid);
jsonWriter.endObject(); // end a product
}
}
jsonWriter.endArray();

jsonWriter.name("lidvid").value(id + "::" + ver);
jsonWriter.endObject(); // end a product
}
jsonWriter.endArray();
jsonWriter.endObject(); // end Product_Context
Expand Down Expand Up @@ -672,7 +660,6 @@ private void copyFile(File source, File dest) throws IOException {
*
* @throws ConfigurationException If an error occurred while querying the configuration file.
*/
@SuppressWarnings("unchecked")
public void query(File configuration) throws ConfigurationException {
try {
Configuration config = null;
Expand Down Expand Up @@ -837,7 +824,6 @@ public void setAdditionalPaths(List<String> additionalPaths) throws MalformedURL
LOG.debug("setAdditionalPaths:additionalPaths {},{}", additionalPaths, additionalPaths.size());
this.alternateReferentialPaths.clear();
while (alternateReferentialPaths.remove("")) {

}
for (String pathEntries : additionalPaths) {
LOG.debug("setAdditionalPaths:pathEntries {}", pathEntries);
Expand Down
6 changes: 3 additions & 3 deletions src/main/resources/validate.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ validate.date=${buildNumber}
# TODO: Is this acceptable for validate.copyright?
validate.copyright=\nCopyright 2019, by the California Institute of Technology ("Caltech").\nAll rights reserved.

validate.search_url=https://pds.nasa.gov/services/search
validate.search_url=https://pds.nasa.gov/api
validate.output_file_name=registered_context_products.json
validate.endpoint=search
validate.query=product_class:Product_Context AND -data_class:Resource AND -data_class:PDS_Affiliate
validate.endpoint=search/1/products
validate.query=product_class eq "Product_Context"

0 comments on commit 0dbd9bc

Please sign in to comment.