From e28c177a72e5288a36f4baa7b270e1a8d1e36ef2 Mon Sep 17 00:00:00 2001 From: varontron Date: Wed, 10 Oct 2018 19:48:29 -0400 Subject: [PATCH 01/17] resolves #84 --- yada-war/src/main/webapp/WEB-INF/web.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/yada-war/src/main/webapp/WEB-INF/web.xml b/yada-war/src/main/webapp/WEB-INF/web.xml index 6b42fe15..f8d5e90a 100644 --- a/yada-war/src/main/webapp/WEB-INF/web.xml +++ b/yada-war/src/main/webapp/WEB-INF/web.xml @@ -41,6 +41,10 @@ + + YADA-init From 7d786ff6b0153c4a5cc972dd378018b66eeb2f80 Mon Sep 17 00:00:00 2001 From: varontron Date: Tue, 16 Oct 2018 22:38:27 -0400 Subject: [PATCH 02/17] #87: removed conditions/setters referencing deprecated overargs and added conditions/setters for oauth --- .../main/java/com/novartis/opensource/yada/Service.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/Service.java b/yada-api/src/main/java/com/novartis/opensource/yada/Service.java index 223ea7e8..d1fb20e2 100755 --- a/yada-api/src/main/java/com/novartis/opensource/yada/Service.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/Service.java @@ -308,13 +308,13 @@ public void handleRequest(String referer, Map paraMap) { getYADARequest().setMethod(paraMap.get(YADARequest.PS_METHOD)); } - if (paraMap.get(YADARequest.PL_OVERARGS) != null) + if (paraMap.get(YADARequest.PL_OAUTH) != null) { - setDeprecatedPlugin(paraMap, YADARequest.PL_OVERARGS); + getYADARequest().setOAuth(paraMap.get(YADARequest.PL_OAUTH)); } - if (paraMap.get(YADARequest.PS_OVERARGS) != null) + if (paraMap.get(YADARequest.PS_OAUTH) != null) { - setDeprecatedPlugin(paraMap, YADARequest.PS_OVERARGS); + getYADARequest().setOAuth(paraMap.get(YADARequest.PS_OAUTH)); } if (paraMap.get(YADARequest.PL_BYPASSARGS) != null) { From 785ea8e98e2730a49880b4695f85c9be5bb64178 Mon Sep 17 00:00:00 2001 From: varontron Date: Tue, 16 Oct 2018 22:39:03 -0400 Subject: [PATCH 03/17] #87: bumped version to 8.7.0 --- pom.xml | 2 +- yada-api/pom.xml | 2 +- yada-war/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 58d2a8f4..7e343248 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.novartis.opensource YADA - 8.6.2-SNAPSHOT + 8.7.0-SNAPSHOT YADA http://github.com/Novartis/YADA YADA (Yet Another Data Abstraction) Thin Server Framework diff --git a/yada-api/pom.xml b/yada-api/pom.xml index 3a69a39b..fdd12e6f 100644 --- a/yada-api/pom.xml +++ b/yada-api/pom.xml @@ -4,7 +4,7 @@ com.novartis.opensource YADA - 8.6.2-SNAPSHOT + 8.7.0-SNAPSHOT yada-api yada-api diff --git a/yada-war/pom.xml b/yada-war/pom.xml index 3a618be9..df360de3 100644 --- a/yada-war/pom.xml +++ b/yada-war/pom.xml @@ -4,7 +4,7 @@ com.novartis.opensource YADA - 8.6.2-SNAPSHOT + 8.7.0-SNAPSHOT yada-war yada-war From 7dea1bf38178206ae3017114fe2093c87f174547 Mon Sep 17 00:00:00 2001 From: varontron Date: Tue, 16 Oct 2018 22:40:16 -0400 Subject: [PATCH 04/17] #87: Added oauth params and handlers, removed overargs --- .../novartis/opensource/yada/YADARequest.java | 60 +++++++++++++------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/YADARequest.java b/yada-api/src/main/java/com/novartis/opensource/yada/YADARequest.java index 7ada6ea4..1095c160 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/YADARequest.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/YADARequest.java @@ -423,6 +423,11 @@ public class YADARequest { * No longer deprecated (4.0.0) as of 8.5.0 */ public static final String PS_METHOD = "m"; + /** + * a constant equals to: {@value} + * @since 8.7.0 + */ + public static final String PS_OAUTH = "o"; /** * A constant equal to: {@value} * @since 4.0.0 (Short param aliases were first added in 4.0.0) @@ -667,6 +672,12 @@ public class YADARequest { * No longer deprecated (4.0.0) as of 8.5.0 */ public static final String PL_METHOD = "method"; + /** + * A constant equals to: {@value}. + * + * @since 8.7.0 + */ + public static final String PL_OAUTH = "oauth"; /** * A constant equal to: {@value}. Use {@link YADARequest#PL_BYPASSARGS} instead. * @deprecated as of 4.0.0 @@ -833,6 +844,7 @@ public class YADARequest { map.put(PS_JOIN, PL_JOIN); map.put(PS_LEFTJOIN, PL_LEFTJOIN); map.put(PS_METHOD,PL_METHOD); + map.put(PS_OAUTH,PL_OAUTH); map.put(PS_OVERARGS,PL_OVERARGS); map.put(PS_PAGE,PL_PAGE); map.put(PS_PAGESIZE,PL_PAGESIZE); @@ -876,6 +888,7 @@ public class YADARequest { map.put(PL_LABELS,PL_LABELS); map.put(PL_MAIL,PL_MAIL); map.put(PL_METHOD,PL_METHOD); + map.put(PL_OAUTH,PL_OAUTH); map.put(PL_OVERARGS,PL_OVERARGS); map.put(PL_PAGE,PL_PAGE); map.put(PL_PAGESIZE,PL_PAGESIZE); @@ -1015,6 +1028,11 @@ public class YADARequest { * The proxy server to use for external REST queries */ private String proxy = null; + /** + * A JSON object containing oauth key/value pairs needed for authenticated requests + * @since 8.7.0 + */ + private JSONObject oauth = null; /** * The name of the query to be executed, when coupled with {@link #params}, or {@link #plugin}. Defaults to {@link #DEFAULT_QNAME} */ @@ -1233,7 +1251,6 @@ public List getRequestParamsForQueries() throws YADAQueryConfiguratio */ public static String getParamValueForKey(List q, String frag) throws YADAQueryConfigurationException { - //for (Iterator iterator = q.iterator(); iterator.hasNext();) for (YADAParam param : q) { try @@ -2214,23 +2231,21 @@ public void setMethod(String[] method) { } /** - * @deprecated As of YADA 4.0.0 - * @param args bypass plugin arguments + * @since 8.7.0 + * @param oauth oauth parameters + * */ - @Deprecated - public void setOverArgs(List args) { - this.bypassArgs = args; - l.debug(getFormattedDebugString("bypassArgs", args.toString())); - } - - /** - * @since 4.0.0 - * @param argArr bypass plugin arguments - * @deprecated as of 7.1.0 - */ - @Deprecated - public void setOverargs(String[] argArr) { - this.setBypassargs(argArr); + public void setOAuth(String[] oauth) throws YADARequestException + { + try + { + this.oauth = new JSONObject(oauth); + } + catch(JSONException e) + { + String msg = "The OAuth parameter JSON string appears to be malformed."; + throw new YADARequestException(msg, e); + } } /** @@ -3038,6 +3053,17 @@ public int getPageSize() { return this.pageSize; } + /** + * Returns an {@link JSONObject} containing oauth parameters such as consumer_key, + * secret, etc. See the YADA OAuth 1.0a specification or YADA parameter documentation + * for details. + * @return the oauth parameters object + * @since 8.7.0 + */ + //TODO Add link to YADA OAuth 1.0a spec + public JSONObject getOAuth() { + return this.oauth; + } /** * Returns the results from page {@code pageStart}, using {@link #getPageSize()} to determine which row should From 0a90dd6c761ec7803c7883fb2196b9703bbc90d6 Mon Sep 17 00:00:00 2001 From: varontron Date: Tue, 16 Oct 2018 22:41:50 -0400 Subject: [PATCH 05/17] #87: removed 'overarg' and 'o' parameter tests --- .../resources/test/plugin_script_bypass_standard.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/yada-api/src/test/resources/test/plugin_script_bypass_standard.txt b/yada-api/src/test/resources/test/plugin_script_bypass_standard.txt index b92d6c94..7d8cfb9e 100644 --- a/yada-api/src/test/resources/test/plugin_script_bypass_standard.txt +++ b/yada-api/src/test/resources/test/plugin_script_bypass_standard.txt @@ -1,18 +1,18 @@ q=YADA test SELECT&pl=com.novartis.opensource.yada.plugin.ScriptBypass&a=scriptPluginBypassTest.pl&c=false -q=YADA test SELECT&pl=com.novartis.opensource.yada.plugin.ScriptBypass&o=scriptPluginBypassTest.pl&c=false +#q=YADA test SELECT&pl=com.novartis.opensource.yada.plugin.ScriptBypass&o=scriptPluginBypassTest.pl&c=false q=YADA test SELECT&pl=com.novartis.opensource.yada.plugin.ScriptBypass&b=scriptPluginBypassTest.pl&c=false q=YADA test SELECT&pl=ScriptBypass&a=scriptPluginBypassTest.pl&c=false -q=YADA test SELECT&pl=ScriptBypass&o=scriptPluginBypassTest.pl&c=false +#q=YADA test SELECT&pl=ScriptBypass&o=scriptPluginBypassTest.pl&c=false q=YADA test SELECT&pl=ScriptBypass&b=scriptPluginBypassTest.pl&c=false -q=YADA test SELECT&o=scriptPluginBypassTest.pl&c=false +#q=YADA test SELECT&o=scriptPluginBypassTest.pl&c=false q=YADA test SELECT&b=scriptPluginBypassTest.pl&c=false qname=YADA test SELECT&plugin=com.novartis.opensource.yada.plugin.ScriptBypass&args=scriptPluginBypassTest.pl&count=false -qname=YADA test SELECT&plugin=com.novartis.opensource.yada.plugin.ScriptBypass&overargs=scriptPluginBypassTest.pl&count=false +#qname=YADA test SELECT&plugin=com.novartis.opensource.yada.plugin.ScriptBypass&overargs=scriptPluginBypassTest.pl&count=false qname=YADA test SELECT&plugin=com.novartis.opensource.yada.plugin.ScriptBypass&bypassargs=scriptPluginBypassTest.pl&count=false qname=YADA test SELECT&plugin=ScriptBypass&args=scriptPluginBypassTest.pl&count=false -qname=YADA test SELECT&plugin=ScriptBypass&overargs=scriptPluginBypassTest.pl&count=false +#qname=YADA test SELECT&plugin=ScriptBypass&overargs=scriptPluginBypassTest.pl&count=false qname=YADA test SELECT&plugin=ScriptBypass&bypassargs=scriptPluginBypassTest.pl&count=false -qname=YADA test SELECT&overargs=scriptPluginBypassTest.pl&count=false +#qname=YADA test SELECT&overargs=scriptPluginBypassTest.pl&count=false qname=YADA test SELECT&bypassargs=scriptPluginBypassTest.pl&count=false q=YADA test SELECT&pl=com.novartis.opensource.yada.plugin.ScriptBypass,scriptPluginBypassTest.pl&c=false From cb5fd0ed9c03ad71a9c6af8cefae9bede97a4c72 Mon Sep 17 00:00:00 2001 From: varontron Date: Tue, 16 Oct 2018 22:42:38 -0400 Subject: [PATCH 06/17] #87: added support for oauth 1.0a with RSA --- .../opensource/yada/adaptor/RESTAdaptor.java | 693 +++++++++++++++--- 1 file changed, 596 insertions(+), 97 deletions(-) diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/RESTAdaptor.java b/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/RESTAdaptor.java index 97e2e899..a3e2ed20 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/RESTAdaptor.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/RESTAdaptor.java @@ -17,7 +17,9 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.io.OutputStream; import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; import java.net.HttpCookie; import java.net.HttpURLConnection; import java.net.InetSocketAddress; @@ -25,21 +27,39 @@ import java.net.Proxy; import java.net.URL; import java.net.URLConnection; +import java.net.URLEncoder; +import java.security.InvalidKeyException; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.SecureRandom; +import java.security.Signature; +import java.security.SignatureException; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.net.ssl.HttpsURLConnection; + import org.apache.commons.codec.binary.Base64; import org.apache.log4j.Logger; import com.novartis.opensource.yada.ConnectionFactory; import com.novartis.opensource.yada.YADAQuery; +import com.novartis.opensource.yada.YADAQueryConfigurationException; import com.novartis.opensource.yada.YADAQueryResult; import com.novartis.opensource.yada.YADARequest; +import org.json.JSONObject; + + /** * For connecting to REST endpoints with YADA. * @author David Varon @@ -51,6 +71,37 @@ public class RESTAdaptor extends Adaptor { * Local logger handle */ private static Logger l = Logger.getLogger(RESTAdaptor.class); + + /** + * Constant equal to: {@value}. The character set name. + * @since 8.7.0 + */ + private final static String CHARSET_UTF8 = "UTF-8"; + + private final static String OAUTH_VERIFIER = "oauth_verifier"; + private final static String OAUTH_TIMESTAMP = "oauth_timestamp"; + private final static String OAUTH_NONCE = "oauth_nonce"; + private final static String OAUTH_SIGNATURE = "oauth_signature"; + private final static String OAUTH_REALM = "OAuth realm"; + private final static String SPACE = " "; + private final static String EQUAL = "="; + private final static String QUOTE = "\""; + private final static String COMMA = ","; + private final static String AMP = "&"; + private final static String CR = "\r"; + private final static String LF = "\n"; + private final static String CRLF = CR+LF; + private final static char URL_QUERY_MARKER = '?'; + private final static String BODY = "body"; + + /** + * Secure random number generator to sign requests. + * @see OAuthParameters.java + * @since 8.7.0 + */ + private final static SecureRandom RANDOM = new SecureRandom(); + private final static String ALGO_RSA = "RSA"; + private final static String ALGO_RSASHA1 = "SHA1withRSA"; /** * Constant equal to: {@value} */ @@ -61,7 +112,7 @@ public class RESTAdaptor extends Adaptor { * {@link JSONParams} key for delivery of {@code HTTP POST, PUT, PATCH} body content * @since 8.5.0 */ - private final static String YADA_PAYLOAD = "YADA_PAYLOAD"; + private final static String YADA_PAYLOAD = "YADA_PAYLOAD"; /** * Constant equal to: {@value} @@ -69,12 +120,18 @@ public class RESTAdaptor extends Adaptor { * @since 8.5.0 */ private final static String X_HTTP_METHOD_OVERRIDE = "X-HTTP-Method-Override"; - - /** + private final static String X_AUTHORIZATION = "X-Authorization"; + private final static String CONTENT_LENGTH = "Content-length"; + /** * Variable to hold the proxy server string if necessary */ private String proxy = null; + /** + * Variable to hold the oauth parameters + */ + private JSONObject oauth = null; + /** * Variable denoting the HTTP method * @since 8.5.0 @@ -91,7 +148,7 @@ public RESTAdaptor() { } /** - * The yadaReq constructor + * The yadaReq constructor called by {@link com.novartis.opensource.yada.QueryManager#endowQuery} * @param yadaReq YADA request configuration */ public RESTAdaptor(YADARequest yadaReq) @@ -106,6 +163,11 @@ public RESTAdaptor(YADARequest yadaReq) { this.proxy = yadaReq.getProxy(); } + + if(yadaReq.getOAuth() != null) + { + this.oauth = yadaReq.getOAuth(); + } } /** @@ -118,7 +180,496 @@ protected boolean hasProxy() return true; return false; } + + /** + * Creates the {@link YADAQueryResult} and adds it to the {@link YADAQuery} + * @param yq The {@link YADAQuery} to process + * @since 8.7.0 + */ + private void setYADAQueryResultForYADAQuery(YADAQuery yq) { + yq.setResult(); + YADAQueryResult yqr = yq.getResult(); + yqr.setApp(yq.getApp()); + } + + /** + * Replaces YADA markup in the url query string with values from the request + * @param yq The {@link YADAQuery} to process + * @param row The current collection of query parameter values + * @return the fully fleshed-out query string + * @since 8.7.0 + */ + private String setPositionalParameterValues(YADAQuery yq, int row) { + String urlStr = yq.getUrl(row); + Matcher m = Pattern.compile(PARAM_SYMBOL_RX).matcher(urlStr); + StringBuffer sb = new StringBuffer(); + int i=0; + while(m.find()) + { + String param = yq.getVals(row).get(i++); + String repl = m.group(1)+param; + m.appendReplacement(sb, repl); + } + m.appendTail(sb); + urlStr = sb.toString(); + l.debug("REST url w/params: ["+urlStr+"]"); + return urlStr; + } + + private void setAuthentication(YADAQuery yq, int row, HttpURLConnection hConn) throws YADAQueryConfigurationException, InvalidKeyException, NoSuchAlgorithmException, InvalidKeySpecException, SignatureException, IOException + { + if(yq.hasParam(YADARequest.PS_OAUTH) || yq.hasParam(YADARequest.PL_OAUTH)) + { + if(hConn.getURL().getUserInfo() != null) + { + String msg = "A query cannot contain both basic and oauth configurations. " + + "When using oauth, make sure the REST data source is not also configured with " + + "basic auth values."; + throw new YADAQueryConfigurationException(msg); + } + String body = yq.getDataRow(row).get(YADA_PAYLOAD)[0]; + setAuthenticationOAuth(body,hConn); + } + else if(hConn.getURL().getUserInfo() != null) + { + setAuthenticatonBasic(hConn.getURL(), hConn); + } + } + + /** + * Generates a random nonce. This method originated at + * developer.pearson.com + * Oauth 1.0a Sample Code + * + * @return A unique identifier for the request + * @since 8.7.0 + */ + private static String getNonce() + { + return Long.toHexString(Math.abs(RANDOM.nextLong())); + } + + /** + * Generates an integer representing the number of seconds since the unix epoch using the + * date/time the request is issued. This method originated at + * developer.pearson.com + * Oauth 1.0a Sample Code + * + * + * @return A timestamp for the request + * @since 8.7.0 + */ + private static String getTimestamp() + { + return Long.toString((System.currentTimeMillis() / 1000)); + } + + /** + * Generates an OAuth 1.0 signature. This method originated at + * developer.pearson.com + * Oauth 1.0a Sample Code + * + * + * @param httpMethod The HTTP method of the request + * @param URL The request URL + * @param oauthParams The associative set of signable oAuth parameters + * @param requestBody The serialized POST/PUT message body + * @param secret Alphanumeric string used to validate the identity of the education partner (Private Key) + * + * @return A string containing the Base64-encoded signature digest + * + * @throws UnsupportedEncodingException + * @throws SignatureException + * @throws InvalidKeySpecException + * @throws NoSuchAlgorithmException + * @throws InvalidKeyException + * @since 8.7.0 + */ + private static String generateSignature( + String httpMethod, + URL url, + Map oauthParams, + byte[] requestBody, + String secret + ) throws UnsupportedEncodingException, InvalidKeyException, NoSuchAlgorithmException, InvalidKeySpecException, SignatureException + { + // Ensure the HTTP Method is upper-cased + httpMethod = httpMethod.toUpperCase(); + + // Construct the URL-encoded OAuth parameter portion of the signature base string + String encodedParams = normalizeParams(httpMethod, url, oauthParams, requestBody); + + // URL-encode the relative URL + String encodedUri = URLEncoder.encode(url.getPath(), CHARSET_UTF8); + + // Build the signature base string to be signed with the Consumer Secret + String baseString = String.format("%s&%s&%s", httpMethod, encodedUri, encodedParams); + + return generateRSA(secret, baseString); + + } + + /** + * Normalizes all OAuth signable parameters and url query parameters according to OAuth 1.0. This method originated at + * developer.pearson.com + * Oauth 1.0a Sample Code + * + * + * @param httpMethod The upper-cased HTTP method + * @param URL The request URL + * @param oauthParams The associative set of signable oAuth parameters + * @param requstBody The serialized POST/PUT message body + * + * @return A string containing normalized and encoded oAuth parameters + * + * @throws UnsupportedEncodingException + * @since 8.7.0 + */ + private static String normalizeParams( + String httpMethod, + URL url, + Map oauthParams, + byte[] requestBody + ) throws UnsupportedEncodingException + { + + // Sort the parameters in lexicographical order, 1st by Key then by Value + Map kvpParams = new TreeMap(String.CASE_INSENSITIVE_ORDER); + kvpParams.putAll(oauthParams); + + // Place any query string parameters into a key value pair using equals ("=") to mark + // the key/value relationship and join each parameter with an ampersand ("&") + if (url.getQuery() != null) + { + for(String keyValue : url.getQuery().split(AMP)) + { + String[] p = keyValue.split(EQUAL); + kvpParams.put(p[0],p[1]); + } + + } + + // Include the body parameter if dealing with a POST or PUT request + if (YADARequest.METHOD_POST.equals(httpMethod) || YADARequest.METHOD_PUT.equals(httpMethod)) + { + String body = Base64.encodeBase64String(requestBody).replaceAll(CRLF, ""); + // url encode the body 2 times now before combining other params + body = URLEncoder.encode(body, CHARSET_UTF8); + body = URLEncoder.encode(body, CHARSET_UTF8); + kvpParams.put(BODY, body); + } + + // separate the key and values with a "=" + // separate the kvp with a "&" + StringBuilder combinedParams = new StringBuilder(); + String delimiter=""; + for(String key : kvpParams.keySet()) { + combinedParams.append(delimiter); + combinedParams.append(key); + combinedParams.append(EQUAL); + combinedParams.append(kvpParams.get(key)); + delimiter=AMP; + } + + // url encode the entire string again before returning + return URLEncoder.encode(combinedParams.toString(), CHARSET_UTF8); + } + +//TODO implement MAC support for Oauth 1.0a +// Leave the commented out MAC stuff in here for now. It's not pressing for the current requirement +// but should be rectified soon. It will require finding or building a provider in order to test, +// which could take some time. +// +// /** +// * Generates a Base64-encoded CMAC-AES digest +// * +// * @param key The secret key used to sign the data +// * @param msg The data to be signed +// * +// * @return A CMAC-AES hash +// * +// * @throws UnsupportedEncodingException +// */ +// private static String generateCmac(String key, String msg) throws UnsupportedEncodingException +// { +// byte[] keyBytes = key.getBytes(CHARSET_UTF8); +// byte[] data = msg.getBytes(CHARSET_UTF8); + +//// Person code: +//// CMac macProvider = new CMac(new AESFastEngine()); +//// macProvider.init(new KeyParameter(keyBytes)); +//// macProvider.reset(); +//// +//// macProvider.update(data, 0, data.length); +//// byte[] output = new byte[macProvider.getMacSize()]; +//// macProvider.doFinal(output, 0); +// YADA stubby: +// Mac mac = Mac.getInstance("HmacSHA1"); +// mac.init(secretKey); +// +// // Convert the CMAC to a Base64 string and remove the new line the Base64 library adds +// String cmac = Base64.encodeBase64String(output).replaceAll("\r\n", ""); +// +// return cmac; +// +// } + + /** + * Generates a Base64-encoded RSA-SHA1 digest + * + * @param key The secret key used to sign the data + * @param msg The data to be signed + * + * @return An RSA-SHA1 hash + * + * @throws UnsupportedEncodingException + * @throws NoSuchAlgorithmException + * @throws InvalidKeySpecException + * @throws InvalidKeyException + * @throws SignatureException + */ + private static String generateRSA(String key, String msg) throws UnsupportedEncodingException, NoSuchAlgorithmException, InvalidKeySpecException, InvalidKeyException, SignatureException + { + byte[] keyBytes = key.getBytes(CHARSET_UTF8); + byte[] data = msg.getBytes(CHARSET_UTF8); + Signature signer = Signature.getInstance(ALGO_RSASHA1); + KeyFactory kf = KeyFactory.getInstance(ALGO_RSA); + PrivateKey privateKey = kf.generatePrivate(new PKCS8EncodedKeySpec(keyBytes)); + signer.initSign(privateKey); + signer.update(data); + byte[] signed = signer.sign(); + return Base64.encodeBase64String(signed); + } + + /** + * This method relies on passage of {@link YADAQuery#getOAuth()} which returns + * a {@link JSONObject} containing: + * + *
    + *
  • oauth_consumer_key
    • + *
  • oauth_signature_method. For now, should always equal RSA
    • + *
  • oauth_token
    • + *
  • oauth_verifier (secret)
    • + *
  • oauth_version. For now, should always equal 1.0a
    • + *
+ * + * The logic of this method is derived from + * developer.pearson.com + * Oauth 1.0a Sample Code + * + * @param body the value associated to the YADA_PAYLOAD key in the current "row" + * @param hConn the {@link HttpURLConnection} used to engage with the web service + * @throws SignatureException + * @throws InvalidKeySpecException + * @throws NoSuchAlgorithmException + * @throws InvalidKeyException + * @throws IOException + * @since 8.7.0 + */ + private void setAuthenticationOAuth(String body, HttpURLConnection hConn) throws InvalidKeyException, NoSuchAlgorithmException, InvalidKeySpecException, SignatureException, IOException + { + + String httpMethod = hConn.getRequestMethod(); + URL url = hConn.getURL(); + byte[] requestBody = null; + BufferedReader in = null; + + // Set the request body if making a POST or PUT request + //TODO handle body content--this is just YADA_PAYLOAD now which is probs totes wrong + if (YADARequest.METHOD_POST.equals(httpMethod) || YADARequest.METHOD_PUT.equals(httpMethod)) + { + requestBody = body.getBytes(CHARSET_UTF8); + } + + // Create the OAuth parameter map from the oauth yada param + Map oauthParams = new LinkedHashMap(); + for(Object key : oauth.keySet()) + { + oauthParams.put((String)key, oauth.getString((String)key)); + } + oauthParams.put(OAUTH_TIMESTAMP, getTimestamp()); + oauthParams.put(OAUTH_NONCE, getNonce()); + + + // Get the OAuth 1.0 Signature + String secret = oauthParams.get(OAUTH_VERIFIER); + String signature = generateSignature(httpMethod, url, oauthParams, requestBody, secret); + + l.debug(String.format("OAuth 1.0 Signature = %s", signature)); + + // Add the oauth_signature parameter to the set of OAuth Parameters + oauthParams.put(OAUTH_SIGNATURE, signature); + + // Generate a string of comma delimited: keyName="URL-encoded(value)" pairs + StringBuilder paramStringBldr = new StringBuilder(); + String delimiter = ""; + for (String keyName : oauthParams.keySet()) { + paramStringBldr.append(delimiter); + String value = oauthParams.get((String) keyName); + paramStringBldr.append(keyName).append(EQUAL+QUOTE).append(URLEncoder.encode(value, CHARSET_UTF8)).append(QUOTE); + delimiter=COMMA; + } + + String urlString = url.toString(); + // omit the queryString from the url + int startOfQueryString = urlString.indexOf(URL_QUERY_MARKER); + if(startOfQueryString != -1) { + urlString = urlString.substring(0,startOfQueryString); + } + + // Build the X-Authorization request header + String xauth = String.format(OAUTH_REALM+EQUAL+QUOTE+"%s"+QUOTE+",%s", urlString, paramStringBldr.toString()); + l.debug(String.format("X-Authorization request header = %s", xauth)); + + // Add the header + hConn.addRequestProperty(X_AUTHORIZATION, xauth); + } + + /** + * Sets the Authorization header for the request with Base64-encoded key/value pair + * @param url the URL object + * @param conn the URLConnection object + * @since 8.7.0 + */ + private void setAuthenticatonBasic(URL url, URLConnection conn) + { + //TODO basic auth and other auth methods should be mutually exclusive + if (url.getUserInfo() != null) + { + //TODO issue with '@' sign in pw, must decode first + String basicAuth = "Basic " + new String(new Base64().encode(url.getUserInfo().getBytes())); + conn.setRequestProperty("Authorization", basicAuth); + } + } + + /** + * Looks for cookie strings passed in the request or stored in the {@link YADAQuery} + * and adds them to the {@link URLConnection} + * @param yq The YADAQuery object + * @param conn The URLConnection object + * @since 8.7.0 + */ + private void setCookies(YADAQuery yq, URLConnection conn) + { + if(yq.getCookies() != null && yq.getCookies().size() > 0) + { + String cookieStr = ""; + for (HttpCookie cookie : yq.getCookies()) + { + cookieStr += cookie.getName()+"="+cookie.getValue()+";"; + } + conn.setRequestProperty("Cookie", cookieStr); + } + } + + /** + * Looks for http header strings passed in the request or stored in the {@link YADAQuery} + * and adds them to the {@link URLConnection} + * @param yq the YADAQuery object + * @param conn the URLConnection object + * @since 8.7.0 + */ + private void setHeaders(YADAQuery yq, URLConnection conn) + { + if(yq.getHttpHeaders() != null && yq.getHttpHeaders().length() > 0) + { + l.debug("Processing custom headers..."); + @SuppressWarnings("unchecked") + Iterator keys = yq.getHttpHeaders().keys(); + while(keys.hasNext()) + { + String name = keys.next(); + String value = yq.getHttpHeaders().getString(name); + l.debug("Custom header: "+name+" : "+value); + conn.setRequestProperty(name, value); + if(name.equals(X_HTTP_METHOD_OVERRIDE) && value.equals(YADARequest.METHOD_PATCH)) + { + l.debug("Resetting method to ["+YADARequest.METHOD_POST+"]"); + this.method = YADARequest.METHOD_POST; + } + } + } + } + + /** + * Injects the payload into the request's {@link OutputStream} if {@link YADARequest#getMethod()} returns + * {@link YADARequest#METHOD_POST}, {@link YADARequest#METHOD_PUT}, or {@link YADARequest#METHOD_PATCH} + * @param yq The current YADAQuery object + * @param row The current data "row" + * @param hConn The HTTPUrlConnection into which the data will be put + * @throws YADAAdaptorExecutionException in the event of an issue writing to the output stream. + * @since 8.7.0 + */ + private void handlePostPutPatch(YADAQuery yq, int row, HttpURLConnection hConn) throws YADAAdaptorExecutionException { + boolean isPostPutPatch = this.method.equals(YADARequest.METHOD_POST) + || this.method.equals(YADARequest.METHOD_PUT) + || this.method.equals(YADARequest.METHOD_PATCH); + + if(!this.method.equals(YADARequest.METHOD_GET)) + { + try + { + hConn.setRequestMethod(this.method); + if(isPostPutPatch) + { + //TODO make YADA_PAYLOAD case-insensitive and create an alias for it, e.g., ypl + // NOTE: YADA_PAYLOAD is a COLUMN NAME found in a JSONParams DATA object. It + // is not a YADA param + String payload = yq.getDataRow(row).get(YADA_PAYLOAD)[0]; + hConn.setDoOutput(true); + OutputStreamWriter writer; + writer = new OutputStreamWriter(hConn.getOutputStream()); + writer.write(payload.toString()); + writer.flush(); + } + } + catch (IOException e) + { + String msg = "There was a problem injecting the payload into the request"; + throw new YADAAdaptorExecutionException(msg, e); + } + } + } + + /** + * Reads the response {@link InputStream} and writes it to a {@link String} + * @param hConn the connection object + * @return a String containing the response data + * @throws YADAAdaptorExecutionException when reading the input stream fails + * @since 8.7.0 + */ + private String processResponse(HttpURLConnection hConn) throws YADAAdaptorExecutionException + { + String result = ""; + try(BufferedReader in = new BufferedReader(new InputStreamReader(hConn.getInputStream()))) + { + String inputLine; + while ((inputLine = in.readLine()) != null) + { + result += String.format("%1s%n",inputLine); + } + } + catch (IOException e) + { + String msg = "There was a problem reading from the response's input stream"; + throw new YADAAdaptorExecutionException(msg, e); + } + return result; + } + + /** + * Writes the header fields to the console or log + * @param conn the current connection object + * @since 8.7.0 + */ + private void logRequest(URLConnection conn) { + Map> map = conn.getHeaderFields(); + for (Map.Entry> entry : map.entrySet()) { + l.debug("Key : " + entry.getKey() + " ,Value : " + entry.getValue()); + } + } + /** * Gets the input stream from the {@link URLConnection} and stores it in * the {@link YADAQueryResult} in {@code yq} @@ -127,9 +678,6 @@ protected boolean hasProxy() @Override public void execute(YADAQuery yq) throws YADAAdaptorExecutionException { - boolean isPostPutPatch = this.method.equals(YADARequest.METHOD_POST) - || this.method.equals(YADARequest.METHOD_PUT) - || this.method.equals(YADARequest.METHOD_PATCH); resetCountParameter(yq); int rows = yq.getData().size() > 0 ? yq.getData().size() : 1; /* @@ -144,35 +692,21 @@ public void execute(YADAQuery yq) throws YADAAdaptorExecutionException * ...j=[{qname:queryname,DATA:[{col1:x,col2:y,col3:z},{col1:a,col2:b,col3:c}]}] * so 2 rows */ + URL url; + URLConnection conn; + HttpURLConnection hConn = null; for(int row=0;row 0) - { - String cookieStr = ""; - for (HttpCookie cookie : yq.getCookies()) - { - cookieStr += cookie.getName()+"="+cookie.getValue()+";"; - } - conn.setRequestProperty("Cookie", cookieStr); - } - - if(yq.getHttpHeaders() != null && yq.getHttpHeaders().length() > 0) - { - l.debug("Processing custom headers..."); - @SuppressWarnings("unchecked") - Iterator keys = yq.getHttpHeaders().keys(); - while(keys.hasNext()) - { - String name = keys.next(); - String value = yq.getHttpHeaders().getString(name); - l.debug("Custom header: "+name+" : "+value); - conn.setRequestProperty(name, value); - if(name.equals(X_HTTP_METHOD_OVERRIDE) && value.equals(YADARequest.METHOD_PATCH)) - { - l.debug("Resetting method to ["+YADARequest.METHOD_POST+"]"); - this.method = YADARequest.METHOD_POST; - } - } - } - - - HttpURLConnection hConn = (HttpURLConnection)conn; - if(!this.method.equals(YADARequest.METHOD_GET)) - { - hConn.setRequestMethod(this.method); - if(isPostPutPatch) - { - //TODO make YADA_PAYLOAD case-insensitive and create an alias for it, e.g., ypl - // NOTE: YADA_PAYLOAD is a COLUMN NAME found in a JSONParams DATA object. It - // is not a YADA param - String payload = yq.getDataRow(row).get(YADA_PAYLOAD)[0]; - hConn.setDoOutput(true); - OutputStreamWriter writer; - writer = new OutputStreamWriter(conn.getOutputStream()); - writer.write(payload.toString()); - writer.flush(); - } - } - + setCookies(yq, hConn); + + // headers + setHeaders(yq, hConn); + + // handle auth if necessary + setAuthentication(yq,row,hConn); + + // inject body content if necessary + handlePostPutPatch(yq, row, hConn); + // debug - Map> map = conn.getHeaderFields(); - for (Map.Entry> entry : map.entrySet()) { - l.debug("Key : " + entry.getKey() + " ,Value : " + entry.getValue()); - } + logRequest(hConn); - try(BufferedReader in = new BufferedReader(new InputStreamReader(hConn.getInputStream()))) - { - String inputLine; - while ((inputLine = in.readLine()) != null) - { - result += String.format("%1s%n",inputLine); - } - } - yqr.addResult(row, result); + // process the response + String result = processResponse(hConn); + + // store the response in the YADAQueryResult + yq.getResult().addResult(row, result); } catch (MalformedURLException e) { @@ -267,7 +751,22 @@ public void execute(YADAQuery yq) throws YADAAdaptorExecutionException { String msg = "Unable to read REST response."; throw new YADAAdaptorExecutionException(msg, e); + } + catch (YADAQueryConfigurationException e) + { + String msg = "The YADAQuery was configured improperly. Check YADA parameters, especially 'oauth'."; + throw new YADAAdaptorExecutionException(msg, e); + } + catch (InvalidKeyException|NoSuchAlgorithmException|InvalidKeySpecException|SignatureException e) + { + String msg = "There was a problem oauth signature encryption. It could be related to key validation, alogorithm spec, etc."; + throw new YADAAdaptorExecutionException(msg, e); + } + finally + { + if(hConn != null) hConn.disconnect(); } + } } From 4b5ea28ab4d802b0b5a833baa6b93c00d134dac6 Mon Sep 17 00:00:00 2001 From: varontron Date: Mon, 22 Oct 2018 07:38:20 -0400 Subject: [PATCH 07/17] added google-client dependencies --- yada-api/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/yada-api/pom.xml b/yada-api/pom.xml index fdd12e6f..6f650243 100644 --- a/yada-api/pom.xml +++ b/yada-api/pom.xml @@ -849,6 +849,16 @@ + + com.google.oauth-client + google-oauth-client + 1.22.0 + + + com.google.http-client + google-http-client-jackson2 + 1.11.0-beta + com.novartis.opensource jsqlparser From cfbdfdf9fac36647b8b39cd80001bd2e826841b8 Mon Sep 17 00:00:00 2001 From: varontron Date: Mon, 22 Oct 2018 07:39:54 -0400 Subject: [PATCH 08/17] default 'autoCommit' to false --- yada-war/src/main/webapp/yada-admin/component/app-mgr.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/yada-war/src/main/webapp/yada-admin/component/app-mgr.js b/yada-war/src/main/webapp/yada-admin/component/app-mgr.js index 98316400..a9dad336 100644 --- a/yada-war/src/main/webapp/yada-admin/component/app-mgr.js +++ b/yada-war/src/main/webapp/yada-admin/component/app-mgr.js @@ -126,8 +126,7 @@ define( '#see https://github.com/brettwooldridge/HikariCP for documentation\n'+ 'jdbcUrl=e.g., jdbc:hsqldb:hsql://localhost/YADADB\n'+ 'username=\n'+ - 'password=\n'+ - 'autoCommit=false\n'+ + 'password=\n'+ 'connectionTimeout=300000\n'+ 'idleTimeout=600000\n'+ 'maxLifetime=1800000\n'+ From a35bff2f51ac22312384d697d57f729b3eb72255 Mon Sep 17 00:00:00 2001 From: varontron Date: Mon, 22 Oct 2018 07:40:21 -0400 Subject: [PATCH 09/17] Added comments regarding CORS filter --- yada-war/src/main/webapp/WEB-INF/web.xml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/yada-war/src/main/webapp/WEB-INF/web.xml b/yada-war/src/main/webapp/WEB-INF/web.xml index f8d5e90a..b4e64c7f 100644 --- a/yada-war/src/main/webapp/WEB-INF/web.xml +++ b/yada-war/src/main/webapp/WEB-INF/web.xml @@ -38,9 +38,22 @@ FORWARD - + + + + From d30db141e2f56bf41b8862929f19c613715cead2 Mon Sep 17 00:00:00 2001 From: varontron Date: Mon, 22 Oct 2018 07:40:53 -0400 Subject: [PATCH 10/17] default omit minimumIdle config --- yada-war/src/main/resources/conf/YADA.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yada-war/src/main/resources/conf/YADA.properties b/yada-war/src/main/resources/conf/YADA.properties index 40a34ceb..138584f2 100644 --- a/yada-war/src/main/resources/conf/YADA.properties +++ b/yada-war/src/main/resources/conf/YADA.properties @@ -9,9 +9,9 @@ autoCommit=${YADA.index.autoCommit} connectionTimeout=300000 idleTimeout=600000 maxLifetime=1800000 -minimumIdle=5 maximumPoolSize=100 poolName=HikariPool-YADA +#minimumIdle=5 #connectionTestQuery=${YADA.index.validationQuery} #initializationFailFast=true #isolateInternalQueries=false From f1b64fcd93846c6655bfdda3d81c7adc0b18eb64 Mon Sep 17 00:00:00 2001 From: varontron Date: Mon, 22 Oct 2018 07:42:41 -0400 Subject: [PATCH 11/17] added PS_OAUTH to request params object --- .../main/java/com/novartis/opensource/yada/YADARequest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/YADARequest.java b/yada-api/src/main/java/com/novartis/opensource/yada/YADARequest.java index 1095c160..d9136129 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/YADARequest.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/YADARequest.java @@ -1189,6 +1189,7 @@ public List getGlobalParamsForQueries() throws YADAQueryConfiguration *
  • {@link #PS_FORMAT}
    • *
  • {@link #PS_HARMONYMAP}
    • *
  • {@link #PS_METHOD} (for backward compatibility)
    • + *
  • {@link #PS_OAUTH}
    • *
  • {@link #PS_PAGESIZE}
    • *
  • {@link #PS_PAGESTART}
    • *
  • {@link #PS_ROW_DELIMITER}
    • @@ -1204,6 +1205,7 @@ public List getRequestParamsForQueries() throws YADAQueryConfiguratio JSONObject jobj = new JSONObject(); try { + jobj.put(PS_OAUTH, getOAuth()); jobj.put(PS_COUNT, getCount()); jobj.put(PS_FILTERS, getFilters()); jobj.put(PS_PAGESIZE, getPageSize()); @@ -2239,7 +2241,7 @@ public void setOAuth(String[] oauth) throws YADARequestException { try { - this.oauth = new JSONObject(oauth); + this.oauth = new JSONObject(oauth[0]); } catch(JSONException e) { From aee598a62837771d22161464f52babbe81b32ef5 Mon Sep 17 00:00:00 2001 From: varontron Date: Mon, 22 Oct 2018 07:43:02 -0400 Subject: [PATCH 12/17] corrected typo in comment --- .../main/java/com/novartis/opensource/yada/adaptor/Adaptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/Adaptor.java b/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/Adaptor.java index bdf0dcea..038ad842 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/Adaptor.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/Adaptor.java @@ -130,7 +130,7 @@ protected YADARequest getServiceParameters() * @param yq the query to which to apply the new settings */ protected void resetCountParameter(YADAQuery yq) { - //TODO run this through the debugger to check the distinction betwen these 2 instances of yadaReq + //TODO run this through the debugger to check the distinction between these 2 instances of yadaReq this.yadaReq.setCount(new String[] {"false"}); yq.replaceParam(YADARequest.PS_COUNT, "false"); } From a3449b158a3d2311995f35b22580f5f5ffc24ad9 Mon Sep 17 00:00:00 2001 From: varontron Date: Mon, 22 Oct 2018 07:43:55 -0400 Subject: [PATCH 13/17] removed 'overargs' param tests --- .../resources/test/plugin_script_bypass_standard.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/yada-api/src/test/resources/test/plugin_script_bypass_standard.txt b/yada-api/src/test/resources/test/plugin_script_bypass_standard.txt index 7d8cfb9e..ae86b6ab 100644 --- a/yada-api/src/test/resources/test/plugin_script_bypass_standard.txt +++ b/yada-api/src/test/resources/test/plugin_script_bypass_standard.txt @@ -1,18 +1,18 @@ q=YADA test SELECT&pl=com.novartis.opensource.yada.plugin.ScriptBypass&a=scriptPluginBypassTest.pl&c=false -#q=YADA test SELECT&pl=com.novartis.opensource.yada.plugin.ScriptBypass&o=scriptPluginBypassTest.pl&c=false + q=YADA test SELECT&pl=com.novartis.opensource.yada.plugin.ScriptBypass&b=scriptPluginBypassTest.pl&c=false q=YADA test SELECT&pl=ScriptBypass&a=scriptPluginBypassTest.pl&c=false -#q=YADA test SELECT&pl=ScriptBypass&o=scriptPluginBypassTest.pl&c=false + q=YADA test SELECT&pl=ScriptBypass&b=scriptPluginBypassTest.pl&c=false -#q=YADA test SELECT&o=scriptPluginBypassTest.pl&c=false + q=YADA test SELECT&b=scriptPluginBypassTest.pl&c=false qname=YADA test SELECT&plugin=com.novartis.opensource.yada.plugin.ScriptBypass&args=scriptPluginBypassTest.pl&count=false -#qname=YADA test SELECT&plugin=com.novartis.opensource.yada.plugin.ScriptBypass&overargs=scriptPluginBypassTest.pl&count=false + qname=YADA test SELECT&plugin=com.novartis.opensource.yada.plugin.ScriptBypass&bypassargs=scriptPluginBypassTest.pl&count=false qname=YADA test SELECT&plugin=ScriptBypass&args=scriptPluginBypassTest.pl&count=false -#qname=YADA test SELECT&plugin=ScriptBypass&overargs=scriptPluginBypassTest.pl&count=false + qname=YADA test SELECT&plugin=ScriptBypass&bypassargs=scriptPluginBypassTest.pl&count=false -#qname=YADA test SELECT&overargs=scriptPluginBypassTest.pl&count=false + qname=YADA test SELECT&bypassargs=scriptPluginBypassTest.pl&count=false q=YADA test SELECT&pl=com.novartis.opensource.yada.plugin.ScriptBypass,scriptPluginBypassTest.pl&c=false From 84c851244110ab72b8583e0db4164013f5a66869 Mon Sep 17 00:00:00 2001 From: varontron Date: Mon, 22 Oct 2018 11:06:51 -0400 Subject: [PATCH 14/17] javadoc updates --- .../java/com/novartis/opensource/yada/YADAQueryResult.java | 2 +- .../main/java/com/novartis/opensource/yada/YADARequest.java | 6 +++--- .../novartis/opensource/yada/adaptor/FileSystemAdaptor.java | 2 +- .../com/novartis/opensource/yada/format/CountResponse.java | 1 + .../java/com/novartis/opensource/yada/util/QueryUtils.java | 2 +- .../com/novartis/opensource/yada/test/YADAAdminTest.java | 1 + 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/YADAQueryResult.java b/yada-api/src/main/java/com/novartis/opensource/yada/YADAQueryResult.java index 45df17f3..227b2a47 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/YADAQueryResult.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/YADAQueryResult.java @@ -36,7 +36,7 @@ public class YADAQueryResult { */ private String qname; /** - * The app to which the {@link qname} is mapped + * The app to which the {@code qname} is mapped * @since 8.6.1 */ private String app; diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/YADARequest.java b/yada-api/src/main/java/com/novartis/opensource/yada/YADARequest.java index d9136129..8b9737c6 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/YADARequest.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/YADARequest.java @@ -734,7 +734,7 @@ public class YADARequest { public static final String PL_PLUGINTYPE = "plugintype"; /** * A constant equal to: {@value} - * @deprecated + * @deprecated as of 7.1.0 */ @Deprecated public static final String PL_POSTARGS = "postargs"; @@ -976,7 +976,7 @@ public class YADARequest { private String mail; /** * YADA execution method. Defaults to {@link #METHOD_GET} - * @since 1.0.0, deprecated in v4.0.0, un-deprecated in 8.5.0 to support different HTTP methods for {@link RESTAdaptor} + * @since 1.0.0, deprecated in v4.0.0, un-deprecated in 8.5.0 to support different HTTP methods for {@link com.novartis.opensource.yada.adaptor.RESTAdaptor} */ private String method = METHOD_GET; /** @@ -2235,7 +2235,7 @@ public void setMethod(String[] method) { /** * @since 8.7.0 * @param oauth oauth parameters - * + * @throws YADARequestException if the parameter string is not malformed */ public void setOAuth(String[] oauth) throws YADARequestException { diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/FileSystemAdaptor.java b/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/FileSystemAdaptor.java index 9d7e3dfd..3ebd4f73 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/FileSystemAdaptor.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/FileSystemAdaptor.java @@ -52,7 +52,7 @@ public class FileSystemAdaptor extends Adaptor */ protected final static String PARAM_SYMBOL_RX = "([\\/=:<])(\\?[idvn])"; /** - * Constant equal to: {@code <{1,2}.*} + * Constant equal to: {@code <{1,2}.*} */ protected final static String NON_READ_SFX_RX = "<{1,2}.*"; /** diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/format/CountResponse.java b/yada-api/src/main/java/com/novartis/opensource/yada/format/CountResponse.java index f5d248eb..791dd6a3 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/format/CountResponse.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/format/CountResponse.java @@ -28,6 +28,7 @@ * compatibility when the deprecated {@link YADARequest#METHOD_UPDATE} parameter is included in the request * @author David Varon */ +@SuppressWarnings("javadoc") public class CountResponse extends AbstractResponse { diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/util/QueryUtils.java b/yada-api/src/main/java/com/novartis/opensource/yada/util/QueryUtils.java index c5974f63..aed0e323 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/util/QueryUtils.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/util/QueryUtils.java @@ -155,7 +155,7 @@ public class QueryUtils */ public static final String RX_DELETE = "^DELETE.*"; /** - * A constant equal to: {@code ^([^<]+)((<{1,2})(.+))*$ } + * A constant equal to: {@code ^([^<]+)((<{1,2})(.+))*$ } * @since PROVISIONAL */ public final static String RX_FILE_URI = "^([^<]+)((<{1,2})(.+))*$"; diff --git a/yada-api/src/test/java/com/novartis/opensource/yada/test/YADAAdminTest.java b/yada-api/src/test/java/com/novartis/opensource/yada/test/YADAAdminTest.java index ed97c8be..1a1c34ca 100644 --- a/yada-api/src/test/java/com/novartis/opensource/yada/test/YADAAdminTest.java +++ b/yada-api/src/test/java/com/novartis/opensource/yada/test/YADAAdminTest.java @@ -335,6 +335,7 @@ public void testNewQueryForNewAppSave() throws InterruptedException /** * Creates a new query for the current app using {@code SELECT 1 AS COL1} * @param qname the query to insert + * @param action the imperative to executed * @throws InterruptedException if the any thread has interrupted the current thread */ private void createQueryForNewApp(String qname, String action) throws InterruptedException From 7818c02497cc5408c208bc400c18a2097dba7fc4 Mon Sep 17 00:00:00 2001 From: varontron Date: Mon, 22 Oct 2018 11:08:21 -0400 Subject: [PATCH 15/17] moved YADASecurityException to main package --- .../main/java/com/novartis/opensource/yada/Service.java | 1 - .../yada/{plugin => }/YADASecurityException.java | 7 +++++-- .../java/com/novartis/opensource/yada/adaptor/Adaptor.java | 4 +++- .../opensource/yada/plugin/AbstractPreprocessor.java | 1 + .../com/novartis/opensource/yada/plugin/ContentPolicy.java | 2 ++ .../novartis/opensource/yada/plugin/ExecutionPolicy.java | 2 ++ .../com/novartis/opensource/yada/plugin/Gatekeeper.java | 2 +- .../java/com/novartis/opensource/yada/plugin/Login.java | 1 + .../novartis/opensource/yada/plugin/SecurityPolicy.java | 2 ++ .../novartis/opensource/yada/plugin/TokenValidator.java | 2 ++ .../com/novartis/opensource/yada/plugin/Validation.java | 2 ++ .../java/com/novartis/opensource/yada/util/YADAUtils.java | 4 +++- 12 files changed, 24 insertions(+), 6 deletions(-) rename yada-api/src/main/java/com/novartis/opensource/yada/{plugin => }/YADASecurityException.java (85%) diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/Service.java b/yada-api/src/main/java/com/novartis/opensource/yada/Service.java index d1fb20e2..973235e3 100755 --- a/yada-api/src/main/java/com/novartis/opensource/yada/Service.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/Service.java @@ -47,7 +47,6 @@ import com.novartis.opensource.yada.plugin.Postprocess; import com.novartis.opensource.yada.plugin.Preprocess; import com.novartis.opensource.yada.plugin.YADAPluginException; -import com.novartis.opensource.yada.plugin.YADASecurityException; import com.novartis.opensource.yada.util.FileUtils; import com.novartis.opensource.yada.util.QueryUtils; import com.novartis.opensource.yada.util.YADAUtils; diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/YADASecurityException.java b/yada-api/src/main/java/com/novartis/opensource/yada/YADASecurityException.java similarity index 85% rename from yada-api/src/main/java/com/novartis/opensource/yada/plugin/YADASecurityException.java rename to yada-api/src/main/java/com/novartis/opensource/yada/YADASecurityException.java index 6139155d..360523ea 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/YADASecurityException.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/YADASecurityException.java @@ -15,10 +15,13 @@ /** * */ -package com.novartis.opensource.yada.plugin; +package com.novartis.opensource.yada; + +import com.novartis.opensource.yada.plugin.YADAPluginException; /** - * Throw from a security preprocessor when authentication fails. + * Throw from a security preprocessor or adaptor when authentication or authorization fails. + * Moved to main package from {@coode plugin} for {@code 8.7.0} * @author Dave Varon * @since 7.0.0 * diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/Adaptor.java b/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/Adaptor.java index 038ad842..e7b9ceae 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/Adaptor.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/Adaptor.java @@ -22,6 +22,7 @@ import com.novartis.opensource.yada.YADAQuery; import com.novartis.opensource.yada.YADAQueryResult; import com.novartis.opensource.yada.YADARequest; +import com.novartis.opensource.yada.YADASecurityException; /** * The abstract Adaptor class is at the root of the package hierarchy. Adaptors @@ -79,8 +80,9 @@ public Adaptor(YADARequest yadaReq) * * @param yq the {@link YADAQuery} containing the code to execute * @throws YADAAdaptorExecutionException if exception is thrown during query execution + * @throws YADASecurityException if there is authentication or authorization error in preparation for or during execution */ - public void execute(YADAQuery yq) throws YADAAdaptorExecutionException + public void execute(YADAQuery yq) throws YADAAdaptorExecutionException, YADASecurityException { // } diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/AbstractPreprocessor.java b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/AbstractPreprocessor.java index 233439f3..5f51b5ab 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/AbstractPreprocessor.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/AbstractPreprocessor.java @@ -35,6 +35,7 @@ import com.novartis.opensource.yada.YADAQuery; import com.novartis.opensource.yada.YADARequest; import com.novartis.opensource.yada.YADASQLException; +import com.novartis.opensource.yada.YADASecurityException; import com.novartis.opensource.yada.util.YADAUtils; /** diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/ContentPolicy.java b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/ContentPolicy.java index f1d81b84..48167783 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/ContentPolicy.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/ContentPolicy.java @@ -14,6 +14,8 @@ */ package com.novartis.opensource.yada.plugin; +import com.novartis.opensource.yada.YADASecurityException; + /** * @author Dave Varon * @since 7.0.0 diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/ExecutionPolicy.java b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/ExecutionPolicy.java index 4549e848..299430e5 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/ExecutionPolicy.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/ExecutionPolicy.java @@ -14,6 +14,8 @@ */ package com.novartis.opensource.yada.plugin; +import com.novartis.opensource.yada.YADASecurityException; + /** * @author Dave Varon * @since 7.0.0 diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/Gatekeeper.java b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/Gatekeeper.java index 764e438c..b5a59408 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/Gatekeeper.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/Gatekeeper.java @@ -44,8 +44,8 @@ import com.novartis.opensource.yada.YADAQuery; import com.novartis.opensource.yada.YADAQueryConfigurationException; import com.novartis.opensource.yada.YADARequest; +import com.novartis.opensource.yada.YADASecurityException; import com.novartis.opensource.yada.plugin.AbstractPreprocessor; -import com.novartis.opensource.yada.plugin.YADASecurityException; import com.novartis.opensource.yada.util.YADAUtils; /** diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/Login.java b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/Login.java index b82183a5..61d51d27 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/Login.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/Login.java @@ -24,6 +24,7 @@ import com.novartis.opensource.yada.YADAQuery; import com.novartis.opensource.yada.YADARequest; +import com.novartis.opensource.yada.YADASecurityException; /** * This plugin is both a {@link Preprocess} and {@link Postprocess} plugin. diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/SecurityPolicy.java b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/SecurityPolicy.java index 3434c3be..ca172d24 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/SecurityPolicy.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/SecurityPolicy.java @@ -16,6 +16,8 @@ import java.util.List; +import com.novartis.opensource.yada.YADASecurityException; + /** * @author Dave Varon * @since 7.0.0 diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/TokenValidator.java b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/TokenValidator.java index 8db48a83..55a2dfbb 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/TokenValidator.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/TokenValidator.java @@ -14,6 +14,8 @@ */ package com.novartis.opensource.yada.plugin; +import com.novartis.opensource.yada.YADASecurityException; + /** * * @author David Varon diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/Validation.java b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/Validation.java index 38aca5ae..8a7c27be 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/plugin/Validation.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/plugin/Validation.java @@ -17,6 +17,8 @@ */ package com.novartis.opensource.yada.plugin; +import com.novartis.opensource.yada.YADASecurityException; + /** * @author Dave Varon * @since 7.0.0 diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/util/YADAUtils.java b/yada-api/src/main/java/com/novartis/opensource/yada/util/YADAUtils.java index 854dca25..7e16746d 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/util/YADAUtils.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/util/YADAUtils.java @@ -39,6 +39,7 @@ import com.novartis.opensource.yada.YADARequestException; import com.novartis.opensource.yada.YADAResourceException; import com.novartis.opensource.yada.YADASQLException; +import com.novartis.opensource.yada.YADASecurityException; import com.novartis.opensource.yada.YADAUnsupportedAdaptorException; import com.novartis.opensource.yada.adaptor.YADAAdaptorException; import com.novartis.opensource.yada.adaptor.YADAAdaptorExecutionException; @@ -359,8 +360,9 @@ public static String executeYADAGet(String[] qname, String[] params) * @param yadaReq YADA request configuration * @return Object result of query execution * @throws YADAExecutionException if multiple query executions are attempted, or other YADA exceptions are thrown internally + * @throws YADASecurityException if there is authentication or authorization error in preparation for or during execution */ - public static Object executeYADAQuery(YADARequest yadaReq) throws YADAExecutionException + public static Object executeYADAQuery(YADARequest yadaReq) throws YADAExecutionException, YADASecurityException { Object result = null; try From 04248f3c6e504603a90ab04fe24269ac3f36be0c Mon Sep 17 00:00:00 2001 From: varontron Date: Mon, 22 Oct 2018 11:10:13 -0400 Subject: [PATCH 16/17] Enabled OAuth 1.0a support. Resolve #87. --- .../opensource/yada/adaptor/RESTAdaptor.java | 752 +++++++----------- 1 file changed, 303 insertions(+), 449 deletions(-) diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/RESTAdaptor.java b/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/RESTAdaptor.java index a3e2ed20..64717d45 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/RESTAdaptor.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/adaptor/RESTAdaptor.java @@ -15,48 +15,49 @@ package com.novartis.opensource.yada.adaptor; import java.io.BufferedReader; +import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.UnsupportedEncodingException; import java.net.HttpCookie; -import java.net.HttpURLConnection; -import java.net.InetSocketAddress; -import java.net.MalformedURLException; -import java.net.Proxy; import java.net.URL; import java.net.URLConnection; -import java.net.URLEncoder; -import java.security.InvalidKeyException; -import java.security.KeyFactory; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; -import java.security.SecureRandom; -import java.security.Signature; -import java.security.SignatureException; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.PKCS8EncodedKeySpec; +import java.security.UnrecoverableEntryException; +import java.security.cert.CertificateException; import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; -import java.util.TreeMap; +import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; -import javax.net.ssl.HttpsURLConnection; - -import org.apache.commons.codec.binary.Base64; import org.apache.log4j.Logger; +import com.google.api.client.auth.oauth.OAuthParameters; +import com.google.api.client.auth.oauth.OAuthRsaSigner; +import com.google.api.client.http.BasicAuthentication; +import com.google.api.client.http.ByteArrayContent; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpContent; +import com.google.api.client.http.HttpHeaders; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.javanet.NetHttpTransport; + import com.novartis.opensource.yada.ConnectionFactory; import com.novartis.opensource.yada.YADAQuery; import com.novartis.opensource.yada.YADAQueryConfigurationException; import com.novartis.opensource.yada.YADAQueryResult; import com.novartis.opensource.yada.YADARequest; +import com.novartis.opensource.yada.YADASecurityException; - +import org.json.JSONException; import org.json.JSONObject; @@ -76,32 +77,50 @@ public class RESTAdaptor extends Adaptor { * Constant equal to: {@value}. The character set name. * @since 8.7.0 */ - private final static String CHARSET_UTF8 = "UTF-8"; + private final static String PROP_KEYSTORE = "javax.net.ssl.keyStore"; + + /** + * Constant equal to: {@value}. The keystore password passed to the jvm. + * @since 8.7.0 + */ + private final static String PROP_KEYSTORE_PASS = "javax.net.ssl.keyStorePassword"; + + /** + * Constant equal to: {@value}. The {@code oauth} request parameter property name. + * @since 8.7.0 + */ + private final static String OAUTH_VERSION = "oauth_version"; + + /** + * Constant equal to: {@value}. The {@code oauth} request parameter property name. + * @since 8.7.0 + */ + private final static String OAUTH_SIGNATURE_METHOD = "oauth_signature_method"; + + /** + * Constant equal to: {@value}. The {@code oauth} request parameter property name. + * @since 8.7.0 + */ + private final static String OAUTH_VERIFIER = "oauth_verifier"; - private final static String OAUTH_VERIFIER = "oauth_verifier"; - private final static String OAUTH_TIMESTAMP = "oauth_timestamp"; - private final static String OAUTH_NONCE = "oauth_nonce"; - private final static String OAUTH_SIGNATURE = "oauth_signature"; - private final static String OAUTH_REALM = "OAuth realm"; - private final static String SPACE = " "; - private final static String EQUAL = "="; - private final static String QUOTE = "\""; - private final static String COMMA = ","; - private final static String AMP = "&"; - private final static String CR = "\r"; - private final static String LF = "\n"; - private final static String CRLF = CR+LF; - private final static char URL_QUERY_MARKER = '?'; - private final static String BODY = "body"; + /** + * Constant equal to: {@value}. The {@code oauth} request parameter property name. + * @since 8.7.0 + */ + private final static String OAUTH_PRIVATE_KEY = "oauth_private_key"; /** - * Secure random number generator to sign requests. - * @see OAuthParameters.java + * Constant equal to: {@value}. The {@code oauth} request parameter property name. * @since 8.7.0 */ - private final static SecureRandom RANDOM = new SecureRandom(); - private final static String ALGO_RSA = "RSA"; - private final static String ALGO_RSASHA1 = "SHA1withRSA"; + private final static String OAUTH_CONSUMER_KEY = "oauth_consumer_key"; + + /** + * Constant equal to: {@value}. The {@code oauth} request parameter property name. + * @since 8.7.0 + */ + private final static String OAUTH_TOKEN = "oauth_token"; + /** * Constant equal to: {@value} */ @@ -109,20 +128,24 @@ public class RESTAdaptor extends Adaptor { /** * Constant equal to: {@value} - * {@link JSONParams} key for delivery of {@code HTTP POST, PUT, PATCH} body content + * {@link com.novartis.opensource.yada.JSONParams} key for delivery of {@code HTTP POST, PUT, PATCH} body content * @since 8.5.0 */ private final static String YADA_PAYLOAD = "YADA_PAYLOAD"; /** - * Constant equal to: {@value} + * Constant equal to: {@value}. * Workaround for requests using {@code HTTP PATCH} * @since 8.5.0 */ - private final static String X_HTTP_METHOD_OVERRIDE = "X-HTTP-Method-Override"; - private final static String X_AUTHORIZATION = "X-Authorization"; - private final static String CONTENT_LENGTH = "Content-length"; - /** + private final static String H_X_HTTP_METHOD_OVERRIDE = "X-HTTP-Method-Override"; + + /** + * Constant equal to: {@value}. For retrieving the default keystore. + */ + private static final String KEYSTORE_TYPE_JKS = "jks"; + + /** * Variable to hold the proxy server string if necessary */ private String proxy = null; @@ -138,7 +161,6 @@ public class RESTAdaptor extends Adaptor { */ private String method = YADARequest.METHOD_GET; - /** * Default constructor */ @@ -216,234 +238,72 @@ private String setPositionalParameterValues(YADAQuery yq, int row) { return urlStr; } - private void setAuthentication(YADAQuery yq, int row, HttpURLConnection hConn) throws YADAQueryConfigurationException, InvalidKeyException, NoSuchAlgorithmException, InvalidKeySpecException, SignatureException, IOException - { + /** + * Determines if OAuth or Basic Authentication should be applied given the {@link YADARequest} + * configuration, and proceeds accordingly. + * @param yq The query object defined in the {@link YADARequest} + * @param url The REST endpoint + * @return The {@link HttpRequestInitializer} to pass to the {@link HttpRequestFactory} + * @throws YADAQueryConfigurationException if both OAuth and Basic are defined in the request + * @throws YADASecurityException if authentication fails + * @since 8.7.0 + */ + private HttpRequestInitializer setAuthentication(YADAQuery yq, GenericUrl url) throws YADAQueryConfigurationException, YADASecurityException + { if(yq.hasParam(YADARequest.PS_OAUTH) || yq.hasParam(YADARequest.PL_OAUTH)) { - if(hConn.getURL().getUserInfo() != null) + if(url.getUserInfo() != null) { String msg = "A query cannot contain both basic and oauth configurations. " + "When using oauth, make sure the REST data source is not also configured with " + "basic auth values."; throw new YADAQueryConfigurationException(msg); } - String body = yq.getDataRow(row).get(YADA_PAYLOAD)[0]; - setAuthenticationOAuth(body,hConn); + return setAuthenticationOAuth(url); } - else if(hConn.getURL().getUserInfo() != null) + else if(url.getUserInfo() != null) { - setAuthenticatonBasic(hConn.getURL(), hConn); + return setAuthenticatonBasic(url); } + return null; } - /** - * Generates a random nonce. This method originated at - * developer.pearson.com - * Oauth 1.0a Sample Code - * - * @return A unique identifier for the request - * @since 8.7.0 - */ - private static String getNonce() - { - return Long.toHexString(Math.abs(RANDOM.nextLong())); - } - /** - * Generates an integer representing the number of seconds since the unix epoch using the - * date/time the request is issued. This method originated at - * developer.pearson.com - * Oauth 1.0a Sample Code - * - * - * @return A timestamp for the request - * @since 8.7.0 + * Generates a {@link PrivateKey} object from the encoded {@code privateKey}string obtained from the + * {@code JKS} keystore. The provided {@link KeyStore} is designated at application boot time by the JVM's + * {@code javax.net.ssl.keyStore} and {@code javax.net.ssl.keyStorePassword} properties.s + * under the provided {@code alias} + * @param alias the alias to the private key stored in the YADA keystore + * @return the private key object + * @throws NoSuchAlgorithmException when the JKS keystore can't be loaded + * @throws KeyStoreException when there is no {@code JKS} keystore available + * @throws IOException when the JKS keystore can't be loaded due to a filesystem issue + * @throws CertificateException when the JKS keystore can't be loaded + * @throws UnrecoverableEntryException when the desired entry can't be extracted + * @since 8.7.0 */ - private static String getTimestamp() - { - return Long.toString((System.currentTimeMillis() / 1000)); - } - - /** - * Generates an OAuth 1.0 signature. This method originated at - * developer.pearson.com - * Oauth 1.0a Sample Code - * - * - * @param httpMethod The HTTP method of the request - * @param URL The request URL - * @param oauthParams The associative set of signable oAuth parameters - * @param requestBody The serialized POST/PUT message body - * @param secret Alphanumeric string used to validate the identity of the education partner (Private Key) - * - * @return A string containing the Base64-encoded signature digest - * - * @throws UnsupportedEncodingException - * @throws SignatureException - * @throws InvalidKeySpecException - * @throws NoSuchAlgorithmException - * @throws InvalidKeyException - * @since 8.7.0 - */ - private static String generateSignature( - String httpMethod, - URL url, - Map oauthParams, - byte[] requestBody, - String secret - ) throws UnsupportedEncodingException, InvalidKeyException, NoSuchAlgorithmException, InvalidKeySpecException, SignatureException + private PrivateKey getPrivateKey(String alias) throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException, UnrecoverableEntryException { - // Ensure the HTTP Method is upper-cased - httpMethod = httpMethod.toUpperCase(); - - // Construct the URL-encoded OAuth parameter portion of the signature base string - String encodedParams = normalizeParams(httpMethod, url, oauthParams, requestBody); - - // URL-encode the relative URL - String encodedUri = URLEncoder.encode(url.getPath(), CHARSET_UTF8); - - // Build the signature base string to be signed with the Consumer Secret - String baseString = String.format("%s&%s&%s", httpMethod, encodedUri, encodedParams); - - return generateRSA(secret, baseString); - - } - - /** - * Normalizes all OAuth signable parameters and url query parameters according to OAuth 1.0. This method originated at - * developer.pearson.com - * Oauth 1.0a Sample Code - * - * - * @param httpMethod The upper-cased HTTP method - * @param URL The request URL - * @param oauthParams The associative set of signable oAuth parameters - * @param requstBody The serialized POST/PUT message body - * - * @return A string containing normalized and encoded oAuth parameters - * - * @throws UnsupportedEncodingException - * @since 8.7.0 - */ - private static String normalizeParams( - String httpMethod, - URL url, - Map oauthParams, - byte[] requestBody - ) throws UnsupportedEncodingException - { - - // Sort the parameters in lexicographical order, 1st by Key then by Value - Map kvpParams = new TreeMap(String.CASE_INSENSITIVE_ORDER); - kvpParams.putAll(oauthParams); - - // Place any query string parameters into a key value pair using equals ("=") to mark - // the key/value relationship and join each parameter with an ampersand ("&") - if (url.getQuery() != null) - { - for(String keyValue : url.getQuery().split(AMP)) - { - String[] p = keyValue.split(EQUAL); - kvpParams.put(p[0],p[1]); - } - - } - - // Include the body parameter if dealing with a POST or PUT request - if (YADARequest.METHOD_POST.equals(httpMethod) || YADARequest.METHOD_PUT.equals(httpMethod)) - { - String body = Base64.encodeBase64String(requestBody).replaceAll(CRLF, ""); - // url encode the body 2 times now before combining other params - body = URLEncoder.encode(body, CHARSET_UTF8); - body = URLEncoder.encode(body, CHARSET_UTF8); - kvpParams.put(BODY, body); - } - - // separate the key and values with a "=" - // separate the kvp with a "&" - StringBuilder combinedParams = new StringBuilder(); - String delimiter=""; - for(String key : kvpParams.keySet()) { - combinedParams.append(delimiter); - combinedParams.append(key); - combinedParams.append(EQUAL); - combinedParams.append(kvpParams.get(key)); - delimiter=AMP; - } - - // url encode the entire string again before returning - return URLEncoder.encode(combinedParams.toString(), CHARSET_UTF8); + String ksPath = System.getProperty(PROP_KEYSTORE); + char[] ksPass = System.getProperty(PROP_KEYSTORE_PASS).toCharArray(); + KeyStore.ProtectionParameter ksProtParam = new KeyStore.PasswordProtection(ksPass); + File ksFile = new File(ksPath); + URL ksURL = ksFile.toURI().toURL(); + KeyStore keyStore = KeyStore.getInstance(KEYSTORE_TYPE_JKS); + try(InputStream is = ksURL.openStream()) + { + keyStore.load(is, null == ksPass ? null : ksPass); + } + KeyStore.PrivateKeyEntry ksEntry = (KeyStore.PrivateKeyEntry) keyStore.getEntry(alias, ksProtParam); + PrivateKey pk = ksEntry.getPrivateKey(); + + return pk; } -//TODO implement MAC support for Oauth 1.0a -// Leave the commented out MAC stuff in here for now. It's not pressing for the current requirement -// but should be rectified soon. It will require finding or building a provider in order to test, -// which could take some time. -// -// /** -// * Generates a Base64-encoded CMAC-AES digest -// * -// * @param key The secret key used to sign the data -// * @param msg The data to be signed -// * -// * @return A CMAC-AES hash -// * -// * @throws UnsupportedEncodingException -// */ -// private static String generateCmac(String key, String msg) throws UnsupportedEncodingException -// { -// byte[] keyBytes = key.getBytes(CHARSET_UTF8); -// byte[] data = msg.getBytes(CHARSET_UTF8); - -//// Person code: -//// CMac macProvider = new CMac(new AESFastEngine()); -//// macProvider.init(new KeyParameter(keyBytes)); -//// macProvider.reset(); -//// -//// macProvider.update(data, 0, data.length); -//// byte[] output = new byte[macProvider.getMacSize()]; -//// macProvider.doFinal(output, 0); - -// YADA stubby: -// Mac mac = Mac.getInstance("HmacSHA1"); -// mac.init(secretKey); -// -// // Convert the CMAC to a Base64 string and remove the new line the Base64 library adds -// String cmac = Base64.encodeBase64String(output).replaceAll("\r\n", ""); -// -// return cmac; -// -// } - - /** - * Generates a Base64-encoded RSA-SHA1 digest - * - * @param key The secret key used to sign the data - * @param msg The data to be signed - * - * @return An RSA-SHA1 hash - * - * @throws UnsupportedEncodingException - * @throws NoSuchAlgorithmException - * @throws InvalidKeySpecException - * @throws InvalidKeyException - * @throws SignatureException - */ - private static String generateRSA(String key, String msg) throws UnsupportedEncodingException, NoSuchAlgorithmException, InvalidKeySpecException, InvalidKeyException, SignatureException - { - byte[] keyBytes = key.getBytes(CHARSET_UTF8); - byte[] data = msg.getBytes(CHARSET_UTF8); - Signature signer = Signature.getInstance(ALGO_RSASHA1); - KeyFactory kf = KeyFactory.getInstance(ALGO_RSA); - PrivateKey privateKey = kf.generatePrivate(new PKCS8EncodedKeySpec(keyBytes)); - signer.initSign(privateKey); - signer.update(data); - byte[] signed = signer.sign(); - return Base64.encodeBase64String(signed); - } /** - * This method relies on passage of {@link YADAQuery#getOAuth()} which returns + * Creates the {@link OAuthParameters} {@link HttpRequestInitializer}. + * This method relies on inclusion of {@link YADARequest#getOAuth()} which returns * a {@link JSONObject} containing: * *
      @@ -452,105 +312,82 @@ private static String generateRSA(String key, String msg) throws UnsupportedEnco *
    • oauth_token
      • *
    • oauth_verifier (secret)
      • *
    • oauth_version. For now, should always equal 1.0a
      • + *
    • oauth_private_key which is the alias + * under which the desired KeyStore.PrivateKeyEntry is stored
      • *
    - * - * The logic of this method is derived from - * developer.pearson.com - * Oauth 1.0a Sample Code * - * @param body the value associated to the YADA_PAYLOAD key in the current "row" - * @param hConn the {@link HttpURLConnection} used to engage with the web service - * @throws SignatureException - * @throws InvalidKeySpecException - * @throws NoSuchAlgorithmException - * @throws InvalidKeyException - * @throws IOException + * @param url the transformed (conformed) REST endpoint + * @return the initializer to pass to the request factory + * @throws YADASecurityException if there is an issue with the keystore or key entry + * @throws YADAQueryConfigurationException if the 'oauth' request param is misconfigured * @since 8.7.0 */ - private void setAuthenticationOAuth(String body, HttpURLConnection hConn) throws InvalidKeyException, NoSuchAlgorithmException, InvalidKeySpecException, SignatureException, IOException + private HttpRequestInitializer setAuthenticationOAuth(GenericUrl url) throws YADASecurityException, YADAQueryConfigurationException { - - String httpMethod = hConn.getRequestMethod(); - URL url = hConn.getURL(); - byte[] requestBody = null; - BufferedReader in = null; + OAuthRsaSigner signer = new OAuthRsaSigner(); - // Set the request body if making a POST or PUT request - //TODO handle body content--this is just YADA_PAYLOAD now which is probs totes wrong - if (YADARequest.METHOD_POST.equals(httpMethod) || YADARequest.METHOD_PUT.equals(httpMethod)) + try { - requestBody = body.getBytes(CHARSET_UTF8); - } - - // Create the OAuth parameter map from the oauth yada param - Map oauthParams = new LinkedHashMap(); - for(Object key : oauth.keySet()) + signer.privateKey = getPrivateKey(oauth.getString(OAUTH_PRIVATE_KEY)); + } + catch (NoSuchAlgorithmException|KeyStoreException|CertificateException|UnrecoverableEntryException|IOException e) + { + String msg = "There was a problem loading the KeyStore or obtaining the designated key."; + throw new YADASecurityException(msg,e); + } + catch (JSONException e) { - oauthParams.put((String)key, oauth.getString((String)key)); - } - oauthParams.put(OAUTH_TIMESTAMP, getTimestamp()); - oauthParams.put(OAUTH_NONCE, getNonce()); - + String msg = "There was a problem obtaining the private key entry alias from the request. Check the 'oauth' request parameter for the 'oauth_private_key' property."; + throw new YADAQueryConfigurationException(msg,e); + } + + OAuthParameters oauthParams = new OAuthParameters(); + oauthParams.consumerKey = oauth.getString(OAUTH_CONSUMER_KEY); + oauthParams.signer = signer; + oauthParams.signatureMethod = oauth.getString(OAUTH_SIGNATURE_METHOD); + oauthParams.token = oauth.getString(OAUTH_TOKEN); + oauthParams.verifier = oauth.getString(OAUTH_VERIFIER); + oauthParams.version = oauth.getString(OAUTH_VERSION); + oauthParams.computeNonce(); + oauthParams.computeTimestamp(); + try - // Get the OAuth 1.0 Signature - String secret = oauthParams.get(OAUTH_VERIFIER); - String signature = generateSignature(httpMethod, url, oauthParams, requestBody, secret); + { + oauthParams.computeSignature(this.method, url); + } + catch (GeneralSecurityException e) + { + String msg = "Unable to compute signature with the information provided. Check 'oauth' request parameter and key pair."; + throw new YADASecurityException(msg, e); + } - l.debug(String.format("OAuth 1.0 Signature = %s", signature)); - - // Add the oauth_signature parameter to the set of OAuth Parameters - oauthParams.put(OAUTH_SIGNATURE, signature); - - // Generate a string of comma delimited: keyName="URL-encoded(value)" pairs - StringBuilder paramStringBldr = new StringBuilder(); - String delimiter = ""; - for (String keyName : oauthParams.keySet()) { - paramStringBldr.append(delimiter); - String value = oauthParams.get((String) keyName); - paramStringBldr.append(keyName).append(EQUAL+QUOTE).append(URLEncoder.encode(value, CHARSET_UTF8)).append(QUOTE); - delimiter=COMMA; - } - - String urlString = url.toString(); - // omit the queryString from the url - int startOfQueryString = urlString.indexOf(URL_QUERY_MARKER); - if(startOfQueryString != -1) { - urlString = urlString.substring(0,startOfQueryString); - } - - // Build the X-Authorization request header - String xauth = String.format(OAUTH_REALM+EQUAL+QUOTE+"%s"+QUOTE+",%s", urlString, paramStringBldr.toString()); - l.debug(String.format("X-Authorization request header = %s", xauth)); - - // Add the header - hConn.addRequestProperty(X_AUTHORIZATION, xauth); + return oauthParams; } /** - * Sets the Authorization header for the request with Base64-encoded key/value pair - * @param url the URL object - * @param conn the URLConnection object + * Creates the {@link BasicAuthentication} {@link HttpRequestInitializer} + * @param url the conformed REST endpoint + * @return the initializer to pass to the request factory * @since 8.7.0 */ - private void setAuthenticatonBasic(URL url, URLConnection conn) + private HttpRequestInitializer setAuthenticatonBasic(GenericUrl url) { - //TODO basic auth and other auth methods should be mutually exclusive - if (url.getUserInfo() != null) - { - //TODO issue with '@' sign in pw, must decode first - String basicAuth = "Basic " + new String(new Base64().encode(url.getUserInfo().getBytes())); - conn.setRequestProperty("Authorization", basicAuth); - } + //TODO issue with '@' sign in pw, must decode first (is this still an issue?) + String[] userinfo = url.getUserInfo().split(":"); + String username = userinfo[0]; + String password = userinfo[1]; + BasicAuthentication ba = new BasicAuthentication(username, password); + return ba; } /** * Looks for cookie strings passed in the request or stored in the {@link YADAQuery} - * and adds them to the {@link URLConnection} + * and adds them to the {@link HttpRequest} * @param yq The YADAQuery object - * @param conn The URLConnection object + * @param request The HttpRequest object * @since 8.7.0 */ - private void setCookies(YADAQuery yq, URLConnection conn) + private void setCookies(YADAQuery yq, HttpRequest request) { if(yq.getCookies() != null && yq.getCookies().size() > 0) { @@ -559,21 +396,24 @@ private void setCookies(YADAQuery yq, URLConnection conn) { cookieStr += cookie.getName()+"="+cookie.getValue()+";"; } - conn.setRequestProperty("Cookie", cookieStr); + HttpHeaders headers = new HttpHeaders(); + headers.setCookie(cookieStr); + request.setHeaders(headers); } } /** * Looks for http header strings passed in the request or stored in the {@link YADAQuery} - * and adds them to the {@link URLConnection} - * @param yq the YADAQuery object - * @param conn the URLConnection object + * and adds them to the {@link HttpRequest} + * @param yq the {@link YADAQuery} object + * @param request the HttpRequest object * @since 8.7.0 */ - private void setHeaders(YADAQuery yq, URLConnection conn) + private void setHeaders(YADAQuery yq, HttpRequest request) { if(yq.getHttpHeaders() != null && yq.getHttpHeaders().length() > 0) { + HttpHeaders headers = new HttpHeaders(); l.debug("Processing custom headers..."); @SuppressWarnings("unchecked") Iterator keys = yq.getHttpHeaders().keys(); @@ -582,67 +422,46 @@ private void setHeaders(YADAQuery yq, URLConnection conn) String name = keys.next(); String value = yq.getHttpHeaders().getString(name); l.debug("Custom header: "+name+" : "+value); - conn.setRequestProperty(name, value); - if(name.equals(X_HTTP_METHOD_OVERRIDE) && value.equals(YADARequest.METHOD_PATCH)) - { - l.debug("Resetting method to ["+YADARequest.METHOD_POST+"]"); - this.method = YADARequest.METHOD_POST; - } + headers.set(name, value); } } } /** - * Injects the payload into the request's {@link OutputStream} if {@link YADARequest#getMethod()} returns - * {@link YADARequest#METHOD_POST}, {@link YADARequest#METHOD_PUT}, or {@link YADARequest#METHOD_PATCH} - * @param yq The current YADAQuery object - * @param row The current data "row" - * @param hConn The HTTPUrlConnection into which the data will be put - * @throws YADAAdaptorExecutionException in the event of an issue writing to the output stream. + * Returns true if method matches POST, PUT, or PATCH + * @param method the http method of the request to check + * @return true if method matches POST, PUT, or PATCH * @since 8.7.0 */ - private void handlePostPutPatch(YADAQuery yq, int row, HttpURLConnection hConn) throws YADAAdaptorExecutionException { - boolean isPostPutPatch = this.method.equals(YADARequest.METHOD_POST) - || this.method.equals(YADARequest.METHOD_PUT) - || this.method.equals(YADARequest.METHOD_PATCH); - - if(!this.method.equals(YADARequest.METHOD_GET)) - { - try - { - hConn.setRequestMethod(this.method); - if(isPostPutPatch) - { - //TODO make YADA_PAYLOAD case-insensitive and create an alias for it, e.g., ypl - // NOTE: YADA_PAYLOAD is a COLUMN NAME found in a JSONParams DATA object. It - // is not a YADA param - String payload = yq.getDataRow(row).get(YADA_PAYLOAD)[0]; - hConn.setDoOutput(true); - OutputStreamWriter writer; - writer = new OutputStreamWriter(hConn.getOutputStream()); - writer.write(payload.toString()); - writer.flush(); - } - } - catch (IOException e) - { - String msg = "There was a problem injecting the payload into the request"; - throw new YADAAdaptorExecutionException(msg, e); - } - } + protected static boolean isPostPutPatch(String method) + { + return method.equals(YADARequest.METHOD_POST) + || method.equals(YADARequest.METHOD_PUT) + || method.equals(YADARequest.METHOD_PATCH); } /** * Reads the response {@link InputStream} and writes it to a {@link String} - * @param hConn the connection object + * @param request the request object * @return a String containing the response data * @throws YADAAdaptorExecutionException when reading the input stream fails * @since 8.7.0 */ - private String processResponse(HttpURLConnection hConn) throws YADAAdaptorExecutionException + private String processResponse(HttpRequest request) throws YADAAdaptorExecutionException { String result = ""; - try(BufferedReader in = new BufferedReader(new InputStreamReader(hConn.getInputStream()))) + HttpResponse response; + try + { + response = request.execute(); + } + catch (IOException e) + { + String msg = "Unable to execute request."; + throw new YADAAdaptorExecutionException(msg,e); + } + + try(BufferedReader in = new BufferedReader(new InputStreamReader(response.getContent()))) { String inputLine; while ((inputLine = in.readLine()) != null) @@ -659,114 +478,149 @@ private String processResponse(HttpURLConnection hConn) throws YADAAdaptorExecut } /** - * Writes the header fields to the console or log - * @param conn the current connection object + * Writes the header fields to the console or log while in DEBUG mode + * @param request the current request object * @since 8.7.0 */ - private void logRequest(URLConnection conn) { - Map> map = conn.getHeaderFields(); - for (Map.Entry> entry : map.entrySet()) { + private void logRequest(HttpRequest request) { + Map map = request.getHeaders(); + for (Entry entry : map.entrySet()) { l.debug("Key : " + entry.getKey() + " ,Value : " + entry.getValue()); } } + /** + * Wrapper for {@link HttpRequestFactory#buildRequest(String, GenericUrl, HttpContent)} + * @param yq The {@link YADAQuery} defined by the {@link YADARequest} + * @param payload the request body or null + * @param url the REST endpoint + * @return the initialized {@link HttpRequest} + * @throws YADAQueryConfigurationException when there is an issue with request construction + * @throws YADASecurityException when authentication fails + */ + private HttpRequest buildRequest(YADAQuery yq, String payload, GenericUrl url) throws YADAQueryConfigurationException, YADASecurityException + { + HttpContent content = null; + HttpRequestInitializer initializer = setAuthentication(yq, url); + HttpRequestFactory requestFactory = new NetHttpTransport().createRequestFactory(initializer); + if(isPostPutPatch(this.method)) + { + content = ByteArrayContent.fromString(null, payload); + if(this.method.equals(YADARequest.METHOD_PATCH)) + this.method = YADARequest.METHOD_PUT; + } + + try + { + return requestFactory.buildRequest(this.method, url, content); + } + catch (IOException e) + { + String msg = "Unable to initialize the "+this.method+" request for ["+url+"]"; + throw new YADAQueryConfigurationException(msg,e); + } + } + /** * Gets the input stream from the {@link URLConnection} and stores it in * the {@link YADAQueryResult} in {@code yq} + * @throws YADASecurityException if authentication fails * @see com.novartis.opensource.yada.adaptor.Adaptor#execute(com.novartis.opensource.yada.YADAQuery) */ @Override - public void execute(YADAQuery yq) throws YADAAdaptorExecutionException + public void execute(YADAQuery yq) throws YADAAdaptorExecutionException, YADASecurityException { + // + // THINGS ORGANIZED IN HERE: + // + // Type of request + // This will determine the 'build' method in the HttpRequestFactory (i.e., buildGet, buildPost, etc) + // + // Type of authentication + // Basic, Oauth, None + // + // Post content + // Cookies + // Headers + // Proxy (config'd in JVM for now) + // + + // 1) Override method type if necessary + if(yq.getHttpHeaders().has(H_X_HTTP_METHOD_OVERRIDE)) + { + this.method = yq.getHttpHeaders().getString(H_X_HTTP_METHOD_OVERRIDE); + l.debug("Resetting method to ["+this.method+"]"); + } + + // 2) set 'count' to false until the algo for counting rest response rows is impl. resetCountParameter(yq); + + // 3) get the data array from the YADAQuery + // + // Remember: + // A row is a set of YADA URL parameter values, e.g., + // + // x,y,z in this: + // ...yada/q/queryname/p/x,y,z + // so 1 row + // + // or each of {col1:x,col2:y,col3:z} and {col1:a,col2:b,col3:c} in this: + // ...j=[{qname:queryname,DATA:[{col1:x,col2:y,col3:z},{col1:a,col2:b,col3:c}]}] + // so 2 rows + // int rows = yq.getData().size() > 0 ? yq.getData().size() : 1; - /* - * Remember: - * A row is an set of YADA URL parameter values, e.g., - * - * x,y,z in this: - * ...yada/q/queryname/p/x,y,z - * so 1 row - * - * or each of {col1:x,col2:y,col3:z} and {col1:a,col2:b,col3:c} in this: - * ...j=[{qname:queryname,DATA:[{col1:x,col2:y,col3:z},{col1:a,col2:b,col3:c}]}] - * so 2 rows - */ - URL url; - URLConnection conn; - HttpURLConnection hConn = null; + + + // 4) process the url + String urlString; + HttpRequest request = null; + for(int row=0;row Date: Mon, 22 Oct 2018 11:42:04 -0400 Subject: [PATCH 17/17] javadoc typo --- .../com/novartis/opensource/yada/YADASecurityException.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yada-api/src/main/java/com/novartis/opensource/yada/YADASecurityException.java b/yada-api/src/main/java/com/novartis/opensource/yada/YADASecurityException.java index 360523ea..d3cec21b 100644 --- a/yada-api/src/main/java/com/novartis/opensource/yada/YADASecurityException.java +++ b/yada-api/src/main/java/com/novartis/opensource/yada/YADASecurityException.java @@ -21,7 +21,7 @@ /** * Throw from a security preprocessor or adaptor when authentication or authorization fails. - * Moved to main package from {@coode plugin} for {@code 8.7.0} + * Moved to main package from {@code plugin} for {@code 8.7.0} * @author Dave Varon * @since 7.0.0 *