diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/Shiftr.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/Shiftr.java index 0b8dc218..f6f308bd 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/Shiftr.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/Shiftr.java @@ -258,11 +258,30 @@ * * * '#' Wildcard - * Valid only on the RHS of the spec, nested in an array, like "[#2]" - * This wildcard is useful if you want to take a JSON map and turn it into a JSON array, and you do not care about the order of the array. + * Valid both on the LHS and RHS, but has different behavior / format on either side. + * They way to think of it, is that it allows you to specify a "synthentic" value, aka a value not found in the input data. * - * While Shiftr is doing its parallel tree walk of the input data and the spec, it tracks how many matched it has processed at each level - * of the spec tree. + * On the RHS of the spec, # is only valid in the the context of an array, like "[#2]". + * What "[#2]" means is, go up the three 2 levels and ask that node how many matches it has had, and then use that as an index + * in the arrays. + * This means that, while Shiftr is doing its parallel tree walk of the input data and the spec, it tracks how many matches it + * has processed at each level of the spec tree. + * + * This useful if you want to take a JSON map and turn it into a JSON array, and you do not care about the order of the array. + * + * On the LHS of the spec, # allows you to specify a hard coded String to be place as a value in the output. + * + * The initial use-case for this feature was to be able to process a Boolean input value, and if the value is + * boolean true write out the string "enabled". Note, this was possible before, but it required two Shiftr steps. + * + * 
+ *      Example
+ *      "hidden" : {
+ *          "true" : {                             // if the value of "hidden" is true
+ *              "#disabled" : "clients.clientId"   // write the word "disabled" to the path "clients.clientId"
+ *          }
+ *      }
+ *
* * * '|' Wildcard @@ -278,7 +297,10 @@ * * * '@' Wildcard - * Valid only on the LHS of the spec. + * Valid only on both sides of the spec. + * + * The basic '@' on the LHS. + * * This wildcard is necessary if you want to do put both the input value and the input key somewhere in the output JSON. * * Example '@' wildcard usage : @@ -298,6 +320,13 @@ * * Thus the '@' wildcard is the mean "copy the value of the data at this level in the tree, to the output". * + * Advanced '@' sign wildcard. + * The format is lools like "@(3,title)", where + * "3" means go up the tree 3 levels and then lookup the key + * "title" and use the value at that key. + * + * See the filter*.json and transpose*.json Unit Test fixtures. + * * * JSON Arrays : * @@ -471,7 +500,7 @@ public Object transform( Object input ) { // Create a root LiteralPathElement so that # is useful at the root level LiteralPathElement rootLpe = new LiteralPathElement( ROOT_KEY ); WalkedPath walkedPath = new WalkedPath(); - walkedPath.add( rootLpe ); + walkedPath.add( input, rootLpe ); rootSpec.apply( ROOT_KEY, input, walkedPath, output ); diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/cardinality/CardinalityCompositeSpec.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/cardinality/CardinalityCompositeSpec.java index 2ed95a97..e0748cfc 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/cardinality/CardinalityCompositeSpec.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/cardinality/CardinalityCompositeSpec.java @@ -144,7 +144,7 @@ public boolean apply( String inputKey, Object input, WalkedPath walkedPath, Obje return false; } - walkedPath.add( thisLevel ); + walkedPath.add( input, thisLevel ); // The specialChild can change the data object that I point to. // Aka, my key had a value that was a List, and that gets changed so that my key points to a ONE value diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/cardinality/CardinalityLeafSpec.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/cardinality/CardinalityLeafSpec.java index 1026453f..654b2ab6 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/cardinality/CardinalityLeafSpec.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/cardinality/CardinalityLeafSpec.java @@ -87,7 +87,7 @@ public Object applyToParentContainer ( String inputKey, Object input, WalkedPath private Object performCardinalityAdjustment( String inputKey, Object input, WalkedPath walkedPath, Map parentContainer, LiteralPathElement thisLevel ) { // Add our the LiteralPathElement for this level, so that write path References can use it as &(0,0) - walkedPath.add( thisLevel ); + walkedPath.add( input, thisLevel ); Object returnValue = null; if ( cardinalityRelationship == CardinalityRelationship.MANY ) { diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/PathStep.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/PathStep.java new file mode 100644 index 00000000..63307334 --- /dev/null +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/PathStep.java @@ -0,0 +1,41 @@ +/* + * Copyright 2014 Bazaarvoice, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.bazaarvoice.jolt.common; + +import com.bazaarvoice.jolt.common.pathelement.LiteralPathElement; + +/** + * A tuple class that contains the data for one level of a + * tree walk, aka a reference to the input for that level, and + * the LiteralPathElement that was matched at that level. + */ +public final class PathStep { + private final Object treeRef; + private final LiteralPathElement literalPathElement; + + public PathStep(Object treeRef, LiteralPathElement literalPathElement) { + this.treeRef = treeRef; + this.literalPathElement = literalPathElement; + } + + public Object getTreeRef() { + return treeRef; + } + + public LiteralPathElement getLiteralPathElement() { + return literalPathElement; + } +} diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/WalkedPath.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/WalkedPath.java index 06484d08..085276e1 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/WalkedPath.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/WalkedPath.java @@ -23,6 +23,10 @@ /** * DataStructure used by a SpecTransform during it's parallel tree walk. * + * Basically this is Stack that records the steps down the tree that have been taken. + * For each level, there is a PathStep, which contains a pointer the data of that level, + * and a pointer to the LiteralPathElement matched at that level. + * * At any given point in time, it represents where in the tree walk a Spec is operating. * It is primarily used to by the ShiftrLeafSpec and CardinalityLeafSpec as a reference * to lookup real values for output "&(1,1)" references. @@ -30,32 +34,43 @@ * It is expected that as the SpecTransform navigates down the tree, LiteralElements will be added and then * removed when that subtree has been walked. */ -public class WalkedPath extends ArrayList { +public class WalkedPath extends ArrayList { public WalkedPath() { super(); } - public WalkedPath( Collection c ) { - super( c ); + public WalkedPath(Collection c) { + super(c); } - public LiteralPathElement removeLast() { - return remove( size() - 1 ); + public WalkedPath( Object treeRef, LiteralPathElement literalPathElement ) { + super(); + this.add( new PathStep( treeRef, literalPathElement ) ); + } + + /** + * Convenience method + */ + public boolean add( Object treeRef, LiteralPathElement literalPathElement ) { + return super.add( new PathStep( treeRef, literalPathElement ) ); + } + + public void removeLast() { + remove(size() - 1); } /** * Method useful to "&", "&1", "&2", etc evaluation. */ - public LiteralPathElement elementFromEnd( int idxFromEnd ) { - if ( isEmpty() ) { + public PathStep elementFromEnd(int idxFromEnd) { + if (isEmpty()) { return null; } - return get( size() - 1 - idxFromEnd ); + return get(size() - 1 - idxFromEnd); } - public LiteralPathElement lastElement() { - return get( size() - 1 ); + public PathStep lastElement() { + return get(size() - 1); } - } diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/AmpPathElement.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/AmpPathElement.java index 7897e1c4..e006ee1f 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/AmpPathElement.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/AmpPathElement.java @@ -114,7 +114,7 @@ public String evaluate( WalkedPath walkedPath ) { } else { AmpReference ref = (AmpReference) token; - LiteralPathElement literalPathElement = walkedPath.elementFromEnd( ref.getPathIndex() ); + LiteralPathElement literalPathElement = walkedPath.elementFromEnd( ref.getPathIndex() ).getLiteralPathElement(); String value = literalPathElement.getSubKeyRef( ref.getKeyGroup() ); output.append( value ); } diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/ArrayPathElement.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/ArrayPathElement.java index 96ce2052..74cdb2ad 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/ArrayPathElement.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/ArrayPathElement.java @@ -98,12 +98,12 @@ public String evaluate( WalkedPath walkedPath ) { return arrayIndex; case HASH: - LiteralPathElement element = walkedPath.elementFromEnd( ref.getPathIndex() ); + LiteralPathElement element = walkedPath.elementFromEnd( ref.getPathIndex() ).getLiteralPathElement(); Integer index = element.getHashCount(); return index.toString(); case REFERENCE: - LiteralPathElement lpe = walkedPath.elementFromEnd( ref.getPathIndex() ); + LiteralPathElement lpe = walkedPath.elementFromEnd( ref.getPathIndex() ).getLiteralPathElement(); String keyPart; if ( ref instanceof PathAndGroupReference ) { @@ -118,7 +118,7 @@ public String evaluate( WalkedPath walkedPath ) { return keyPart; } catch ( NumberFormatException nfe ) { - throw new RuntimeException( " Evaluating canonical ReferencePathElement:" + this.getCanonicalForm() + ", and got a non integer result for reference:" + ref.getCanonicalForm() ); + throw new RuntimeException( " Evaluating canonical ReferencePathElement:" + this.getCanonicalForm() + ", and got a non integer result:(" + keyPart + "), for reference:" + ref.getCanonicalForm() ); } default: throw new IllegalStateException( "ArrayPathType enum added two without updating this switch statement." ); diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/AtPathElement.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/AtPathElement.java index dd06c3d3..a7500c05 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/AtPathElement.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/AtPathElement.java @@ -28,7 +28,7 @@ public AtPathElement( String key ) { } public LiteralPathElement match( String dataKey, WalkedPath walkedPath ) { - return walkedPath.lastElement(); // copy what our parent was so that write keys of &0 and &1 both work. + return walkedPath.lastElement().getLiteralPathElement(); // copy what our parent was so that write keys of &0 and &1 both work. } @Override diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/DollarPathElement.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/DollarPathElement.java index 17eff15c..da054671 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/DollarPathElement.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/DollarPathElement.java @@ -35,7 +35,7 @@ public String getCanonicalForm() { @Override public String evaluate( WalkedPath walkedPath ) { - LiteralPathElement pe = walkedPath.elementFromEnd( dRef.getPathIndex() ); + LiteralPathElement pe = walkedPath.elementFromEnd( dRef.getPathIndex() ).getLiteralPathElement(); return pe.getSubKeyRef( dRef.getKeyGroup() ); } diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/HashPathElement.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/HashPathElement.java new file mode 100644 index 00000000..a87a2f8f --- /dev/null +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/HashPathElement.java @@ -0,0 +1,68 @@ +/* + * Copyright 2013 Bazaarvoice, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.bazaarvoice.jolt.common.pathelement; + +import com.bazaarvoice.jolt.common.WalkedPath; +import com.bazaarvoice.jolt.exception.SpecException; +import com.bazaarvoice.jolt.utils.StringTools; + +/** + * For use on the LHS, allows the user to specify an explicit string to write out. + * Aka given a input that is boolean, would want to write something out other than "true" / "false". + */ +public class HashPathElement extends BasePathElement implements MatchablePathElement { + + private final String keyValue; + + public HashPathElement( String key ) { + super(key); + + if ( StringTools.isBlank( key ) ) { + throw new SpecException( "HashPathElement cannot have empty String as input." ); + } + + if ( ! key.startsWith( "#" ) ) { + throw new SpecException( "LHS # should start with a # : " + key ); + } + + if ( key.length() <= 1 ) { + throw new SpecException( "HashPathElement input is too short : " + key ); + } + + + if ( key.charAt( 1 ) == '(' ) { + if ( key.charAt( key.length() -1 ) == ')' ) { + keyValue = key.substring( 2, key.length() -1 ); + } + else { + throw new SpecException( "HashPathElement, mismatched parens : " + key ); + } + } + else { + keyValue = key.substring( 1 ); + } + } + + @Override + public String getCanonicalForm() { + return "#(" + keyValue + ")"; + } + + @Override + public LiteralPathElement match( String dataKey, WalkedPath walkedPath ) { + return new LiteralPathElement( keyValue ); + } +} diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/LiteralPathElement.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/LiteralPathElement.java index a9d9964e..e8aea9f5 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/LiteralPathElement.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/LiteralPathElement.java @@ -56,7 +56,7 @@ public String evaluate( WalkedPath walkedPath ) { } @Override - public com.bazaarvoice.jolt.common.pathelement.LiteralPathElement match( String dataKey, WalkedPath walkedPath ) { + public LiteralPathElement match( String dataKey, WalkedPath walkedPath ) { return getRawKey().equals( dataKey ) ? this : null ; } diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/TransposePathElement.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/TransposePathElement.java new file mode 100644 index 00000000..c250fb33 --- /dev/null +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/common/pathelement/TransposePathElement.java @@ -0,0 +1,237 @@ +/* + * Copyright 2014 Bazaarvoice, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.bazaarvoice.jolt.common.pathelement; + +import com.bazaarvoice.jolt.common.PathStep; +import com.bazaarvoice.jolt.common.WalkedPath; +import com.bazaarvoice.jolt.exception.SpecException; +import com.bazaarvoice.jolt.shiftr.TransposeReader; +import com.bazaarvoice.jolt.utils.StringTools; + +/** + * This PathElement is used by Shiftr to Transpose data. + * + * It can be used on the Left and Right hand sides of the spec. + * + * Input + * { + * "author" : "Stephen Hawking", + * "book" : "A Brief History of Time" + * } + * + * Wanted + * { + * "Stephen Hawking" : "A Brief History of Time" + * } + * + * The first part of the process is to allow a CompositeShiftr node to look down the input JSON tree. + * + * Spec + * { + * "@author" : "@book" + * } + * + * + * Secondly, we can look up the tree, and come down a different path to locate data. + * + * For example of this see the following ShiftrUnit tests : + * LHS Lookup : json/shiftr/filterParents.json + * RHS Lookup : json/shiftr/transposeComplex6_rhs-complex-at.json + * + * + * CanonicalForm Expansion + * Sugar + * "@2 -> "@(2,) + * "@(2) -> "@(2,) + * "@author" -> "@(0,author)" + * "@(author)" -> "@(0,author)" + * + * Splenda + * "@(a.b)" -> "@(0,a.b)" + * "@(a.&2.c)" -> "@(0,a.&(2,0).c)" + */ +public class TransposePathElement extends BasePathElement implements MatchablePathElement, EvaluatablePathElement { + + private final int upLevel; + private final TransposeReader subPathReader; + private final String canonicalForm; + + /** + * Parse a text value from a Spec, into a TransposePathElement. + * + * @param key rawKey from a Jolt Spec file + * @return a TransposePathElement + */ + public static TransposePathElement parse( String key ) { + + if ( key == null || key.length() < 2 ) { + throw new SpecException( "'Transpose Input' key '@', can not be null or of length 1. Offending key : " + key ); + } + if ( '@' != key.charAt( 0 ) ) { + throw new SpecException( "'Transpose Input' key must start with an '@'. Offending key : " + key ); + } + + // Strip off the leading '@' as we don't need it anymore. + String meat = key.substring( 1 ); + + if ( meat.contains( "@" ) ) { + throw new SpecException( "@ pathElement can not contain a nested @." ); + } + if ( meat.contains( "*" ) || meat.contains( "[]" ) ) { + throw new SpecException( "'Transpose Input' can not contain expansion wildcards (* and []). Offending key : " + key ); + } + + // Check to see if the key is wrapped by parens + if ( meat.startsWith( "(" ) ) { + if ( meat.endsWith( ")" ) ) { + meat = meat.substring( 1, meat.length() - 1 ); + } + else { + throw new SpecException( "@ path element that starts with '(' must have a matching ')'. Offending key : " + key ); + } + } + + return innerParse( key, meat ); + } + + /** + * Parse the core of the TransposePathElement key, once basic errors have been checked and + * syntax has been handled. + * + * @param originalKey The original text for reference. + * @param meat The string to actually parse into a TransposePathElement + * @return TransposePathElement + */ + private static TransposePathElement innerParse( String originalKey, String meat ) { + + char first = meat.charAt( 0 ); + if ( Character.isDigit( first ) ) { + // loop until we find a comma or end of string + StringBuilder sb = new StringBuilder().append( first ); + for ( int index = 1; index < meat.length(); index++ ) { + char c = meat.charAt( index ); + + // when we find a / the first comma, stop looking for integers, and just assume the rest is a String path + if( ',' == c ) { + + int upLevel; + try { + upLevel = Integer.valueOf( sb.toString() ); + } + catch ( NumberFormatException nfe ) { + // I don't know how this exception would get thrown, as all the chars were checked by isDigit, but oh well + throw new SpecException( "@ path element with non/mixed numeric key is not valid, key=" + originalKey ); + } + + return new TransposePathElement( originalKey, upLevel, meat.substring( index + 1 ) ); + } + else if ( Character.isDigit( c ) ) { + sb.append( c ); + } + else { + throw new SpecException( "@ path element with non/mixed numeric key is not valid, key=" + originalKey ); + } + } + + // if we got out of the for loop, then the whole thing was a number. + return new TransposePathElement( originalKey, Integer.valueOf( sb.toString() ), null ); + } + else { + return new TransposePathElement( originalKey, 0, meat ); + } + } + + /** + * Private constructor used after parsing is done. + * + * @param originalKey for reference + * @param upLevel How far up the tree to go + * @param subPath Where to go down the tree + */ + private TransposePathElement( String originalKey, int upLevel, String subPath ) { + super(originalKey); + this.upLevel = upLevel; + if ( StringTools.isEmpty( subPath ) ) { + this.subPathReader = null; + canonicalForm = "@(" + upLevel + ",)"; + } + else { + subPathReader = new TransposeReader(subPath); + canonicalForm = "@(" + upLevel + "," + subPathReader.getCanonicalForm() + ")"; + } + } + + /** + * This method is used when the TransposePathElement is used on the LFH as data. + * + * Aka, normal "evaluate" returns either a Number or a String. + * + * @param walkedPath WalkedPath to evaluate against + * @return The data specified by this TransposePathElement, or null if it can't find anything. + */ + public Object objectEvaluate( WalkedPath walkedPath ) { + // Grap the data we need from however far up the tree we are supposed to go + PathStep pathStep = walkedPath.elementFromEnd( upLevel ); + + Object treeRef = pathStep.getTreeRef(); + + // Now walk down from that level using the subPathReader + if ( subPathReader == null ) { + return treeRef; + } + else { + Object data = subPathReader.read( treeRef, walkedPath ); + return data; + } + } + + @Override + public String evaluate( WalkedPath walkedPath ) { + + Object dataFromTranspose = objectEvaluate( walkedPath ); + + // Coerce a number into a String + if ( dataFromTranspose instanceof Number ) { + // the idea here being we are looking for an array index value + int val = ((Number) dataFromTranspose).intValue(); + return Integer.toString( val ); + } + + // Coerce a boolean into a String + if ( dataFromTranspose instanceof Boolean ) { + return Boolean.toString( (Boolean) dataFromTranspose ); + } + + if ( dataFromTranspose == null || ! ( dataFromTranspose instanceof String ) ) { + + // If this output path has a TransposePathElement, and when we evaluate it + // it does not resolve to a String, then return null + return null; + } + + return (String) dataFromTranspose; + } + + + public LiteralPathElement match( String dataKey, WalkedPath walkedPath ) { + return walkedPath.lastElement().getLiteralPathElement(); // copy what our parent was so that write keys of &0 and &1 both work. + } + + @Override + public String getCanonicalForm() { + return canonicalForm; + } +} diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/PathEvaluatingTraversal.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/PathEvaluatingTraversal.java new file mode 100644 index 00000000..7cfc8c85 --- /dev/null +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/PathEvaluatingTraversal.java @@ -0,0 +1,153 @@ +/* + * Copyright 2014 Bazaarvoice, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.bazaarvoice.jolt.shiftr; + +import com.bazaarvoice.jolt.common.WalkedPath; +import com.bazaarvoice.jolt.common.pathelement.EvaluatablePathElement; +import com.bazaarvoice.jolt.common.pathelement.PathElement; +import com.bazaarvoice.jolt.exception.SpecException; +import com.bazaarvoice.jolt.shiftr.spec.ShiftrSpec; +import com.bazaarvoice.jolt.traversr.Traversr; +import com.bazaarvoice.jolt.utils.StringTools; + +import java.util.*; + +/** + * Combines a Traversr with the ability to evaluate References against a WalkedPath. + * + * Convenience class for path based off a single dot notation String, + * like "rating.&1(2).&.value". + * + * This processes the dot notation path into internal data structures, so + * that the String processing only happens once. + */ +public abstract class PathEvaluatingTraversal { + + private final List elements; + private final Traversr traversr; + + public PathEvaluatingTraversal( String dotNotation ) { + + if ( dotNotation.contains("*") || dotNotation.contains("$")) { + throw new SpecException("DotNotation (write key) can not contain '*' or '$'."); + } + + List paths; + Traversr trav; + + if ( StringTools.isNotBlank( dotNotation ) ) { + + // Compute the path elements. + paths = ShiftrSpec.parseDotNotationRHS( dotNotation ); + + // Use the canonical versions of the path elements to create the Traversr + List traversrPaths = new ArrayList( paths.size() ); + for ( PathElement pe : paths ) { + traversrPaths.add( pe.getCanonicalForm() ); + } + trav = createTraversr( traversrPaths ); + } + else { + paths = Collections.emptyList(); + trav = createTraversr( Arrays.asList( "" ) ); + } + + List evalPaths = new ArrayList( paths.size() ); + for( PathElement pe : paths ) { + if ( ! ( pe instanceof EvaluatablePathElement ) ) { + throw new SpecException( "RHS key=" + pe.getRawKey() + " is not a valid RHS key." ); + } + + evalPaths.add( (EvaluatablePathElement) pe ); + } + + this.elements = Collections.unmodifiableList( evalPaths ); + this.traversr = trav; + } + + protected abstract Traversr createTraversr(List paths); + + /** + * Use the supplied WalkedPath, in the evaluation of each of our PathElements to + * build a concrete output path. Then use that output path to write the given + * data to the output. + * + * @param data data to write + * @param output data structure we are going to write the data to + * @param walkedPath reference used to lookup reference values like "&1(2)" + */ + public void write( Object data, Map output, WalkedPath walkedPath ) { + List evaledPaths = evaluate( walkedPath ); + if ( evaledPaths != null ) { + traversr.set( output, evaledPaths, data ); + } + } + + public Object read( Object data, WalkedPath walkedPath ) { + List evaledPaths = evaluate( walkedPath ); + if ( evaledPaths == null ) { + return null; + } + return traversr.get( data, evaledPaths ); + } + + /** + * Use the supplied WalkedPath, in the evaluation of each of our PathElements. + * + * If our PathElements contained a TransposePathElement, we may return null. + * + * @param walkedPath used to lookup/evaluate PathElement references values like "&1(2)" + * @return null or fully evaluated Strings, possibly with concrete array references like "photos.[3]" + */ + // Visible for testing + List evaluate( WalkedPath walkedPath ) { + + List strings = new ArrayList( elements.size() ); + for ( EvaluatablePathElement pathElement : elements ) { + + String evaledLeafOutput = pathElement.evaluate( walkedPath ); + if ( evaledLeafOutput == null ) { + // If this output path contains a TransposePathElement, and when evaluated, + // return null, then bail + return null; + } + strings.add( evaledLeafOutput ); + } + + return strings; + } + + public int size() { + return elements.size(); + } + + public PathElement get( int index ) { + return elements.get( index ); + } + + /** + * Testing method. + */ + public String getCanonicalForm() { + StringBuilder buf = new StringBuilder(); + + for ( PathElement pe : elements ) { + buf.append( "." ).append( pe.getCanonicalForm() ); + } + + return buf.substring( 1 ); // strip the leading "." + } +} diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/ShiftrTraversr.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/ShiftrTraversr.java index b548fc8f..aea0b7bc 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/ShiftrTraversr.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/ShiftrTraversr.java @@ -21,12 +21,19 @@ import com.bazaarvoice.jolt.traversr.SimpleTraversr; import com.bazaarvoice.jolt.traversr.traversal.TraversalStep; +/** + * Traverser that does not overwrite data. + */ public class ShiftrTraversr extends SimpleTraversr { public ShiftrTraversr( String humanPath ) { super( humanPath ); } + public ShiftrTraversr( List paths ) { + super( paths ); + } + /** * Do a Shift style insert : * 1) if there is no data "there", then just set it diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/ShiftrWriter.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/ShiftrWriter.java index 4e725298..f1908747 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/ShiftrWriter.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/ShiftrWriter.java @@ -15,112 +15,22 @@ */ package com.bazaarvoice.jolt.shiftr; -import com.bazaarvoice.jolt.common.WalkedPath; -import com.bazaarvoice.jolt.exception.SpecException; -import com.bazaarvoice.jolt.common.pathelement.EvaluatablePathElement; -import com.bazaarvoice.jolt.common.pathelement.PathElement; -import com.bazaarvoice.jolt.shiftr.spec.ShiftrSpec; import com.bazaarvoice.jolt.traversr.Traversr; -import com.bazaarvoice.jolt.utils.StringTools; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Map; /** - * Convenience class for path based off a single dot notation String, - * like "rating.&1(2).&.value". - * - * This processes the dot notation path into internal data structures, so - * that the String processing only happens once. + * Shiftr Specific version of a PathEvaluatingTraversal, where we supply a + * ShiftrTraversr. */ -public class ShiftrWriter { - - private final List elements; - private final Traversr traversr; +public class ShiftrWriter extends PathEvaluatingTraversal { public ShiftrWriter( String dotNotation ) { - - if ( dotNotation.contains("@") || dotNotation.contains("*") || dotNotation.contains("$")) { - throw new SpecException("DotNotation (write key) can not contain '@', '*', or '$'."); - } - - List paths; - Traversr trav; - - if ( StringTools.isNotBlank(dotNotation) ) { - String[] split = dotNotation.split( "\\." ); - - paths = ShiftrSpec.parse( split ); - trav = new ShiftrTraversr( dotNotation ); - } - else { - paths = Collections.emptyList(); - trav = new ShiftrTraversr( "" ); - } - - List evalPaths = new ArrayList( paths.size() ); - for( PathElement pe : paths ) { - if ( ! ( pe instanceof EvaluatablePathElement ) ) { - throw new SpecException( "RHS key=" + pe.getRawKey() + " is not a valid RHS key." ); - } - - evalPaths.add( (EvaluatablePathElement) pe ); - } - - this.elements = Collections.unmodifiableList( evalPaths ); - this.traversr = trav; - } - - /** - * Use the supplied WalkedPath, in the evaluation of each of our PathElements to - * build a concrete output path. Then use that output path to write the given - * data to the output. - * - * @param data data to write - * @param output data structure we are going to write the data to - * @param walkedPath reference used to lookup reference values like "&1(2)" - */ - public void write( Object data, Map output, WalkedPath walkedPath ) { - traversr.set( output, evaluate( walkedPath ), data ); + super( dotNotation ); } - /** - * Use the supplied WalkedPath, in the evaluation of each of our PathElements - * @param walkedPath used to lookup/evaluate PathElement references values like "&1(2)" - * @return fully evaluated Strings, possibly with concrete array references like "photos.[3]" - */ - // Visible for testing - List evaluate( WalkedPath walkedPath ) { - - List strings = new ArrayList(elements.size()); - for ( EvaluatablePathElement pathElement : elements ) { - String evaledLeafOutput = pathElement.evaluate( walkedPath ); - strings.add( evaledLeafOutput ); - } - - return strings; - } - - public int size() { - return elements.size(); - } - - public PathElement get( int index ) { - return elements.get( index ); - } - - /** - * Testing method. - */ - public String getCanonicalForm() { - StringBuilder buf = new StringBuilder(); - - for ( PathElement pe : elements ) { - buf.append( "." ).append( pe.getCanonicalForm() ); - } - - return buf.substring( 1 ); // strip the leading "." + @Override + protected Traversr createTraversr( List paths ) { + return new ShiftrTraversr( paths ); } } diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/TransposeReader.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/TransposeReader.java new file mode 100644 index 00000000..b9562882 --- /dev/null +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/TransposeReader.java @@ -0,0 +1,40 @@ +/* + * Copyright 2014 Bazaarvoice, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.bazaarvoice.jolt.shiftr; + +import com.bazaarvoice.jolt.traversr.SimpleTraversr; +import com.bazaarvoice.jolt.traversr.Traversr; + +import java.util.List; + +/** + * The TransposeReader uses a PathEvaluatingTraversal with a SimpleTraversr. + * + * This means that as it walks a path in a tree structure (PathEvaluatingTraversal), + * it uses the behavior of the SimpleTraversr for tree traversal operations like + * get, set, and final set. + */ +public class TransposeReader extends PathEvaluatingTraversal { + + public TransposeReader( String dotNotation ) { + super( dotNotation ); + } + + @Override + protected Traversr createTraversr( List paths ) { + return new SimpleTraversr( paths ); + } +} diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/spec/ShiftrCompositeSpec.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/spec/ShiftrCompositeSpec.java index 69a7e426..9a1b886d 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/spec/ShiftrCompositeSpec.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/spec/ShiftrCompositeSpec.java @@ -82,8 +82,10 @@ public ShiftrCompositeSpec(String rawKey, Map spec ) { literals.put( child.pathElement.getRawKey(), child ); } // special is it is "@" or "$" - else if ( child.pathElement instanceof AtPathElement || ( - child.pathElement instanceof DollarPathElement ) ) { + else if ( child.pathElement instanceof AtPathElement || + child.pathElement instanceof HashPathElement || + child.pathElement instanceof DollarPathElement || + child.pathElement instanceof TransposePathElement ) { special.add( child ); } else { // star || (& with children) @@ -167,7 +169,7 @@ public boolean apply( String inputKey, Object input, WalkedPath walkedPath, Map< } // add ourselves to the path, so that our children can reference us - walkedPath.add( thisLevel ); + walkedPath.add( input, thisLevel ); // Handle any special / key based children first, but don't have them block anything for( ShiftrSpec subSpec : specialChildren ) { @@ -181,7 +183,7 @@ public boolean apply( String inputKey, Object input, WalkedPath walkedPath, Map< walkedPath.removeLast(); // we matched so increment the matchCount of our parent - walkedPath.lastElement().incrementHashCount(); + walkedPath.lastElement().getLiteralPathElement().incrementHashCount(); return true; } diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/spec/ShiftrLeafSpec.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/spec/ShiftrLeafSpec.java index 1d6b419d..ca722efb 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/spec/ShiftrLeafSpec.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/spec/ShiftrLeafSpec.java @@ -16,6 +16,8 @@ package com.bazaarvoice.jolt.shiftr.spec; import com.bazaarvoice.jolt.Shiftr; +import com.bazaarvoice.jolt.common.pathelement.HashPathElement; +import com.bazaarvoice.jolt.common.pathelement.TransposePathElement; import com.bazaarvoice.jolt.shiftr.ShiftrWriter; import com.bazaarvoice.jolt.utils.StringTools; import com.bazaarvoice.jolt.exception.SpecException; @@ -68,7 +70,7 @@ else if ( rhs instanceof List ) { private static ShiftrWriter parseOutputDotNotation( Object rawObj ) { if ( ! ( rawObj instanceof String ) ) { - throw new SpecException( "Invalid Shiftr spec RHS. Should be a string or array of Strings. Value in question : " + rawObj ); + throw new SpecException( "Invalid Shiftr spec, RHS should be a String or array of Strings. Value in question : " + rawObj ); } // Prepend "root" to each output path. @@ -99,20 +101,29 @@ public boolean apply( String inputKey, Object input, WalkedPath walkedPath, Map< } Object data; - boolean realChild; + boolean realChild = false; // by default don't block further Shiftr matches - if ( this.pathElement instanceof DollarPathElement ) { - DollarPathElement subRef = (DollarPathElement) this.pathElement; + if ( this.pathElement instanceof DollarPathElement || + this.pathElement instanceof HashPathElement ) { - // The data is the parent key, so evaluate against the parent's path - data = subRef.evaluate( walkedPath ); - realChild = false; // don't block further Shiftr matches + // The data is already encoded in the thisLevel object created by the pathElement.match called above + data = thisLevel.getCanonicalForm(); } else if ( this.pathElement instanceof AtPathElement ) { // The data is our parent's data data = input; - realChild = false; // don't block further Shiftr matches + } + else if ( this.pathElement instanceof TransposePathElement ) { + // We try to walk down the tree to find the value / data we want + TransposePathElement tpe = (TransposePathElement) this.pathElement; + + // Note the data found may not be a String, thus we have to call the special objectEvaluate + data = tpe.objectEvaluate( walkedPath ); + if ( data == null ) { + // if we could not find the value we want looking down the tree, bail + return false; + } } else { // the data is the input @@ -122,7 +133,7 @@ else if ( this.pathElement instanceof AtPathElement ) { } // Add our the LiteralPathElement for this level, so that write path References can use it as &(0,0) - walkedPath.add( thisLevel ); + walkedPath.add( input, thisLevel ); // Write out the data for ( ShiftrWriter outputPath : shiftrWriters ) { @@ -133,7 +144,7 @@ else if ( this.pathElement instanceof AtPathElement ) { if ( realChild ) { // we were a "real" child, so increment the matchCount of our parent - walkedPath.lastElement().incrementHashCount(); + walkedPath.lastElement().getLiteralPathElement().incrementHashCount(); } return realChild; diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/spec/ShiftrSpec.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/spec/ShiftrSpec.java index f0f75478..3994c53a 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/spec/ShiftrSpec.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/shiftr/spec/ShiftrSpec.java @@ -21,8 +21,10 @@ import com.bazaarvoice.jolt.utils.StringTools; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Iterator; +import java.util.NoSuchElementException; import java.util.List; +import java.util.LinkedList; import java.util.Map; /** @@ -50,7 +52,7 @@ * The tree structure of formed by the CompositeSpecs is what is used during Shiftr transforms * to do the parallel tree walk with the input data tree. * - * During the parallel tree walk, a Path is maintained, and used when + * During the parallel tree walk a stack of data (a WalkedPath) is maintained, and used when * a tree walk encounters an Outputting spec to evaluate the wildcards in the write DotNotationPath. */ public abstract class ShiftrSpec { @@ -59,13 +61,9 @@ public abstract class ShiftrSpec { protected final MatchablePathElement pathElement; public ShiftrSpec(String rawJsonKey) { - List pathElements = parse( rawJsonKey ); - if ( pathElements.size() != 1 ){ - throw new SpecException( "Shiftr invalid LHS:" + rawJsonKey + " can not contain '.'" ); - } + PathElement pe = parseSingleKeyLHS( rawJsonKey ); - PathElement pe = pathElements.get( 0 ); if ( ! ( pe instanceof MatchablePathElement ) ) { throw new SpecException( "Spec LHS key=" + rawJsonKey + " is not a valid LHS key." ); } @@ -73,17 +71,28 @@ public ShiftrSpec(String rawJsonKey) { this.pathElement = (MatchablePathElement) pe; } - // once all the shiftr specific logic is extracted. - public static List parse( String key ) { - - - if ( key.contains("@") ) { + /** + * Visible for Testing. + * + * Inspects the key in a particular order to determine the correct sublass of + * PathElement to create. + * + * @param key String that should represent a single PathElement + * @return a concrete implementation of PathElement + */ + public static PathElement parseSingleKeyLHS( String key ) { - return Arrays.asList( new AtPathElement( key ) ); + if ( "@".equals( key ) ) { + return new AtPathElement( key ); + } + else if ( key.startsWith("@") || key.contains( "@(" ) ) { + return TransposePathElement.parse( key ); + } + else if ( key.contains( "@" ) ) { + throw new SpecException( "Invalid key:" + key + " can not have an @ other than at the front." ); } else if ( key.contains("$") ) { - - return Arrays.asList( new DollarPathElement( key ) ); + return new DollarPathElement( key ); } else if ( key.contains("[") ) { @@ -91,71 +100,260 @@ else if ( key.contains("[") ) { throw new SpecException( "Invalid key:" + key + " has too many [] references."); } - // is canonical array? - if ( key.charAt( 0 ) == '[' && key.charAt( key.length() - 1 ) == ']') { - return Arrays.asList( new ArrayPathElement( key ) ); - } - - // Split syntactic sugar of "photos[]" --> [ "photos", "[]" ] - // or "bob-&(3,1)-smith[&0]" --> [ "bob-&(3,1)-smith", "[&(0,0)]" ] - - String canonicalKey = key.replace( "[", ".[" ); - String[] subkeys = canonicalKey.split( "\\." ); - - List subElements = parse( subkeys ); // at this point each sub key should be a valid key, so just recall parse - - for ( int index = 0; index < subElements.size() - 1; index++ ) { - PathElement v = subElements.get( index ); - if ( v instanceof ArrayPathElement ) { - throw new SpecException( "Array [..] must be the last thing in the key, was:" + key ); - } - } - - return subElements; + return new ArrayPathElement( key ); } - else if ( key.contains("&") ) { + else if ( key.contains( "&" ) ) { if ( key.contains("*") ) { throw new SpecException("Can't mix * with & ) "); } - return Arrays.asList( new AmpPathElement( key ) ); + return new AmpPathElement( key ); } else if ( "*".equals( key ) ) { - return Arrays.asList( new StarAllPathElement( key ) ); + return new StarAllPathElement( key ); } - else if (key.contains("*" ) ) { + else if ( key.contains("*" ) ) { int numOfStars = StringTools.countMatches(key, "*"); if(numOfStars == 1){ - return Arrays.asList( new StarSinglePathElement( key ) ); + return new StarSinglePathElement( key ); } else if(numOfStars == 2){ - return Arrays.asList( new StarDoublePathElement( key ) ); + return new StarDoublePathElement( key ); } else { - return Arrays.asList( new StarRegexPathElement( key ) ); + return new StarRegexPathElement( key ); } } + else if ( key.contains("#" ) ) { + return new HashPathElement( key ); + } else { - return Arrays.asList( new LiteralPathElement( key ) ); + return new LiteralPathElement( key ); + } + } + + + /** + * Helper method to turn a String into an Iterator + */ + private static Iterator stringIterator(final String string) { + // Ensure the error is found as soon as possible. + if (string == null) + throw new NullPointerException(); + + return new Iterator() { + private int index = 0; + + public boolean hasNext() { + return index < string.length(); + } + + public Character next() { + + // Throw NoSuchElementException as defined by the Iterator contract, + // not IndexOutOfBoundsException. + if (!hasNext()) + throw new NoSuchElementException(); + return string.charAt(index++); + } + + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + /** + * Given a dotNotation style outputPath like "data[2].&(1,1)", this method fixes the syntactic sugar + * of "data[2]" --> "data.[2]" + * + * This makes all the rest of the String processing easier once we know that we can always + * split on the '.' character. + * + * @param dotNotaton Output path dot notation + * @return + */ + // TODO Unit Test this + private static String fixLeadingBracketSugar( String dotNotaton ) { + + if ( dotNotaton == null || dotNotaton.length() == 0 ) { + return ""; + } + + char prev = dotNotaton.charAt( 0 ); + StringBuilder sb = new StringBuilder(); + sb.append( prev ); + + for ( int index = 1; index < dotNotaton.length(); index++ ) { + char curr = dotNotaton.charAt( index ); + + if ( curr == '[' ) { + if ( prev == '@' || prev == '.' ) { + // no need to add an extra '.' + } + else { + sb.append( '.' ); + } + } + + sb.append( curr ); + prev = curr; + } + + return sb.toString(); + } + + + /** + * Parse RHS Transpose @ logic. + * "@(a.b)" or + * "@a.b + * + * This method expects that the the '@' character has already been seen. + * Thus if ther are + * + * @param iter iterator to pull data from + * @param dotNotationRef the original dotNotation string used for error messages + */ + // TODO Unit Test this + private static String parseAtPathElement( Iterator iter, String dotNotationRef ) { + + if ( ! iter.hasNext() ) { + return ""; + } + + StringBuilder sb = new StringBuilder(); + + // Strategy here is to walk thru the string looking for matching parenthesis. + // '(' increments the count, while ')' decrements it + // If we ever get negative there is a problem. + boolean isParensAt = false; + int atParensCount = 0; + + char c = iter.next(); + if ( c == '(' ) { + isParensAt = true; + atParensCount++; + } + else if ( c == '.' ) { + throw new SpecException( "Unable to parse dotNotation, invalid TransposePathElement : " + dotNotationRef ); + } + + sb.append( c ); + + while( iter.hasNext() ) { + c = iter.next(); + sb.append( c ); + + // Parsing "@(a.b.[&2])" + if ( isParensAt ) { + if ( c == '(' ) { + throw new SpecException( "Unable to parse dotNotation, too many open parens '(' : " + dotNotationRef ); + } + else if ( c == ')' ) { + atParensCount--; + } + + if ( atParensCount == 0 ) { + return sb.toString(); + } + else if ( atParensCount < 0 ) { + throw new SpecException( "Unable to parse dotNotation, specifically the '@()' part : " + dotNotationRef ); + } + } + // Parsing "@abc.def + else if ( c == '.' ) { + return sb.toString(); + } + } + + // if we got to the end of the String and we have mismatched parenthesis throw an exception. + if ( isParensAt && atParensCount != 0 ) { + throw new SpecException( "Invalid @() pathElement from : " + dotNotationRef ); + } + // Parsing "@abc" + return sb.toString(); + } + + /** + * Method that recursively parses a dotNotation String based on an iterator. + * + * This method will call out to parseAtPathElement + * + * @param pathStrings List to store parsed Strings that each represent a PathElement + * @param iter the iterator to pull characters from + * @param dotNotationRef the original dotNotation string used for error messages + * @return + */ + private static List parseDotNotation( List pathStrings, Iterator iter, String dotNotationRef ) { + + if ( ! iter.hasNext() ) { + return pathStrings; + } + + StringBuilder sb = new StringBuilder(); + + char c; + while( iter.hasNext() ) { + + c = iter.next(); + + if( c == '@' ) { + sb.append( '@' ); + sb.append( parseAtPathElement( iter, dotNotationRef ) ); + pathStrings.add( sb.toString() ); + sb = new StringBuilder(); + } + else { + if ( c == '.' ) { + if ( sb.length() != 0 ) { + pathStrings.add( sb.toString() ); + } + return parseDotNotation( pathStrings, iter, dotNotationRef ); + } + + sb.append( c ); + } } + + if ( sb.length() != 0 ) { + pathStrings.add( sb.toString() ); + } + return pathStrings; } - public static List parse( String[] keys ) { + /** + * @param refDotNotation the original dotNotation string used for error messages + * @return List of PathElements based on the provided List keys + */ + private static List parseList( List keys, String refDotNotation ) { ArrayList paths = new ArrayList(); for( String key: keys ) { - List subPaths = parse( key ); - for ( PathElement path : subPaths ) { - paths.add( path ); + PathElement path = parseSingleKeyLHS( key ); + if ( path instanceof AtPathElement ) { + throw new SpecException( "'.@.' is not valid on the RHS: " + refDotNotation ); } + paths.add( path ); } return paths; } + /** + * Parse the dotNotation of the RHS. + */ + public static List parseDotNotationRHS( String dotNotation ) { + String fixedNotation = fixLeadingBracketSugar( dotNotation ); + List pathStrs = parseDotNotation( new LinkedList(), stringIterator( fixedNotation ), dotNotation ); + + return parseList( pathStrs, dotNotation ); + } + + /** * This is the main recursive method of the Shiftr parallel "spec" and "input" tree walk. * diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/traversr/SimpleTraversr.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/traversr/SimpleTraversr.java index 5ceb9062..da90876b 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/traversr/SimpleTraversr.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/traversr/SimpleTraversr.java @@ -17,6 +17,8 @@ import com.bazaarvoice.jolt.traversr.traversal.TraversalStep; +import java.util.List; + /** * Simple Traversr that * @@ -29,6 +31,10 @@ public SimpleTraversr( String humanPath ) { super( humanPath ); } + public SimpleTraversr( List paths ) { + super( paths ); + } + @Override public Object handleFinalSet( TraversalStep traversalStep, Object tree, String key, Object data ) { return traversalStep.overwriteSet( tree, key, data ); diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/traversr/Traversr.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/traversr/Traversr.java index 62b92898..f4e6f934 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/traversr/Traversr.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/traversr/Traversr.java @@ -82,6 +82,19 @@ public Traversr ( String humanPath ) { root = rooty; } + /** + * Constructor where we provide a known good set of pathElement Strings in a list. + * Aka, no need to extract it from a "Human Readable" form. + */ + public Traversr( List paths ) { + TraversalStep rooty = null; + for ( int index = paths.size() -1 ; index >= 0; index--) { + rooty = makePathElement( paths.get(index), rooty ); + } + traversalLength = paths.size(); + root = rooty; + } + private TraversalStep makePathElement(String path, TraversalStep child) { if ( "[]".equals( path ) ) { diff --git a/jolt-core/src/main/java/com/bazaarvoice/jolt/traversr/traversal/BaseTraversalStep.java b/jolt-core/src/main/java/com/bazaarvoice/jolt/traversr/traversal/BaseTraversalStep.java index 943926aa..dbece56e 100644 --- a/jolt-core/src/main/java/com/bazaarvoice/jolt/traversr/traversal/BaseTraversalStep.java +++ b/jolt-core/src/main/java/com/bazaarvoice/jolt/traversr/traversal/BaseTraversalStep.java @@ -73,7 +73,7 @@ public final Object traverse( Object tree, Operation op, Iterator keys, // Do nothing // Throw Exception // "make" the type ok by overwriting with a new container object (got list, wanted map, so trash the list by overwriting with a new map) - throw new TraversrException( "Type mismatch on parent." ); + return null; } } } diff --git a/jolt-core/src/test/java/com/bazaarvoice/jolt/ChainrTest.java b/jolt-core/src/test/java/com/bazaarvoice/jolt/ChainrTest.java index b7e672cd..a8e64593 100644 --- a/jolt-core/src/test/java/com/bazaarvoice/jolt/ChainrTest.java +++ b/jolt-core/src/test/java/com/bazaarvoice/jolt/ChainrTest.java @@ -208,6 +208,11 @@ public void process_itBlowsUp_fromTransform(Object spec) { public Object[][] getTestCaseNames() { return new Object[][] { {"firstSample", true}, + {"wolfermann1", false}, + {"wolfermann2", false}, + {"wolfermann2", false}, + {"andrewkcarter1", false}, + {"andrewkcarter2", false}, }; } diff --git a/jolt-core/src/test/java/com/bazaarvoice/jolt/JoltTestUtil.java b/jolt-core/src/test/java/com/bazaarvoice/jolt/JoltTestUtil.java index 76950d26..8d0c2d9b 100644 --- a/jolt-core/src/test/java/com/bazaarvoice/jolt/JoltTestUtil.java +++ b/jolt-core/src/test/java/com/bazaarvoice/jolt/JoltTestUtil.java @@ -22,16 +22,30 @@ public class JoltTestUtil { private static Diffy diffy = new Diffy(); + private static Diffy arrayOrderObliviousDiffy = new ArrayOrderObliviousDiffy(); public static void runDiffy( String failureMessage, Object expected, Object actual ) throws IOException { + runDiffy( diffy, failureMessage, expected, actual ); + } + + public static void runDiffy( Object expected, Object actual ) throws IOException { + runDiffy( diffy, "Failed", expected, actual ); + } + + public static void runArrayOrderObliviousDiffy( String failureMessage, Object expected, Object actual ) throws IOException { + runDiffy( arrayOrderObliviousDiffy, failureMessage, expected, actual ); + } + + public static void runArrayOrderObliviousDiffy( Object expected, Object actual ) throws IOException { + runDiffy( arrayOrderObliviousDiffy, "Failed", expected, actual ); + } + + + private static void runDiffy( Diffy diffy, String failureMessage, Object expected, Object actual ) throws IOException { Diffy.Result result = diffy.diff( expected, actual ); if (!result.isEmpty()) { AssertJUnit.fail( failureMessage + ".\nhere is a diff:\nexpected: " + JsonUtils.toJsonString(result.expected) + "\n actual: " + JsonUtils.toJsonString(result.actual)); } } - - public static void runDiffy( Object expected, Object actual ) throws IOException { - runDiffy( "Failed", expected, actual ); - } } diff --git a/jolt-core/src/test/java/com/bazaarvoice/jolt/ShiftrTest.java b/jolt-core/src/test/java/com/bazaarvoice/jolt/ShiftrTest.java index 7564267e..6609e336 100644 --- a/jolt-core/src/test/java/com/bazaarvoice/jolt/ShiftrTest.java +++ b/jolt-core/src/test/java/com/bazaarvoice/jolt/ShiftrTest.java @@ -32,7 +32,11 @@ public Object[][] getTestCaseUnits() { {"bucketToPrefixSoup"}, {"declaredOutputArray"}, {"explicitArrayKey"}, + {"filterParallelArrays"}, + {"filterParents1"}, + {"filterParents2"}, {"firstSample"}, + {"hashDefault"}, {"identity"}, {"inputArrayToPrefix"}, {"invertMap"}, @@ -41,6 +45,9 @@ public Object[][] getTestCaseUnits() { {"listKeys"}, {"mapToList"}, {"mapToList2"}, + {"mergeParallelArrays1_and-transpose"}, + {"mergeParallelArrays2_and-do-not-transpose"}, + {"mergeParallelArrays3_and-filter"}, {"multiPlacement"}, {"objectToArray"}, {"passThru"}, @@ -50,9 +57,27 @@ public Object[][] getTestCaseUnits() { {"queryMappingXform"}, {"singlePlacement"}, {"specialKeys"}, + {"transposeArrayContents1"}, + {"transposeArrayContents2"}, + {"transposeComplex1"}, + {"transposeComplex2"}, + {"transposeComplex3_both-sides-multipart"}, + {"transposeComplex4_lhs-multipart-rhs-sugar"}, + {"transposeComplex5_at-logic-with-embedded-array-lookups"}, + {"transposeComplex6_rhs-complex-at"}, + {"transposeComplex7_coerce-int-string-conversion"}, + {"transposeComplex8_coerce-boolean-string-conversion"}, + {"transposeInverseMap1"}, + {"transposeInverseMap2"}, + {"transposeLHS1"}, + {"transposeLHS2"}, + {"transposeLHS3"}, + {"transposeSimple1"}, + {"transposeSimple2"}, + {"transposeSimple3"}, {"wildcards"}, {"wildcardSelfAndRef"}, - {"wildcardsWithOr"}, + {"wildcardsWithOr"} }; } diff --git a/jolt-core/src/test/java/com/bazaarvoice/jolt/shiftr/PathElementTest.java b/jolt-core/src/test/java/com/bazaarvoice/jolt/shiftr/PathElementTest.java index b61b02d9..4a33a15b 100644 --- a/jolt-core/src/test/java/com/bazaarvoice/jolt/shiftr/PathElementTest.java +++ b/jolt-core/src/test/java/com/bazaarvoice/jolt/shiftr/PathElementTest.java @@ -27,7 +27,6 @@ import org.testng.AssertJUnit; import org.testng.annotations.Test; -import java.util.Arrays; import java.util.List; // Todo Now that the PathElement classes have been split out (no longer inner classes) @@ -105,8 +104,8 @@ public void arrayRefTest() { @Test public void calculateOutputTest_refsOnly() { - MatchablePathElement pe1 = (MatchablePathElement) ShiftrSpec.parse( "tuna-*-marlin-*" ).get( 0 ); - MatchablePathElement pe2 = (MatchablePathElement) ShiftrSpec.parse( "rating-*" ).get( 0 ); + MatchablePathElement pe1 = (MatchablePathElement) ShiftrSpec.parseSingleKeyLHS( "tuna-*-marlin-*" ); + MatchablePathElement pe2 = (MatchablePathElement) ShiftrSpec.parseSingleKeyLHS( "rating-*" ); LiteralPathElement lpe = pe1.match( "tuna-marlin", new WalkedPath() ); AssertJUnit.assertNull( lpe ); @@ -118,26 +117,28 @@ public void calculateOutputTest_refsOnly() { AssertJUnit.assertEquals( "A" , lpe.getSubKeyRef( 1 ) ); AssertJUnit.assertEquals( "AAA" , lpe.getSubKeyRef( 2 ) ); - LiteralPathElement lpe2 = pe2.match( "rating-BBB", new WalkedPath( Arrays.asList( lpe ) ) ); + LiteralPathElement lpe2 = pe2.match( "rating-BBB", new WalkedPath( null, lpe ) ); AssertJUnit.assertEquals( "rating-BBB", lpe2.getRawKey() ); AssertJUnit.assertEquals( "rating-BBB", lpe2.getSubKeyRef( 0 ) ); AssertJUnit.assertEquals( 2, lpe2.getSubKeyCount() ); AssertJUnit.assertEquals( "BBB" , lpe2.getSubKeyRef( 1 ) ); ShiftrWriter outputPath = new ShiftrWriter( "&(1,2).&.value" ); + WalkedPath twoSteps = new WalkedPath( null, lpe ); + twoSteps.add( null, lpe2 ); { EvaluatablePathElement outputElement = (EvaluatablePathElement) outputPath.get( 0 ); - String evaledLeafOutput = outputElement.evaluate( new WalkedPath( Arrays.asList( lpe, lpe2 ) ) ); + String evaledLeafOutput = outputElement.evaluate( twoSteps ); AssertJUnit.assertEquals( "AAA", evaledLeafOutput ); } { EvaluatablePathElement outputElement = (EvaluatablePathElement) outputPath.get( 1 ); - String evaledLeafOutput = outputElement.evaluate( new WalkedPath( Arrays.asList( lpe, lpe2 ) ) ); + String evaledLeafOutput = outputElement.evaluate( twoSteps ); AssertJUnit.assertEquals( "rating-BBB", evaledLeafOutput ); } { EvaluatablePathElement outputElement = (EvaluatablePathElement) outputPath.get( 2 ); - String evaledLeafOutput = outputElement.evaluate( new WalkedPath( Arrays.asList( lpe, lpe2 ) ) ); + String evaledLeafOutput = outputElement.evaluate( twoSteps ); AssertJUnit.assertEquals( "value", evaledLeafOutput ); } } @@ -146,15 +147,15 @@ public void calculateOutputTest_refsOnly() { public void calculateOutputTest_arrayIndexes() { // simulate Shiftr LHS specs - MatchablePathElement pe1 = (MatchablePathElement) ShiftrSpec.parse( "tuna-*-marlin-*" ).get( 0 ); - MatchablePathElement pe2 = (MatchablePathElement) ShiftrSpec.parse( "rating-*" ).get( 0 ); + MatchablePathElement pe1 = (MatchablePathElement) ShiftrSpec.parseSingleKeyLHS( "tuna-*-marlin-*" ); + MatchablePathElement pe2 = (MatchablePathElement) ShiftrSpec.parseSingleKeyLHS( "rating-*" ); // match them against some data to get LiteralPathElements with captured values LiteralPathElement lpe = pe1.match( "tuna-2-marlin-3", new WalkedPath() ); AssertJUnit.assertEquals( "2" , lpe.getSubKeyRef( 1 ) ); AssertJUnit.assertEquals( "3" , lpe.getSubKeyRef( 2 ) ); - LiteralPathElement lpe2 = pe2.match( "rating-BBB", new WalkedPath( Arrays.asList( lpe ) ) ); + LiteralPathElement lpe2 = pe2.match( "rating-BBB", new WalkedPath( null, lpe ) ); AssertJUnit.assertEquals( 2, lpe2.getSubKeyCount() ); AssertJUnit.assertEquals( "BBB" , lpe2.getSubKeyRef( 1 ) ); @@ -165,8 +166,9 @@ public void calculateOutputTest_arrayIndexes() { AssertJUnit.assertEquals( "tuna.[&(1,1)].marlin.[&(1,2)].&(0,1)", shiftrWriter.getCanonicalForm() ); // Evaluate the write path against the LiteralPath elements we build above ( like Shiftr does ) - WalkedPath walkedPath = new WalkedPath( Arrays.asList( lpe, lpe2 ) ); - List stringPath = shiftrWriter.evaluate( walkedPath ); + WalkedPath twoSteps = new WalkedPath( null, lpe ); + twoSteps.add( null, lpe2 ); + List stringPath = shiftrWriter.evaluate( twoSteps ); AssertJUnit.assertEquals( "tuna", stringPath.get( 0 ) ); AssertJUnit.assertEquals( "2", stringPath.get( 1 ) ); diff --git a/jolt-core/src/test/java/com/bazaarvoice/jolt/shiftr/ShiftrUnitTest.java b/jolt-core/src/test/java/com/bazaarvoice/jolt/shiftr/ShiftrUnitTest.java index 54a98719..eaeb3ece 100644 --- a/jolt-core/src/test/java/com/bazaarvoice/jolt/shiftr/ShiftrUnitTest.java +++ b/jolt-core/src/test/java/com/bazaarvoice/jolt/shiftr/ShiftrUnitTest.java @@ -18,12 +18,17 @@ import com.bazaarvoice.jolt.JoltTestUtil; import com.bazaarvoice.jolt.JsonUtils; import com.bazaarvoice.jolt.Shiftr; +import com.bazaarvoice.jolt.common.pathelement.PathElement; import com.bazaarvoice.jolt.exception.SpecException; +import com.bazaarvoice.jolt.shiftr.spec.ShiftrSpec; +import com.google.common.base.Joiner; +import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.IOException; import java.util.ArrayList; +import java.util.List; import java.util.Map; public class ShiftrUnitTest { @@ -104,9 +109,13 @@ public Object[][] badSpecs() throws IOException { JsonUtils.jsonToMap( "{ \"tuna-*-marlin-*\" : { \"rating-@\" : \"&(1,2).&.value\" } }" ), }, { - "RHS @", + "RHS @ by itself", JsonUtils.jsonToMap( "{ \"tuna-*-marlin-*\" : { \"rating-*\" : \"&(1,2).@.value\" } }" ), }, + { + "RHS @ with bad Parens", + JsonUtils.jsonToMap( "{ \"tuna-*-marlin-*\" : { \"rating-*\" : \"&(1,2).@(data.&(1,1).value\" } }" ), + }, { "RHS *", JsonUtils.jsonToMap( "{ \"tuna-*-marlin-*\" : { \"rating-*\" : \"&(1,2).*.value\" } }" ), @@ -130,4 +139,58 @@ public Object[][] badSpecs() throws IOException { public void failureUnitTest(String testName, Object spec) { new Shiftr( spec ); } + + /** + * @return canonical dotNotation String built from the given paths + */ + public String buildCanonicalString( List paths ) { + + List pathStrs = new ArrayList( paths.size() ); + for( PathElement pe : paths ) { + pathStrs.add( pe.getCanonicalForm() ); + } + + return Joiner.on(".").join( paths ); + } + + + @DataProvider + public Object[][] validRHS() throws IOException { + return new Object[][]{ + { "@a", "@(0,a)" }, + { "@abc", "@(0,abc)" }, + { "@a.b.c", "@(0,a).b.c" }, + { "@a.b.@c", "@(0,a).b.@(0,c)" }, + { "@(a[2].&).b.@c", "@(0,a.[2].&(0,0)).b.@(0,c)" }, + { "a[&2].@b[1].c", "a.[&(2,0)].@(0,b).[1].c" } + }; + } + + @Test(dataProvider = "validRHS" ) + public void validRHSTests( String dotNotation, String expected ) { + List paths = ShiftrSpec.parseDotNotationRHS( dotNotation ); + String actualCanonicalForm = buildCanonicalString( paths ); + + Assert.assertEquals( actualCanonicalForm, expected, "TestCase: " + dotNotation ); + } + + + @DataProvider + public Object[][] badRHS() throws IOException { + return new Object[][]{ + { "@" }, + { "a@" }, + { "@a@b" }, + { "@(a.b.&(2,2)" }, // missing trailing ) + { "@(a.b.&(2,2).d" }, // missing trailing ) + { "@(a.b.@c).d" }, + { "@(a.*.c)" }, // @ can not contain a * + { "@(a.$2.c)" }, // @ can not contain a $ + }; + } + + @Test(dataProvider = "badRHS", expectedExceptions = SpecException.class) + public void failureRHSTests( String dotNotation ) { + ShiftrSpec.parseDotNotationRHS( dotNotation ); + } } diff --git a/jolt-core/src/test/resources/json/chainr/integration/andrewkcarter1.json b/jolt-core/src/test/resources/json/chainr/integration/andrewkcarter1.json new file mode 100644 index 00000000..f31d3d35 --- /dev/null +++ b/jolt-core/src/test/resources/json/chainr/integration/andrewkcarter1.json @@ -0,0 +1,53 @@ +{ + // Test from Jolt Issue 98 : Question one from andrewkcarter + // Summary : Bucket data from an Array, based on a leaf level value + + "input": { + "entities": [ + { + "type": "alpha", + "data": "foo" + }, + { + "type": "beta", + "data": "bar" + }, + { + "type": "alpha", + "data": "baz" + } + ] + }, + + "spec": [ + { + "operation": "shift", + "spec": { + "entities": { + // The "*" matches each Map pair of "type" and "data". + // We then write the Map pair, to the value of the "type" key, in a forced array + "*" : "@type[]" + } + } + } + ], + + "expected": { + "alpha":[ + { + "type":"alpha", + "data":"foo" + }, + { + "type":"alpha", + "data":"baz" + } + ], + "beta":[ + { + "type":"beta", + "data":"bar" + } + ] + } +} diff --git a/jolt-core/src/test/resources/json/chainr/integration/andrewkcarter2.json b/jolt-core/src/test/resources/json/chainr/integration/andrewkcarter2.json new file mode 100644 index 00000000..9a93425b --- /dev/null +++ b/jolt-core/src/test/resources/json/chainr/integration/andrewkcarter2.json @@ -0,0 +1,66 @@ +{ + // Test from Jolt Issue 98 : Question two from andrewkcarter + // Summary : Filter books by there availability type. In particular just want Paperback books. + + "input": { + "books": [ + { + "title": "foo", + "availability": [ + "online" + ] + }, + { + "title": "bar", + "availability": [ + "online", + "paperback" + ] + }, + { + "title": "baz", + "availability": [ + "paperback" + ] + } + ] + }, + + "spec": [ + { + "operation": "shift", + "spec": { + + "books": { + "*": { + "availability": { + "*": { + "paperback": { + "@3": "PaperBooks[]" + } + } + } + } + } + } + } + ], + + "expected": { + "PaperBooks":[ + { + "title":"bar", + "availability":[ + "online", + "paperback" + ] + }, + { + "title":"baz", + "availability":[ + "paperback" + ] + } + ] + } +} diff --git a/jolt-core/src/test/resources/json/chainr/integration/wolfermann1.json b/jolt-core/src/test/resources/json/chainr/integration/wolfermann1.json new file mode 100644 index 00000000..8921ec9d --- /dev/null +++ b/jolt-core/src/test/resources/json/chainr/integration/wolfermann1.json @@ -0,0 +1,54 @@ +{ + // Test from Jolt Issue 104 : Question one from ThorstenWolfermann + // Summary : Take nested input, and "flatten" it + + "input": { + "level1": { + "L1Attribute": "6643287c-4800-49dd-b5cb-e0cf3ea637a9", + "L1SecondAttribtue": 543, + "level2": { + "L2Attribute": "d1bc9284-3087-4a92-904a-42f041f4fc23", + "L2SecondAttribute": 1234, + "leveln": { + "L3Attribute": "d1bc9284-3087-4a92-904a-42f041f4fc23", + "L3SecondAttribute": 1234 + } + } + } + }, + + "spec": [ + { + "operation": "shift", + "spec": { + "level*": { + "*Attribute": "&1.&0", + "*SecondAttribtue": "&1.&0", + "level*": { + "*Attribute": "&1.&0", + "*SecondAttribute": "&1.&0", + "level*": { + "*Attribute": "&1.&0", + "*SecondAttribute": "&1.&0" + } + } + } + } + } + ], + + "expected": { + "level1": { + "L1Attribute": "6643287c-4800-49dd-b5cb-e0cf3ea637a9", + "L1SecondAttribtue": 543 + }, + "level2": { + "L2Attribute": "d1bc9284-3087-4a92-904a-42f041f4fc23", + "L2SecondAttribute": 1234 + }, + "leveln": { + "L3Attribute": "d1bc9284-3087-4a92-904a-42f041f4fc23", + "L3SecondAttribute": 1234 + } + } +} diff --git a/jolt-core/src/test/resources/json/chainr/integration/wolfermann2.json b/jolt-core/src/test/resources/json/chainr/integration/wolfermann2.json new file mode 100644 index 00000000..687367e0 --- /dev/null +++ b/jolt-core/src/test/resources/json/chainr/integration/wolfermann2.json @@ -0,0 +1,88 @@ +{ + // Test from Jolt Issue 104 : Question two from ThorstenWolfermann + // Summary : Take nested input with arrays, and "flatten" it + // + // The first pass gets all the data into a single array, but with extra stuff in each slot. + // The second pass extracts and passes thru the desired values in the array. + // The "[]" in the Right Hand Side spec means place the data into an auto expanding array. + + "input": { + "environment": { + "value": 123, + "orderLevel": [ + { + "rate": "low", + "value": 666 + }, + { + "rate": "low", + "start": 321, + "orderLevel-1": [ + { + "rate1": "OL", + "value": 444 + }, + { + "rate": "OL3", + "start": 444 + } + ] + } + ] + } + }, + + "spec": [ + { + "operation": "shift", + "spec": { + "environment": { + "orderLevel": { + "*": { + "@": "Data[]", + "orderLevel-1": { + "*": "Data[]" + } + } + } + } + } + }, + { + "operation": "shift", + "spec": { + "Data": { + "*": { + "rate": "Data2.Data[&1].rate", + "rate*": "Data2.Data[&1].&", + "value": "Data2.Data[&1].value", + "start": "Data2.Data[&1].start" + } + } + } + } + ], + + "expected": { + "Data2": { + "Data": [ + { + "rate": "low", + "value": 666 + }, + { + "rate": "low", + "start": 321 + }, + { + "rate1": "OL", + "value": 444 + }, + { + "rate": "OL3", + "start": 444 + } + ] + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/filterParallelArrays.json b/jolt-core/src/test/resources/json/shiftr/filterParallelArrays.json new file mode 100644 index 00000000..763f7821 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/filterParallelArrays.json @@ -0,0 +1,24 @@ +{ + "input": { + "states" : [ "Alabama", "Alaska", "Arizona", "Arkansas" ], + "capitals" : [ "Montgomery", "Juneau", "Phoenix" , "Little Rock" ] + }, + + "spec": { + "states": { + "*": { + "Ar*": { // only match states that start with "Ar" + // for those states, grab the captital, and use that as the output value + "@(3,capitals[&1])": "capitals[]", + "$" : "states[]" + } + } + } + }, + + + "expected": { + "states" : [ "Arizona", "Arkansas" ], + "capitals" : [ "Phoenix" , "Little Rock" ] + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/filterParents1.json b/jolt-core/src/test/resources/json/shiftr/filterParents1.json new file mode 100644 index 00000000..de53d32b --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/filterParents1.json @@ -0,0 +1,49 @@ +{ + // Identify all the book titles that are available in Paperback. + "input": { + "books": [ + { + "title": "JavaScript", + "availability": [ + "online" + ] + }, + { + "title": "Scala", + "availability": [ + "online", + "paperback" + ] + }, + { + "title": "Java", + "availability": [ + "paperback" + ] + } + ] + }, + + "spec": { + "books": { + "*": { + "availability": { + "*": { // match all elements of the availability array + "paperback": { // if the word paperback exists match it + "@(3,title)": "PaperBacks" // Look up the tree 3 levels, then back down and grab the value for the "title" + // and write it to PaperBacks in the output + }, + "online" : { + "@(3,title)": "Online" + } + } + } + } + } + }, + + "expected": { + "PaperBacks": [ "Scala", "Java" ], + "Online" : [ "JavaScript", "Scala" ] + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/filterParents2.json b/jolt-core/src/test/resources/json/shiftr/filterParents2.json new file mode 100644 index 00000000..bbfeb241 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/filterParents2.json @@ -0,0 +1,55 @@ +{ + // Identify all the book titles that are available in Paperback. + "input": { + "books": [ + { + "book": { + "title": "JavaScript", + "ISBN": "1234" + }, + "availability": [ + "online" + ] + }, + { + "book": { + "title": "Scala", + "ISBN": "9876" + }, + "availability": [ + "online", + "paperback" + ] + }, + { + "book": { + "title": "Java", + "ISBN": "5555" + }, + "availability": [ + "paperback" + ] + } + ] + }, + + "spec": { + "books": { + "*": { + "availability": { + "*": { // match all elements of the availability array + "*": { // match any availability type + "@(3,book.title)": "&[]" // Look up the tree 3 levels, then back down and grab the value for the "title" + // and write out to the top level + } + } + } + } + } + }, + + "expected": { + "paperback": [ "Scala", "Java" ], + "online": [ "JavaScript", "Scala" ] + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/hashDefault.json b/jolt-core/src/test/resources/json/shiftr/hashDefault.json new file mode 100644 index 00000000..3341457c --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/hashDefault.json @@ -0,0 +1,37 @@ +{ + "input": { + "data" : { + "1234" : { + "clientId": "12", + "hidden" : true + }, + "1235" : { + "clientId": "35", + "hidden" : false + } + } + }, + + "spec": { + "data" : { + "*" : { + "hidden" : { + "true" : { // if hidden is true, then write the value disabled to the RHS output path + // Also @(3,clientId) means go up 3 levels, to the "1234" or "1235" level, then lookup / down the tree for the value of "clientId" + "#disabled" : "clients.@(3,clientId)" + }, + "false" : { + "#enabled" : "clients.@(3,clientId)" + } + } + } + } + }, + + "expected": { + "clients" : { + "12" : "disabled", + "35" : "enabled" + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/mergeParallelArrays1_and-transpose.json b/jolt-core/src/test/resources/json/shiftr/mergeParallelArrays1_and-transpose.json new file mode 100644 index 00000000..05bf9d1d --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/mergeParallelArrays1_and-transpose.json @@ -0,0 +1,25 @@ +{ + "input": { + "states" : [ "Alabama", "Alaska", "Arizona", "Arkansas" ], + "capitals" : [ "Montgomery", "Juneau", "Phoenix" , "Little Rock" ] + }, + + "spec": { // Level 2 : the root + "capitals": { // Level 1 : capitals + + // Write out the "value" of each array index, aka 0->Montgomery, so the "Montgomery" part + // to "states.Alabama" where "Alabama" is 2 levels up the tree, + // then down into the "states" array, indexing using the same index value we matched + "*": "states.@(2,states[&])" + } + }, + + "expected": { + "states" : { + "Alabama" : "Montgomery", + "Alaska" : "Juneau", + "Arizona" : "Phoenix", + "Arkansas" : "Little Rock" + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/mergeParallelArrays2_and-do-not-transpose.json b/jolt-core/src/test/resources/json/shiftr/mergeParallelArrays2_and-do-not-transpose.json new file mode 100644 index 00000000..a3b27c28 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/mergeParallelArrays2_and-do-not-transpose.json @@ -0,0 +1,45 @@ +{ + "input": { + "states" : [ "Alabama", "Alaska", "Arizona", "Arkansas" ], + "capitals" : [ "Montgomery", "Juneau", "Phoenix" , "Little Rock" ] + }, + + "spec": { + "states": { + "*": { + "*": { + "$": "states[&2].state" + } + } + }, + "capitals": { + "*": { + "*": { + "$": "states[&2].capital" + } + } + } + }, + + + "expected": { + "states" : [ + { + "state" : "Alabama", + "capital" : "Montgomery" + }, + { + "state" : "Alaska", + "capital" : "Juneau" + }, + { + "state" : "Arizona", + "capital" : "Phoenix" + }, + { + "state" : "Arkansas", + "capital" : "Little Rock" + } + ] + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/mergeParallelArrays3_and-filter.json b/jolt-core/src/test/resources/json/shiftr/mergeParallelArrays3_and-filter.json new file mode 100644 index 00000000..f6ca9386 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/mergeParallelArrays3_and-filter.json @@ -0,0 +1,25 @@ +{ + "input": { + "states" : [ "Alabama", "Alaska", "Arizona", "Arkansas" ], + "capitals" : [ "Montgomery", "Juneau", "Phoenix" , "Little Rock" ] + }, + + "spec": { + "states": { + "*": { + "Ar*": { // only match states that start with "Ar" + // for those states, grab the captital, and use that as the output value + "@(3,capitals[&1])": "states.&1" + } + } + } + }, + + + "expected": { + "states" : { + "Arizona" : "Phoenix", + "Arkansas" : "Little Rock" + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeArrayContents1.json b/jolt-core/src/test/resources/json/shiftr/transposeArrayContents1.json new file mode 100644 index 00000000..0c74b3d8 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeArrayContents1.json @@ -0,0 +1,33 @@ +{ + "input": { + "photos" : [ + { + "id": "1234", + "fileName": "Acme.jpg" + }, + { + "id": "9876", + "fileName": "Boxy.jpg" + } + ] + }, + + "spec": { + "photos" : { + "*" : { + "@fileName": "photos[&1].@id" + } + } + }, + + "expected": { + "photos" : [ + { + "1234" : "Acme.jpg" + }, + { + "9876" : "Boxy.jpg" + } + ] + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeArrayContents2.json b/jolt-core/src/test/resources/json/shiftr/transposeArrayContents2.json new file mode 100644 index 00000000..58a771bf --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeArrayContents2.json @@ -0,0 +1,29 @@ +{ + "input": { + "photos" : [ + { + "id": "1234", + "fileName": "Acme.jpg" + }, + { + "id": "9876", + "fileName": "Boxy.jpg" + } + ] + }, + + "spec": { + "photos" : { + "*" : { + "@fileName": "photos.@id" + } + } + }, + + "expected": { + "photos" : { + "1234" : "Acme.jpg", + "9876" : "Boxy.jpg" + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeComplex1.json b/jolt-core/src/test/resources/json/shiftr/transposeComplex1.json new file mode 100644 index 00000000..2e812b80 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeComplex1.json @@ -0,0 +1,23 @@ +{ + "input": { + "data" : { + "clientId": "1234", + "clientNameStuff" : { + "clientName": "Acme", + "otherClientStuff" : "pants" + } + } + }, + + "spec": { + "data" : { + "@clientNameStuff.clientName" : "bookMap.@clientId" + } + }, + + "expected": { + "bookMap" : { + "1234" : "Acme" + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeComplex2.json b/jolt-core/src/test/resources/json/shiftr/transposeComplex2.json new file mode 100644 index 00000000..564126c7 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeComplex2.json @@ -0,0 +1,24 @@ +{ + "input": { + "data" : { + "clientId": "1234", + "clientNameStuff" : { + "clientName": "Acme", + "otherClientStuff" : "pants" + } + } + }, + + "spec": { + "data" : { + // Verify LHS canonical form + "@(clientNameStuff.clientName)" : "bookMap.@clientId" + } + }, + + "expected": { + "bookMap" : { + "1234" : "Acme" + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeComplex3_both-sides-multipart.json b/jolt-core/src/test/resources/json/shiftr/transposeComplex3_both-sides-multipart.json new file mode 100644 index 00000000..ccffc815 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeComplex3_both-sides-multipart.json @@ -0,0 +1,29 @@ +{ + "input": { + "data" : { + "clientIdStuff" : { + "clientId": "1234", + "orderIdStuff" : "shoes" + }, + "clientNameStuff" : { + "clientName": "Acme", + "otherClientStuff" : "pants" + } + } + }, + + "spec": { + "data" : { + // Verify we can write data out to a "literal.@().literal" location + "@(clientNameStuff.clientName)" : "data.@(clientIdStuff.clientId).clientName" + } + }, + + "expected": { + "data" : { + "1234": { + "clientName": "Acme" + } + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeComplex4_lhs-multipart-rhs-sugar.json b/jolt-core/src/test/resources/json/shiftr/transposeComplex4_lhs-multipart-rhs-sugar.json new file mode 100644 index 00000000..3f53db3e --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeComplex4_lhs-multipart-rhs-sugar.json @@ -0,0 +1,26 @@ +{ + "input": { + "data" : { + "clientId": "1234", + "clientNameStuff" : { + "clientName": "Acme", + "otherClientStuff" : "pants" + } + } + }, + + "spec": { + "data" : { + // Verify we can write data out to a RHS with syntactic sugar @ in the middle of two literals: "literal.@SUGAR.literal" + "@(clientNameStuff.clientName)" : "data.@clientId.clientName" + } + }, + + "expected": { + "data" : { + "1234": { + "clientName": "Acme" + } + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeComplex5_at-logic-with-embedded-array-lookups.json b/jolt-core/src/test/resources/json/shiftr/transposeComplex5_at-logic-with-embedded-array-lookups.json new file mode 100644 index 00000000..50e2980b --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeComplex5_at-logic-with-embedded-array-lookups.json @@ -0,0 +1,23 @@ +{ + "input": { + "sillyPhotoData" : { + "captions" : [ "Hi!", "Sunny" ], + "fileNames" : [ "hola.jpg", "sunny.jpg"] + } + }, + + "spec": { + "sillyPhotoData" : { + // This is a rather silly test, but it does prove you can have [0] inside an @() + "@(fileNames[0])" : "sillyPhotoData.@(captions[0])", + "@(fileNames[1])" : "sillyPhotoData.@(captions[1])" + } + }, + + "expected": { + "sillyPhotoData" : { + "Hi!" : "hola.jpg", + "Sunny" : "sunny.jpg" + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeComplex6_rhs-complex-at.json b/jolt-core/src/test/resources/json/shiftr/transposeComplex6_rhs-complex-at.json new file mode 100644 index 00000000..8f1a9b8c --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeComplex6_rhs-complex-at.json @@ -0,0 +1,34 @@ +{ + "input": { + "data" : { + "clientIdStuff" : { + "clientId": "1234" + }, + "clientNameStuff" : { + "clientName": "Acme", + "otherClientStuff" : "pants" + } + } + }, + + "spec": { + "data" : { + "clientNameStuff" : { + // walk down the tree till we find a clientName + // 1) go two levels up the tree the "2" from "@(2,...)" + // 2) then walk down to find a clientId : "1234" + // 3) use the "1234" value as part of the output path + "clientName" : "data.@(2,clientIdStuff.clientId).clientName" + } + } + }, + + "expected": { + "data" : { + "1234": { // in this case we want the clientId (1234) to be a level in the output tree + // rather than just a key/value pair. + "clientName": "Acme" + } + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeComplex7_coerce-int-string-conversion.json b/jolt-core/src/test/resources/json/shiftr/transposeComplex7_coerce-int-string-conversion.json new file mode 100644 index 00000000..1293dd6a --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeComplex7_coerce-int-string-conversion.json @@ -0,0 +1,34 @@ +{ + "input": { + "data" : { + "clientIdStuff" : { + "clientId": 1234 + }, + "clientNameStuff" : { + "clientName": "Acme", + "otherClientStuff" : "pants" + } + } + }, + + "spec": { + "data" : { + "clientNameStuff" : { + // walk down the tree till we find a clientName + // 1) go two levels up the tree the "2" from "@(2,...)" + // 2) then walk down to find a clientId : 1234 + // 3) coerce the number 1234, into a String "1234", and use that as part of the output path + "clientName" : "data.@(2,clientIdStuff.clientId).clientName" + } + } + }, + + "expected": { + "data" : { + "1234": { // in this case we want the clientId (1234) to be a level in the output tree + // rather than just a key/value pair. + "clientName": "Acme" + } + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeComplex8_coerce-boolean-string-conversion.json b/jolt-core/src/test/resources/json/shiftr/transposeComplex8_coerce-boolean-string-conversion.json new file mode 100644 index 00000000..2a58dfef --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeComplex8_coerce-boolean-string-conversion.json @@ -0,0 +1,33 @@ +{ + "input": { + "clients" : { + "Acme" : { + "clientId": "guid", + "enabled" : true + }, + "Axe" : { + "clientId": 2, + "enabled" : false + }, + "Bob's Burgers" : { + "clientId": 3, + "enabled" : true + } + } + }, + + "spec": { + "clients" : { + "*": { // clientName + "clientId": "clientsById.@(1,enabled)[]" // coerce the boolean into the string "true" or "false" + } + } + }, + + "expected": { + "clientsById" : { + "true" : [ "guid", 3 ], // Note the Ids, whatever they are get passed along, because they are data + "false" : [ 2 ] + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeInverseMap1.json b/jolt-core/src/test/resources/json/shiftr/transposeInverseMap1.json new file mode 100644 index 00000000..1c55d438 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeInverseMap1.json @@ -0,0 +1,38 @@ +{ + "input": { + "data" : [ + { + "clientId": "1", + "clientName": "Acme", + "otherStuff" : "Boom" + }, + { + "clientId": "2", + "clientName": "Bob's", + "otherStuff" : "Burgers" + } + ] + }, + + "spec": { + "data" : { + // We can use the RHS @ sign to look down the tree to find a key for the output + "*" : "data.@clientId" + } + }, + + "expected": { + "data" : { + "1" : { + "clientId": "1", + "clientName": "Acme", + "otherStuff" : "Boom" + }, + "2" : { + "clientId": "2", + "clientName": "Bob's", + "otherStuff" : "Burgers" + } + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeInverseMap2.json b/jolt-core/src/test/resources/json/shiftr/transposeInverseMap2.json new file mode 100644 index 00000000..15cc196a --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeInverseMap2.json @@ -0,0 +1,25 @@ +{ + "input": { + "data" : [ + { + "clientId": "1", + "clientName": "Acme", + "otherStuff" : "Boom" + }, + { + "clientId": "2", + "clientName": "Bob's", + "otherStuff" : "Burgers" + } + ] + }, + + "spec": { + "data" : { + // Verify that if the RHS @ does not find anything, nothing get written + "*" : "data.@pants" + } + }, + + "expected": null +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeLHS1.json b/jolt-core/src/test/resources/json/shiftr/transposeLHS1.json new file mode 100644 index 00000000..d7c15b84 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeLHS1.json @@ -0,0 +1,18 @@ +{ + "input": { + "data" : { + "clientId": "1234", + "clientName": "Acme" + } + }, + + "spec": { + "data" : { + "@clientName" : "CLIENTNAME" // verify that we can look down the tree + } + }, + + "expected": { + "CLIENTNAME" : "Acme" + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeLHS2.json b/jolt-core/src/test/resources/json/shiftr/transposeLHS2.json new file mode 100644 index 00000000..e24b470c --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeLHS2.json @@ -0,0 +1,18 @@ +{ + "input": { + "data" : { + "clientId": "1234", + "data": "Acme" + } + }, + + "spec": { + "data" : { + "@&" : "CLIENTNAME" // verify that we can look down the tree, using refs + } + }, + + "expected": { + "CLIENTNAME" : "Acme" + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeLHS3.json b/jolt-core/src/test/resources/json/shiftr/transposeLHS3.json new file mode 100644 index 00000000..867bb238 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeLHS3.json @@ -0,0 +1,22 @@ +{ + "input": { + "data" : [ + { + "clientId": "1234", + "data": "Acme" + } + ] + }, + + "spec": { + "*" : { // catch the data + "*": { // for all array elements + "@&1": "CLIENTNAME" // match the key "data" based on what the key was two levels up + } + } + }, + + "expected": { + "CLIENTNAME" : "Acme" + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeSimple1.json b/jolt-core/src/test/resources/json/shiftr/transposeSimple1.json new file mode 100644 index 00000000..e48120d4 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeSimple1.json @@ -0,0 +1,18 @@ +{ + "input": { + "data" : { + "clientId": "1234", + "clientName": "Acme" + } + }, + + "spec": { + "data" : { + "@clientName" : "@clientId" + } + }, + + "expected": { + "1234" : "Acme" + } +} \ No newline at end of file diff --git a/jolt-core/src/test/resources/json/shiftr/transposeSimple2.json b/jolt-core/src/test/resources/json/shiftr/transposeSimple2.json new file mode 100644 index 00000000..dc0d6988 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeSimple2.json @@ -0,0 +1,20 @@ +{ + "input": { + "data": { + "clientId": "1234", + "clientName": "Acme" + } + }, + + "spec": { + "data": { + "@clientName": "bookMap.@clientId" + } + }, + + "expected": { + "bookMap": { + "1234": "Acme" + } + } +} diff --git a/jolt-core/src/test/resources/json/shiftr/transposeSimple3.json b/jolt-core/src/test/resources/json/shiftr/transposeSimple3.json new file mode 100644 index 00000000..5f1153d9 --- /dev/null +++ b/jolt-core/src/test/resources/json/shiftr/transposeSimple3.json @@ -0,0 +1,14 @@ +{ + "input": { + "clientId": "1234", + "clientName": "Acme" + }, + + "spec": { + "@clientName" : "@clientId" // verify that data can be transposed at the root level + }, + + "expected": { + "1234" : "Acme" + } +} \ No newline at end of file