diff --git a/atlas-core/src/main/scala/com/netflix/atlas/core/util/SmallHashMap.scala b/atlas-core/src/main/scala/com/netflix/atlas/core/util/SmallHashMap.scala index 3bcb36bd5..3c61d6c92 100644 --- a/atlas-core/src/main/scala/com/netflix/atlas/core/util/SmallHashMap.scala +++ b/atlas-core/src/main/scala/com/netflix/atlas/core/util/SmallHashMap.scala @@ -23,7 +23,8 @@ object SmallHashMap { } def apply[K <: AnyRef, V <: AnyRef](ts: Iterable[(K, V)]): SmallHashMap[K, V] = { - apply(ts.toSeq) + val seq = ts.toSeq + apply(seq.size, seq.iterator) } def apply[K <: AnyRef, V <: AnyRef](length: Int, iter: Iterator[(K, V)]): SmallHashMap[K, V] = { @@ -95,6 +96,35 @@ object SmallHashMap { } } } + + class EntryIterator[K <: AnyRef, V <: AnyRef](map: SmallHashMap[K, V]) extends Iterator[(K, V)] { + private final val len = map.data.length + var pos = 0 + skipEmptyEntries() + + def hasNext: Boolean = pos < len + + def next(): (K, V) = { + val t = map.data(pos).asInstanceOf[K] -> map.data(pos + 1).asInstanceOf[V] + nextEntry() + t + } + + def nextEntry(): Unit = { + pos += 2 + skipEmptyEntries() + } + + private def skipEmptyEntries(): Unit = { + while (pos < len && map.data(pos) == null) { + pos += 2 + } + } + + def key: K = map.data(pos).asInstanceOf[K] + + def value: V = map.data(pos + 1).asInstanceOf[V] + } } /** @@ -110,8 +140,7 @@ object SmallHashMap { * it may be a good fit. * * @param data array with the items - * @param dataLength number of pairs contained within the array starting at index 0. Everything - * in the array after 2 * dataLength will be ignored. + * @param dataLength number of pairs contained within the array starting at index 0. */ class SmallHashMap[K <: AnyRef, V <: AnyRef](val data: Array[AnyRef], dataLength: Int) extends scala.collection.immutable.Map[K, V] { @@ -157,48 +186,40 @@ class SmallHashMap[K <: AnyRef, V <: AnyRef](val data: Array[AnyRef], dataLength } } - def iterator: Iterator[(K, V)] = new Iterator[(K, V)] { + def find(f: (K, V) => Boolean): Option[(K, V)] = { var i = 0 - var pos = 0 - def hasNext: Boolean = i < dataLength - def next(): (K, V) = { - while (data(pos) == null) { - pos += 2 + while (i < data.length) { + if (data(i) != null && f(data(i).asInstanceOf[K], data(i + 1).asInstanceOf[V])) { + return Some(data(i).asInstanceOf[K] -> data(i + 1).asInstanceOf[V]) } - val t = data(pos).asInstanceOf[K] -> data(pos + 1).asInstanceOf[V] - pos += 2 - i += 1 - t + i += 2 } + None + } + + def entriesIterator: SmallHashMap.EntryIterator[K, V] = { + new SmallHashMap.EntryIterator[K, V](this) } + def iterator: Iterator[(K, V)] = entriesIterator + override def keysIterator: Iterator[K] = new Iterator[K] { - var i = 0 - var pos = 0 - def hasNext: Boolean = i < dataLength + val iter = entriesIterator + def hasNext: Boolean = iter.hasNext def next(): K = { - while (data(pos) == null) { - pos += 2 - } - val t = data(pos).asInstanceOf[K] - pos += 2 - i += 1 - t + val k = iter.key + iter.nextEntry() + k } } override def valuesIterator: Iterator[V] = new Iterator[V] { - var i = 0 - var pos = 0 - def hasNext: Boolean = i < dataLength + val iter = entriesIterator + def hasNext: Boolean = iter.hasNext def next(): V = { - while (data(pos) == null) { - pos += 2 - } - val t = data(pos + 1).asInstanceOf[V] - pos += 2 - i += 1 - t + val v = iter.value + iter.nextEntry() + v } } diff --git a/atlas-core/src/main/scala/com/netflix/atlas/core/validation/HasKeyRule.scala b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/HasKeyRule.scala new file mode 100644 index 000000000..84a9c12fe --- /dev/null +++ b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/HasKeyRule.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.netflix.atlas.core.util.SmallHashMap +import com.typesafe.config.Config + +/** + * Verifies that the tags contain a specified key. Sample config: + * + * ``` + * key = name + * ``` + */ +class HasKeyRule(config: Config) extends Rule { + private val key = config.getString("key") + + def validate(tags: SmallHashMap[String, String]): ValidationResult = { + if (tags.contains(key)) ValidationResult.Pass else failure(s"missing '$key': ${tags.keys}") + } +} + diff --git a/atlas-core/src/main/scala/com/netflix/atlas/core/validation/KeyLengthRule.scala b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/KeyLengthRule.scala new file mode 100644 index 000000000..bc6cbcdde --- /dev/null +++ b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/KeyLengthRule.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.typesafe.config.Config + +/** + * Verifies that the keys are within the specified length bounds. Sample config: + * + * ``` + * min-length = 2 + * max-length = 60 + * ``` + */ +class KeyLengthRule(config: Config) extends TagRule { + private val minLength = config.getInt("min-length") + private val maxLength = config.getInt("max-length") + + def validate(k: String, v: String): ValidationResult = { + k.length match { + case len if len > maxLength => + failure(s"key too long: [$k] ($len > $maxLength)") + case len if len < minLength => + failure(s"key too short: [$k] ($len < $minLength)") + case _ => + ValidationResult.Pass + } + } +} diff --git a/atlas-core/src/main/scala/com/netflix/atlas/core/validation/KeyPatternRule.scala b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/KeyPatternRule.scala new file mode 100644 index 000000000..d7b71eb11 --- /dev/null +++ b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/KeyPatternRule.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import java.util.regex.Pattern + +import com.typesafe.config.Config + +/** + * Verifies that the keys match a specified pattern. Sample config: + * + * ``` + * pattern = "^[-_.a-zA-Z0-9]{4,60}$" + * ``` + */ +class KeyPatternRule(config: Config) extends TagRule { + + private val pattern = Pattern.compile(config.getString("pattern")) + + def validate(k: String, v: String): ValidationResult = { + if (pattern.matcher(k).matches()) ValidationResult.Pass else { + failure(s"key doesn't match pattern '$pattern': [$k]") + } + } +} diff --git a/atlas-core/src/main/scala/com/netflix/atlas/core/validation/MaxUserTagsRule.scala b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/MaxUserTagsRule.scala new file mode 100644 index 000000000..0abe9e332 --- /dev/null +++ b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/MaxUserTagsRule.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.netflix.atlas.core.model.TagKey +import com.netflix.atlas.core.util.SmallHashMap +import com.typesafe.config.Config + +/** + * Verifies that the number of custom user tags are within a specified limit. Sample config: + * + * ``` + * limit = 10 + * ``` + */ +class MaxUserTagsRule(config: Config) extends Rule { + + private val limit = config.getInt("limit") + + override def validate(tags: SmallHashMap[String, String]): ValidationResult = { + var count = 0 + val iter = tags.entriesIterator + while (iter.hasNext) { + if (!TagKey.isRestricted(iter.key)) count += 1 + iter.nextEntry() + } + if (count <= limit) ValidationResult.Pass else { + failure(s"too many user tags: $count > $limit") + } + } +} diff --git a/atlas-core/src/main/scala/com/netflix/atlas/core/validation/NonStandardKeyRule.scala b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/NonStandardKeyRule.scala new file mode 100644 index 000000000..1a527fd8a --- /dev/null +++ b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/NonStandardKeyRule.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.netflix.atlas.core.model.TagKey +import com.typesafe.config.Config + +/** + * Verifies that no non-standard keys are used with a standardized prefix ("nf." and "atlas."). + */ +class NonStandardKeyRule(config: Config) extends TagRule { + + override def validate(k: String, v: String): ValidationResult = { + if (TagKey.isValid(k)) ValidationResult.Pass else failure(s"non-standard key: $k") + } +} diff --git a/atlas-core/src/main/scala/com/netflix/atlas/core/validation/Rule.scala b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/Rule.scala new file mode 100644 index 000000000..cf01da53e --- /dev/null +++ b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/Rule.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.netflix.atlas.core.util.SmallHashMap +import com.typesafe.config.Config + +/** + * Base type for validation rules. + */ +trait Rule { + + private val ruleName = getClass.getSimpleName + + /** + * Validates that the tag map matches the rule. + */ + def validate(tags: Map[String, String]): ValidationResult = { + tags match { + case m: SmallHashMap[String, String] => validate(m) + case _ => validate(SmallHashMap(tags)) + } + } + + /** + * Validates that the tag map matches the rule. + */ + def validate(tags: SmallHashMap[String, String]): ValidationResult + + /** + * Helper for generating the failure response. + */ + protected def failure(reason: String): ValidationResult = { + ValidationResult.Fail(ruleName, reason) + } +} + +object Rule { + def load(ruleConfigs: java.util.List[_ <: Config]): List[Rule] = { + import scala.collection.JavaConverters._ + load(ruleConfigs.asScala.toList) + } + + def load(ruleConfigs: List[_ <: Config]): List[Rule] = { + ruleConfigs.map { cfg => + val cls = Class.forName(cfg.getString("class")) + cls.getConstructor(classOf[Config]).newInstance(cfg).asInstanceOf[Rule] + } + } + + @scala.annotation.tailrec + def validate(tags: Map[String, String], rules: List[Rule]): ValidationResult = { + if (rules.isEmpty) ValidationResult.Pass else { + val res = rules.head.validate(tags) + if (res.isFailure) res else validate(tags, rules.tail) + } + } +} + diff --git a/atlas-core/src/main/scala/com/netflix/atlas/core/validation/TagRule.scala b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/TagRule.scala new file mode 100644 index 000000000..2ec4e89de --- /dev/null +++ b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/TagRule.scala @@ -0,0 +1,36 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.netflix.atlas.core.util.SmallHashMap + +/** + * Helper for rules that can be checked using a single key and value pair. + */ +trait TagRule extends Rule { + + def validate(tags: SmallHashMap[String, String]): ValidationResult = { + val iter = tags.entriesIterator + while (iter.hasNext) { + val res = validate(iter.key, iter.value) + if (res.isFailure) return res + iter.nextEntry() + } + ValidationResult.Pass + } + + def validate(k: String, v: String): ValidationResult +} diff --git a/atlas-core/src/main/scala/com/netflix/atlas/core/validation/ValidCharactersRule.scala b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/ValidCharactersRule.scala new file mode 100644 index 000000000..3163f195c --- /dev/null +++ b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/ValidCharactersRule.scala @@ -0,0 +1,66 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.typesafe.config.Config + +/** + * Verifies that the keys and values only use alpha-numeric, underscore, dash, and period. + */ +class ValidCharactersRule(config: Config) extends TagRule { + + import ValidCharactersRule._ + + def validate(k: String, v: String): ValidationResult = { + var i = 0 + var length = k.length + while (i < length) { + if (!isSupported(k.charAt(i))) { + return failure(s"invalid characters in key: [$k] ([-_.A-Za-z0-9] are allowed)") + } + i += 1 + } + + i = 0 + length = v.length + while (i < length) { + if (!isSupported(v.charAt(i))) { + return failure(s"invalid characters in value: $k = [$v] ([-_.A-Za-z0-9] are allowed)") + } + i += 1 + } + + ValidationResult.Pass + } +} + +object ValidCharactersRule { + private final val supported = { + val cs = new Array[Boolean](128) + (0 until 128).foreach { i => cs(i) = false } + ('A' to 'Z').foreach { c => cs(c) = true } + ('a' to 'z').foreach { c => cs(c) = true } + ('0' to '9').foreach { c => cs(c) = true } + cs('-') = true + cs('_') = true + cs('.') = true + cs + } + + private final def isSupported(c: Char): Boolean = { + c >=0 && c < 128 && supported(c) + } +} diff --git a/atlas-core/src/main/scala/com/netflix/atlas/core/validation/ValidationResult.scala b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/ValidationResult.scala new file mode 100644 index 000000000..d3306930a --- /dev/null +++ b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/ValidationResult.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +/** + * Created by brharrington on 3/14/15. + */ +sealed trait ValidationResult { + def isSuccess: Boolean + def isFailure: Boolean = !isSuccess +} + +object ValidationResult { + case object Pass extends ValidationResult { + def isSuccess: Boolean = true + } + + case class Fail(rule: String, reason: String) extends ValidationResult { + def isSuccess: Boolean = false + } +} + diff --git a/atlas-core/src/main/scala/com/netflix/atlas/core/validation/ValueLengthRule.scala b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/ValueLengthRule.scala new file mode 100644 index 000000000..d76c621d0 --- /dev/null +++ b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/ValueLengthRule.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.typesafe.config.Config + +/** + * Verifies that the values are within the specified length bounds. Sample config: + * + * ``` + * min-length = 2 + * max-length = 60 + * ``` + */ +class ValueLengthRule(config: Config) extends TagRule { + private val minLength = config.getInt("min-length") + private val maxLength = config.getInt("max-length") + + def validate(k: String, v: String): ValidationResult = { + v.length match { + case len if len > maxLength => + failure(s"value too long: $k = [$v] ($len > $maxLength)") + case len if len < minLength => + failure(s"value too short: $k = [$v] ($len < $minLength)") + case _ => + ValidationResult.Pass + } + } +} diff --git a/atlas-core/src/main/scala/com/netflix/atlas/core/validation/ValuePatternRule.scala b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/ValuePatternRule.scala new file mode 100644 index 000000000..05e015ebb --- /dev/null +++ b/atlas-core/src/main/scala/com/netflix/atlas/core/validation/ValuePatternRule.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import java.util.regex.Pattern + +import com.typesafe.config.Config + +/** + * Verifies that the values match a specified pattern. Sample config: + * + * ``` + * pattern = "^[-_.a-zA-Z0-9]{4,60}$" + * ``` + */ +class ValuePatternRule(config: Config) extends TagRule { + + private val pattern = Pattern.compile(config.getString("pattern")) + + def validate(k: String, v: String): ValidationResult = { + if (pattern.matcher(v).matches()) ValidationResult.Pass else { + failure(s"value doesn't match pattern '$pattern': [$v]") + } + } +} diff --git a/atlas-core/src/test/scala/com/netflix/atlas/core/validation/HasKeyRuleSuite.scala b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/HasKeyRuleSuite.scala new file mode 100644 index 000000000..1a3fb94b6 --- /dev/null +++ b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/HasKeyRuleSuite.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.typesafe.config.ConfigFactory +import org.scalatest.FunSuite + + +class HasKeyRuleSuite extends FunSuite { + + private val config = ConfigFactory.parseString("key = name") + private val rule = new HasKeyRule(config) + + test("has key") { + assert(rule.validate(Map("name" -> "foo")) === ValidationResult.Pass) + } + + test("missing key") { + val res = rule.validate(Map("cluster" -> "foo")) + assert(res.isFailure) + } +} diff --git a/atlas-core/src/test/scala/com/netflix/atlas/core/validation/KeyLengthRuleSuite.scala b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/KeyLengthRuleSuite.scala new file mode 100644 index 000000000..50f1b3268 --- /dev/null +++ b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/KeyLengthRuleSuite.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.typesafe.config.ConfigFactory +import org.scalatest.FunSuite + + +class KeyLengthRuleSuite extends FunSuite { + + private val config = ConfigFactory.parseString( + """ + |min-length = 2 + |max-length = 4 + """.stripMargin) + + private val rule = new KeyLengthRule(config) + + test("valid") { + assert(rule.validate("ab", "def") === ValidationResult.Pass) + assert(rule.validate("abc", "def") === ValidationResult.Pass) + assert(rule.validate("abcd", "def") === ValidationResult.Pass) + } + + test("too short") { + val res = rule.validate("a", "def") + assert(res.isFailure) + } + + test("too long") { + val res = rule.validate("abcde", "def") + assert(res.isFailure) + } +} diff --git a/atlas-core/src/test/scala/com/netflix/atlas/core/validation/KeyPatternRuleSuite.scala b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/KeyPatternRuleSuite.scala new file mode 100644 index 000000000..a6f58b7e5 --- /dev/null +++ b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/KeyPatternRuleSuite.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.typesafe.config.ConfigFactory +import org.scalatest.FunSuite + + +class KeyPatternRuleSuite extends FunSuite { + + private val config = ConfigFactory.parseString("""pattern = "^[a-c]+$" """) + + private val rule = new KeyPatternRule(config) + + test("valid") { + assert(rule.validate("abc", "ab") === ValidationResult.Pass) + assert(rule.validate("aaa", "abc") === ValidationResult.Pass) + } + + test("invalid") { + val res = rule.validate("abcd", "a") + assert(res.isFailure) + } +} diff --git a/atlas-core/src/test/scala/com/netflix/atlas/core/validation/MaxUserTagsRuleSuite.scala b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/MaxUserTagsRuleSuite.scala new file mode 100644 index 000000000..a83c5988b --- /dev/null +++ b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/MaxUserTagsRuleSuite.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.typesafe.config.ConfigFactory +import org.scalatest.FunSuite + + +class MaxUserTagsRuleSuite extends FunSuite { + + private val config = ConfigFactory.parseString("limit = 2") + private val rule = new MaxUserTagsRule(config) + + test("ok") { + assert(rule.validate(Map("name" -> "foo")) === ValidationResult.Pass) + assert(rule.validate(Map("name" -> "foo", "foo" -> "bar")) === ValidationResult.Pass) + assert(rule.validate(Map("name" -> "foo", "foo" -> "bar", "nf.region" -> "west")) === ValidationResult.Pass) + } + + test("too many") { + val res = rule.validate(Map("name" -> "foo", "foo" -> "bar", "abc" -> "def")) + assert(res.isFailure) + } +} diff --git a/atlas-core/src/test/scala/com/netflix/atlas/core/validation/NonStandardKeyRuleSuite.scala b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/NonStandardKeyRuleSuite.scala new file mode 100644 index 000000000..150ba7824 --- /dev/null +++ b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/NonStandardKeyRuleSuite.scala @@ -0,0 +1,36 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.typesafe.config.ConfigFactory +import org.scalatest.FunSuite + + +class NonStandardKeyRuleSuite extends FunSuite { + + private val config = ConfigFactory.parseString("") + private val rule = new NonStandardKeyRule(config) + + test("valid") { + assert(rule.validate("nf.region", "def") === ValidationResult.Pass) + } + + test("invalid") { + val res = rule.validate("nf.foo", "def") + assert(res.isFailure) + } + +} diff --git a/atlas-core/src/test/scala/com/netflix/atlas/core/validation/RuleSuite.scala b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/RuleSuite.scala new file mode 100644 index 000000000..a12c5a0e8 --- /dev/null +++ b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/RuleSuite.scala @@ -0,0 +1,76 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.typesafe.config.ConfigFactory +import org.scalatest.FunSuite + + +class RuleSuite extends FunSuite { + + private val config = ConfigFactory.parseString( + """ + |rules = [ + | { + | class = "com.netflix.atlas.core.validation.HasKeyRule" + | key = "name" + | }, + | { + | class = "com.netflix.atlas.core.validation.KeyLengthRule" + | min-length = 2 + | max-length = 60 + | }, + | { + | class = "com.netflix.atlas.core.validation.ValueLengthRule" + | min-length = 1 + | max-length = 120 + | }, + | { + | class = "com.netflix.atlas.core.validation.ValidCharactersRule" + | }, + | { + | class = "com.netflix.atlas.core.validation.KeyPatternRule" + | pattern = "^[-_.a-zA-Z0-9]+$" + | }, + | { + | class = "com.netflix.atlas.core.validation.ValuePatternRule" + | pattern = "^[-_.a-zA-Z0-9]+$" + | }, + | { + | class = "com.netflix.atlas.core.validation.MaxUserTagsRule" + | limit = 20 + | } + |] + """.stripMargin) + + test("load") { + val rules = Rule.load(config.getConfigList("rules")) + assert(rules.size === 7) + } + + test("validate ok") { + val rules = Rule.load(config.getConfigList("rules")) + val res = Rule.validate(Map("name" -> "foo", "status" -> "2xx"), rules) + assert(res.isSuccess) + } + + test("validate failure") { + val rules = Rule.load(config.getConfigList("rules")) + val res = Rule.validate(Map("name" -> "foo", "status" -> "2 xx"), rules) + assert(res.isFailure) + } + +} diff --git a/atlas-core/src/test/scala/com/netflix/atlas/core/validation/ValidCharactersRuleSuite.scala b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/ValidCharactersRuleSuite.scala new file mode 100644 index 000000000..ca3f3a584 --- /dev/null +++ b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/ValidCharactersRuleSuite.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.typesafe.config.ConfigFactory +import org.scalatest.FunSuite + + +class ValidCharactersRuleSuite extends FunSuite { + + private val config = ConfigFactory.parseString("") + + private val alpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._" + + test("valid") { + val rule = new ValidCharactersRule(config) + assert(rule.validate(alpha, alpha) === ValidationResult.Pass) + } + + test("invalid key") { + val rule = new ValidCharactersRule(config) + val res = rule.validate("spaces not allowed", alpha) + assert(res.isFailure) + } + + test("invalid value") { + val rule = new ValidCharactersRule(config) + val res = rule.validate(alpha, "spaces not allowed") + assert(res.isFailure) + } +} diff --git a/atlas-core/src/test/scala/com/netflix/atlas/core/validation/ValueLengthRuleSuite.scala b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/ValueLengthRuleSuite.scala new file mode 100644 index 000000000..c78ae2c38 --- /dev/null +++ b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/ValueLengthRuleSuite.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.typesafe.config.ConfigFactory +import org.scalatest.FunSuite + + +class ValueLengthRuleSuite extends FunSuite { + + private val config = ConfigFactory.parseString( + """ + |min-length = 2 + |max-length = 4 + """.stripMargin) + + private val rule = new ValueLengthRule(config) + + test("valid") { + assert(rule.validate("def", "ab") === ValidationResult.Pass) + assert(rule.validate("def", "abc") === ValidationResult.Pass) + assert(rule.validate("def", "abcd") === ValidationResult.Pass) + } + + test("too short") { + val res = rule.validate("def", "a") + assert(res.isFailure) + } + + test("too long") { + val res = rule.validate("def", "abcde") + assert(res.isFailure) + } +} diff --git a/atlas-core/src/test/scala/com/netflix/atlas/core/validation/ValuePatternRuleSuite.scala b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/ValuePatternRuleSuite.scala new file mode 100644 index 000000000..548356451 --- /dev/null +++ b/atlas-core/src/test/scala/com/netflix/atlas/core/validation/ValuePatternRuleSuite.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.core.validation + +import com.typesafe.config.ConfigFactory +import org.scalatest.FunSuite + + +class ValuePatternRuleSuite extends FunSuite { + + private val config = ConfigFactory.parseString("""pattern = "^[a-c]+$" """) + + private val rule = new ValuePatternRule(config) + + test("valid") { + assert(rule.validate("def", "abc") === ValidationResult.Pass) + assert(rule.validate("def", "aaa") === ValidationResult.Pass) + } + + test("invalid") { + val res = rule.validate("def", "abcd") + assert(res.isFailure) + } +} diff --git a/atlas-webapi/src/main/resources/reference.conf b/atlas-webapi/src/main/resources/reference.conf index c3c900d11..5df36f250 100644 --- a/atlas-webapi/src/main/resources/reference.conf +++ b/atlas-webapi/src/main/resources/reference.conf @@ -66,6 +66,35 @@ atlas { publish { // Should we try to intern strings and tag values while parsing the request? intern-while-parsing = true + + // Validation rules to apply before accepting input data + rules = [ + { + class = "com.netflix.atlas.core.validation.HasKeyRule" + key = "name" + }, + { + class = "com.netflix.atlas.core.validation.KeyLengthRule" + min-length = 2 + max-length = 40 + }, + { + class = "com.netflix.atlas.core.validation.ValueLengthRule" + min-length = 1 + max-length = 80 + }, + { + class = "com.netflix.atlas.core.validation.ValidCharactersRule" + }, + { + class = "com.netflix.atlas.core.validation.MaxUserTagsRule" + limit = 20 + } + ] + + // Max age for a datapoint. By default it is one step interval. If the timestamps are + // normalized locally on the client 2 times the step is likely more appropriate. + max-age = ${atlas.core.model.step} } } diff --git a/atlas-webapi/src/main/scala/com/netflix/atlas/webapi/ApiSettings.scala b/atlas-webapi/src/main/scala/com/netflix/atlas/webapi/ApiSettings.scala index 8873f90bb..720ffcd95 100644 --- a/atlas-webapi/src/main/scala/com/netflix/atlas/webapi/ApiSettings.scala +++ b/atlas-webapi/src/main/scala/com/netflix/atlas/webapi/ApiSettings.scala @@ -15,15 +15,21 @@ */ package com.netflix.atlas.webapi +import java.util.concurrent.TimeUnit + import com.netflix.atlas.chart.GraphEngine import com.netflix.atlas.config.ConfigManager import com.netflix.atlas.core.db.Database import com.netflix.atlas.core.model.DefaultSettings +import com.netflix.atlas.core.model.StyleVocabulary +import com.netflix.atlas.core.stacklang.Vocabulary +import com.netflix.atlas.core.validation.Rule import com.typesafe.config.Config -object ApiSettings { +object ApiSettings extends ApiSettings(ConfigManager.current) + +class ApiSettings(root: => Config) { - private def root = ConfigManager.current private def config = root.getConfig("atlas.webapi") def newDbInstance: Database = { @@ -64,4 +70,15 @@ object ApiSettings { cls.newInstance().asInstanceOf[GraphEngine] } } + + def graphVocabulary: Vocabulary = { + config.getString("graph.vocabulary") match { + case "default" => StyleVocabulary + case cls => Class.forName(cls).newInstance().asInstanceOf[Vocabulary] + } + } + + def maxDatapointAge: Long = config.getDuration("publish.max-age", TimeUnit.MILLISECONDS) + + def validationRules: List[Rule] = Rule.load(config.getConfigList("publish.rules")) } diff --git a/atlas-webapi/src/main/scala/com/netflix/atlas/webapi/LocalPublishActor.scala b/atlas-webapi/src/main/scala/com/netflix/atlas/webapi/LocalPublishActor.scala index 9dbe3c0a9..edb2c1473 100644 --- a/atlas-webapi/src/main/scala/com/netflix/atlas/webapi/LocalPublishActor.scala +++ b/atlas-webapi/src/main/scala/com/netflix/atlas/webapi/LocalPublishActor.scala @@ -19,11 +19,13 @@ import java.util.concurrent.TimeUnit import akka.actor.Actor import akka.actor.ActorLogging +import com.netflix.atlas.akka.DiagnosticMessage import com.netflix.atlas.core.db.MemoryDatabase import com.netflix.atlas.core.model.Datapoint import com.netflix.atlas.core.model.DefaultSettings import com.netflix.atlas.core.model.TagKey import com.netflix.atlas.core.norm.NormalizationCache +import com.netflix.atlas.core.validation.ValidationResult import com.netflix.spectator.api.Spectator import com.netflix.spectator.sandbox.BucketCounter import com.netflix.spectator.sandbox.BucketFunctions @@ -43,12 +45,32 @@ class LocalPublishActor(db: MemoryDatabase) extends Actor with ActorLogging { BucketCounter.get(registry, registry.createId("atlas.db.numMetricsReceived"), f) } + // Number of invalid datapoints received + private val numInvalid = Spectator.registry.createId("atlas.db.numInvalid") + private val cache = new NormalizationCache(DefaultSettings.stepSize, db.update) def receive = { - case PublishRequest(vs) => - update(vs) + case PublishRequest(Nil, Nil) => + DiagnosticMessage.sendError(sender(), StatusCodes.BadRequest, "empty payload") + case PublishRequest(Nil, failures) => + updateStats(failures) + val msg = FailureMessage.error(failures) + DiagnosticMessage.sendError(sender(), StatusCodes.BadRequest, msg.toJson) + case PublishRequest(values, Nil) => + update(values) sender() ! HttpResponse(StatusCodes.OK) + case PublishRequest(values, failures) => + update(values) + updateStats(failures) + val msg = FailureMessage.partial(failures) + sender() ! HttpResponse(StatusCodes.Accepted, msg.toJson) + } + + private def updateStats(failures: List[ValidationResult]): Unit = { + failures.foreach { case ValidationResult.Fail(error, _) => + Spectator.registry.counter(numInvalid.withTag("error", error)) + } } private def update(vs: List[Datapoint]): Unit = { diff --git a/atlas-webapi/src/main/scala/com/netflix/atlas/webapi/PublishApi.scala b/atlas-webapi/src/main/scala/com/netflix/atlas/webapi/PublishApi.scala index d8d7bb2af..bef66e10f 100644 --- a/atlas-webapi/src/main/scala/com/netflix/atlas/webapi/PublishApi.scala +++ b/atlas-webapi/src/main/scala/com/netflix/atlas/webapi/PublishApi.scala @@ -18,6 +18,7 @@ package com.netflix.atlas.webapi import akka.actor.ActorRefFactory import com.fasterxml.jackson.core.JsonGenerator import com.fasterxml.jackson.core.JsonParser +import com.netflix.atlas.akka.DiagnosticMessage import com.netflix.atlas.akka.WebApi import com.netflix.atlas.config.ConfigManager import com.netflix.atlas.core.model.Datapoint @@ -26,9 +27,10 @@ import com.netflix.atlas.core.model.TaggedItem import com.netflix.atlas.core.util.Interner import com.netflix.atlas.core.util.SmallHashMap import com.netflix.atlas.core.util.Streams +import com.netflix.atlas.core.validation.Rule +import com.netflix.atlas.core.validation.ValidationResult import com.netflix.atlas.json.Json -import spray.http.HttpResponse -import spray.http.StatusCodes +import com.netflix.atlas.json.JsonSupport import spray.routing.RequestContext class PublishApi(implicit val actorRefFactory: ActorRefFactory) extends WebApi { @@ -41,6 +43,8 @@ class PublishApi(implicit val actorRefFactory: ActorRefFactory) extends WebApi { private val internWhileParsing = config.getBoolean("intern-while-parsing") + private val rules = ApiSettings.validationRules + def routes: RequestContext => Unit = { post { path("api" / "v1" / "publish") { ctx => @@ -58,23 +62,34 @@ class PublishApi(implicit val actorRefFactory: ActorRefFactory) extends WebApi { getJsonParser(ctx.request) match { case Some(parser) => val data = decodeBatch(parser, internWhileParsing) - validate(data) - publishRef.tell(PublishRequest(data), ctx.responder) + val req = validate(data) + publishRef.tell(req, ctx.responder) case None => throw new IllegalArgumentException("empty request body") } } catch handleException(ctx) } - private def validate(vs: List[Datapoint]): Unit = { - // Temporary until rules get moved to oss. Just include basic sanity check on timestamps + private def validate(vs: List[Datapoint]): PublishRequest = { + val validDatapoints = List.newBuilder[Datapoint] + val failures = List.newBuilder[ValidationResult] val now = System.currentTimeMillis() - val step = DefaultSettings.stepSize + val limit = ApiSettings.maxDatapointAge vs.foreach { v => val diff = now - v.timestamp - if (diff > step) - throw new IllegalArgumentException("data is too old") + val result = diff match { + case d if d > limit => + val msg = s"data is too old: now = $now, timestamp = ${v.timestamp}, $d > $limit" + ValidationResult.Fail("DataTooOld", msg) + case d if d < -limit => + val msg = s"data is from future: now = $now, timestamp = ${v.timestamp}" + ValidationResult.Fail("DataFromFuture", msg) + case _ => + Rule.validate(v.tags, rules) + } + if (result.isSuccess) validDatapoints += v else failures += result } + PublishRequest(validDatapoints.result(), failures.result()) } } @@ -220,5 +235,21 @@ object PublishApi { } } - case class PublishRequest(values: List[Datapoint]) + case class PublishRequest(values: List[Datapoint], failures: List[ValidationResult]) + + case class FailureMessage(`type`: String, message: List[String]) extends JsonSupport { + def typeName: String = `type` + } + + object FailureMessage { + def error(message: List[ValidationResult]): FailureMessage = { + val failures = message.collect { case ValidationResult.Fail(_, reason) => reason } + new FailureMessage(DiagnosticMessage.Error, failures) + } + + def partial(message: List[ValidationResult]): FailureMessage = { + val failures = message.collect { case ValidationResult.Fail(_, reason) => reason } + new FailureMessage("partial", failures) + } + } } diff --git a/atlas-webapi/src/test/scala/com/netflix/atlas/webapi/ApiSettingsSuite.scala b/atlas-webapi/src/test/scala/com/netflix/atlas/webapi/ApiSettingsSuite.scala new file mode 100644 index 000000000..2c17d5156 --- /dev/null +++ b/atlas-webapi/src/test/scala/com/netflix/atlas/webapi/ApiSettingsSuite.scala @@ -0,0 +1,48 @@ +/* + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.atlas.webapi + +import com.netflix.atlas.core.model.StyleVocabulary +import com.netflix.atlas.core.stacklang.Vocabulary +import com.netflix.atlas.core.stacklang.Word +import com.typesafe.config.ConfigFactory +import org.scalatest.FunSuite + +class ApiSettingsSuite extends FunSuite { + + test("graphVocabulary default") { + val cfg = new ApiSettings(ConfigFactory.parseString("atlas.webapi.graph.vocabulary=default")) + assert(cfg.graphVocabulary === StyleVocabulary) + } + + test("graphVocabulary class") { + val cls = classOf[TestVocabulary].getName + val cfg = new ApiSettings(ConfigFactory.parseString(s"atlas.webapi.graph.vocabulary=$cls")) + assert(cfg.graphVocabulary.isInstanceOf[TestVocabulary]) + } + + test("load validation rules") { + // Throws if there is a problem loading the rules + ApiSettings.validationRules + } +} + +class TestVocabulary extends Vocabulary { + override def name: String = "test" + override def words: List[Word] = Nil + override def dependsOn: List[Vocabulary] = Nil +} + diff --git a/atlas-webapi/src/test/scala/com/netflix/atlas/webapi/PublishApiSuite.scala b/atlas-webapi/src/test/scala/com/netflix/atlas/webapi/PublishApiSuite.scala index d0dbaae35..9dbf732e5 100644 --- a/atlas-webapi/src/test/scala/com/netflix/atlas/webapi/PublishApiSuite.scala +++ b/atlas-webapi/src/test/scala/com/netflix/atlas/webapi/PublishApiSuite.scala @@ -15,23 +15,13 @@ */ package com.netflix.atlas.webapi -import java.io.PrintStream -import java.net.URI - import akka.actor.Actor import akka.actor.Props -import com.netflix.atlas.core.db.StaticDatabase import com.netflix.atlas.core.model.Datapoint -import com.netflix.atlas.core.util.Hash -import com.netflix.atlas.core.util.PngImage -import com.netflix.atlas.core.util.Streams -import com.netflix.atlas.core.util.Strings -import com.netflix.atlas.test.GraphAssertions +import com.netflix.atlas.webapi.PublishApi.FailureMessage import com.netflix.atlas.webapi.PublishApi.PublishRequest import org.scalatest.FunSuite import spray.http.HttpResponse -import spray.http.MediaTypes._ -import spray.http.HttpEntity import spray.http.StatusCodes import spray.testkit.ScalatestRouteTest @@ -55,7 +45,7 @@ class PublishApiSuite extends FunSuite with ScalatestRouteTest { test("publish empty object") { Post("/api/v1/publish", "{}") ~> endpoint.routes ~> check { - assert(response.status === StatusCodes.OK) + assert(response.status === StatusCodes.BadRequest) assert(lastUpdate === Nil) } } @@ -95,6 +85,28 @@ class PublishApiSuite extends FunSuite with ScalatestRouteTest { } } + test("partial failure") { + val json = s"""{ + "metrics": [ + { + "tags": {"name": "cpuUser"}, + "timestamp": ${System.currentTimeMillis() / 1000}, + "value": 42.0 + }, + { + "tags": {"name": "cpuSystem"}, + "timestamp": ${System.currentTimeMillis()}, + "value": 42.0 + } + ] + }""" + Post("/api/v1/publish", json) ~> endpoint.routes ~> check { + assert(response.status === StatusCodes.Accepted) + assert(lastUpdate === PublishApi.decodeBatch(json).tail) + println(responseAs[String]) + } + } + test("publish bad json") { Post("/api/v1/publish", "fubar") ~> endpoint.routes ~> check { assert(response.status === StatusCodes.BadRequest) @@ -102,9 +114,22 @@ class PublishApiSuite extends FunSuite with ScalatestRouteTest { } test("publish-fast alias") { - Post("/api/v1/publish-fast", "{}") ~> endpoint.routes ~> check { + val json = s"""{ + "tags": { + "cluster": "foo", + "node": "i-123" + }, + "metrics": [ + { + "tags": {"name": "cpuUser"}, + "timestamp": ${System.currentTimeMillis()}, + "value": 42.0 + } + ] + }""" + Post("/api/v1/publish-fast", json) ~> endpoint.routes ~> check { assert(response.status === StatusCodes.OK) - assert(lastUpdate === Nil) + assert(lastUpdate === PublishApi.decodeBatch(json)) } } } @@ -115,9 +140,20 @@ object PublishApiSuite { class TestActor extends Actor { def receive = { - case PublishRequest(vs) => - lastUpdate = vs + case PublishRequest(Nil, Nil) => + lastUpdate = Nil + sender() ! HttpResponse(StatusCodes.BadRequest) + case PublishRequest(Nil, failures) => + lastUpdate = Nil + val msg = FailureMessage.error(failures) + sender() ! HttpResponse(StatusCodes.BadRequest, msg.toJson) + case PublishRequest(values, Nil) => + lastUpdate = values sender() ! HttpResponse(StatusCodes.OK) + case PublishRequest(values, failures) => + lastUpdate = values + val msg = FailureMessage.partial(failures) + sender() ! HttpResponse(StatusCodes.Accepted, msg.toJson) } } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 81f9d6b23..4d00c2dd9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -8,7 +8,7 @@ object Dependencies { val jackson = "2.5.0" val log4j = "2.1" val lucene = "4.10.2" - val scala = "2.11.5" + val scala = "2.11.7" val slf4j = "1.7.10" val spectator = "0.19" val spray = "1.3.2"