From 2bd72d689a993bc6735d8ae81525bbae7df1fe48 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Fri, 5 Jun 2015 18:03:01 -0700 Subject: [PATCH 01/68] Improve query performance in the AggregationIterator by calling .hasNext() on the downsampler instead of letting it build a giant exception string that we're just tossing in the bit bucket. --- src/core/AggregationIterator.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/core/AggregationIterator.java b/src/core/AggregationIterator.java index 2c6f6e3433..f806f7b1d2 100644 --- a/src/core/AggregationIterator.java +++ b/src/core/AggregationIterator.java @@ -276,17 +276,12 @@ private AggregationIterator(final SeekableView[] iterators, SeekableView it = iterators[i]; it.seek(start_time); final DataPoint dp; - try { - dp = it.next(); - } catch (NoSuchElementException e) { - // It should be rare but could happen after downsampling when - // we throw away some data points at the beginning after aligning - // start time by downsmpling interval and there are no data points - // left for the current span. + if (!it.hasNext()) { ++num_empty_spans; endReached(i); continue; } + dp = it.next(); //LOG.debug("Creating iterator #" + i); if (dp.timestamp() >= start_time) { //LOG.debug("First DP in range for #" + i + ": " From ef22de8f23e42f9dcb514c4a124bf76712003981 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 12 Sep 2015 12:23:36 -0700 Subject: [PATCH 02/68] Bump AsyncHBase to the release version and remove old md5s --- third_party/hbase/asynchbase-1.5.0.jar.md5 | 1 - third_party/hbase/asynchbase-1.6.0.jar.md5 | 1 - third_party/hbase/asynchbase-1.7.0-20150517.200244-1.jar.md5 | 1 - third_party/hbase/asynchbase-1.7.0-20150904.040751-2.jar.md5 | 1 - third_party/hbase/asynchbase-1.7.0.jar.md5 | 1 + third_party/hbase/include.mk | 4 ++-- 6 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 third_party/hbase/asynchbase-1.5.0.jar.md5 delete mode 100644 third_party/hbase/asynchbase-1.6.0.jar.md5 delete mode 100644 third_party/hbase/asynchbase-1.7.0-20150517.200244-1.jar.md5 delete mode 100644 third_party/hbase/asynchbase-1.7.0-20150904.040751-2.jar.md5 create mode 100644 third_party/hbase/asynchbase-1.7.0.jar.md5 diff --git a/third_party/hbase/asynchbase-1.5.0.jar.md5 b/third_party/hbase/asynchbase-1.5.0.jar.md5 deleted file mode 100644 index e20d2ff219..0000000000 --- a/third_party/hbase/asynchbase-1.5.0.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -12c61569f04eb88229c90dde9fa51848 diff --git a/third_party/hbase/asynchbase-1.6.0.jar.md5 b/third_party/hbase/asynchbase-1.6.0.jar.md5 deleted file mode 100644 index 7fcfbd8ec4..0000000000 --- a/third_party/hbase/asynchbase-1.6.0.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -6738dd73fd48d30cbf5c78f62bc18852 diff --git a/third_party/hbase/asynchbase-1.7.0-20150517.200244-1.jar.md5 b/third_party/hbase/asynchbase-1.7.0-20150517.200244-1.jar.md5 deleted file mode 100644 index 4e13899eda..0000000000 --- a/third_party/hbase/asynchbase-1.7.0-20150517.200244-1.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -78c317f4457b6add9671bc2a30df7bd1 \ No newline at end of file diff --git a/third_party/hbase/asynchbase-1.7.0-20150904.040751-2.jar.md5 b/third_party/hbase/asynchbase-1.7.0-20150904.040751-2.jar.md5 deleted file mode 100644 index 00f09b9593..0000000000 --- a/third_party/hbase/asynchbase-1.7.0-20150904.040751-2.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -af3e0778bf5b94f1302bc8a01504680a diff --git a/third_party/hbase/asynchbase-1.7.0.jar.md5 b/third_party/hbase/asynchbase-1.7.0.jar.md5 new file mode 100644 index 0000000000..5cb6466209 --- /dev/null +++ b/third_party/hbase/asynchbase-1.7.0.jar.md5 @@ -0,0 +1 @@ +f1aed41b7f16345d2f58797ffa77f36a \ No newline at end of file diff --git a/third_party/hbase/include.mk b/third_party/hbase/include.mk index d04740e54c..e2e33dcc32 100644 --- a/third_party/hbase/include.mk +++ b/third_party/hbase/include.mk @@ -13,9 +13,9 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see . -ASYNCHBASE_VERSION := 1.7.0-20150910.030815-3 +ASYNCHBASE_VERSION := 1.7.0 ASYNCHBASE := third_party/hbase/asynchbase-$(ASYNCHBASE_VERSION).jar -ASYNCHBASE_BASE_URL := https://oss.sonatype.org/content/repositories/snapshots/org/hbase/asynchbase/1.7.0-SNAPSHOT/ +ASYNCHBASE_BASE_URL := http://central.maven.org/maven2/org/hbase/asynchbase/$(ASYNCHBASE_VERSION) $(ASYNCHBASE): $(ASYNCHBASE).md5 set dummy "$(ASYNCHBASE_BASE_URL)" "$(ASYNCHBASE)"; shift; $(FETCH_DEPENDENCY) From a30211c23149182390cab1663560f60b9674b20e Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 12 Sep 2015 12:46:59 -0700 Subject: [PATCH 03/68] Release 2.1.1 --- NEWS | 9 +++++++++ THANKS | 1 + configure.ac | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index b45f3dccef..8013e6daf1 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,14 @@ OpenTSDB - User visible changes. +* Version 2.1.1 (2015-09-12) + +Bug Fixes: + - Relax the pgrep regex to correctly find and kill the java process in the RPM init.d + script. + - Improve query performance slightly when aggregating multiple series. + - Fix the /api/search/lookup API call to properly handle the limit parameter. + - Fix the /api/query/last endpoint to properly handle missing tsdb-meta tables. + * Version 2.1.0 (2015-05-06) Bug Fixes: diff --git a/THANKS b/THANKS index b8005c0884..c53931ddea 100644 --- a/THANKS +++ b/THANKS @@ -32,6 +32,7 @@ Josh Thomas Kieren Hynd Kimoon Kim Kris Beevers +Lex Herbert Liangliang He Matt Jibson Mark Smith diff --git a/configure.ac b/configure.ac index 66ab5753b8..082b2d07b9 100644 --- a/configure.ac +++ b/configure.ac @@ -14,7 +14,7 @@ # along with this library. If not, see . # Semantic Versioning (see http://semver.org/). -AC_INIT([opentsdb], [2.1.0], [opentsdb@googlegroups.com]) +AC_INIT([opentsdb], [2.1.1], [opentsdb@googlegroups.com]) AC_CONFIG_AUX_DIR([build-aux]) AM_INIT_AUTOMAKE([foreign]) From 8e7eb329a45ae5de48776c89419fc1be34498e8f Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 12 Sep 2015 12:33:33 -0700 Subject: [PATCH 04/68] Pull in the updated NEWS and THANKS files --- NEWS | 27 ++++++++++++++++++++++++--- THANKS | 1 + 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 02d24359e8..eb54e0282f 100644 --- a/NEWS +++ b/NEWS @@ -1,11 +1,32 @@ OpenTSDB - User visible changes. -* Version 2.1.0 RC1 (2015-04-04) +* Version 2.1.1 (2015-09-12) + +Bug Fixes: + - Relax the pgrep regex to correctly find and kill the java process in the RPM init.d + script. + - Improve query performance slightly when aggregating multiple series. + - Fix the /api/search/lookup API call to properly handle the limit parameter. + - Fix the /api/query/last endpoint to properly handle missing tsdb-meta tables. + +* Version 2.1.0 (2015-05-06) + +Bug Fixes: + - FSCK was not handling compacted and floating point duplicates properly. Now they + are merged correctly. + - TSMeta data updates were not loading the latest data from storage on response + - The config class will now trim spaces from booleans and integers + - On shutdown, the idle state handler could prevent the TSD from shutting down + gracefully. A new thread factory sets that thread as a daemon thread. + - TSMeta objects were not generated if multiple writes for the same data point arrived + in succession due to buffering atomic increments. Increments are no longer buffered. + - Updated paths to the deprecated Google Code repo for dependencies. + +* Version 2.1.0 RC2 (2015-04-04) Noteworthy Changes: - Handle idle connections in Netty by closing them after some period of inactivity - Support compressed HTTP responses - - Bug Fixes: - Various RPM script and package fixes @@ -166,4 +187,4 @@ along with this library. If not, see . Local Variables: mode: outline -End: +End: \ No newline at end of file diff --git a/THANKS b/THANKS index b8005c0884..c53931ddea 100644 --- a/THANKS +++ b/THANKS @@ -32,6 +32,7 @@ Josh Thomas Kieren Hynd Kimoon Kim Kris Beevers +Lex Herbert Liangliang He Matt Jibson Mark Smith From 5388ff3f87af4c717b56f5d76a3696c8eaad6793 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 12 Sep 2015 13:48:45 -0700 Subject: [PATCH 05/68] Cut 2.2.0 RC1 --- NEWS | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ THANKS | 17 +++++++++++++++++ configure.ac | 2 +- 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index eb54e0282f..ac24f3fd81 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,54 @@ OpenTSDB - User visible changes. +* Version 2.2.0 RC1 (2015-09-12) + +Noteworthy Changes: + - Add the option to randomly assign UIDs to metrics to improve distribution across + HBase region servers. + - Introduce salting of data to improve distribution of high cardinality regions + across region servers. + - Introduce query stats for tracking various timings related to TSD queries. + - Add more stats endpoints including /threads, /jvm and /region_clients + - Allow for deleting UID mappings via CLI or the API + - Name the various threads for easier debugging, particularly for distinguishing + between TSD and AsyncHBase threads. + - Allow for pre-fetching all of the meta information for the tables to improve + performance. + - Update to the latest AsyncHBase with support for secure HBase clusters and RPC + timeouts. + - Allow for overriding metric and tag widths via the config file. (Be careful!) + - URLs from the API are now relative instead of absolute, allowing for easier reverse + proxy use. + - Allow for percent deviation in the Nagios check + - Let queries skip over unknown tag values that may not exist yet (via config) + - Add various query filters such as case (in)sensitive pipes, wildcards and pipes + over tag values. Filters do not work over metrics at this time. + - Add support for writing data points using Appends in HBase as a way of writing + compacted data without having to read and re-write at the top of each hour. + - Introduce an option to emit NaNs or Nulls in the JSON output when downsampling and + a bucket is missing values. + - Introduce query time flags to show the original query along with some timing stats + in the response. + - Introduce a storage exception handler plugin that will allow users to spool or + requeue data points that fail writes to HBase due to various issues. + - Rework the HTTP pipeline to support plugins with RPC implementations. + - Allow for some style options in the Gnuplot graphs. + - Allow for timing out long running HTTP queries. + - Text importer will now log and continue bad rows instead of failing. + - New percentile and count aggregators. + - Add the /api/annotations endpoint to fetch multiple annotations in one call. + - Add a class to support improved bulk imports by batching requests in memory for a + full hour before writing. + +Bug Fixes: + - Modify the .rpm build to allow dashes in the name. + - Allow the Nagios check script to handle 0 values properly in checks. + - Fix FSCK where floating point values were not processed correctly (#430) + - Fix missing information from the /appi/uid/tsmeta calls (#498) + - Fix more issues with the FSCK around deleting columns that were in the list (#436) + - Avoid OOM issues over Telnet when the sending client isn't reading errors off it's + socket fast enough by blocking writes. + * Version 2.1.1 (2015-09-12) Bug Fixes: diff --git a/THANKS b/THANKS index c53931ddea..33898d5163 100644 --- a/THANKS +++ b/THANKS @@ -11,20 +11,27 @@ copyright assignment. Adrian Muraru Adrien Mogenet Alex Ioffe +Andre Pech Andrey Stepachev Aravind Gottipati Arvind Jayaprakash Berk D. Demir +Bikrant Neupane Bryan Zubrod Chris McClymont +Cristian Sechel Christophe Furmaniak Dave Barr Filippo Giunchedi +Gabriel Nicolas Avellaneda Guenther Schmuelling Hugo Trippaers Jacek Masiulaniec Jari Takkala +James Royalty Jan Mangs +Jason Harvey +Jim Scott Jesse Chang Johan Zeeck Jonathan Works @@ -34,25 +41,35 @@ Kimoon Kim Kris Beevers Lex Herbert Liangliang He +Loïs Burg Matt Jibson +Matt Schallert +Marc Tamsky Mark Smith Martin Jansen +Michal Kimle Mike Bryant Mike Kobyakov +Nathan Owens Nicole Nagele Nikhil Benesch +Nitin Aggarwal Paula Keezer Peter Gotz Pradeep Chhetri +Rajesh G Ryan Berdeen +Sean Miller Siddartha Guthikonda Simon Matic Langford Slawek Ligus Sy Le Tay Ray Chuan +Thomas Krajca Thomas Sanchez Tibor Vass Tristan Colgate-McFarlane Tony Landells Vasiliy Kiryanov +Yulai Fu Zachary Kurey \ No newline at end of file diff --git a/configure.ac b/configure.ac index 29d2615325..3d43a30c24 100644 --- a/configure.ac +++ b/configure.ac @@ -14,7 +14,7 @@ # along with this library. If not, see . # Semantic Versioning (see http://semver.org/). -AC_INIT([opentsdb], [2.2.0-SNAPSHOT], [opentsdb@googlegroups.com]) +AC_INIT([opentsdb], [2.2.0RC1], [opentsdb@googlegroups.com]) AC_CONFIG_AUX_DIR([build-aux]) AM_INIT_AUTOMAKE([foreign]) From 93b37981a17b3f7e10f5bfc284f1e41f6bccbd08 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 26 Sep 2015 14:42:20 -0700 Subject: [PATCH 06/68] Add the Exceptions utility class for parsing out deferred group exception causes. --- Makefile.am | 2 + src/utils/Exceptions.java | 41 +++++++++++++++++++ test/utils/TestExceptions.java | 73 ++++++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 src/utils/Exceptions.java create mode 100644 test/utils/TestExceptions.java diff --git a/Makefile.am b/Makefile.am index f7307fe915..97bdd24876 100644 --- a/Makefile.am +++ b/Makefile.am @@ -143,6 +143,7 @@ tsdb_SRC := \ src/utils/ByteArrayPair.java \ src/utils/Config.java \ src/utils/DateTime.java \ + src/utils/Exceptions.java \ src/utils/FileSystem.java \ src/utils/JSON.java \ src/utils/JSONException.java \ @@ -252,6 +253,7 @@ test_SRC := \ test/utils/TestByteArrayPair.java \ test/utils/TestConfig.java \ test/utils/TestDateTime.java \ + test/utils/TestExceptions.java \ test/utils/TestJSON.java \ test/utils/TestPair.java \ test/utils/TestPluginLoader.java diff --git a/src/utils/Exceptions.java b/src/utils/Exceptions.java new file mode 100644 index 0000000000..4f52111329 --- /dev/null +++ b/src/utils/Exceptions.java @@ -0,0 +1,41 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.utils; + +import com.stumbleupon.async.DeferredGroupException; + +/** + * A class with utility methods for dealing with Exceptions in OpenTSDB + * @since 2.2 + */ +public class Exceptions { + + /** + * Iterates through the stack trace, looking for the actual cause of the + * deferred group exception. These traces can be huge and truncated in the + * logs so it's really useful to be able to spit out the source. + * @param e A DeferredGroupException to parse + * @return The root cause of the exception if found. + */ + public static Throwable getCause(final DeferredGroupException e) { + Throwable ex = e; + while (ex.getClass().equals(DeferredGroupException.class)) { + if (ex.getCause() == null) { + break; + } else { + ex = ex.getCause(); + } + } + return ex; + } +} diff --git a/test/utils/TestExceptions.java b/test/utils/TestExceptions.java new file mode 100644 index 0000000000..d83ffa715a --- /dev/null +++ b/test/utils/TestExceptions.java @@ -0,0 +1,73 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.utils; + +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; + +import java.util.ArrayList; + +import org.junit.Before; +import org.junit.Test; + +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; + +public class TestExceptions { + private ArrayList> deferreds; + + @Before + public void before() { + deferreds = new ArrayList>(1); + } + + @Test + public void oneLevel() throws Exception { + final RuntimeException ex = new RuntimeException("Boo!"); + deferreds.add(Deferred.fromError(ex)); + try { + Deferred.group(deferreds).join(); + fail("Expected a DeferredGroupException"); + } catch (DeferredGroupException dge) { + assertSame(ex, Exceptions.getCause(dge)); + } + } + + @Test + public void nested() throws Exception { + final RuntimeException ex = new RuntimeException("Boo!"); + deferreds.add(Deferred.fromError(ex)); + + final ArrayList> deferreds2 = + new ArrayList>(1); + deferreds2.add(Deferred.fromResult(null)); + + class LOne implements + Callback>, ArrayList> { + @Override + public Deferred> call(final ArrayList piff) + throws Exception { + return Deferred.group(deferreds); + } + } + + try { + Deferred.group(deferreds2).addCallbackDeferring(new LOne()).join(); + fail("Expected a DeferredGroupException"); + } catch (DeferredGroupException dge) { + assertSame(ex, Exceptions.getCause(dge)); + } + } + +} From 5fc5179fdc3de1f7a805421e630c88843f91ecd6 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 26 Sep 2015 15:07:16 -0700 Subject: [PATCH 07/68] Add the QueryUtil class that pulls some methods out of the TsdbQuery class so they can be shared elsewhere. Also make queries a tiny bit more efficient with salting by compiling the row key regex once instead of once for each bucket. --- Makefile.am | 1 + src/core/TsdbQuery.java | 215 ++++--------------------------- src/query/QueryUtil.java | 267 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 290 insertions(+), 193 deletions(-) create mode 100644 src/query/QueryUtil.java diff --git a/Makefile.am b/Makefile.am index 97bdd24876..f9395a0504 100644 --- a/Makefile.am +++ b/Makefile.am @@ -73,6 +73,7 @@ tsdb_SRC := \ src/meta/TSMeta.java \ src/meta/TSUIDQuery.java \ src/meta/UIDMeta.java \ + src/query/QueryUtil.java \ src/query/filter/TagVFilter.java \ src/query/filter/TagVLiteralOrFilter.java \ src/query/filter/TagVNotKeyFilter.java \ diff --git a/src/core/TsdbQuery.java b/src/core/TsdbQuery.java index c9266d9c35..cef3748f15 100644 --- a/src/core/TsdbQuery.java +++ b/src/core/TsdbQuery.java @@ -23,7 +23,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.Map.Entry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +38,7 @@ import com.stumbleupon.async.Deferred; import com.stumbleupon.async.DeferredGroupException; +import net.opentsdb.query.QueryUtil; import net.opentsdb.query.filter.TagVFilter; import net.opentsdb.stats.Histogram; import net.opentsdb.uid.NoSuchUniqueId; @@ -86,7 +86,10 @@ final class TsdbQuery implements Query { /** ID of the metric being looked up. */ private byte[] metric; - + + /** Row key regex to pass to HBase if we have tags or TSUIDs */ + private String regex; + /** * Tags by which we must group the results. * Each element is a tag ID. @@ -841,14 +844,12 @@ protected Scanner getScanner() throws HBaseException { */ protected Scanner getScanner(final int salt_bucket) throws HBaseException { final short metric_width = tsdb.metrics.width(); - final int metric_salt_width = metric_width + Const.SALT_WIDTH(); - final byte[] start_row = new byte[metric_salt_width + Const.TIMESTAMP_BYTES]; - final byte[] end_row = new byte[metric_salt_width + Const.TIMESTAMP_BYTES]; - if (Const.SALT_WIDTH() > 0) { - final byte[] salt = RowKey.getSaltBytes(salt_bucket); - System.arraycopy(salt, 0, start_row, 0, Const.SALT_WIDTH()); - System.arraycopy(salt, 0, end_row, 0, Const.SALT_WIDTH()); + // set the metric UID based on the TSUIDs if given, or the metric UID + if (tsuids != null && !tsuids.isEmpty()) { + final String tsuid = tsuids.get(0); + final String metric_uid = tsuid.substring(0, metric_width * 2); + metric = UniqueId.stringToUid(metric_uid); } // We search at least one row before and one row after the start & end @@ -857,33 +858,15 @@ protected Scanner getScanner(final int salt_bucket) throws HBaseException { // rely on having a few extra data points before & after the exact start // & end dates in order to do proper rate calculation or downsampling near // the "edges" of the graph. - Bytes.setInt(start_row, (int) getScanStartTimeSeconds(), metric_salt_width); - Bytes.setInt(end_row, (end_time == UNSET - ? -1 // Will scan until the end (0xFFF...). - : (int) getScanEndTimeSeconds()), - metric_salt_width); - - // set the metric UID based on the TSUIDs if given, or the metric UID - if (tsuids != null && !tsuids.isEmpty()) { - final String tsuid = tsuids.get(0); - final String metric_uid = tsuid.substring(0, metric_width * 2); - metric = UniqueId.stringToUid(metric_uid); - System.arraycopy(metric, 0, start_row, Const.SALT_WIDTH(), metric_width); - System.arraycopy(metric, 0, end_row, Const.SALT_WIDTH(), metric_width); - } else { - System.arraycopy(metric, 0, start_row, Const.SALT_WIDTH(), metric_width); - System.arraycopy(metric, 0, end_row, Const.SALT_WIDTH(), metric_width); - } - - final Scanner scanner = tsdb.client.newScanner(tsdb.table); - scanner.setStartKey(start_row); - scanner.setStopKey(end_row); + final Scanner scanner = QueryUtil.getMetricScanner(tsdb, salt_bucket, metric, + (int) getScanStartTimeSeconds(), end_time == UNSET + ? -1 // Will scan until the end (0xFFF...). + : (int) getScanEndTimeSeconds(), tsdb.table, TSDB.FAMILY()); if (tsuids != null && !tsuids.isEmpty()) { createAndSetTSUIDFilter(scanner); } else if (filters.size() > 0) { createAndSetFilter(scanner); } - scanner.setFamily(TSDB.FAMILY); return scanner; } @@ -972,113 +955,14 @@ private long getScanEndTimeSeconds() { * @param scanner The scanner on which to add the filter. */ private void createAndSetFilter(final Scanner scanner) { - if (group_bys != null) { - Collections.sort(group_bys, Bytes.MEMCMP); + if (regex == null) { + regex = QueryUtil.getRowKeyUIDRegex(group_bys, row_key_literals); } - final short name_width = tsdb.tag_names.width(); - final short value_width = tsdb.tag_values.width(); - final short tagsize = (short) (name_width + value_width); - // Generate a regexp for our tags. Say we have 2 tags: { 0 0 1 0 0 2 } - // and { 4 5 6 9 8 7 }, the regexp will be: - // "^.{7}(?:.{6})*\\Q\000\000\001\000\000\002\\E(?:.{6})*\\Q\004\005\006\011\010\007\\E(?:.{6})*$" - final StringBuilder buf = new StringBuilder( - 15 // "^.{N}" + "(?:.{M})*" + "$" - + ((13 + tagsize) // "(?:.{M})*\\Q" + tagsize bytes + "\\E" - * ((row_key_literals == null ? 0 : row_key_literals.size()) + - (group_bys == null ? 0 : group_bys.size() * 3)))); - // In order to avoid re-allocations, reserve a bit more w/ groups ^^^ - - // Alright, let's build this regexp. From the beginning... - buf.append("(?s)" // Ensure we use the DOTALL flag. - + "^.{") - // ... start by skipping the salt, metric ID and timestamp. - .append(Const.SALT_WIDTH() + tsdb.metrics.width() + Const.TIMESTAMP_BYTES) - .append("}"); - - final Iterator> it = row_key_literals == null ? - new ByteMap().iterator() : row_key_literals.iterator(); - - while(it.hasNext()) { - Entry entry = it.hasNext() ? it.next() : null; - // TODO - This look ahead may be expensive. We need to get some data around - // whether it's faster for HBase to scan with a look ahead or simply pass - // the rows back to the TSD for filtering. - final boolean not_key = - entry.getValue() != null && entry.getValue().length == 0; - - // Skip any number of tags. - buf.append("(?:.{").append(tagsize).append("})*"); - if (not_key) { - // start the lookahead as we have a key we expliclty do not want in the - // results - buf.append("(?!"); - } - buf.append("\\Q"); - - addId(buf, entry.getKey()); - if (entry.getValue() != null && entry.getValue().length > 0) { // Add a group_by. - // We want specific IDs. List them: /(AAA|BBB|CCC|..)/ - buf.append("(?:"); - for (final byte[] value_id : entry.getValue()) { - if (value_id == null) { - continue; - } - buf.append("\\Q"); - addId(buf, value_id); - buf.append('|'); - } - // Replace the pipe of the last iteration. - buf.setCharAt(buf.length() - 1, ')'); - } else { - buf.append(".{").append(value_width).append('}'); // Any value ID. - } - - if (not_key) { - // be sure to close off the look ahead - buf.append(")"); - } - } - // Skip any number of tags before the end. - buf.append("(?:.{").append(tagsize).append("})*$"); - scanner.setKeyRegexp(buf.toString(), CHARSET); + scanner.setKeyRegexp(regex, CHARSET); if (LOG.isDebugEnabled()) { - logRegexScanner(buf.toString()); + LOG.debug("Scanner regex: " + QueryUtil.byteRegexToString(regex)); } } - - /** - * Little helper to print out the regular expression by converting the UID - * bytes to an array. - * @param regexp The regex string to print to the debug log - * @since 2.2 - */ - void logRegexScanner(final String regexp) { - final StringBuilder buf = new StringBuilder(); - for (int i = 0; i < regexp.length(); i++) { - if (i > 0 && regexp.charAt(i - 1) == 'Q') { - if (regexp.charAt(i - 3) == '*') { - // tagk - byte[] tagk = new byte[TSDB.tagk_width()]; - for (int x = 0; x < TSDB.tagk_width(); x++) { - tagk[x] = (byte)regexp.charAt(i + x); - } - i += TSDB.tagk_width(); - buf.append(Arrays.toString(tagk)); - } else { - // tagv - byte[] tagv = new byte[TSDB.tagv_width()]; - for (int x = 0; x < TSDB.tagv_width(); x++) { - tagv[x] = (byte)regexp.charAt(i + x); - } - i += TSDB.tagv_width(); - buf.append(Arrays.toString(tagv)); - } - } else { - buf.append(regexp.charAt(i)); - } - } - LOG.debug("Scanner regex: " + buf.toString()); - } /** * Sets the server-side regexp filter on the scanner. @@ -1088,67 +972,12 @@ void logRegexScanner(final String regexp) { * @since 2.0 */ private void createAndSetTSUIDFilter(final Scanner scanner) { - Collections.sort(tsuids); - - // first, convert the tags to byte arrays and count up the total length - // so we can allocate the string builder - final short metric_width = tsdb.metrics.width(); - int tags_length = 0; - final ArrayList uids = new ArrayList(tsuids.size()); - for (final String tsuid : tsuids) { - final String tags = tsuid.substring(metric_width * 2); - final byte[] tag_bytes = UniqueId.stringToUid(tags); - tags_length += tag_bytes.length; - uids.add(tag_bytes); - } - - // Generate a regexp for our tags based on any metric and timestamp (since - // those are handled by the row start/stop) and the list of TSUID tagk/v - // pairs. The generated regex will look like: ^.{7}(tags|tags|tags)$ - // where each "tags" is similar to \\Q\000\000\001\000\000\002\\E - final StringBuilder buf = new StringBuilder( - 13 // "(?s)^.{N}(" + ")$" - + (tsuids.size() * 11) // "\\Q" + "\\E|" - + tags_length); // total # of bytes in tsuids tagk/v pairs - - // Alright, let's build this regexp. From the beginning... - buf.append("(?s)" // Ensure we use the DOTALL flag. - + "^.{") - // ... start by skipping the metric ID and timestamp. - .append(Const.SALT_WIDTH() + tsdb.metrics.width() + Const.TIMESTAMP_BYTES) - .append("}("); - - for (final byte[] tags : uids) { - // quote the bytes - buf.append("\\Q"); - addId(buf, tags); - buf.append('|'); + if (regex == null) { + regex = QueryUtil.getRowKeyTSUIDRegex(tsuids); } - - // Replace the pipe of the last iteration, close and set - buf.setCharAt(buf.length() - 1, ')'); - buf.append("$"); - scanner.setKeyRegexp(buf.toString(), CHARSET); + scanner.setKeyRegexp(regex, CHARSET); } - - /** - * Appends the given ID to the given buffer, followed by "\\E". - */ - private static void addId(final StringBuilder buf, final byte[] id) { - boolean backslash = false; - for (final byte b : id) { - buf.append((char) (b & 0xFF)); - if (b == 'E' && backslash) { // If we saw a `\' and now we have a `E'. - // So we just terminated the quoted section because we just added \E - // to `buf'. So let's put a litteral \E now and start quoting again. - buf.append("\\\\E\\Q"); - } else { - backslash = b == '\\'; - } - } - buf.append("\\E"); - } - + @Override public String toString() { final StringBuilder buf = new StringBuilder(); diff --git a/src/query/QueryUtil.java b/src/query/QueryUtil.java new file mode 100644 index 0000000000..ab168536ea --- /dev/null +++ b/src/query/QueryUtil.java @@ -0,0 +1,267 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2010-2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; + +import net.opentsdb.core.Const; +import net.opentsdb.core.RowKey; +import net.opentsdb.core.TSDB; +import net.opentsdb.uid.UniqueId; + +import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; +import org.hbase.async.Scanner; + +/** + * A simple class with utility methods for executing queries against the storage + * layer. + * @since 2.2 + */ +public class QueryUtil { + + /** + * Crafts a regular expression for scanning over data table rows and filtering + * time series that the user doesn't want. At least one of the parameters + * must be set and have values. + * NOTE: This method will sort the group bys. + * @param group_bys An optional list of tag keys that we want to group on. May + * be null. + * @param row_key_literals An optional list of key value pairs to filter on. + * May be null. + * @return A regular expression string to pass to the storage layer. + */ + public static String getRowKeyUIDRegex(final List group_bys, + final ByteMap row_key_literals) { + if (group_bys != null) { + Collections.sort(group_bys, Bytes.MEMCMP); + } + final short name_width = TSDB.tagk_width(); + final short value_width = TSDB.tagv_width(); + final short tagsize = (short) (name_width + value_width); + // Generate a regexp for our tags. Say we have 2 tags: { 0 0 1 0 0 2 } + // and { 4 5 6 9 8 7 }, the regexp will be: + // "^.{7}(?:.{6})*\\Q\000\000\001\000\000\002\\E(?:.{6})*\\Q\004\005\006\011\010\007\\E(?:.{6})*$" + final StringBuilder buf = new StringBuilder( + 15 // "^.{N}" + "(?:.{M})*" + "$" + + ((13 + tagsize) // "(?:.{M})*\\Q" + tagsize bytes + "\\E" + * ((row_key_literals == null ? 0 : row_key_literals.size()) + + (group_bys == null ? 0 : group_bys.size() * 3)))); + // In order to avoid re-allocations, reserve a bit more w/ groups ^^^ + + // Alright, let's build this regexp. From the beginning... + buf.append("(?s)" // Ensure we use the DOTALL flag. + + "^.{") + // ... start by skipping the salt, metric ID and timestamp. + .append(Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES) + .append("}"); + + final Iterator> it = row_key_literals == null ? + new ByteMap().iterator() : row_key_literals.iterator(); + + while(it.hasNext()) { + Entry entry = it.hasNext() ? it.next() : null; + // TODO - This look ahead may be expensive. We need to get some data around + // whether it's faster for HBase to scan with a look ahead or simply pass + // the rows back to the TSD for filtering. + final boolean not_key = + entry.getValue() != null && entry.getValue().length == 0; + + // Skip any number of tags. + buf.append("(?:.{").append(tagsize).append("})*"); + if (not_key) { + // start the lookahead as we have a key we explicitly do not want in the + // results + buf.append("(?!"); + } + buf.append("\\Q"); + + addId(buf, entry.getKey(), true); + if (entry.getValue() != null && entry.getValue().length > 0) { // Add a group_by. + // We want specific IDs. List them: /(AAA|BBB|CCC|..)/ + buf.append("(?:"); + for (final byte[] value_id : entry.getValue()) { + if (value_id == null) { + continue; + } + buf.append("\\Q"); + addId(buf, value_id, true); + buf.append('|'); + } + // Replace the pipe of the last iteration. + buf.setCharAt(buf.length() - 1, ')'); + } else { + buf.append(".{").append(value_width).append('}'); // Any value ID. + } + + if (not_key) { + // be sure to close off the look ahead + buf.append(")"); + } + } + // Skip any number of tags before the end. + buf.append("(?:.{").append(tagsize).append("})*$"); + return buf.toString(); + } + + /** + * Creates a regular expression with a list of or'd TUIDs to compare + * against the rows in storage. + * @param tsuids The list of TSUIDs to scan for + * @return A regular expression string to pass to the storage layer. + */ + public static String getRowKeyTSUIDRegex(final List tsuids) { + Collections.sort(tsuids); + + // first, convert the tags to byte arrays and count up the total length + // so we can allocate the string builder + final short metric_width = TSDB.metrics_width(); + int tags_length = 0; + final ArrayList uids = new ArrayList(tsuids.size()); + for (final String tsuid : tsuids) { + final String tags = tsuid.substring(metric_width * 2); + final byte[] tag_bytes = UniqueId.stringToUid(tags); + tags_length += tag_bytes.length; + uids.add(tag_bytes); + } + + // Generate a regexp for our tags based on any metric and timestamp (since + // those are handled by the row start/stop) and the list of TSUID tagk/v + // pairs. The generated regex will look like: ^.{7}(tags|tags|tags)$ + // where each "tags" is similar to \\Q\000\000\001\000\000\002\\E + final StringBuilder buf = new StringBuilder( + 13 // "(?s)^.{N}(" + ")$" + + (tsuids.size() * 11) // "\\Q" + "\\E|" + + tags_length); // total # of bytes in tsuids tagk/v pairs + + // Alright, let's build this regexp. From the beginning... + buf.append("(?s)" // Ensure we use the DOTALL flag. + + "^.{") + // ... start by skipping the metric ID and timestamp. + .append(Const.SALT_WIDTH() + metric_width + Const.TIMESTAMP_BYTES) + .append("}("); + + for (final byte[] tags : uids) { + // quote the bytes + buf.append("\\Q"); + addId(buf, tags, true); + buf.append('|'); + } + + // Replace the pipe of the last iteration, close and set + buf.setCharAt(buf.length() - 1, ')'); + buf.append("$"); + return buf.toString(); + } + + /** + * Compiles an HBase scanner against the main data table + * @param tsdb The TSDB with a configured HBaseClient + * @param salt_bucket An optional salt bucket ID for salting the start/stop + * keys. + * @param metric The metric to scan for + * @param start The start time stamp in seconds + * @param stop The stop timestamp in seconds + * @param table The table name to scan over + * @param family The table family to scan over + * @return A scanner ready for processing. + */ + public static Scanner getMetricScanner(final TSDB tsdb, final int salt_bucket, + final byte[] metric, final int start, final int stop, + final byte[] table, final byte[] family) { + final short metric_width = TSDB.metrics_width(); + final int metric_salt_width = metric_width + Const.SALT_WIDTH(); + final byte[] start_row = new byte[metric_salt_width + Const.TIMESTAMP_BYTES]; + final byte[] end_row = new byte[metric_salt_width + Const.TIMESTAMP_BYTES]; + + if (Const.SALT_WIDTH() > 0) { + final byte[] salt = RowKey.getSaltBytes(salt_bucket); + System.arraycopy(salt, 0, start_row, 0, Const.SALT_WIDTH()); + System.arraycopy(salt, 0, end_row, 0, Const.SALT_WIDTH()); + } + + Bytes.setInt(start_row, start, metric_salt_width); + Bytes.setInt(end_row, stop, metric_salt_width); + + System.arraycopy(metric, 0, start_row, Const.SALT_WIDTH(), metric_width); + System.arraycopy(metric, 0, end_row, Const.SALT_WIDTH(), metric_width); + + final Scanner scanner = tsdb.getClient().newScanner(table); + scanner.setStartKey(start_row); + scanner.setStopKey(end_row); + scanner.setFamily(family); + return scanner; + } + + /** + * Appends the given UID to the given regular expression buffer + * @param buf The String buffer to modify + * @param id The UID to add + * @param close Whether or not to append "\\E" to the end + */ + public static void addId(final StringBuilder buf, final byte[] id, + final boolean close) { + boolean backslash = false; + for (final byte b : id) { + buf.append((char) (b & 0xFF)); + if (b == 'E' && backslash) { // If we saw a `\' and now we have a `E'. + // So we just terminated the quoted section because we just added \E + // to `buf'. So let's put a litteral \E now and start quoting again. + buf.append("\\\\E\\Q"); + } else { + backslash = b == '\\'; + } + } + if (close) { + buf.append("\\E"); + } + } + + /** + * Little helper to print out the regular expression by converting the UID + * bytes to an array. + * @param regexp The regex string to print to the debug log + */ + public static String byteRegexToString(final String regexp) { + final StringBuilder buf = new StringBuilder(); + for (int i = 0; i < regexp.length(); i++) { + if (i > 0 && regexp.charAt(i - 1) == 'Q') { + if (regexp.charAt(i - 3) == '*') { + // tagk + byte[] tagk = new byte[TSDB.tagk_width()]; + for (int x = 0; x < TSDB.tagk_width(); x++) { + tagk[x] = (byte)regexp.charAt(i + x); + } + i += TSDB.tagk_width(); + buf.append(Arrays.toString(tagk)); + } else { + // tagv + byte[] tagv = new byte[TSDB.tagv_width()]; + for (int x = 0; x < TSDB.tagv_width(); x++) { + tagv[x] = (byte)regexp.charAt(i + x); + } + i += TSDB.tagv_width(); + buf.append(Arrays.toString(tagv)); + } + } else { + buf.append(regexp.charAt(i)); + } + } + return buf.toString(); + } +} From 6bc6b3f373cdba2a420a5b1aec58e0c8026223d8 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 26 Sep 2015 15:08:17 -0700 Subject: [PATCH 08/68] Fix for #568, modifying the /api/search/lookup method to properly handle salted tables. --- src/search/TimeSeriesLookup.java | 559 ++++++++++++++------ test/search/TestTimeSeriesLookup.java | 164 +++--- test/search/TestTimeSeriesLookupSalted.java | 40 ++ 3 files changed, 484 insertions(+), 279 deletions(-) create mode 100644 test/search/TestTimeSeriesLookupSalted.java diff --git a/src/search/TimeSeriesLookup.java b/src/search/TimeSeriesLookup.java index 5683f352df..6c244b90f4 100644 --- a/src/search/TimeSeriesLookup.java +++ b/src/search/TimeSeriesLookup.java @@ -14,20 +14,25 @@ import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.regex.Pattern; import net.opentsdb.core.Const; +import net.opentsdb.core.Internal; import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.core.Tags; import net.opentsdb.meta.TSMeta; +import net.opentsdb.query.QueryUtil; import net.opentsdb.uid.NoSuchUniqueId; +import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; import net.opentsdb.uid.UniqueId.UniqueIdType; import net.opentsdb.utils.ByteArrayPair; +import net.opentsdb.utils.Exceptions; import net.opentsdb.utils.Pair; import org.hbase.async.Bytes; @@ -36,6 +41,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; + /** * Lookup series related to a metric, tagk, tagv or any combination thereof. * This class doesn't handle wild-card searching yet. @@ -87,6 +96,21 @@ public class TimeSeriesLookup { /** The TSD to use for lookups */ private final TSDB tsdb; + /** The metric UID if given by the query, post resolution */ + private byte[] metric_uid; + + /** Tag UID pairs if given in the query. Key or value may be null. */ + private List pairs; + + /** The compiled row key regex for HBase filtering */ + private String rowkey_regex; + + /** Post scan filtering if we have a lot of values to look at */ + private String tagv_filter; + + /** The results to send to the caller */ + private final List tsuids; + /** * Default ctor * @param tsdb The TSD to which we belong @@ -96,6 +120,7 @@ public class TimeSeriesLookup { public TimeSeriesLookup(final TSDB tsdb, final SearchQuery query) { this.tsdb = tsdb; this.query = query; + tsuids = Collections.synchronizedList(new ArrayList()); } /** @@ -110,26 +135,89 @@ public TimeSeriesLookup(final TSDB tsdb, final SearchQuery query) { * UID. */ public List lookup() { - LOG.info(query.toString()); - boolean limit_reached = false; - final StringBuilder tagv_filter = new StringBuilder(); - final Scanner scanner = getScanner(tagv_filter); - final List tsuids = new ArrayList(); - final Pattern tagv_regex = tagv_filter.length() > 1 ? - Pattern.compile(tagv_filter.toString()) : null; + try { + return lookupAsync().join(); + } catch (InterruptedException e) { + LOG.error("Interrupted performing lookup", e); + Thread.currentThread().interrupt(); + return null; + } catch (DeferredGroupException e) { + final Throwable ex = Exceptions.getCause(e); + if (ex instanceof NoSuchUniqueName) { + throw (NoSuchUniqueName)ex; + } + throw new RuntimeException("Unexpected exception", ex); + } catch (NoSuchUniqueName e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Unexpected exception", e); + } + } + + /** + * Lookup time series associated with the given metric, tagk, tagv or tag + * pairs. Either the meta table or the data table will be scanned. If no + * metric is given, a full table scan must be performed and this call may take + * a long time to complete. + * When dumping to stdout, if an ID can't be looked up, it will be logged and + * skipped. + * @return A list of TSUIDs matching the given lookup query. + * @throws NoSuchUniqueName if any of the given names fail to resolve to a + * UID. + * @since 2.2 + */ + public Deferred> lookupAsync() { + final Pattern tagv_regex = tagv_filter != null ? + Pattern.compile(tagv_filter) : null; + // we don't really know what size the UIDs will resolve to so just grab // a decent amount. final StringBuffer buf = to_stdout ? new StringBuffer(2048) : null; final long start = System.currentTimeMillis(); - - ArrayList> rows; - byte[] last_tsuid = null; // used to avoid dupes when scanning the data table - - try { - // synchronous to avoid stack overflows when scanning across the main data - // table. - while ((rows = scanner.nextRows().joinUninterruptibly()) != null) { + final int limit; + if (query.getLimit() > 0) { + if (query.useMeta() || Const.SALT_WIDTH() < 1) { + limit = query.getLimit(); + } else if (query.getLimit() < Const.SALT_BUCKETS()) { + limit = 1; + } else { + limit = query.getLimit() / Const.SALT_BUCKETS(); + } + } else { + limit = 0; + } + + class ScannerCB implements Callback, ArrayList>> { + private final Scanner scanner; + // used to avoid dupes when scanning the data table + private byte[] last_tsuid = null; + private int rows_read; + + ScannerCB(final Scanner scanner) { + this.scanner = scanner; + } + + Deferred> scan() { + return scanner.nextRows().addCallback(this); + } + + @Override + public List call(final ArrayList> rows) + throws Exception { + if (rows == null) { + scanner.close(); + if (query.useMeta() || Const.SALT_WIDTH() < 1) { + LOG.debug("Lookup query matched " + tsuids.size() + " time series in " + + (System.currentTimeMillis() - start) + " ms"); + } + return tsuids; + } + for (final ArrayList row : rows) { + if (limit > 0 && rows_read >= limit) { + // little recursion to close the scanner and log above. + return call(null); + } final byte[] tsuid = query.useMeta() ? row.get(0).key() : UniqueId.getTSUIDFromKey(row.get(0).key(), TSDB.metrics_width(), Const.TIMESTAMP_BYTES); @@ -161,203 +249,326 @@ public List lookup() { buf.append(tag_pair.getKey()).append("=") .append(tag_pair.getValue()).append(" "); } - System.out.println(buf.toString()); } catch (NoSuchUniqueId nsui) { LOG.error("Unable to resolve UID in TSUID (" + UniqueId.uidToString(tsuid) + ") " + nsui.getMessage()); } - buf.setLength(0); // reset the buffer so we can re-use it + buf.setLength(0); // reset the buffer so we can re-use it } else { - if(tsuids.size() < query.getLimit()) { - tsuids.add(tsuid); - } else { - limit_reached = true; - break; - } + tsuids.add(tsuid); } + ++rows_read; } - if(limit_reached) { - break; + + scan(); + return tsuids; + } + + @Override + public String toString() { + return "Scanner callback"; + } + } + + class CompleteCB implements Callback, ArrayList>> { + @Override + public List call(final ArrayList> unused) throws Exception { + LOG.debug("Lookup query matched " + tsuids.size() + " time series in " + + (System.currentTimeMillis() - start) + " ms"); + return tsuids; + } + @Override + public String toString() { + return "Final async lookup callback"; + } + } + + class UIDCB implements Callback>, Object> { + @Override + public Deferred> call(Object arg0) throws Exception { + if (!query.useMeta() && Const.SALT_WIDTH() > 0 && metric_uid != null) { + final ArrayList>> deferreds = + new ArrayList>>(Const.SALT_BUCKETS()); + for (int i = 0; i < Const.SALT_BUCKETS(); i++) { + deferreds.add(new ScannerCB(getScanner(i)).scan()); + } + return Deferred.group(deferreds).addCallback(new CompleteCB()); + } else { + return new ScannerCB(getScanner(0)).scan(); } } - } catch (Exception e) { - throw new RuntimeException("Shouldn't be here", e); - } finally { - scanner.close(); + @Override + public String toString() { + return "UID resolution callback"; + } } - LOG.debug("Lookup query matched " + tsuids.size() + " time series in " + - (System.currentTimeMillis() - start) + " ms"); - return tsuids; + return resolveUIDs().addCallbackDeferring(new UIDCB()); } /** - * Configures the scanner for iterating over the meta or data tables. If the - * metric has been set, then we scan a small slice of the table where the - * metric lies, otherwise we have to scan the whole table. If tags are - * given then we setup a row key regex - * @return A configured scanner + * Resolves the metric and tag strings to their UIDs + * @return A deferred to wait on for resolution to complete. */ - private Scanner getScanner(final StringBuilder tagv_filter) { - final Scanner scanner = tsdb.getClient().newScanner( - query.useMeta() ? tsdb.metaTable() : tsdb.dataTable()); - scanner.setFamily( - query.useMeta() ? TSMeta.FAMILY : TSDB.FAMILY()); + private Deferred resolveUIDs() { - // if a metric is given, we need to resolve it's UID and set the start key - // to the UID and the stop key to the next row by incrementing the UID. - if (query.getMetric() != null && !query.getMetric().isEmpty() && - !query.getMetric().equals("*")) { - final byte[] metric_uid = tsdb.getUID(UniqueIdType.METRIC, - query.getMetric()); - LOG.debug("Found UID (" + UniqueId.uidToString(metric_uid) + - ") for metric (" + query.getMetric() + ")"); - scanner.setStartKey(metric_uid); - long uid = UniqueId.uidToLong(metric_uid, TSDB.metrics_width()); - uid++; // TODO - see what happens when this rolls over - scanner.setStopKey(UniqueId.longToUID(uid, TSDB.metrics_width())); - } else { - LOG.debug("Performing full table scan, no metric provided"); + class TagsCB implements Callback> { + @Override + public Object call(final ArrayList ignored) throws Exception { + rowkey_regex = getRowKeyRegex(); + return null; + } } - if (query.getTags() != null && !query.getTags().isEmpty()) { - final List pairs = - new ArrayList(query.getTags().size()); - for (Pair tag : query.getTags()) { - final byte[] tagk = tag.getKey() != null && !tag.getKey().equals("*")? - tsdb.getUID(UniqueIdType.TAGK, tag.getKey()) : null; - final byte[] tagv = tag.getValue() != null && !tag.getValue().equals("*")? - tsdb.getUID(UniqueIdType.TAGV, tag.getValue()) : null; - pairs.add(new ByteArrayPair(tagk, tagv)); - } - // remember, tagks are sorted in the row key so we need to supply a sorted - // regex or matching will fail. - Collections.sort(pairs); - - final short name_width = TSDB.tagk_width(); - final short value_width = TSDB.tagv_width(); - final short tagsize = (short) (name_width + value_width); - - int index = 0; - final StringBuilder buf = new StringBuilder( - 22 // "^.{N}" + "(?:.{M})*" + "$" + wiggle - + ((13 + tagsize) // "(?:.{M})*\\Q" + tagsize bytes + "\\E" - * (pairs.size()))); - buf.append("(?s)^.{").append(TSDB.metrics_width()) - .append("}"); - if (!query.useMeta()) { - buf.append("(?:.{").append(Const.TIMESTAMP_BYTES).append("})*"); - } - buf.append("(?:.{").append(tagsize).append("})*"); - - // at the top of the list will be the null=tagv pairs. We want to compile - // a separate regex for them. - for (; index < pairs.size(); index++) { - if (pairs.get(index).getKey() != null) { - break; + class PairResolution implements Callback> { + @Override + public Object call(final ArrayList tags) throws Exception { + if (tags.size() < 2) { + throw new IllegalArgumentException("Somehow we received an array " + + "that wasn't two bytes in size! " + tags); } - - if (index > 0) { - buf.append("|"); - } - buf.append("(?:.{").append(name_width).append("})"); - buf.append("\\Q"); - addId(buf, pairs.get(index).getValue()); - buf.append("\\E"); - } - buf.append("(?:.{").append(tagsize).append("})*") - .append("$"); - - if (index > 0 && index < pairs.size()) { - // we had one or more tagvs to lookup AND we have tagk or tag pairs to - // filter on, so we dump the previous regex into the tagv_filter and - // continue on with a row key - tagv_filter.append(buf.toString()); - LOG.debug("Setting tagv filter: " + buf.toString()); - } else if (index >= pairs.size()) { - // in this case we don't have any tagks to deal with so we can just - // pass the previously compiled regex to the rowkey filter of the - // scanner - scanner.setKeyRegexp(buf.toString(), CHARSET); - LOG.debug("Setting scanner row key filter with tagvs only: " + - buf.toString()); + pairs.add(new ByteArrayPair(tags.get(0), tags.get(1))); + return Deferred.fromResult(null); } - - // catch any left over tagk/tag pairs - if (index < pairs.size()){ - buf.setLength(0); - buf.append("(?s)^.{").append(TSDB.metrics_width()) - .append("}"); - if (!query.useMeta()) { - buf.append("(?:.{").append(Const.TIMESTAMP_BYTES).append("})*"); + } + + class TagResolution implements Callback, Object> { + @Override + public Deferred call(final Object unused) throws Exception { + if (query.getTags() == null || query.getTags().isEmpty()) { + return Deferred.fromResult(null); } - ByteArrayPair last_pair = null; - for (; index < pairs.size(); index++) { - if (last_pair != null && last_pair.getValue() == null && - Bytes.memcmp(last_pair.getKey(), pairs.get(index).getKey()) == 0) { - // tagk=null is a wildcard so we don't need to bother adding - // tagk=tagv pairs with the same tagk. - LOG.debug("Skipping pair due to wildcard: " + pairs.get(index)); - } else if (last_pair != null && - Bytes.memcmp(last_pair.getKey(), pairs.get(index).getKey()) == 0) { - // in this case we're ORing e.g. "host=web01|host=web02" - buf.append("|\\Q"); - addId(buf, pairs.get(index).getKey()); - addId(buf, pairs.get(index).getValue()); - buf.append("\\E"); + pairs = Collections.synchronizedList( + new ArrayList(query.getTags().size())); + final ArrayList> deferreds = + new ArrayList>(pairs.size()); + + for (final Pair tags : query.getTags()) { + final ArrayList> deferred_tags = + new ArrayList>(2); + if (tags.getKey() != null && !tags.getKey().equals("*")) { + deferred_tags.add(tsdb.getUIDAsync(UniqueIdType.TAGK, tags.getKey())); } else { - if (last_pair != null) { - buf.append(")"); - } - // moving on to the next tagk set - buf.append("(?:.{6})*"); // catch tag pairs in between - buf.append("(?:"); - if (pairs.get(index).getKey() != null && - pairs.get(index).getValue() != null) { - buf.append("\\Q"); - addId(buf, pairs.get(index).getKey()); - addId(buf, pairs.get(index).getValue()); - buf.append("\\E"); - } else { - buf.append("\\Q"); - addId(buf, pairs.get(index).getKey()); - buf.append("\\E"); - buf.append("(?:.{").append(value_width).append("})+"); - } + deferred_tags.add(Deferred.fromResult(null)); + } + if (tags.getValue() != null && !tags.getValue().equals("*")) { + deferred_tags.add(tsdb.getUIDAsync(UniqueIdType.TAGV, tags.getValue())); + } else { + deferred_tags.add(Deferred.fromResult(null)); } - last_pair = pairs.get(index); + deferreds.add(Deferred.groupInOrder(deferred_tags) + .addCallback(new PairResolution())); } - buf.append(")(?:.{").append(tagsize).append("})*").append("$"); - - scanner.setKeyRegexp(buf.toString(), CHARSET); - LOG.debug("Setting scanner row key filter: " + buf.toString()); + return Deferred.group(deferreds).addCallback(new TagsCB()); + } + } + + class MetricCB implements Callback, byte[]> { + @Override + public Deferred call(final byte[] uid) throws Exception { + metric_uid = uid; + LOG.debug("Found UID (" + UniqueId.uidToString(metric_uid) + + ") for metric (" + query.getMetric() + ")"); + return new TagResolution().call(null); + } + } + + if (query.getMetric() != null && !query.getMetric().isEmpty() && + !query.getMetric().equals("*")) { + return tsdb.getUIDAsync(UniqueIdType.METRIC, query.getMetric()) + .addCallbackDeferring(new MetricCB()); + } else { + try { + return new TagResolution().call(null); + } catch (Exception e) { + return Deferred.fromError(e); + } + } + } + + /** Compiles a scanner with the given salt ID if salting is enabled AND we're + * not scanning the meta table. + * @param salt An ID for the salt bucket + * @return A scanner to send to HBase. + */ + private Scanner getScanner(final int salt) { + final Scanner scanner = tsdb.getClient().newScanner( + query.useMeta() ? tsdb.metaTable() : tsdb.dataTable()); + scanner.setFamily(query.useMeta() ? TSMeta.FAMILY : TSDB.FAMILY()); + + if (metric_uid != null) { + byte[] key; + if (query.useMeta() || Const.SALT_WIDTH() < 1) { + key = metric_uid; + } else { + key = new byte[Const.SALT_WIDTH() + TSDB.metrics_width()]; + key[0] = (byte)salt; + System.arraycopy(metric_uid, 0, key, Const.SALT_WIDTH(), metric_uid.length); + } + scanner.setStartKey(key); + long uid = UniqueId.uidToLong(metric_uid, TSDB.metrics_width()); + uid++; + if (uid < Internal.getMaxUnsignedValueOnBytes(TSDB.metrics_width())) { + // if random metrics are enabled we could see a metric with the max UID + // value. If so, we need to leave the stop key as null + if (query.useMeta() || Const.SALT_WIDTH() < 1) { + key = UniqueId.longToUID(uid, TSDB.metrics_width()); + } else { + key = new byte[Const.SALT_WIDTH() + TSDB.metrics_width()]; + key[0] = (byte)salt; + System.arraycopy(UniqueId.longToUID(uid, TSDB.metrics_width()), 0, + key, Const.SALT_WIDTH(), metric_uid.length); + } + scanner.setStopKey(key); } } + + if (rowkey_regex != null) { + scanner.setKeyRegexp(rowkey_regex, CHARSET); + if (LOG.isDebugEnabled()) { + LOG.debug("Scanner regex: " + QueryUtil.byteRegexToString(rowkey_regex)); + } + } + return scanner; } /** - * Appends the given ID to the given buffer, escaping where appropriate - * @param buf The string buffer to append to - * @param id The ID to append + * Constructs a row key regular expression to pass to HBase if the user gave + * some tags in the query + * @return The regular expression to use. */ - private static void addId(final StringBuilder buf, final byte[] id) { - boolean backslash = false; - for (final byte b : id) { - buf.append((char) (b & 0xFF)); - if (b == 'E' && backslash) { // If we saw a `\' and now we have a `E'. - // So we just terminated the quoted section because we just added \E - // to `buf'. So let's put a litteral \E now and start quoting again. - buf.append("\\\\E\\Q"); - } else { - backslash = b == '\\'; + private String getRowKeyRegex() { + final StringBuilder tagv_buffer = new StringBuilder(); + // remember, tagks are sorted in the row key so we need to supply a sorted + // regex or matching will fail. + Collections.sort(pairs); + + final short name_width = TSDB.tagk_width(); + final short value_width = TSDB.tagv_width(); + final short tagsize = (short) (name_width + value_width); + + int index = 0; + final StringBuilder buf = new StringBuilder( + 22 // "^.{N}" + "(?:.{M})*" + "$" + wiggle + + ((13 + tagsize) // "(?:.{M})*\\Q" + tagsize bytes + "\\E" + * (pairs.size()))); + buf.append("(?s)^.{").append(query.useMeta() ? TSDB.metrics_width() : + TSDB.metrics_width() + Const.SALT_WIDTH()) + .append("}"); + if (!query.useMeta()) { + buf.append("(?:.{").append(Const.TIMESTAMP_BYTES).append("})*"); + } + buf.append("(?:.{").append(tagsize).append("})*"); + + // at the top of the list will be the null=tagv pairs. We want to compile + // a separate regex for them. + for (; index < pairs.size(); index++) { + if (pairs.get(index).getKey() != null) { + break; } + + if (index > 0) { + buf.append("|"); + } + buf.append("(?:.{").append(name_width).append("})"); + buf.append("\\Q"); + QueryUtil.addId(buf, pairs.get(index).getValue(), true); } + buf.append("(?:.{").append(tagsize).append("})*") + .append("$"); + + if (index > 0 && index < pairs.size()) { + // we had one or more tagvs to lookup AND we have tagk or tag pairs to + // filter on, so we dump the previous regex into the tagv_filter and + // continue on with a row key + tagv_buffer.append(buf.toString()); + LOG.debug("Setting tagv filter: " + QueryUtil.byteRegexToString(buf.toString())); + } else if (index >= pairs.size()) { + // in this case we don't have any tagks to deal with so we can just + // pass the previously compiled regex to the rowkey filter of the + // scanner + LOG.debug("Setting scanner row key filter with tagvs only: " + + QueryUtil.byteRegexToString(buf.toString())); + if (tagv_buffer.length() > 0) { + tagv_filter = tagv_buffer.toString(); + } + return buf.toString(); + } + + // catch any left over tagk/tag pairs + if (index < pairs.size()){ + buf.setLength(0); + buf.append("(?s)^.{").append(query.useMeta() ? TSDB.metrics_width() : + TSDB.metrics_width() + Const.SALT_WIDTH()) + .append("}"); + if (!query.useMeta()) { + buf.append("(?:.{").append(Const.TIMESTAMP_BYTES).append("})*"); + } + + ByteArrayPair last_pair = null; + for (; index < pairs.size(); index++) { + if (last_pair != null && last_pair.getValue() == null && + Bytes.memcmp(last_pair.getKey(), pairs.get(index).getKey()) == 0) { + // tagk=null is a wildcard so we don't need to bother adding + // tagk=tagv pairs with the same tagk. + LOG.debug("Skipping pair due to wildcard: " + pairs.get(index)); + } else if (last_pair != null && + Bytes.memcmp(last_pair.getKey(), pairs.get(index).getKey()) == 0) { + // in this case we're ORing e.g. "host=web01|host=web02" + buf.append("|\\Q"); + QueryUtil.addId(buf, pairs.get(index).getKey(), false); + QueryUtil.addId(buf, pairs.get(index).getValue(), true); + } else { + if (last_pair != null) { + buf.append(")"); + } + // moving on to the next tagk set + buf.append("(?:.{6})*"); // catch tag pairs in between + buf.append("(?:"); + if (pairs.get(index).getKey() != null && + pairs.get(index).getValue() != null) { + buf.append("\\Q"); + QueryUtil.addId(buf, pairs.get(index).getKey(), false); + QueryUtil.addId(buf, pairs.get(index).getValue(), true); + } else { + buf.append("\\Q"); + QueryUtil.addId(buf, pairs.get(index).getKey(), true); + buf.append("(?:.{").append(value_width).append("})+"); + } + } + last_pair = pairs.get(index); + } + buf.append(")(?:.{").append(tagsize).append("})*").append("$"); + } + if (tagv_buffer.length() > 0) { + tagv_filter = tagv_buffer.toString(); + } + return buf.toString(); } /** @param to_stdout Whether or not to dump to standard out as we scan */ public void setToStdout(final boolean to_stdout) { this.to_stdout = to_stdout; } + + @Override + public String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("query={") + .append(query) + .append("}, to_stdout=") + .append(to_stdout) + .append(", metric_uid=") + .append(metric_uid == null ? "null" : Arrays.toString(metric_uid)) + .append(", pairs=") + .append(pairs) + .append(", rowkey_regex=") + .append(rowkey_regex) + .append(", tagv_filter=") + .append(tagv_filter); + return buf.toString(); + + } } diff --git a/test/search/TestTimeSeriesLookup.java b/test/search/TestTimeSeriesLookup.java index af0b73fd17..cdf3e3de22 100644 --- a/test/search/TestTimeSeriesLookup.java +++ b/test/search/TestTimeSeriesLookup.java @@ -16,13 +16,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.mock; -import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import net.opentsdb.core.BaseTsdbTest; import net.opentsdb.core.Const; +import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.meta.TSMeta; import net.opentsdb.storage.MockBase; @@ -31,6 +32,7 @@ import net.opentsdb.utils.Config; import net.opentsdb.utils.Pair; +import org.hbase.async.Bytes; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; import org.hbase.async.Scanner; @@ -49,14 +51,7 @@ "com.sum.*", "org.xml.*"}) @PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, KeyValue.class, Scanner.class, TimeSeriesLookup.class}) -public class TestTimeSeriesLookup { - private Config config; - private TSDB tsdb = null; - private HBaseClient client = mock(HBaseClient.class); - private UniqueId metrics = mock(UniqueId.class); - private UniqueId tag_names = mock(UniqueId.class); - private UniqueId tag_values = mock(UniqueId.class); - private MockBase storage = null; +public class TestTimeSeriesLookup extends BaseTsdbTest { // tsuids private static List test_tsuids = new ArrayList(7); @@ -64,70 +59,26 @@ public class TestTimeSeriesLookup { test_tsuids.add(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); test_tsuids.add(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }); test_tsuids.add(new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 1 }); - test_tsuids.add(new byte[] { 0, 0, 3, 0, 0, 1, 0, 0, 1, 0, 0, 4, 0, 0, 5}); - test_tsuids.add(new byte[] { 0, 0, 3, 0, 0, 1, 0, 0, 2, 0, 0, 4, 0, 0, 5}); - test_tsuids.add(new byte[] { 0, 0, 3, 0, 0, 6, 0, 0, 7, 0, 0, 8, 0, 0, 1, + test_tsuids.add(new byte[] { 0, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 3, 0, 0, 5}); + test_tsuids.add(new byte[] { 0, 0, 4, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 5}); + test_tsuids.add(new byte[] { 0, 0, 4, 0, 0, 6, 0, 0, 7, 0, 0, 8, 0, 0, 1, 0, 0, 9, 0, 0, 3}); - test_tsuids.add(new byte[] { 0, 0, 3, 0, 0, 6, 0, 0, 7, 0, 0, 8, 0, 0, 10, + test_tsuids.add(new byte[] { 0, 0, 4, 0, 0, 6, 0, 0, 7, 0, 0, 8, 0, 0, 10, 0, 0, 9, 0, 0, 3}); } @Before - public void before() throws Exception { - config = new Config(false); - tsdb = new TSDB(client, config); - - // replace the "real" field objects with mocks - Field met = tsdb.getClass().getDeclaredField("metrics"); - met.setAccessible(true); - met.set(tsdb, metrics); - - Field tagk = tsdb.getClass().getDeclaredField("tag_names"); - tagk.setAccessible(true); - tagk.set(tsdb, tag_names); - - Field tagv = tsdb.getClass().getDeclaredField("tag_values"); - tagv.setAccessible(true); - tagv.set(tsdb, tag_values); - - // mock UniqueId - when(metrics.getIdAsync("sys.cpu.user")) - .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(metrics.getIdAsync("sys.cpu.system")) - .thenReturn(Deferred.fromError( - new NoSuchUniqueName("sys.cpu.system", "metric"))); - when(metrics.getIdAsync("sys.cpu.nice")) - .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 2 })); - when(metrics.getIdAsync("sys.cpu.idle")) - .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 3 })); + public void beforeLocal() { when(metrics.getIdAsync("no.values")) .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 11 })); - - when(tag_names.getIdAsync("host")) - .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_names.getIdAsync("dc")) - .thenReturn(Deferred.fromError( - new NoSuchUniqueName("dc", "metric"))); - when(tag_names.getIdAsync("owner")) + when(metrics.getIdAsync("filtered")) .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 4 })); - - when(tag_values.getIdAsync("web01")) - .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 1 })); - when(tag_values.getIdAsync("web02")) - .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 2 })); - when(tag_values.getIdAsync("web03")) - .thenReturn(Deferred.fromError( - new NoSuchUniqueName("web03", "metric"))); - - when(metrics.width()).thenReturn((short)3); - when(tag_names.width()).thenReturn((short)3); - when(tag_values.width()).thenReturn((short)3); } - + @Test public void metricOnlyMeta() throws Exception { generateMeta(); - final SearchQuery query = new SearchQuery("sys.cpu.user"); + final SearchQuery query = new SearchQuery(METRIC_STRING); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); assertNotNull(tsuids); @@ -150,7 +101,7 @@ public void metricOnlyMetaStar() throws Exception { @Test public void metricOnlyData() throws Exception { generateData(); - final SearchQuery query = new SearchQuery("sys.cpu.user"); + final SearchQuery query = new SearchQuery(METRIC_STRING); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -163,7 +114,7 @@ public void metricOnlyData() throws Exception { @Test public void metricOnly2Meta() throws Exception { generateMeta(); - final SearchQuery query = new SearchQuery("sys.cpu.nice"); + final SearchQuery query = new SearchQuery(METRIC_B_STRING); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); assertNotNull(tsuids); @@ -174,7 +125,7 @@ public void metricOnly2Meta() throws Exception { @Test public void metricOnly2Data() throws Exception { generateData(); - final SearchQuery query = new SearchQuery("sys.cpu.nice"); + final SearchQuery query = new SearchQuery(METRIC_B_STRING); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -186,7 +137,7 @@ public void metricOnly2Data() throws Exception { @Test (expected = NoSuchUniqueName.class) public void noSuchMetricMeta() throws Exception { storage = new MockBase(tsdb, client, true, true, true, true); - final SearchQuery query = new SearchQuery("sys.cpu.system"); + final SearchQuery query = new SearchQuery(NSUN_METRIC); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); lookup.lookup(); } @@ -217,7 +168,7 @@ public void tagkOnlyMeta() throws Exception { generateMeta(); final List> tags = new ArrayList>(1); - tags.add(new Pair("host", null)); + tags.add(new Pair(TAGK_STRING, null)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -233,7 +184,7 @@ public void tagkOnlyMetaStar() throws Exception { generateMeta(); final List> tags = new ArrayList>(1); - tags.add(new Pair("host", "*")); + tags.add(new Pair(TAGK_STRING, "*")); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -249,11 +200,12 @@ public void tagkOnlyData() throws Exception { generateData(); final List> tags = new ArrayList>(1); - tags.add(new Pair("host", null)); + tags.add(new Pair(TAGK_STRING, null)); final SearchQuery query = new SearchQuery(tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); + Collections.sort(tsuids, Bytes.MEMCMP); // for salting assertNotNull(tsuids); assertEquals(5, tsuids.size()); for (int i = 0; i < 5; i++) { @@ -266,7 +218,7 @@ public void tagkOnly2Meta() throws Exception { generateMeta(); final List> tags = new ArrayList>(1); - tags.add(new Pair("owner", null)); + tags.add(new Pair(TAGK_B_STRING, null)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -281,11 +233,12 @@ public void tagkOnly2Data() throws Exception { generateData(); final List> tags = new ArrayList>(1); - tags.add(new Pair("owner", null)); + tags.add(new Pair(TAGK_B_STRING, null)); final SearchQuery query = new SearchQuery(tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); + Collections.sort(tsuids, Bytes.MEMCMP); // for salting assertNotNull(tsuids); assertEquals(2, tsuids.size()); assertArrayEquals(test_tsuids.get(3), tsuids.get(0)); @@ -297,7 +250,7 @@ public void noSuchTagkMeta() throws Exception { storage = new MockBase(tsdb, client, true, true, true, true); final List> tags = new ArrayList>(1); - tags.add(new Pair("dc", null)); + tags.add(new Pair(NSUN_TAGK, null)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); lookup.lookup(); @@ -308,7 +261,7 @@ public void tagvOnlyMeta() throws Exception { generateMeta(); final List> tags = new ArrayList>(1); - tags.add(new Pair(null, "web01")); + tags.add(new Pair(null, TAGV_STRING)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -325,7 +278,7 @@ public void tagvOnlyMetaStar() throws Exception { generateMeta(); final List> tags = new ArrayList>(1); - tags.add(new Pair("*", "web01")); + tags.add(new Pair("*", TAGV_STRING)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -342,11 +295,12 @@ public void tagvOnlyData() throws Exception { generateData(); final List> tags = new ArrayList>(1); - tags.add(new Pair(null, "web01")); + tags.add(new Pair(null, TAGV_STRING)); final SearchQuery query = new SearchQuery(tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); + Collections.sort(tsuids, Bytes.MEMCMP); // for salting assertNotNull(tsuids); assertEquals(4, tsuids.size()); assertArrayEquals(test_tsuids.get(0), tsuids.get(0)); @@ -360,7 +314,7 @@ public void tagvOnly2Meta() throws Exception { generateMeta(); final List> tags = new ArrayList>(1); - tags.add(new Pair(null, "web02")); + tags.add(new Pair(null, TAGV_B_STRING)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -375,11 +329,12 @@ public void tagvOnly2Data() throws Exception { generateData(); final List> tags = new ArrayList>(1); - tags.add(new Pair(null, "web02")); + tags.add(new Pair(null, TAGV_B_STRING)); final SearchQuery query = new SearchQuery(tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); + Collections.sort(tsuids, Bytes.MEMCMP); // for salting assertNotNull(tsuids); assertEquals(2, tsuids.size()); assertArrayEquals(test_tsuids.get(1), tsuids.get(0)); @@ -391,7 +346,7 @@ public void noSuchTagvMeta() throws Exception { storage = new MockBase(tsdb, client, true, true, true, true); final List> tags = new ArrayList>(1); - tags.add(new Pair(null, "web03")); + tags.add(new Pair(null, NSUN_TAGV)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); lookup.lookup(); @@ -402,8 +357,8 @@ public void metricAndTagkMeta() throws Exception { generateMeta(); final List> tags = new ArrayList>(1); - tags.add(new Pair("host", null)); - final SearchQuery query = new SearchQuery("sys.cpu.nice", + tags.add(new Pair(TAGK_STRING, null)); + final SearchQuery query = new SearchQuery(METRIC_B_STRING, tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -417,8 +372,8 @@ public void metricAndTagkMetaStar() throws Exception { generateMeta(); final List> tags = new ArrayList>(1); - tags.add(new Pair("host", "*")); - final SearchQuery query = new SearchQuery("sys.cpu.nice", + tags.add(new Pair(TAGK_STRING, "*")); + final SearchQuery query = new SearchQuery(METRIC_B_STRING, tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -432,8 +387,8 @@ public void metricAndTagkData() throws Exception { generateData(); final List> tags = new ArrayList>(1); - tags.add(new Pair("host", null)); - final SearchQuery query = new SearchQuery("sys.cpu.nice", + tags.add(new Pair(TAGK_STRING, null)); + final SearchQuery query = new SearchQuery(METRIC_B_STRING, tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); @@ -448,9 +403,8 @@ public void metricAndTagvMeta() throws Exception { generateMeta(); final List> tags = new ArrayList>(1); - tags.add(new Pair(null, "web02")); - final SearchQuery query = new SearchQuery("sys.cpu.idle", - tags); + tags.add(new Pair(null, TAGV_B_STRING)); + final SearchQuery query = new SearchQuery("filtered", tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); assertNotNull(tsuids); @@ -463,9 +417,8 @@ public void metricAndTagvMetaStar() throws Exception { generateMeta(); final List> tags = new ArrayList>(1); - tags.add(new Pair("*", "web02")); - final SearchQuery query = new SearchQuery("sys.cpu.idle", - tags); + tags.add(new Pair("*", TAGV_B_STRING)); + final SearchQuery query = new SearchQuery("filtered",tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); assertNotNull(tsuids); @@ -478,9 +431,8 @@ public void metricAndTagvData() throws Exception { generateData(); final List> tags = new ArrayList>(1); - tags.add(new Pair(null, "web02")); - final SearchQuery query = new SearchQuery("sys.cpu.idle", - tags); + tags.add(new Pair(null, TAGV_B_STRING)); + final SearchQuery query = new SearchQuery("filtered", tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -494,9 +446,8 @@ public void metricAndTagPairMeta() throws Exception { generateMeta(); final List> tags = new ArrayList>(1); - tags.add(new Pair("host", "web01")); - final SearchQuery query = new SearchQuery("sys.cpu.idle", - tags); + tags.add(new Pair(TAGK_STRING, TAGV_STRING)); + final SearchQuery query = new SearchQuery("filtered", tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); assertNotNull(tsuids); @@ -509,9 +460,8 @@ public void metricAndTagPairData() throws Exception { generateData(); final List> tags = new ArrayList>(1); - tags.add(new Pair("host", "web01")); - final SearchQuery query = new SearchQuery("sys.cpu.idle", - tags); + tags.add(new Pair(TAGK_STRING, TAGV_STRING)); + final SearchQuery query = new SearchQuery("filtered", tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); query.setUseMeta(false); @@ -526,7 +476,7 @@ public void tagPairOnlyMeta() throws Exception { generateMeta(); final List> tags = new ArrayList>(1); - tags.add(new Pair("host", "web01")); + tags.add(new Pair(TAGK_STRING, TAGV_STRING)); final SearchQuery query = new SearchQuery(tags); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -542,11 +492,12 @@ public void tagPairOnlyData() throws Exception { generateData(); final List> tags = new ArrayList>(1); - tags.add(new Pair("host", "web01")); + tags.add(new Pair(TAGK_STRING, TAGV_STRING)); final SearchQuery query = new SearchQuery(tags); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); + Collections.sort(tsuids, Bytes.MEMCMP); // for salting assertNotNull(tsuids); assertEquals(3, tsuids.size()); assertArrayEquals(test_tsuids.get(0), tsuids.get(0)); @@ -559,7 +510,7 @@ public void limitVerification() throws Exception { generateData(); final List> tags = new ArrayList>(1); - tags.add(new Pair("host", "web01")); + tags.add(new Pair(TAGK_STRING, TAGV_STRING)); final SearchQuery query = new SearchQuery(tags); query.setUseMeta(false); query.setLimit(1); @@ -598,11 +549,14 @@ private void generateData() { final byte[] qual = new byte[] { 0, 0 }; final byte[] val = new byte[] { 1 }; for (final byte[] tsuid : test_tsuids) { - byte[] row_key = new byte[tsuid.length + Const.TIMESTAMP_BYTES]; - System.arraycopy(tsuid, 0, row_key, 0, TSDB.metrics_width()); + byte[] row_key = new byte[Const.SALT_WIDTH() + tsuid.length + + Const.TIMESTAMP_BYTES]; + System.arraycopy(tsuid, 0, row_key, Const.SALT_WIDTH(), + TSDB.metrics_width()); System.arraycopy(tsuid, TSDB.metrics_width(), row_key, - TSDB.metrics_width() + Const.TIMESTAMP_BYTES, + Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES, tsuid.length - TSDB.metrics_width()); + RowKey.prefixKeyWithSalt(row_key); storage.addColumn(row_key, qual, val); } } diff --git a/test/search/TestTimeSeriesLookupSalted.java b/test/search/TestTimeSeriesLookupSalted.java new file mode 100644 index 0000000000..a8f1109125 --- /dev/null +++ b/test/search/TestTimeSeriesLookupSalted.java @@ -0,0 +1,40 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2015 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.search; + +import net.opentsdb.core.Const; +import net.opentsdb.core.TSDB; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.Config; + +import org.hbase.async.HBaseClient; +import org.hbase.async.KeyValue; +import org.hbase.async.Scanner; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({TSDB.class, Config.class, UniqueId.class, HBaseClient.class, + KeyValue.class, Scanner.class, TimeSeriesLookup.class, Const.class }) +public class TestTimeSeriesLookupSalted extends TestTimeSeriesLookup { + + @Before + public void beforeLocalSalted() throws Exception { + PowerMockito.mockStatic(Const.class); + PowerMockito.when(Const.SALT_WIDTH()).thenReturn(1); + PowerMockito.when(Const.SALT_BUCKETS()).thenReturn(4); + } +} From fcb2f72a1a6420ea16c1eda97c2b90263a68c5d8 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 27 Sep 2015 13:54:53 -0700 Subject: [PATCH 09/68] Fix up the search API for lookups to be fully asynchronous. Thanks to to @Dieken for his work on the salting patch. --- src/tsd/SearchRpc.java | 145 ++++++++++---- test/search/TestTimeSeriesLookup.java | 68 ++++--- test/tsd/TestSearchRpc.java | 262 +++++++++++++++++--------- 3 files changed, 320 insertions(+), 155 deletions(-) diff --git a/src/tsd/SearchRpc.java b/src/tsd/SearchRpc.java index 12431a0abe..5d8a59b09e 100644 --- a/src/tsd/SearchRpc.java +++ b/src/tsd/SearchRpc.java @@ -16,10 +16,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; +import com.stumbleupon.async.DeferredGroupException; + import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.core.Tags; @@ -29,6 +34,7 @@ import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.Exceptions; import net.opentsdb.utils.Pair; /** @@ -166,44 +172,119 @@ private void processLookup(final TSDB tsdb, final HttpQuery query, "Missing metric and tags. Please supply at least one value."); } final long start = System.currentTimeMillis(); - try { - final List tsuids = - new TimeSeriesLookup(tsdb, search_query).lookup(); - - search_query.setTotalResults(tsuids.size()); - // TODO maybe track in nanoseconds so we can get a floating point. But most - // lookups will probably take a fair amount of time. - search_query.setTime(System.currentTimeMillis() - start); + + class MetricCB implements Callback { + final Map series; + MetricCB(final Map series) { + this.series = series; + } - final List results = new ArrayList(tsuids.size()); + @Override + public Object call(final String name) throws Exception { + series.put("metric", name); + return null; + } + } + + class TagsCB implements Callback> { + final Map series; + TagsCB(final Map series) { + this.series = series; + } - Map series; - List tag_ids; + @Override + public Object call(final HashMap names) throws Exception { + series.put("tags", names); + return null; + } + } + + class Serialize implements Callback> { + final List results; + Serialize(final List results) { + this.results = results; + } - // TODO - honor limit and pagination - for (final byte[] tsuid : tsuids) { - series = new HashMap((tsuid.length / 2) + 1); - try { + @Override + public Object call(final ArrayList ignored) throws Exception { + search_query.setResults(results); + search_query.setTime(System.currentTimeMillis() - start); + query.sendReply(query.serializer().formatSearchResultsV1(search_query)); + return null; + } + } + + class LookupCB implements Callback, List> { + @Override + public Deferred call(final List tsuids) throws Exception { + final List results = new ArrayList(tsuids.size()); + search_query.setTotalResults(tsuids.size()); + + final ArrayList> deferreds = + new ArrayList>(tsuids.size()); + + for (final byte[] tsuid : tsuids) { + // has to be concurrent if the uid table is split across servers + final Map series = + new ConcurrentHashMap(3); + results.add(series); + series.put("tsuid", UniqueId.uidToString(tsuid)); - series.put("metric", RowKey.metricNameAsync(tsdb, tsuid) - .joinUninterruptibly()); - tag_ids = UniqueId.getTagPairsFromTSUID(tsuid); - series.put("tags", Tags.resolveIdsAsync(tsdb, tag_ids) - .joinUninterruptibly()); - } catch (NoSuchUniqueId nsui) { - throw new BadRequestException(HttpResponseStatus.NOT_FOUND, - "Unable to resolve one or more UIDs", nsui); - } catch (Exception e) { - throw new RuntimeException("Shouldn't be here", e); + deferreds.add(RowKey.metricNameAsync(tsdb, tsuid) + .addCallback(new MetricCB(series))); + + final List tag_ids = UniqueId.getTagPairsFromTSUID(tsuid); + deferreds.add(Tags.resolveIdsAsync(tsdb, tag_ids) + .addCallback(new TagsCB(series))); } - results.add(series); + + return Deferred.group(deferreds).addCallback(new Serialize(results)); } - - search_query.setResults(results); - query.sendReply(query.serializer().formatSearchResultsV1(search_query)); - } catch (NoSuchUniqueName nsun) { - throw new BadRequestException(HttpResponseStatus.NOT_FOUND, - "Unable to resolve one or more names", nsun); } + + class ErrCB implements Callback { + @Override + public Object call(final Exception e) throws Exception { + if (e instanceof NoSuchUniqueId) { + query.sendReply(HttpResponseStatus.NOT_FOUND, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.NOT_FOUND, + "Unable to resolve one or more TSUIDs", (NoSuchUniqueId)e))); + } else if (e instanceof NoSuchUniqueName) { + query.sendReply(HttpResponseStatus.NOT_FOUND, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.NOT_FOUND, + "Unable to resolve one or more UIDs", (NoSuchUniqueName)e))); + } else if (e instanceof DeferredGroupException) { + final Throwable ex = Exceptions.getCause((DeferredGroupException)e); + if (ex instanceof NoSuchUniqueId) { + query.sendReply(HttpResponseStatus.NOT_FOUND, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.NOT_FOUND, + "Unable to resolve one or more TSUIDs", (NoSuchUniqueId)ex))); + } else if (ex instanceof NoSuchUniqueName) { + query.sendReply(HttpResponseStatus.NOT_FOUND, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.NOT_FOUND, + "Unable to resolve one or more UIDs", (NoSuchUniqueName)ex))); + } else { + query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.INTERNAL_SERVER_ERROR, + "Unexpected exception", ex))); + } + } else { + query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, + query.serializer().formatErrorV1( + new BadRequestException(HttpResponseStatus.INTERNAL_SERVER_ERROR, + "Unexpected exception", e))); + } + return null; + } + } + + new TimeSeriesLookup(tsdb, search_query).lookupAsync() + .addCallback(new LookupCB()) + .addErrback(new ErrCB()); } } diff --git a/test/search/TestTimeSeriesLookup.java b/test/search/TestTimeSeriesLookup.java index cdf3e3de22..d07c14501b 100644 --- a/test/search/TestTimeSeriesLookup.java +++ b/test/search/TestTimeSeriesLookup.java @@ -54,7 +54,7 @@ public class TestTimeSeriesLookup extends BaseTsdbTest { // tsuids - private static List test_tsuids = new ArrayList(7); + public static List test_tsuids = new ArrayList(7); static { test_tsuids.add(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); test_tsuids.add(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }); @@ -69,6 +69,7 @@ public class TestTimeSeriesLookup extends BaseTsdbTest { @Before public void beforeLocal() { + storage = new MockBase(tsdb, client, true, true, true, true); when(metrics.getIdAsync("no.values")) .thenReturn(Deferred.fromResult(new byte[] { 0, 0, 11 })); when(metrics.getIdAsync("filtered")) @@ -77,7 +78,7 @@ public void beforeLocal() { @Test public void metricOnlyMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final SearchQuery query = new SearchQuery(METRIC_STRING); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -90,7 +91,7 @@ public void metricOnlyMeta() throws Exception { // returns everything @Test public void metricOnlyMetaStar() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final SearchQuery query = new SearchQuery("*"); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -100,7 +101,7 @@ public void metricOnlyMetaStar() throws Exception { @Test public void metricOnlyData() throws Exception { - generateData(); + generateData(tsdb, storage); final SearchQuery query = new SearchQuery(METRIC_STRING); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); @@ -113,7 +114,7 @@ public void metricOnlyData() throws Exception { @Test public void metricOnly2Meta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final SearchQuery query = new SearchQuery(METRIC_B_STRING); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -124,7 +125,7 @@ public void metricOnly2Meta() throws Exception { @Test public void metricOnly2Data() throws Exception { - generateData(); + generateData(tsdb, storage); final SearchQuery query = new SearchQuery(METRIC_B_STRING); query.setUseMeta(false); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); @@ -136,7 +137,6 @@ public void metricOnly2Data() throws Exception { @Test (expected = NoSuchUniqueName.class) public void noSuchMetricMeta() throws Exception { - storage = new MockBase(tsdb, client, true, true, true, true); final SearchQuery query = new SearchQuery(NSUN_METRIC); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); lookup.lookup(); @@ -144,7 +144,7 @@ public void noSuchMetricMeta() throws Exception { @Test public void metricOnlyNoValuesMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final SearchQuery query = new SearchQuery("no.values"); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); final List tsuids = lookup.lookup(); @@ -154,7 +154,7 @@ public void metricOnlyNoValuesMeta() throws Exception { @Test public void metricOnlyNoValuesData() throws Exception { - generateData(); + generateData(tsdb, storage); final SearchQuery query = new SearchQuery("no.values"); final TimeSeriesLookup lookup = new TimeSeriesLookup(tsdb, query); query.setUseMeta(false); @@ -165,7 +165,7 @@ public void metricOnlyNoValuesData() throws Exception { @Test public void tagkOnlyMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_STRING, null)); @@ -181,7 +181,7 @@ public void tagkOnlyMeta() throws Exception { @Test public void tagkOnlyMetaStar() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_STRING, "*")); @@ -197,7 +197,7 @@ public void tagkOnlyMetaStar() throws Exception { @Test public void tagkOnlyData() throws Exception { - generateData(); + generateData(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_STRING, null)); @@ -215,7 +215,7 @@ public void tagkOnlyData() throws Exception { @Test public void tagkOnly2Meta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_B_STRING, null)); @@ -230,7 +230,7 @@ public void tagkOnly2Meta() throws Exception { @Test public void tagkOnly2Data() throws Exception { - generateData(); + generateData(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_B_STRING, null)); @@ -247,7 +247,6 @@ public void tagkOnly2Data() throws Exception { @Test (expected = NoSuchUniqueName.class) public void noSuchTagkMeta() throws Exception { - storage = new MockBase(tsdb, client, true, true, true, true); final List> tags = new ArrayList>(1); tags.add(new Pair(NSUN_TAGK, null)); @@ -258,7 +257,7 @@ public void noSuchTagkMeta() throws Exception { @Test public void tagvOnlyMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(null, TAGV_STRING)); @@ -275,7 +274,7 @@ public void tagvOnlyMeta() throws Exception { @Test public void tagvOnlyMetaStar() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair("*", TAGV_STRING)); @@ -292,7 +291,7 @@ public void tagvOnlyMetaStar() throws Exception { @Test public void tagvOnlyData() throws Exception { - generateData(); + generateData(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(null, TAGV_STRING)); @@ -311,7 +310,7 @@ public void tagvOnlyData() throws Exception { @Test public void tagvOnly2Meta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(null, TAGV_B_STRING)); @@ -326,7 +325,7 @@ public void tagvOnly2Meta() throws Exception { @Test public void tagvOnly2Data() throws Exception { - generateData(); + generateData(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(null, TAGV_B_STRING)); @@ -343,7 +342,6 @@ public void tagvOnly2Data() throws Exception { @Test (expected = NoSuchUniqueName.class) public void noSuchTagvMeta() throws Exception { - storage = new MockBase(tsdb, client, true, true, true, true); final List> tags = new ArrayList>(1); tags.add(new Pair(null, NSUN_TAGV)); @@ -354,7 +352,7 @@ public void noSuchTagvMeta() throws Exception { @Test public void metricAndTagkMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_STRING, null)); @@ -369,7 +367,7 @@ public void metricAndTagkMeta() throws Exception { @Test public void metricAndTagkMetaStar() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_STRING, "*")); @@ -384,7 +382,7 @@ public void metricAndTagkMetaStar() throws Exception { @Test public void metricAndTagkData() throws Exception { - generateData(); + generateData(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_STRING, null)); @@ -400,7 +398,7 @@ public void metricAndTagkData() throws Exception { @Test public void metricAndTagvMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(null, TAGV_B_STRING)); @@ -414,7 +412,7 @@ public void metricAndTagvMeta() throws Exception { @Test public void metricAndTagvMetaStar() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair("*", TAGV_B_STRING)); @@ -428,7 +426,7 @@ public void metricAndTagvMetaStar() throws Exception { @Test public void metricAndTagvData() throws Exception { - generateData(); + generateData(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(null, TAGV_B_STRING)); @@ -443,7 +441,7 @@ public void metricAndTagvData() throws Exception { @Test public void metricAndTagPairMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_STRING, TAGV_STRING)); @@ -457,7 +455,7 @@ public void metricAndTagPairMeta() throws Exception { @Test public void metricAndTagPairData() throws Exception { - generateData(); + generateData(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_STRING, TAGV_STRING)); @@ -473,7 +471,7 @@ public void metricAndTagPairData() throws Exception { @Test public void tagPairOnlyMeta() throws Exception { - generateMeta(); + generateMeta(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_STRING, TAGV_STRING)); @@ -489,7 +487,7 @@ public void tagPairOnlyMeta() throws Exception { @Test public void tagPairOnlyData() throws Exception { - generateData(); + generateData(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_STRING, TAGV_STRING)); @@ -507,7 +505,7 @@ public void tagPairOnlyData() throws Exception { @Test public void limitVerification() throws Exception { - generateData(); + generateData(tsdb, storage); final List> tags = new ArrayList>(1); tags.add(new Pair(TAGK_STRING, TAGV_STRING)); @@ -526,8 +524,7 @@ public void limitVerification() throws Exception { /** * Stores some data in the mock tsdb-meta table for unit testing */ - private void generateMeta() { - storage = new MockBase(tsdb, client, true, true, true, true); + public static void generateMeta(final TSDB tsdb, final MockBase storage) { final List families = new ArrayList(1); families.add(TSMeta.FAMILY); storage.addTable("tsdb-meta".getBytes(), families); @@ -542,8 +539,7 @@ private void generateMeta() { /** * Stores some data in the mock tsdb data table for unit testing */ - private void generateData() { - storage = new MockBase(tsdb, client, true, true, true, true); + public static void generateData(final TSDB tsdb, final MockBase storage) { storage.setFamily("t".getBytes(MockBase.ASCII())); final byte[] qual = new byte[] { 0, 0 }; diff --git a/test/tsd/TestSearchRpc.java b/test/tsd/TestSearchRpc.java index 5232fdf512..a881b59990 100644 --- a/test/tsd/TestSearchRpc.java +++ b/test/tsd/TestSearchRpc.java @@ -16,6 +16,7 @@ import static org.mockito.Matchers.anyChar; import static org.mockito.Matchers.anyList; import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mock; import static org.junit.Assert.assertEquals; @@ -24,26 +25,29 @@ import java.lang.reflect.Field; import java.nio.charset.Charset; -import java.util.AbstractMap; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; +import net.opentsdb.core.BaseTsdbTest; import net.opentsdb.core.RowKey; import net.opentsdb.core.TSDB; import net.opentsdb.core.Tags; import net.opentsdb.meta.Annotation; import net.opentsdb.meta.TSMeta; import net.opentsdb.meta.UIDMeta; +import net.opentsdb.search.SearchPlugin; import net.opentsdb.search.SearchQuery; +import net.opentsdb.search.TestTimeSeriesLookup; import net.opentsdb.search.TimeSeriesLookup; +import net.opentsdb.storage.MockBase; +import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.UniqueId; import net.opentsdb.uid.UniqueId.UniqueIdType; import net.opentsdb.utils.Config; -import net.opentsdb.utils.JSON; import net.opentsdb.utils.Pair; +import org.hbase.async.Bytes; import org.jboss.netty.handler.codec.http.DefaultHttpRequest; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpRequest; @@ -54,31 +58,26 @@ import org.junit.runner.RunWith; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; import com.stumbleupon.async.Deferred; +import com.sun.java_cup.internal.runtime.Scanner; @RunWith(PowerMockRunner.class) -@PrepareForTest({TSDB.class, Config.class, HttpQuery.class, UniqueId.class, - RowKey.class, Tags.class, TimeSeriesLookup.class, SearchRpc.class}) -public final class TestSearchRpc { - private TSDB tsdb = null; +@PrepareForTest({ TSDB.class, Config.class, HttpQuery.class, UniqueId.class, + RowKey.class, Tags.class, TimeSeriesLookup.class, SearchRpc.class, + SearchPlugin.class, Scanner.class }) +public final class TestSearchRpc extends BaseTsdbTest { + private SearchPlugin mock_plugin; private SearchRpc rpc = new SearchRpc(); private SearchQuery search_query = null; - private TimeSeriesLookup mock_lookup = null; private static final Charset UTF = Charset.forName("UTF-8"); - private static List test_tsuids = new ArrayList(3); - static { - test_tsuids.add(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); - test_tsuids.add(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 2 }); - test_tsuids.add(new byte[] { 0, 0, 2, 0, 0, 1, 0, 0, 1 }); - } @Before - public void before() throws Exception { - tsdb = NettyMocks.getMockedHTTPTSDB(); + public void beforeLocal() throws Exception { + HttpQuery.initializeSerializerMaps(tsdb); } @Test @@ -221,9 +220,7 @@ public void searchMissingQuery() throws Exception { @Test (expected = BadRequestException.class) public void searchPluginNotEnabled() throws Exception { - when(tsdb.executeSearch((SearchQuery)any())) - .thenThrow(new IllegalStateException( - "Searching has not been enabled on this TSD")); + Whitebox.setInternalState(tsdb, "search", (SearchPlugin)null); final HttpQuery query = NettyMocks.getQuery(tsdb, "/api/search/tsmeta?query=*"); rpc.execute(tsdb, query); @@ -244,48 +241,122 @@ public void searchInvalidStartIndex() throws Exception { } @Test - public void searchLookup() throws Exception { - setupAnswerLookupQuery(); + public void searchLookupTagkOnlyMeta() throws Exception { + setupLookup(true); final HttpQuery query = NettyMocks.getQuery(tsdb, "/api/search/lookup?m={host=}"); rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); final String result = query.response().getContent().toString(UTF); assertTrue(result.contains("\"host\":\"web01\"")); - assertTrue(result.contains("\"totalResults\":3")); + assertTrue(result.contains("\"totalResults\":5")); + assertTrue(result.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(result.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(result.contains("\"tsuid\":\"000002000001000001\"")); + assertTrue(result.contains("\"tsuid\":\"000004000001000001000003000005\"")); + assertTrue(result.contains("\"tsuid\":\"000004000001000002000003000005\"")); } @Test - public void searchLookupPOST() throws Exception { - setupAnswerLookupQuery(); - SearchQuery q = new SearchQuery(); - q.setTags(new ArrayList>(2)); - q.getTags().add(new Pair("host", "web01")); - q.getTags().add(new Pair("dc", "phx")); - + public void searchLookupPOSTTagkOnlyMeta() throws Exception { + setupLookup(true); final HttpQuery query = NettyMocks.postQuery(tsdb, - "/api/search/lookup", "{\"tags\":[{\"key\":\"host\",\"value\":\"web01\"}]}"); + "/api/search/lookup", "{\"tags\":[{\"key\":\"host\",\"value\":null}]}"); + query.setSerializer(); + rpc.execute(tsdb, query); + + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); + final String result = query.response().getContent().toString(UTF); + assertTrue(result.contains("\"host\":\"web01\"")); + assertTrue(result.contains("\"totalResults\":5")); + assertTrue(result.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(result.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(result.contains("\"tsuid\":\"000002000001000001\"")); + assertTrue(result.contains("\"tsuid\":\"000004000001000001000003000005\"")); + assertTrue(result.contains("\"tsuid\":\"000004000001000002000003000005\"")); + } + + @Test + public void searchLookupPOSTTagkOnlyData() throws Exception { + setupLookup(false); + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/search/lookup", + "{\"tags\":[{\"key\":\"host\",\"value\":null}],\"useMeta\":false}"); rpc.execute(tsdb, query); + assertEquals(HttpResponseStatus.OK, query.response().getStatus()); final String result = query.response().getContent().toString(UTF); assertTrue(result.contains("\"host\":\"web01\"")); - assertTrue(result.contains("\"totalResults\":3")); + assertTrue(result.contains("\"totalResults\":5")); + assertTrue(result.contains("\"tsuid\":\"000001000001000001\"")); + assertTrue(result.contains("\"tsuid\":\"000001000001000002\"")); + assertTrue(result.contains("\"tsuid\":\"000002000001000001\"")); + assertTrue(result.contains("\"tsuid\":\"000004000001000001000003000005\"")); + assertTrue(result.contains("\"tsuid\":\"000004000001000002000003000005\"")); + } + + @Test + public void searchLookupNoMetaTable() throws Exception { + setupLookup(false); + final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/search/lookup", + "{\"tags\":[{\"key\":\"host\",\"value\":null}]}"); + rpc.execute(tsdb, query); + + assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR, + query.response().getStatus()); + final String result = query.response().getContent().toString(UTF); + assertTrue(result.contains("\"code\":500")); + assertTrue(result.contains("\"message\":\"Unexpected exception\"")); } @Test (expected = BadRequestException.class) public void searchLookupMissingQuery() throws Exception { - setupAnswerLookupQuery(); - final HttpQuery query = NettyMocks.getQuery(tsdb, - "/api/search/lookup"); + setupLookup(true); + final HttpQuery query = NettyMocks.getQuery(tsdb, "/api/search/lookup"); rpc.execute(tsdb, query); } @Test (expected = BadRequestException.class) public void searchLookupBadQuery() throws Exception { - setupAnswerLookupQuery(); + setupLookup(true); + final HttpQuery query = NettyMocks.getQuery(tsdb, "/api/search/lookup?m={"); + rpc.execute(tsdb, query); + } + + @Test + public void searchLookupNSUN() throws Exception { + setupLookup(true); final HttpQuery query = NettyMocks.getQuery(tsdb, - "/api/search/lookup?m={"); + "/api/search/lookup?m=" + NSUN_METRIC); rpc.execute(tsdb, query); + + assertEquals(HttpResponseStatus.NOT_FOUND, query.response().getStatus()); + final String result = query.response().getContent().toString(UTF); + assertTrue(result.contains("\"code\":404")); + assertTrue(result.contains("\"details\":\"No such name")); + } + + @Test + public void searchLookupNSUI() throws Exception { + setupLookup(true); + when(metrics.getNameAsync(new byte[] { 0, 0, 4 })) + .thenAnswer(new Answer>() { + @Override + public Deferred answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromError( + new NoSuchUniqueId("metrics", new byte[] { 0, 0, 4 })); + } + }); + final HttpQuery query = NettyMocks.postQuery(tsdb, + "/api/search/lookup", "{\"tags\":[{\"key\":\"host\",\"value\":null}]}"); + query.setSerializer(); + rpc.execute(tsdb, query); + + assertEquals(HttpResponseStatus.NOT_FOUND, query.response().getStatus()); + final String result = query.response().getContent().toString(UTF); + assertTrue(result.contains("\"code\":404")); + assertTrue(result.contains("\"details\":\"No such unique ID")); } /** @@ -294,7 +365,10 @@ public void searchLookupBadQuery() throws Exception { * responses for parsing tests. */ private void setupAnswerSearchQuery() { - when(tsdb.executeSearch((SearchQuery)any())).thenAnswer( + mock_plugin = mock(SearchPlugin.class); + Whitebox.setInternalState(tsdb, "search", mock_plugin); + + when(mock_plugin.executeQuery((SearchQuery)any())).thenAnswer( new Answer>() { @Override @@ -392,56 +466,70 @@ public Deferred answer(InvocationOnMock invocation) }); } - - @SuppressWarnings("unchecked") - private void setupAnswerLookupQuery() throws Exception { - PowerMockito.mockStatic(RowKey.class); - when(RowKey.metricNameAsync(tsdb, test_tsuids.get(0))) - .thenReturn(Deferred.fromResult("sys.cpu.user")); - when(RowKey.metricNameAsync(tsdb, test_tsuids.get(1))) - .thenReturn(Deferred.fromResult("sys.cpu.user")); - when(RowKey.metricNameAsync(tsdb, test_tsuids.get(2))) - .thenReturn(Deferred.fromResult("sys.cpu.nice")); - PowerMockito.mockStatic(UniqueId.class); - final List pair_a = new ArrayList(2); - pair_a.add(new byte[] { 0, 0, 1 }); - pair_a.add(new byte[] { 0, 0, 1 }); + private void setupLookup(final boolean use_meta) { + storage = new MockBase(tsdb, client, true, true, true, true); + if (use_meta) { + TestTimeSeriesLookup.generateMeta(tsdb, storage); + } else { + TestTimeSeriesLookup.generateData(tsdb, storage); + } - final List pair_b = new ArrayList(2); - pair_b.add(new byte[] { 0, 0, 1 }); - pair_b.add(new byte[] { 0, 0, 2 }); - - when(UniqueId.getTagPairsFromTSUID(test_tsuids.get(0))) - .thenReturn(pair_a); - when(UniqueId.getTagPairsFromTSUID(test_tsuids.get(1))) - .thenReturn(pair_b); - when(UniqueId.getTagPairsFromTSUID(test_tsuids.get(2))) - .thenReturn(pair_a); - when(UniqueId.uidToString((byte[])any())).thenCallRealMethod(); - - PowerMockito.mockStatic(Tags.class); - final HashMap tags_a = new HashMap(1); - tags_a.put("host", "web01"); - - final HashMap tags_b = new HashMap(1); - tags_b.put("host", "web02"); - - when(Tags.resolveIdsAsync(tsdb, pair_a)) - .thenReturn(Deferred.fromResult(tags_a)); - when(Tags.resolveIdsAsync(tsdb, pair_b)) - .thenReturn(Deferred.fromResult(tags_b)); - - when(Tags.parseWithMetric(anyString(), anyList())).thenCallRealMethod(); - when(Tags.splitString(anyString(), anyChar())).thenCallRealMethod(); - PowerMockito.doCallRealMethod().when(Tags.class, "parse", - anyList(), anyString()); - - mock_lookup = mock(TimeSeriesLookup.class); - PowerMockito.whenNew(TimeSeriesLookup.class) - .withArguments((TSDB)any(), (SearchQuery)any()) - .thenReturn(mock_lookup); - - when(mock_lookup.lookup()).thenReturn(test_tsuids); + when(metrics.getNameAsync(new byte[] { 0, 0, 4 })) + .thenAnswer(new Answer>() { + @Override + public Deferred answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("filtered"); + } + }); + when(tag_names.getNameAsync(new byte[] { 0, 0, 6 })) + .thenAnswer(new Answer>() { + @Override + public Deferred answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("6"); + } + }); + when(tag_names.getNameAsync(new byte[] { 0, 0, 8 })) + .thenAnswer(new Answer>() { + @Override + public Deferred answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("8"); + } + }); + when(tag_names.getNameAsync(new byte[] { 0, 0, 9 })) + .thenAnswer(new Answer>() { + @Override + public Deferred answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("9"); + } + }); + when(tag_values.getNameAsync(new byte[] { 0, 0, 7 })) + .thenAnswer(new Answer>() { + @Override + public Deferred answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("7"); + } + }); + when(tag_values.getNameAsync(new byte[] { 0, 0, 5 })) + .thenAnswer(new Answer>() { + @Override + public Deferred answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("5"); + } + }); + when(tag_values.getNameAsync(new byte[] { 0, 0, 10 })) + .thenAnswer(new Answer>() { + @Override + public Deferred answer(InvocationOnMock invocation) + throws Throwable { + return Deferred.fromResult("10"); + } + }); } } From 7681bc3225958fae9949ec9e6295b3b90e52db6e Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 3 Oct 2015 19:00:57 -0700 Subject: [PATCH 10/68] Bump the AsyncHBase version to 1.7.1-SNAPSHOT to fix decoding bugs with pre 0.96 region servers. --- third_party/hbase/asynchbase-1.7.1-20151004.015637-1.jar.md5 | 1 + third_party/hbase/include.mk | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 third_party/hbase/asynchbase-1.7.1-20151004.015637-1.jar.md5 diff --git a/third_party/hbase/asynchbase-1.7.1-20151004.015637-1.jar.md5 b/third_party/hbase/asynchbase-1.7.1-20151004.015637-1.jar.md5 new file mode 100644 index 0000000000..75abc13db6 --- /dev/null +++ b/third_party/hbase/asynchbase-1.7.1-20151004.015637-1.jar.md5 @@ -0,0 +1 @@ +898d34a463b52e570addf0f0160add48 \ No newline at end of file diff --git a/third_party/hbase/include.mk b/third_party/hbase/include.mk index e2e33dcc32..0e25495aba 100644 --- a/third_party/hbase/include.mk +++ b/third_party/hbase/include.mk @@ -13,9 +13,9 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see . -ASYNCHBASE_VERSION := 1.7.0 +ASYNCHBASE_VERSION := 1.7.1-20151004.015637-1 ASYNCHBASE := third_party/hbase/asynchbase-$(ASYNCHBASE_VERSION).jar -ASYNCHBASE_BASE_URL := http://central.maven.org/maven2/org/hbase/asynchbase/$(ASYNCHBASE_VERSION) +ASYNCHBASE_BASE_URL := https://oss.sonatype.org/content/repositories/snapshots/org/hbase/asynchbase/1.7.1-SNAPSHOT/ $(ASYNCHBASE): $(ASYNCHBASE).md5 set dummy "$(ASYNCHBASE_BASE_URL)" "$(ASYNCHBASE)"; shift; $(FETCH_DEPENDENCY) From 5cc9e6512aef3a826c62dcaeb37053cafca30a36 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 3 Oct 2015 19:11:16 -0700 Subject: [PATCH 11/68] Version to 2.2.0RC2-SNAPSHOT --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 3d43a30c24..b9d6c136f9 100644 --- a/configure.ac +++ b/configure.ac @@ -14,7 +14,7 @@ # along with this library. If not, see . # Semantic Versioning (see http://semver.org/). -AC_INIT([opentsdb], [2.2.0RC1], [opentsdb@googlegroups.com]) +AC_INIT([opentsdb], [2.2.0RC2-SNAPSHOT], [opentsdb@googlegroups.com]) AC_CONFIG_AUX_DIR([build-aux]) AM_INIT_AUTOMAKE([foreign]) From 5d2479c844a02154ce83bdcd2adbd64e94efcd88 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Tue, 27 Oct 2015 16:46:00 -0700 Subject: [PATCH 12/68] Fix test dependencies where DeleteRequest was pulled from Zookeeper instead of AsyncHBase. --- test/tools/TestDumpSeries.java | 2 +- test/tools/TestTextImporter.java | 2 +- test/tools/TestUID.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/tools/TestDumpSeries.java b/test/tools/TestDumpSeries.java index 85100f7852..375f3c4710 100644 --- a/test/tools/TestDumpSeries.java +++ b/test/tools/TestDumpSeries.java @@ -30,8 +30,8 @@ import net.opentsdb.uid.UniqueId; import net.opentsdb.utils.Config; -import org.apache.zookeeper.proto.DeleteRequest; import org.hbase.async.Bytes; +import org.hbase.async.DeleteRequest; import org.hbase.async.GetRequest; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; diff --git a/test/tools/TestTextImporter.java b/test/tools/TestTextImporter.java index 991562a748..851a9af0f4 100644 --- a/test/tools/TestTextImporter.java +++ b/test/tools/TestTextImporter.java @@ -37,8 +37,8 @@ import net.opentsdb.uid.UniqueId; import net.opentsdb.utils.Config; -import org.apache.zookeeper.proto.DeleteRequest; import org.hbase.async.Bytes; +import org.hbase.async.DeleteRequest; import org.hbase.async.GetRequest; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; diff --git a/test/tools/TestUID.java b/test/tools/TestUID.java index ab271dc131..e7a0bc939c 100644 --- a/test/tools/TestUID.java +++ b/test/tools/TestUID.java @@ -23,8 +23,8 @@ import net.opentsdb.storage.MockBase; import net.opentsdb.utils.Config; -import org.apache.zookeeper.proto.DeleteRequest; import org.hbase.async.Bytes; +import org.hbase.async.DeleteRequest; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; import org.hbase.async.Scanner; From 384aaf9bc008f0f9820e42b44a693225734dcc9c Mon Sep 17 00:00:00 2001 From: Hong Dai Thanh Date: Wed, 28 Oct 2015 10:22:31 +0700 Subject: [PATCH 13/68] s/space/comma/ in instruction for specifying ZooKeeper quorum --- build-aux/rpm/opentsdb.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-aux/rpm/opentsdb.conf b/build-aux/rpm/opentsdb.conf index 11f66ca6cf..a515418a7e 100644 --- a/build-aux/rpm/opentsdb.conf +++ b/build-aux/rpm/opentsdb.conf @@ -58,6 +58,6 @@ tsd.core.plugin_path = /usr/share/opentsdb/plugins # Path under which the znode for the -ROOT- region is located, default is "/hbase" #tsd.storage.hbase.zk_basedir = /hbase -# A space separated list of Zookeeper hosts to connect to, with or without +# A comma separated list of Zookeeper hosts to connect to, with or without # port specifiers, default is "localhost" #tsd.storage.hbase.zk_quorum = localhost From 5933ac28ef9ad952b402ec57b2221049734066d8 Mon Sep 17 00:00:00 2001 From: Hong Dai Thanh Date: Wed, 28 Oct 2015 10:22:31 +0700 Subject: [PATCH 14/68] s/space/comma/ in instruction for specifying ZooKeeper quorum --- build-aux/deb/opentsdb.conf | 2 +- build-aux/rpm/opentsdb.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-aux/deb/opentsdb.conf b/build-aux/deb/opentsdb.conf index d95b65efe2..f58d5cf14c 100644 --- a/build-aux/deb/opentsdb.conf +++ b/build-aux/deb/opentsdb.conf @@ -58,6 +58,6 @@ tsd.core.plugin_path = /usr/share/opentsdb/plugins # Path under which the znode for the -ROOT- region is located, default is "/hbase" #tsd.storage.hbase.zk_basedir = /hbase -# A space separated list of Zookeeper hosts to connect to, with or without +# A comma separated list of Zookeeper hosts to connect to, with or without # port specifiers, default is "localhost" #tsd.storage.hbase.zk_quorum = localhost diff --git a/build-aux/rpm/opentsdb.conf b/build-aux/rpm/opentsdb.conf index 11f66ca6cf..a515418a7e 100644 --- a/build-aux/rpm/opentsdb.conf +++ b/build-aux/rpm/opentsdb.conf @@ -58,6 +58,6 @@ tsd.core.plugin_path = /usr/share/opentsdb/plugins # Path under which the znode for the -ROOT- region is located, default is "/hbase" #tsd.storage.hbase.zk_basedir = /hbase -# A space separated list of Zookeeper hosts to connect to, with or without +# A comma separated list of Zookeeper hosts to connect to, with or without # port specifiers, default is "localhost" #tsd.storage.hbase.zk_quorum = localhost From fcaf2d472c0ccc2587cb20b6ea905413e335bb24 Mon Sep 17 00:00:00 2001 From: Yubao Liu Date: Fri, 23 Oct 2015 22:36:08 +0800 Subject: [PATCH 15/68] remove debug log with wrong INFO level to save disk space --- src/meta/TSMeta.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/meta/TSMeta.java b/src/meta/TSMeta.java index 765ac0eb16..7c34cbfc2a 100644 --- a/src/meta/TSMeta.java +++ b/src/meta/TSMeta.java @@ -525,7 +525,6 @@ final class TSMetaCB implements Callback, Long> { @Override public Deferred call(final Long incremented_value) throws Exception { -LOG.info("Value: " + incremented_value); if (incremented_value > 1) { // TODO - maybe update the search index every X number of increments? // Otherwise the search engine would only get last_updated/count From 99f9abe7b660bb5f87dd8d62b4ed5e5c7c5f3f60 Mon Sep 17 00:00:00 2001 From: Jim Westfall Date: Mon, 19 Oct 2015 17:05:58 -0700 Subject: [PATCH 16/68] QueryUi: URL.decode() url/query string before use Some browsers (aka firefox) like to encode { and } as %7B and %7D. This causes problem when parsing the query string since its using { and } to figure out the metric and tags. Without this the UI thows an error like the following: Request failed: Bad Request: No such name for 'metrics': 'server.nic.usage.mbit%7Bhost=host1%7D' --- src/tsd/client/QueryUi.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsd/client/QueryUi.java b/src/tsd/client/QueryUi.java index 4e2eccdd9d..cf78c6feca 100644 --- a/src/tsd/client/QueryUi.java +++ b/src/tsd/client/QueryUi.java @@ -780,7 +780,7 @@ private static QueryString getQueryString(final String qs) { } private void refreshFromQueryString() { - final QueryString qs = getQueryString(History.getToken()); + final QueryString qs = getQueryString(URL.decode(History.getToken())); maybeSetTextbox(qs, "start", start_datebox.getTextBox()); maybeSetTextbox(qs, "end", end_datebox.getTextBox()); @@ -961,7 +961,7 @@ public void got(final JSONValue json) { if (autoreload.getValue()) { history += "&autoreload=" + autoreoload_interval.getText(); } - if (!history.equals(History.getToken())) { + if (!history.equals(URL.decode(History.getToken()))) { History.newItem(history, false); } From 601209bfaa6ea5c4ff757d023e55464db4fe4cc2 Mon Sep 17 00:00:00 2001 From: Yubao Liu Date: Thu, 29 Oct 2015 23:46:42 +0800 Subject: [PATCH 17/68] fix stuck metasync when salting is enabled MetaScanner extracts (salt_width + metric_width) bytes from tsuid to metric uid, this triggers IllegalArgumentException in UniqueId.getNameAsync(id), then MetaScanner.call() won't never call result.callback(null), and result.joinUninterruptibly() in MetaSync.run() never returns. BTW, I checked all usages of Const.SALT_WIDTH() and fixed some other similar wrong calculations. --- src/core/BatchedDataPoints.java | 2 +- src/search/TimeSeriesLookup.java | 4 ++-- src/tools/MetaSync.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/BatchedDataPoints.java b/src/core/BatchedDataPoints.java index 3fcea6018c..c56143f234 100644 --- a/src/core/BatchedDataPoints.java +++ b/src/core/BatchedDataPoints.java @@ -304,7 +304,7 @@ public Deferred metricNameAsync() { if (row_key == null) { throw new IllegalStateException("Instance was not properly constructed!"); } - final byte[] id = Arrays.copyOfRange(row_key, 0, + final byte[] id = Arrays.copyOfRange(row_key, Const.SALT_WIDTH(), tsdb.metrics.width() + Const.SALT_WIDTH()); return tsdb.metrics.getNameAsync(id); } diff --git a/src/search/TimeSeriesLookup.java b/src/search/TimeSeriesLookup.java index 6c244b90f4..d640644c04 100644 --- a/src/search/TimeSeriesLookup.java +++ b/src/search/TimeSeriesLookup.java @@ -403,7 +403,7 @@ private Scanner getScanner(final int salt) { key = metric_uid; } else { key = new byte[Const.SALT_WIDTH() + TSDB.metrics_width()]; - key[0] = (byte)salt; + System.arraycopy(RowKey.getSaltBytes(salt), 0, key, 0, Const.SALT_WIDTH()); System.arraycopy(metric_uid, 0, key, Const.SALT_WIDTH(), metric_uid.length); } scanner.setStartKey(key); @@ -416,7 +416,7 @@ private Scanner getScanner(final int salt) { key = UniqueId.longToUID(uid, TSDB.metrics_width()); } else { key = new byte[Const.SALT_WIDTH() + TSDB.metrics_width()]; - key[0] = (byte)salt; + System.arraycopy(RowKey.getSaltBytes(salt), 0, key, 0, Const.SALT_WIDTH()); System.arraycopy(UniqueId.longToUID(uid, TSDB.metrics_width()), 0, key, Const.SALT_WIDTH(), metric_uid.length); } diff --git a/src/tools/MetaSync.java b/src/tools/MetaSync.java index 2303650a0e..b3c7510d23 100644 --- a/src/tools/MetaSync.java +++ b/src/tools/MetaSync.java @@ -392,7 +392,7 @@ public Object call(ArrayList> rows) // now process the UID metric meta data final byte[] metric_uid_bytes = - Arrays.copyOfRange(tsuid, 0, Const.SALT_WIDTH() + TSDB.metrics_width()); + Arrays.copyOfRange(tsuid, 0, TSDB.metrics_width()); final String metric_uid = UniqueId.uidToString(metric_uid_bytes); Long last_get = metric_uids.get(metric_uid); From 4111c06d993606b9bc759738077d3d2ae438bf85 Mon Sep 17 00:00:00 2001 From: louyl Date: Mon, 9 Nov 2015 16:53:31 +0800 Subject: [PATCH 18/68] Fix IncomingDataPoint toString, separate tags with space --- src/core/IncomingDataPoint.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/IncomingDataPoint.java b/src/core/IncomingDataPoint.java index 0a7c70970d..dced8077ef 100644 --- a/src/core/IncomingDataPoint.java +++ b/src/core/IncomingDataPoint.java @@ -94,10 +94,10 @@ public String toString() { final StringBuilder buf = new StringBuilder(); buf.append("metric=").append(this.metric); buf.append(" ts=").append(this.timestamp); - buf.append(" value=").append(this.value).append(" "); + buf.append(" value=").append(this.value); if (this.tags != null) { for (Map.Entry entry : this.tags.entrySet()) { - buf.append(entry.getKey()).append("=").append(entry.getValue()); + buf.append(" ").append(entry.getKey()).append("=").append(entry.getValue()); } } return buf.toString(); From ec783c674f73fa96830c80ce4905ee35da7b356b Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Fri, 9 Oct 2015 17:38:44 +0800 Subject: [PATCH 19/68] Release 2.2.0RC2. Thanks for the bug fixes! --- NEWS | 23 +++++++++++++++++++++++ THANKS | 4 ++++ configure.ac | 2 +- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index ac24f3fd81..04393452ea 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,19 @@ OpenTSDB - User visible changes. +* Version 2.2.0 RC2 (2015-11-09) + +Noteworthy Changes: + - Allow overriding the metric and tag UID widths via config file instead of + having to modify the source code. + +Bug Fixes: + - OOM handling script now handles multiple TSDs installed on the same host. + - Fix a bug where queries never return if an exception is thrown from the + storage layer. + - Fix random metric UID assignment in the CLI tool. + - Fix for meta data sync when salting is enabled. + - + * Version 2.2.0 RC1 (2015-09-12) Noteworthy Changes: @@ -49,6 +63,15 @@ Bug Fixes: - Avoid OOM issues over Telnet when the sending client isn't reading errors off it's socket fast enough by blocking writes. +* Version 2.1.2 (2015-11-09) + +Bug Fixes: + - Fix the built-in UI to handle query parameter parsing properly (found when Firefox + changed their URI behavior) + - Fix comments about the Zookeeper quorum setting in various config files. + - Fix quoting in the Makefile when installing. + - Make sure builds write files in the proper location on FreeBSD. + * Version 2.1.1 (2015-09-12) Bug Fixes: diff --git a/THANKS b/THANKS index 33898d5163..4761912619 100644 --- a/THANKS +++ b/THANKS @@ -25,6 +25,7 @@ Dave Barr Filippo Giunchedi Gabriel Nicolas Avellaneda Guenther Schmuelling +Hong Dai Thanh Hugo Trippaers Jacek Masiulaniec Jari Takkala @@ -33,7 +34,9 @@ Jan Mangs Jason Harvey Jim Scott Jesse Chang +Jim Westfall Johan Zeeck +Johannes Meixner Jonathan Works Josh Thomas Kieren Hynd @@ -42,6 +45,7 @@ Kris Beevers Lex Herbert Liangliang He Loïs Burg +Lou Yunlong Matt Jibson Matt Schallert Marc Tamsky diff --git a/configure.ac b/configure.ac index b9d6c136f9..41c5cac7d3 100644 --- a/configure.ac +++ b/configure.ac @@ -14,7 +14,7 @@ # along with this library. If not, see . # Semantic Versioning (see http://semver.org/). -AC_INIT([opentsdb], [2.2.0RC2-SNAPSHOT], [opentsdb@googlegroups.com]) +AC_INIT([opentsdb], [2.2.0RC2], [opentsdb@googlegroups.com]) AC_CONFIG_AUX_DIR([build-aux]) AM_INIT_AUTOMAKE([foreign]) From 6d151e3d8fa922a83ad2ebc7734ea2db70bf5f7b Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Wed, 11 Nov 2015 12:43:42 -0800 Subject: [PATCH 20/68] Release 2.1.3 to fix the static file bug copying introduced in 8dcd77d8907e3a0eaebde0eede81b2f76150dfd7 --- Makefile.am | 4 ++-- NEWS | 5 +++++ configure.ac | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 20f9509a46..5d13365f50 100644 --- a/Makefile.am +++ b/Makefile.am @@ -389,8 +389,8 @@ install-data-local: staticroot install-data-lib install-data-tools \ install-data-bin install-data-etc @$(NORMAL_INSTALL) test -z "$(staticdir)" || $(mkdir_p) "$(DESTDIR)$(staticdir)" - @set -e; pwd; ls -lFh; (cd "$(DEV_TSD_STATICROOT)"; \ - list=`find -L . ! -type d`); for p in $$list; do \ + @set -e; pwd; ls -lFh; cd "$(DEV_TSD_STATICROOT)"; \ + list=`find -L . ! -type d`; for p in $$list; do \ p=$${p#./}; \ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ dstdir=`dirname "$(DESTDIR)$(staticdir)/$$p"`; \ diff --git a/NEWS b/NEWS index 3948ef925a..029a4d509a 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,10 @@ OpenTSDB - User visible changes. +* Version 2.1.3 (2015-11-11) + +Bug Fixes: + - Fix build issues where the static files were not copied into the proper location. + * Version 2.1.2 (2015-11-09) Bug Fixes: diff --git a/configure.ac b/configure.ac index 36e407eb9f..e953a43527 100644 --- a/configure.ac +++ b/configure.ac @@ -14,7 +14,7 @@ # along with this library. If not, see . # Semantic Versioning (see http://semver.org/). -AC_INIT([opentsdb], [2.1.2], [opentsdb@googlegroups.com]) +AC_INIT([opentsdb], [2.1.3], [opentsdb@googlegroups.com]) AC_CONFIG_AUX_DIR([build-aux]) AM_INIT_AUTOMAKE([foreign]) From ad0ccad10bc49b5579ae71063250ef57be8f0b8e Mon Sep 17 00:00:00 2001 From: Vitaliy Fuks Date: Sat, 21 Nov 2015 16:40:09 -0500 Subject: [PATCH 21/68] Silence stray logging output in TSMeta.call() This line was added in ab276c823ee082bc216951e371062d1d019652d6 and based on indentation is likely an accidental commit. It causes continuous spamming in our logs after upgrade. Changing log level to "debug" to quiet it down. --- src/meta/TSMeta.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meta/TSMeta.java b/src/meta/TSMeta.java index 9abc89e463..a9d6dd2a69 100644 --- a/src/meta/TSMeta.java +++ b/src/meta/TSMeta.java @@ -525,7 +525,7 @@ final class TSMetaCB implements Callback, Long> { @Override public Deferred call(final Long incremented_value) throws Exception { -LOG.info("Value: " + incremented_value); + LOG.debug("Value: " + incremented_value); if (incremented_value > 1) { // TODO - maybe update the search index every X number of increments? // Otherwise the search engine would only get last_updated/count From eab97099cc605668e1c6f69eb0731c29089cab44 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Mon, 14 Dec 2015 20:13:15 -0800 Subject: [PATCH 22/68] A couple of additional UTs for TSSubQuery --- test/core/TestTSSubQuery.java | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/core/TestTSSubQuery.java b/test/core/TestTSSubQuery.java index 3336f18827..743a47fbaa 100644 --- a/test/core/TestTSSubQuery.java +++ b/test/core/TestTSSubQuery.java @@ -185,6 +185,46 @@ public void validateWithGroupByFilter() { assertEquals(300000, sub.downsampleInterval()); } + @Test + public void validateWithFilterAndGroupByFilter() { + TSSubQuery sub = getMetricForValidate(); + final List filters = new ArrayList(1); + filters.add(new TagVWildcardFilter("colo", "lga*")); + sub.setFilters(filters); + Map tags = new HashMap(); + tags.put("host", TagVWildcardFilter.FILTER_NAME + "(*nari)"); + sub.setTags(tags); + + sub.validateAndSetQuery(); + assertEquals("sys.cpu.0", sub.getMetric()); + assertEquals(TagVWildcardFilter.FILTER_NAME + "(*nari)", + sub.getTags().get("host")); + assertEquals(1, sub.getFilters().size()); + assertEquals(Aggregators.SUM, sub.aggregator()); + assertEquals(Aggregators.AVG, sub.downsampler()); + assertEquals(300000, sub.downsampleInterval()); + } + + @Test + public void validateWithFilterAndGroupByFilterSameTag() { + TSSubQuery sub = getMetricForValidate(); + final List filters = new ArrayList(1); + filters.add(new TagVWildcardFilter("host", "veti*")); + sub.setFilters(filters); + Map tags = new HashMap(); + tags.put("host", TagVWildcardFilter.FILTER_NAME + "(*nari)"); + sub.setTags(tags); + + sub.validateAndSetQuery(); + assertEquals("sys.cpu.0", sub.getMetric()); + assertEquals(TagVWildcardFilter.FILTER_NAME + "(*nari)", + sub.getTags().get("host")); + assertEquals(1, sub.getFilters().size()); + assertEquals(Aggregators.SUM, sub.aggregator()); + assertEquals(Aggregators.AVG, sub.downsampler()); + assertEquals(300000, sub.downsampleInterval()); + } + // NOTE: Each of the hash and equals tests should make sure that we the code // doesn't change after validation. From 1279d45f1f11758265dc247720bb110716ad6e93 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Mon, 14 Dec 2015 21:18:47 -0800 Subject: [PATCH 23/68] Fix #642 by sorting the tags properly on the bytes, NOT the string values. Sheesh. Thanks @wuxuehong214 --- src/tsd/UniqueIdRpc.java | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/tsd/UniqueIdRpc.java b/src/tsd/UniqueIdRpc.java index 5f49d0af6c..b1c2ddaf67 100644 --- a/src/tsd/UniqueIdRpc.java +++ b/src/tsd/UniqueIdRpc.java @@ -22,6 +22,7 @@ import java.util.TreeMap; import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; import org.hbase.async.PutRequest; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponseStatus; @@ -549,6 +550,7 @@ private TSMeta parseTSMetaQS(final HttpQuery query) { * @param data_query The query we're building * @throws BadRequestException if we are unable to parse the query or it is * missing components + * @todo - make this asynchronous */ private String getTSUIDForMetric(final String query_string, TSDB tsdb) { if (query_string == null || query_string.isEmpty()) { @@ -565,17 +567,23 @@ private String getTSUIDForMetric(final String query_string, TSDB tsdb) { } catch (IllegalArgumentException e) { throw new BadRequestException(e); } - final TreeMap sortedTags = new TreeMap(tags); + + // sort the UIDs on tagk values + final ByteMap tag_uids = new ByteMap(); + for (final Entry pair : tags.entrySet()) { + tag_uids.put(tsdb.getUID(UniqueIdType.TAGK, pair.getKey()), + tsdb.getUID(UniqueIdType.TAGV, pair.getValue())); + } + // Byte Buffer to generate TSUID, pre allocated to the size of the TSUID final ByteArrayOutputStream buf = new ByteArrayOutputStream( - TSDB.metrics_width() + sortedTags.size() * + TSDB.metrics_width() + tag_uids.size() * (TSDB.tagk_width() + TSDB.tagv_width())); try { - buf.write(tsdb.getUID(UniqueIdType.METRIC, metric)); - for (Entry e: sortedTags.entrySet()) { - // Fix for net.opentsdb.tsd.TestUniqueIdRpc.tsuidPostByM() - buf.write(tsdb.getUID(UniqueIdType.TAGK, e.getKey()), 0, TSDB.tagk_width()); - buf.write(tsdb.getUID(UniqueIdType.TAGV, e.getValue()), 0, TSDB.tagv_width()); + buf.write(tsdb.getUID(UniqueIdType.METRIC, metric)); + for (final Entry uids: tag_uids.entrySet()) { + buf.write(uids.getKey()); + buf.write(uids.getValue()); } } catch (IOException e) { throw new BadRequestException(e); From 87b4933c9d55745b3cf8fce8170e041a0bac8b49 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Wed, 16 Dec 2015 18:11:07 -0800 Subject: [PATCH 24/68] Fix case on the new UI header filename. --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index a2bc819fb6..8ccf3c2f46 100644 --- a/Makefile.am +++ b/Makefile.am @@ -307,7 +307,7 @@ httpui_DEPS = src/tsd/QueryUi.gwt.xml #dist_pkgdata_DATA = src/logback.xml dist_static_DATA = \ src/tsd/static/favicon.ico \ - src/tsd/static/openTSDB_header.jpg + src/tsd/static/opentsdb_header.jpg EXTRA_DIST = tsdb.in $(tsdb_SRC) $(test_SRC) \ $(test_plugin_SRC) $(test_plugin_MF) $(test_plugin_SVCS:%=test/%) \ From a6dbb75032ecb00de1b265b5b2682091820e3563 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Mon, 25 Jan 2016 19:40:21 -0800 Subject: [PATCH 25/68] Fix up the Makefile for debian creation with the updated UI files --- Makefile.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index 8ccf3c2f46..9963574c64 100644 --- a/Makefile.am +++ b/Makefile.am @@ -736,8 +736,8 @@ debian: dist staticroot chmod 755 $(distdir)/debian/DEBIAN/* cp $(top_srcdir)/build-aux/deb/init.d/opentsdb $(distdir)/debian/etc/init.d cp $(jar) $(distdir)/debian/usr/share/opentsdb/lib - cp -r staticroot/icon.ico $(distdir)/debian/usr/share/opentsdb/static - cp -r staticroot/openTSDB_header.jpg $(distdir)/debian/usr/share/opentsdb/static + cp -r staticroot/favicon.ico $(distdir)/debian/usr/share/opentsdb/static + cp -r staticroot/opentsdb_header.jpg $(distdir)/debian/usr/share/opentsdb/static cp -r gwt/queryui/* $(distdir)/debian/usr/share/opentsdb/static `for dep_jar in $(tsdb_DEPS); do cp $$dep_jar \ $(distdir)/debian/usr/share/opentsdb/lib; done;` From a9e463ade28e794e49ccbaacd8c7cf984a20a3e2 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Tue, 2 Feb 2016 19:15:01 -0800 Subject: [PATCH 26/68] Set tsd.query.allow_simultaneous_duplicates = true by default. This was causing some conflicts and confusion so we'll let folks enable it if they have problems with abuse. --- src/utils/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/Config.java b/src/utils/Config.java index 3344ff2e3c..2227800017 100644 --- a/src/utils/Config.java +++ b/src/utils/Config.java @@ -492,7 +492,7 @@ protected void setDefaults() { default_map.put("tsd.core.uid.random_metrics", "false"); default_map.put("tsd.query.filter.expansion_limit", "4096"); default_map.put("tsd.query.skip_unresolved_tagvs", "false"); - default_map.put("tsd.query.allow_simultaneous_duplicates", "false"); + default_map.put("tsd.query.allow_simultaneous_duplicates", "true"); default_map.put("tsd.rtpublisher.enable", "false"); default_map.put("tsd.rtpublisher.plugin", ""); default_map.put("tsd.search.enable", "false"); From 9a36e67cbff629895973ce901afc28a831aff9eb Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 13 Feb 2016 12:54:27 -0800 Subject: [PATCH 27/68] Bump to AsyncHbase 1.7.1 release --- third_party/hbase/asynchbase-1.7.0-20150910.030815-3.jar.md5 | 1 - third_party/hbase/asynchbase-1.7.1.jar.md5 | 1 + third_party/hbase/include.mk | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 third_party/hbase/asynchbase-1.7.0-20150910.030815-3.jar.md5 create mode 100644 third_party/hbase/asynchbase-1.7.1.jar.md5 diff --git a/third_party/hbase/asynchbase-1.7.0-20150910.030815-3.jar.md5 b/third_party/hbase/asynchbase-1.7.0-20150910.030815-3.jar.md5 deleted file mode 100644 index 9d3a066783..0000000000 --- a/third_party/hbase/asynchbase-1.7.0-20150910.030815-3.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -84b8410ba9003ecadbeececb02943ee1 \ No newline at end of file diff --git a/third_party/hbase/asynchbase-1.7.1.jar.md5 b/third_party/hbase/asynchbase-1.7.1.jar.md5 new file mode 100644 index 0000000000..45ad0e9669 --- /dev/null +++ b/third_party/hbase/asynchbase-1.7.1.jar.md5 @@ -0,0 +1 @@ +f236854721eac6d40b6710ec7d59f4a8 diff --git a/third_party/hbase/include.mk b/third_party/hbase/include.mk index 0e25495aba..bb25220311 100644 --- a/third_party/hbase/include.mk +++ b/third_party/hbase/include.mk @@ -13,9 +13,9 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see . -ASYNCHBASE_VERSION := 1.7.1-20151004.015637-1 +ASYNCHBASE_VERSION := 1.7.1 ASYNCHBASE := third_party/hbase/asynchbase-$(ASYNCHBASE_VERSION).jar -ASYNCHBASE_BASE_URL := https://oss.sonatype.org/content/repositories/snapshots/org/hbase/asynchbase/1.7.1-SNAPSHOT/ +ASYNCHBASE_BASE_URL := http://central.maven.org/maven2/org/hbase/asynchbase/$(ASYNCHBASE_VERSION) $(ASYNCHBASE): $(ASYNCHBASE).md5 set dummy "$(ASYNCHBASE_BASE_URL)" "$(ASYNCHBASE)"; shift; $(FETCH_DEPENDENCY) From 4fc6f52b25d63f54c1c1a94dca0d459aa1c1f508 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Fri, 20 Nov 2015 17:06:22 -0800 Subject: [PATCH 28/68] Add a try/catch to the FSCK utility to log problems found when printing row information. --- src/tools/Fsck.java | 62 ++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/src/tools/Fsck.java b/src/tools/Fsck.java index 45dde4539f..6295ff32f4 100644 --- a/src/tools/Fsck.java +++ b/src/tools/Fsck.java @@ -643,36 +643,40 @@ private void fsckDataPoints(final Map> datapoints) dp_index++) { duplicates.getAndIncrement(); DP dp = time_map.getValue().get(dp_index); - final byte flags = (byte)Internal.getFlagsFromQualifier(dp.kv.qualifier()); - buf.append(" ") - .append("write time: (") - .append(dp.kv.timestamp()) - .append(" - ") - .append(new Date(dp.kv.timestamp())) - .append(") ") - .append(" compacted: (") - .append(dp.compacted) - .append(") qualifier: ") - .append(Arrays.toString(dp.kv.qualifier())) - .append(" value: ") - .append(Internal.isFloat(dp.kv.qualifier()) ? - Internal.extractFloatingPointValue(dp.value(), 0, flags) : - Internal.extractIntegerValue(dp.value(), 0, flags)) - .append("\n"); - unique_columns.put(dp.kv.qualifier(), dp.kv.value()); - if (options.fix() && options.resolveDupes()) { - if (compact_row) { - // Scheduled for deletion by compaction. - duplicates_fixed_comp.getAndIncrement(); - } else if (!dp.compacted) { - LOG.debug("Removing duplicate data point: " + dp.kv); - tsdb.getClient().delete( - new DeleteRequest( - tsdb.dataTable(), dp.kv.key(), dp.kv.family(), dp.qualifier() - ) - ); - duplicates_fixed.getAndIncrement(); + try { + final byte flags = (byte)Internal.getFlagsFromQualifier(dp.kv.qualifier()); + buf.append(" ") + .append("write time: (") + .append(dp.kv.timestamp()) + .append(" - ") + .append(new Date(dp.kv.timestamp())) + .append(") ") + .append(" compacted: (") + .append(dp.compacted) + .append(") qualifier: ") + .append(Arrays.toString(dp.kv.qualifier())) + .append(" value: ") + .append(Internal.isFloat(dp.kv.qualifier()) ? + Internal.extractFloatingPointValue(dp.value(), 0, flags) : + Internal.extractIntegerValue(dp.value(), 0, flags)) + .append("\n"); + unique_columns.put(dp.kv.qualifier(), dp.kv.value()); + if (options.fix() && options.resolveDupes()) { + if (compact_row) { + // Scheduled for deletion by compaction. + duplicates_fixed_comp.getAndIncrement(); + } else if (!dp.compacted) { + LOG.debug("Removing duplicate data point: " + dp.kv); + tsdb.getClient().delete( + new DeleteRequest( + tsdb.dataTable(), dp.kv.key(), dp.kv.family(), dp.qualifier() + ) + ); + duplicates_fixed.getAndIncrement(); + } } + } catch (Exception e) { + LOG.error("Unexpected exception processing DP: " + dp); } } if (options.lastWriteWins()) { From 27c54b20833289cd76983c4814d7c40f285fbb7d Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 14 Feb 2016 12:35:58 -0800 Subject: [PATCH 29/68] Update the logback.xml config file with disabled configs for the query log. Also add the file appender to the /src/logback.xml file. --- build-aux/deb/logback.xml | 33 +++++++++++++++++++-- build-aux/rpm/logback.xml | 33 +++++++++++++++++++-- src/logback.xml | 60 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 118 insertions(+), 8 deletions(-) diff --git a/build-aux/deb/logback.xml b/build-aux/deb/logback.xml index 7f0fb57694..7ae2c3fcfc 100644 --- a/build-aux/deb/logback.xml +++ b/build-aux/deb/logback.xml @@ -9,10 +9,15 @@ + 1024 + /var/log/opentsdb/opentsdb.log true @@ -27,16 +32,40 @@ 128MB - %d{HH:mm:ss.SSS} %-5level [%logger{0}.%M] - %msg%n + + + + /var/log/opentsdb/queries.log + true + + + /var/log/opentsdb/queries.log.%i + 1 + 4 + + + + 128MB + + + %date{ISO8601} [%logger.%M] %msg%n + + + + + + + + diff --git a/build-aux/rpm/logback.xml b/build-aux/rpm/logback.xml index 7f0fb57694..7ae2c3fcfc 100644 --- a/build-aux/rpm/logback.xml +++ b/build-aux/rpm/logback.xml @@ -9,10 +9,15 @@ + 1024 + /var/log/opentsdb/opentsdb.log true @@ -27,16 +32,40 @@ 128MB - %d{HH:mm:ss.SSS} %-5level [%logger{0}.%M] - %msg%n + + + + /var/log/opentsdb/queries.log + true + + + /var/log/opentsdb/queries.log.%i + 1 + 4 + + + + 128MB + + + %date{ISO8601} [%logger.%M] %msg%n + + + + + + + + diff --git a/src/logback.xml b/src/logback.xml index b06776504a..ff97a50889 100644 --- a/src/logback.xml +++ b/src/logback.xml @@ -8,15 +8,67 @@ + + 1024 + + + + /var/log/opentsdb/opentsdb.log + true + + /home/y/logs/opentsdb2/opentsdb.log.%i + 1 + 4 + + + 512MB + + + + %date{ISO8601} [%thread] %-5level [%logger{0}.%M] - %msg%n + + + + + + /var/log/opentsdb/queries.log + true - - - - + + /var/log/opentsdb/queries.log.%i + 1 + 4 + + + + 128MB + + + %date{ISO8601} [%logger.%M] %msg%n + + + + + + + + + + + + + + + From 5577f7eb56c6c39f4442378510abe11d6f73c745 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 14 Feb 2016 12:59:59 -0800 Subject: [PATCH 30/68] Rollback 6d2102af7d7391b759698979cab9ae74bea15d80 as I was mistaken in that on Linux hosts it did indeed point to the wrong config directory when installing locally and via package. We'll need to revisit this for FreeBSD. --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index b43e01c093..9963574c64 100644 --- a/Makefile.am +++ b/Makefile.am @@ -356,7 +356,7 @@ printdeps: # This is kind of a hack, but I couldn't find a better way to adjust the paths # in the script before it gets installed... install-exec-hook: - script=tsdb; pkgdatadir='$(pkgdatadir)'; configdir='$(prefix)/etc/opentsdb'; \ + script=tsdb; pkgdatadir='$(pkgdatadir)'; configdir='$(sysconfigdir)/etc/opentsdb'; \ abs_srcdir=''; abs_builddir=''; $(edit_tsdb_script) cat tsdb.tmp >"$(DESTDIR)$(bindir)/tsdb" rm -f tsdb.tmp From c0612fcf15b8adcd9d6c2ea0388b5311284f848e Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 14 Feb 2016 13:34:44 -0800 Subject: [PATCH 31/68] Cut 2.2.0 Update News and Thanks! --- NEWS | 27 +++++++++++++++++++++++++++ THANKS | 6 ++++++ configure.ac | 2 +- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 1a7515e1e9..5378eb836c 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,32 @@ OpenTSDB - User visible changes. +* Version 2.2.0 (2016-02-14) + +Noteworthy Changes + - Rework the QueryStats output to be a bit more useful and add timings from the + various scanners and query components. + - Modify the UI to allow for group by or aggregate per tag (use the new query feature) + - Rework the UI skin with the new TSDB logo and color scheme. + - Add the QueryLog config to logback.xml so users can optionally enable logging of + all queries along with their stats. + +Buf Fixes: + - Properly handle append data points in the FSCK utility. + - Fix FSCK to handle salting properly. + - Fix the IncomingDataPoints class for the CLI import tool to properly account for + salting. + - Fix the QueryStats maps by making sure the hash accounts for an unmodified list of + filters (return copies to callers so sorting won't break the hash code). + - Fix the case-insensitive wildcard filter to properly ignore the case. + - Fix the CLI dumper/scan utility to properly handle salted data. + - Fix a case where the compaction queue could grow unbounded when salting was enabled. + - Allow duplicate queries by default (as we did in the past) and users must now block + them explicitly. + - Fix the /api/stats endpoint to allow for returning a value if the max UID width is + set to 8 for any type. Previously it would throw an exception. + - Add a try catch to FSCK to debug issues where printing a problematic row would cause + a hangup or no output. + * Version 2.2.0 RC3 (2015-11-11) Bug Fixes: diff --git a/THANKS b/THANKS index 4761912619..05e9ae58ff 100644 --- a/THANKS +++ b/THANKS @@ -22,11 +22,14 @@ Chris McClymont Cristian Sechel Christophe Furmaniak Dave Barr +Davide D Amico Filippo Giunchedi Gabriel Nicolas Avellaneda Guenther Schmuelling +Hari Krishna Dara Hong Dai Thanh Hugo Trippaers +Ivan Babrou Jacek Masiulaniec Jari Takkala James Royalty @@ -42,8 +45,10 @@ Josh Thomas Kieren Hynd Kimoon Kim Kris Beevers +Kyle Brandt Lex Herbert Liangliang He +Liu Yubao Loïs Burg Lou Yunlong Matt Jibson @@ -74,6 +79,7 @@ Thomas Sanchez Tibor Vass Tristan Colgate-McFarlane Tony Landells +Utkarsh Bhatnagar Vasiliy Kiryanov Yulai Fu Zachary Kurey \ No newline at end of file diff --git a/configure.ac b/configure.ac index ad2b2b4893..9b8c7f5c75 100644 --- a/configure.ac +++ b/configure.ac @@ -14,7 +14,7 @@ # along with this library. If not, see . # Semantic Versioning (see http://semver.org/). -AC_INIT([opentsdb], [2.2.0RC3], [opentsdb@googlegroups.com]) +AC_INIT([opentsdb], [2.2.0], [opentsdb@googlegroups.com]) AC_CONFIG_AUX_DIR([build-aux]) AM_INIT_AUTOMAKE([foreign]) From fb453a5801f8a2c6749f08f840ebc4dac3e6fd47 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 14 Feb 2016 13:36:29 -0800 Subject: [PATCH 32/68] Cut 2.1.4 --- NEWS | 11 +++++++++++ configure.ac | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 029a4d509a..d94c1df182 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,16 @@ OpenTSDB - User visible changes. +* Version 2.1.4 (2016-02-14) + +Bug Fixes: + - Fix the meta table where the UID/TSMeta APIs were not sorting tags properly + prior to creating the row key, thus allowing for duplicates if the caller changed + the order of tags. + - Fix a situation where meta sync could hang forever if a routine threw an exception. + - Fix an NPE thrown when accessing the /logs endpoint if the Cyclic appender is not + enabled in the logback config. + - Remove an overly chatty log line in TSMeta on new time series creation. + * Version 2.1.3 (2015-11-11) Bug Fixes: diff --git a/configure.ac b/configure.ac index e953a43527..eccf64827d 100644 --- a/configure.ac +++ b/configure.ac @@ -14,7 +14,7 @@ # along with this library. If not, see . # Semantic Versioning (see http://semver.org/). -AC_INIT([opentsdb], [2.1.3], [opentsdb@googlegroups.com]) +AC_INIT([opentsdb], [2.1.4], [opentsdb@googlegroups.com]) AC_CONFIG_AUX_DIR([build-aux]) AM_INIT_AUTOMAKE([foreign]) From f49d5d3d9202071b9688e94a7bd620a5b86461c4 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 14 Feb 2016 13:59:03 -0800 Subject: [PATCH 33/68] Fix --- src/core/TSDB.java | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/src/core/TSDB.java b/src/core/TSDB.java index 55b562ab5f..64cf75c779 100644 --- a/src/core/TSDB.java +++ b/src/core/TSDB.java @@ -428,30 +428,6 @@ public Deferred getUIDAsync(final UniqueIdType type, final String name) } } - /** - * Attempts to find the UID matching a given name - * @param type The type of UID - * @param name The name to search for - * @throws IllegalArgumentException if the type is not valid - * @throws NoSuchUniqueName if the name was not found - * @since 2.1 - */ - public Deferred getUIDAsync(final UniqueIdType type, final String name) { - if (name == null || name.isEmpty()) { - throw new IllegalArgumentException("Missing UID name"); - } - switch (type) { - case METRIC: - return metrics.getIdAsync(name); - case TAGK: - return tag_names.getIdAsync(name); - case TAGV: - return tag_values.getIdAsync(name); - default: - throw new IllegalArgumentException("Unrecognized UID type"); - } - } - /** * Verifies that the data and UID tables exist in HBase and optionally the * tree and meta data tables if the user has enabled meta tracking or tree From 3be2e6339f279a8710e76776e090ad969bc3060e Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 21 Feb 2016 14:26:19 -0800 Subject: [PATCH 34/68] Add HBase appends and iddle connections closed stats. --- src/core/TSDB.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/TSDB.java b/src/core/TSDB.java index 64cf75c779..c38bc229d9 100644 --- a/src/core/TSDB.java +++ b/src/core/TSDB.java @@ -548,12 +548,14 @@ public void collectStats(final StatsCollector collector) { collector.record("hbase.rpcs", stats.deletes(), "type=delete"); collector.record("hbase.rpcs", stats.gets(), "type=get"); collector.record("hbase.rpcs", stats.puts(), "type=put"); + collector.record("hbase.rpcs", stats.appends(), "type=append"); collector.record("hbase.rpcs", stats.rowLocks(), "type=rowLock"); collector.record("hbase.rpcs", stats.scannersOpened(), "type=openScanner"); collector.record("hbase.rpcs", stats.scans(), "type=scan"); collector.record("hbase.rpcs.batched", stats.numBatchedRpcSent()); collector.record("hbase.flushes", stats.flushes()); collector.record("hbase.connections.created", stats.connectionsCreated()); + collector.record("hbase.connections.idle_closed", stats.idleConnectionsClosed()); collector.record("hbase.nsre", stats.noSuchRegionExceptions()); collector.record("hbase.nsre.rpcs_delayed", stats.numRpcDelayedDueToNSRE()); From 1c73ea838dd870114ed101107516b84c31775458 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Mon, 18 Apr 2016 16:54:58 -0700 Subject: [PATCH 35/68] Add comments and UTs to 931242b5db4f8ab79c0346d876c53cc96349ba3d as well as clean up the deferreds a bit. --- src/meta/TSMeta.java | 24 +++++++++++--------- test/meta/TestTSMeta.java | 46 ++++++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/src/meta/TSMeta.java b/src/meta/TSMeta.java index 5fdaa21b0c..65e0030dd0 100644 --- a/src/meta/TSMeta.java +++ b/src/meta/TSMeta.java @@ -587,9 +587,7 @@ public Deferred call(Boolean success) throws Exception { } LOG.info("Successfullly created new TSUID entry for: " + meta); - return Deferred.fromResult(meta) - .addCallbackDeferring( - new LoadUIDs(tsdb, UniqueId.uidToString(tsuid))) + return new LoadUIDs(tsdb, UniqueId.uidToString(tsuid)).call(meta) .addCallbackDeferring(new FetchNewCB()); } @@ -613,7 +611,16 @@ public Deferred call(Boolean success) throws Exception { new TSMetaCB()); } - public static void storeIfNecessary(final TSDB tsdb, final byte[] tsuid) { + /** + * Attempts to fetch the meta column and if null, attempts to write a new + * column using {@link #storeNew}. + * @param tsdb The TSDB instance to use for access. + * @param tsuid The TSUID of the time series. + * @return A deferred with a true if the meta exists or was created, false + * if the meta did not exist and writing failed. + */ + public static Deferred storeIfNecessary(final TSDB tsdb, + final byte[] tsuid) { final GetRequest get = new GetRequest(tsdb.metaTable(), tsuid); get.family(FAMILY); get.qualifier(META_QUALIFIER); @@ -649,9 +656,7 @@ public Deferred call(Boolean success) throws Exception { } LOG.info("Successfullly created new TSUID entry for: " + meta); - return Deferred.fromResult(meta) - .addCallbackDeferring( - new LoadUIDs(tsdb, UniqueId.uidToString(tsuid))) + return new LoadUIDs(tsdb, UniqueId.uidToString(tsuid)).call(meta) .addCallbackDeferring(new FetchNewCB()); } } @@ -665,14 +670,13 @@ final class ExistsCB implements Callback, ArrayList> @Override public Deferred call(ArrayList row) throws Exception { if (row == null || row.isEmpty() || row.get(0).value() == null) { - return Deferred.fromResult(new Object()) - .addCallbackDeferring(new CreateNewCB()); + return new CreateNewCB().call(null); } return Deferred.fromResult(true); } } - tsdb.getClient().get(get).addCallbackDeferring(new ExistsCB()); + return tsdb.getClient().get(get).addCallbackDeferring(new ExistsCB()); } /** diff --git a/test/meta/TestTSMeta.java b/test/meta/TestTSMeta.java index 49c6e2f390..3984495aee 100644 --- a/test/meta/TestTSMeta.java +++ b/test/meta/TestTSMeta.java @@ -62,6 +62,7 @@ public final class TestTSMeta { private final static byte[] NAME_FAMILY = "name".getBytes(MockBase.ASCII()); private final static byte[] META_TABLE = "tsdb-meta".getBytes(MockBase.ASCII()); private final static byte[] UID_TABLE = "tsdb-uid".getBytes(MockBase.ASCII()); + private final static byte[] TSUID = new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }; private TSDB tsdb; private Config config; private HBaseClient client = mock(HBaseClient.class); @@ -120,7 +121,7 @@ public void before() throws Exception { "1328140801,\"displayName\":\"Web server 1\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(META_TABLE, new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, + storage.addColumn(META_TABLE, TSUID, NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()), ("{\"tsuid\":\"000001000001000001\",\"" + @@ -128,7 +129,7 @@ public void before() throws Exception { "\"custom\":null,\"units\":\"\",\"retention\":42,\"max\":1.0,\"min\":" + "\"NaN\",\"displayName\":\"Display\",\"dataType\":\"Data\"}") .getBytes(MockBase.ASCII())); - storage.addColumn(META_TABLE, new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, + storage.addColumn(META_TABLE, TSUID, TSMeta.FAMILY, "ts_ctr".getBytes(MockBase.ASCII()), Bytes.fromLong(1L)); @@ -260,7 +261,7 @@ public void deleteNull() throws Exception { @Test public void syncToStorage() throws Exception { - meta = new TSMeta(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, 1357300800000L); + meta = new TSMeta(TSUID, 1357300800000L); meta.setDisplayName("New DN"); meta.syncToStorage(tsdb, false).joinUninterruptibly(); assertEquals("New DN", meta.getDisplayName()); @@ -269,7 +270,7 @@ public void syncToStorage() throws Exception { @Test public void syncToStorageOverwrite() throws Exception { - meta = new TSMeta(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, 1357300800000L); + meta = new TSMeta(TSUID, 1357300800000L); meta.setDisplayName("New DN"); meta.syncToStorage(tsdb, true).joinUninterruptibly(); assertEquals("New DN", meta.getDisplayName()); @@ -290,14 +291,14 @@ public void syncToStorageNullTSUID() throws Exception { @Test (expected = IllegalArgumentException.class) public void syncToStorageDoesNotExist() throws Exception { - storage.flushRow(META_TABLE, new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); - meta = new TSMeta(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, 1357300800000L); + storage.flushRow(META_TABLE, TSUID); + meta = new TSMeta(TSUID, 1357300800000L); meta.syncToStorage(tsdb, false).joinUninterruptibly(); } @Test public void storeNew() throws Exception { - meta = new TSMeta(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, 1357300800000L); + meta = new TSMeta(TSUID, 1357300800000L); meta.setDisplayName("New DN"); meta.storeNew(tsdb); assertEquals("New DN", meta.getDisplayName()); @@ -323,7 +324,7 @@ public void metaExistsInStorage() throws Exception { @Test public void metaExistsInStorageNot() throws Exception { - storage.flushRow(META_TABLE, new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); + storage.flushRow(META_TABLE, TSUID); assertFalse(TSMeta.metaExistsInStorage(tsdb, "000001000001000001") .joinUninterruptibly()); } @@ -331,14 +332,14 @@ public void metaExistsInStorageNot() throws Exception { @Test public void counterExistsInStorage() throws Exception { assertTrue(TSMeta.counterExistsInStorage(tsdb, - new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }).joinUninterruptibly()); + TSUID).joinUninterruptibly()); } @Test public void counterExistsInStorageNot() throws Exception { - storage.flushRow(META_TABLE, new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); + storage.flushRow(META_TABLE, TSUID); assertFalse(TSMeta.counterExistsInStorage(tsdb, - new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }).joinUninterruptibly()); + TSUID).joinUninterruptibly()); } @Test @@ -378,12 +379,27 @@ public void COUNTER_QUALIFIER() throws Exception { TSMeta.COUNTER_QUALIFIER()); } + @Test + public void storeIfNecessaryExists() throws Exception { + assertTrue(TSMeta.storeIfNecessary(tsdb, TSUID).join()); + } + + @Test + public void storeIfNecessaryMissing() throws Exception { + storage.flushRow(META_TABLE, TSUID); + assertNull(storage.getColumn(META_TABLE, TSUID, NAME_FAMILY, + TSMeta.META_QUALIFIER())); + assertTrue(TSMeta.storeIfNecessary(tsdb, TSUID).join()); + assertNotNull(storage.getColumn(META_TABLE, TSUID, NAME_FAMILY, + TSMeta.META_QUALIFIER())); + } + @Test public void parseFromColumn() throws Exception { final KeyValue column = mock(KeyValue.class); - when(column.key()).thenReturn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); + when(column.key()).thenReturn(TSUID); when(column.value()).thenReturn(storage.getColumn(META_TABLE, - new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, + TSUID, NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()))); final TSMeta meta = TSMeta.parseFromColumn(tsdb, column, false) @@ -396,9 +412,9 @@ public void parseFromColumn() throws Exception { @Test public void parseFromColumnWithUIDMeta() throws Exception { final KeyValue column = mock(KeyValue.class); - when(column.key()).thenReturn(new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }); + when(column.key()).thenReturn(TSUID); when(column.value()).thenReturn(storage.getColumn(META_TABLE, - new byte[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 }, + TSUID, NAME_FAMILY, "ts_meta".getBytes(MockBase.ASCII()))); final TSMeta meta = TSMeta.parseFromColumn(tsdb, column, true) From e1140c763ac2c557028831b3859267a81d76d73c Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Mon, 18 Apr 2016 17:20:28 -0700 Subject: [PATCH 36/68] Fix TSDB write data around the TSMeta and TSUID incrementation. Also update news. --- NEWS | 9 +++++++++ src/core/TSDB.java | 23 ++++++++++++----------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/NEWS b/NEWS index 277824cc13..14bac38ef2 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,14 @@ OpenTSDB - User visible changes. +* Version 2.2.1 (2015-?-?) + +Noteworthy Changes + - Generate an incrementing TSMeta request only if both enable_tsuid_incrementing and + tsd.core.meta.enable_realtime_ts are enabled. Previously, increments would run + regardless of whether or not the real time ts setting was enabled. If tsuid + incrementing is disabled then a get and optional put is executed each time without + modifying the meta counter field. + * Version 2.2.0 (2016-02-14) Noteworthy Changes diff --git a/src/core/TSDB.java b/src/core/TSDB.java index 703d219ca1..6eab7220c6 100644 --- a/src/core/TSDB.java +++ b/src/core/TSDB.java @@ -822,17 +822,18 @@ private Deferred addPointInternal(final String metric, final byte[] tsuid = UniqueId.getTSUIDFromKey(row, METRICS_WIDTH, Const.TIMESTAMP_BYTES); - // for busy TSDs we may only enable TSUID tracking, storing a 1 in the - // counter field for a TSUID with the proper timestamp. If the user would - // rather have TSUID incrementing enabled, that will trump the PUT - if (config.enable_tsuid_tracking() && !config.enable_tsuid_incrementing()) { - final PutRequest tracking = new PutRequest(meta_table, tsuid, - TSMeta.FAMILY(), TSMeta.COUNTER_QUALIFIER(), Bytes.fromLong(1)); - client.put(tracking); - } else if (config.enable_tsuid_incrementing() && config.enable_realtime_ts()) { - TSMeta.incrementAndGetCounter(TSDB.this, tsuid); - } else if (!config.enable_tsuid_incrementing() && config.enable_realtime_ts()) { - TSMeta.storeIfNecessary(TSDB.this, tsuid); + if (config.enable_tsuid_tracking()) { + if (config.enable_realtime_ts()) { + if (config.enable_tsuid_incrementing()) { + TSMeta.incrementAndGetCounter(TSDB.this, tsuid); + } else { + TSMeta.storeIfNecessary(TSDB.this, tsuid); + } + } else { + final PutRequest tracking = new PutRequest(meta_table, tsuid, + TSMeta.FAMILY(), TSMeta.COUNTER_QUALIFIER(), Bytes.fromLong(1)); + client.put(tracking); + } } if (rt_publisher != null) { From 4087cd116b2ecdf7f512d691534871cd713d1020 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Mon, 18 Apr 2016 17:23:20 -0700 Subject: [PATCH 37/68] Bump to 2.2.1-SNAPSHOT --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 9b8c7f5c75..0ca046cf10 100644 --- a/configure.ac +++ b/configure.ac @@ -14,7 +14,7 @@ # along with this library. If not, see . # Semantic Versioning (see http://semver.org/). -AC_INIT([opentsdb], [2.2.0], [opentsdb@googlegroups.com]) +AC_INIT([opentsdb], [2.2.1-SNAPSHOT], [opentsdb@googlegroups.com]) AC_CONFIG_AUX_DIR([build-aux]) AM_INIT_AUTOMAKE([foreign]) From 9f6a58512520e1e8e3c1c54647136561f551c584 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Mon, 18 Apr 2016 17:39:23 -0700 Subject: [PATCH 38/68] Fix the filter metric and tag resolution chain by making sure the first callback is attached as a deferring callback and returns the proper type --- src/core/TsdbQuery.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/TsdbQuery.java b/src/core/TsdbQuery.java index 5d96c4e1c3..257d3cf018 100644 --- a/src/core/TsdbQuery.java +++ b/src/core/TsdbQuery.java @@ -361,14 +361,14 @@ class FilterCB implements Callback> { @Override public Object call(final ArrayList results) throws Exception { findGroupBys(); - return Deferred.fromResult(null); + return null; } } /** Resolve and group by tags after resolving the metric */ - class MetricCB implements Callback { + class MetricCB implements Callback, byte[]> { @Override - public Object call(final byte[] uid) throws Exception { + public Deferred call(final byte[] uid) throws Exception { metric = uid; if (filters != null) { final List> deferreds = @@ -385,7 +385,7 @@ public Object call(final byte[] uid) throws Exception { // fire off the callback chain by resolving the metric first return tsdb.metrics.getIdAsync(sub_query.getMetric()) - .addCallback(new MetricCB()); + .addCallbackDeferring(new MetricCB()); } } From 4c2fd437990bed4cbed0814d09d7fd6a221f60e7 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Tue, 19 Apr 2016 13:01:44 -0700 Subject: [PATCH 39/68] Add TagVFilter.getCopy() for creating a duplicate of the filter. --- src/query/filter/TagVFilter.java | 12 ++++++++++++ test/query/filter/TestTagVFilter.java | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/query/filter/TagVFilter.java b/src/query/filter/TagVFilter.java index 0c5e797b22..611a6c8266 100644 --- a/src/query/filter/TagVFilter.java +++ b/src/query/filter/TagVFilter.java @@ -498,11 +498,23 @@ public byte[] getTagkBytes() { return tagk_bytes; } + /** @return a non-null list of tag value UIDs. May be empty. */ @JsonIgnore public List getTagVUids() { return tagv_uids == null ? Collections.emptyList() : tagv_uids; } + /** @return A copy of this filter BEFORE tag resolution, as a new object. */ + @JsonIgnore + public TagVFilter getCopy() { + return Builder() + .setFilter(filter) + .setTagk(tagk) + .setType(getType()) + .setGroupBy(group_by) + .build(); + } + /** @return whether or not to group by the results of this filter */ @JsonIgnore public boolean isGroupBy() { diff --git a/test/query/filter/TestTagVFilter.java b/test/query/filter/TestTagVFilter.java index 477f65aaa9..5c952c83df 100644 --- a/test/query/filter/TestTagVFilter.java +++ b/test/query/filter/TestTagVFilter.java @@ -15,6 +15,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -400,5 +401,21 @@ public void tagsToFiltersSameTagDiffValues() throws Exception { assertEquals(2, filters.size()); } + @Test + public void getCopy() { + final TagVFilter filter = TagVFilter.Builder() + .setFilter("*") + .setTagk(TAGK_STRING) + .setType("wildcard") + .setGroupBy(true) + .build(); + final TagVFilter copy = filter.getCopy(); + assertNotSame(filter, copy); + assertEquals(filter.filter, copy.filter); + assertEquals(filter.tagk, copy.tagk); + assertEquals(filter.getType(), copy.getType()); + assertEquals(filter.group_by, copy.group_by); + } + // TODO - test the plugin loader similar to the other plugins } From 07ff47e9f736709edd368c6926db6d401672efb8 Mon Sep 17 00:00:00 2001 From: HugoMFernandes Date: Fri, 15 Apr 2016 15:12:39 +0100 Subject: [PATCH 40/68] Fix #773 - Add global_annotations to graphs Fixed a bug in which global_annotations were ignored by GraphHandler.java (and thus, were not rendered by gnuplot). Added async call to hbase to retrieve global annotations when required in a similar fashion as that of QueryRpc.java. Note: Since both GraphHandler.java and QueryRpc.java implement (similar) query-processing logic, all of this should be unified in the future (requires a large refactor to split all query logic with visualization logic). --- src/tsd/GraphHandler.java | 71 +++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/src/tsd/GraphHandler.java b/src/tsd/GraphHandler.java index 6f02526a3b..3df3086a90 100644 --- a/src/tsd/GraphHandler.java +++ b/src/tsd/GraphHandler.java @@ -45,11 +45,15 @@ import net.opentsdb.core.TSDB; import net.opentsdb.core.TSQuery; import net.opentsdb.graph.Plot; +import net.opentsdb.meta.Annotation; import net.opentsdb.stats.Histogram; import net.opentsdb.stats.StatsCollector; import net.opentsdb.utils.DateTime; import net.opentsdb.utils.JSON; +import com.stumbleupon.async.Callback; +import com.stumbleupon.async.Deferred; + /** * Stateless handler of HTTP graph requests (the {@code /q} endpoint). */ @@ -123,6 +127,10 @@ public void execute(final TSDB tsdb, final HttpQuery query) { } } + // TODO(HugoMFernandes): Most of this (query-related) logic is implemented in + // net.opentsdb.tsd.QueryRpc.java (which actually does this asynchronously), + // so we should refactor both classes to split the actual logic used to + // generate the data from the actual visualization (removing all duped code). private void doGraph(final TSDB tsdb, final HttpQuery query) throws IOException { final String basepath = getGnuplotBasePath(tsdb, query); @@ -154,10 +162,15 @@ private void doGraph(final TSDB tsdb, final HttpQuery query) if (!nocache && isDiskCacheHit(query, end_time, max_age, basepath)) { return; } - Query[] tsdbqueries; - List options; - tsdbqueries = parseQuery(tsdb, query); - options = query.getQueryStringParams("o"); + + // Parse TSQuery from HTTP query + final TSQuery tsquery = QueryRpc.parseQuery(tsdb, query); + tsquery.validateAndSetQuery(); + + // Build the queries for the parsed TSQuery + Query[] tsdbqueries = tsquery.buildQueries(tsdb); + + List options = query.getQueryStringParams("o"); if (options == null) { options = new ArrayList(tsdbqueries.length); for (int i = 0; i < tsdbqueries.length; i++) { @@ -212,9 +225,37 @@ private void doGraph(final TSDB tsdb, final HttpQuery query) return; } + final RunGnuplot rungnuplot = new RunGnuplot(query, max_age, plot, basepath, + aggregated_tags, npoints); + + class ErrorCB implements Callback { + public Object call(final Exception e) throws Exception { + LOG.info("Failed to retrieve global annotations: ", e); + throw e; + } + } + + class GlobalCB implements Callback> { + public Object call(final List globalAnnotations) throws Exception { + rungnuplot.plot.setGlobals(globalAnnotations); + execGnuplot(rungnuplot, query); + + return null; + } + } + + // Fetch global annotations, if needed + if (!tsquery.getNoAnnotations() && tsquery.getGlobalAnnotations()) { + Annotation.getGlobalAnnotations(tsdb, start_time, end_time) + .addCallback(new GlobalCB()).addErrback(new ErrorCB()); + } else { + execGnuplot(rungnuplot, query); + } + } + + private void execGnuplot(RunGnuplot rungnuplot, HttpQuery query) { try { - gnuplot.execute(new RunGnuplot(query, max_age, plot, basepath, - aggregated_tags, npoints)); + gnuplot.execute(rungnuplot); } catch (RejectedExecutionException e) { query.internalError(new Exception("Too many requests pending," + " please try again later", e)); @@ -354,7 +395,7 @@ private String getGnuplotBasePath(final TSDB tsdb, final HttpQuery query) { qs.remove("png"); qs.remove("json"); qs.remove("ascii"); - return tsdb.getConfig().getDirectoryName("tsd.http.cachedir") + + return tsdb.getConfig().getDirectoryName("tsd.http.cachedir") + Integer.toHexString(qs.hashCode()); } @@ -841,21 +882,7 @@ private static void printMetricHeader(final PrintWriter writer, final String met writer.print(timestamp / 1000L); writer.print(' '); } - - /** - * Parses the {@code /q} query in a list of {@link Query} objects. - * @param tsdb The TSDB to use. - * @param query The HTTP query for {@code /q}. - * @return The corresponding {@link Query} objects. - * @throws BadRequestException if the query was malformed. - * @throws IllegalArgumentException if the metric or tags were malformed. - */ - private static Query[] parseQuery(final TSDB tsdb, final HttpQuery query) { - final TSQuery q = QueryRpc.parseQuery(tsdb, query); - q.validateAndSetQuery(); - return q.buildQueries(tsdb); - } - + private static final PlotThdFactory thread_factory = new PlotThdFactory(); private static final class PlotThdFactory implements ThreadFactory { From 667fccf493d234150c5b69cbebf1794e2ffd59c5 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 23 Apr 2016 14:25:49 -0700 Subject: [PATCH 41/68] Add annotation flags to the UI to pass URI params properly for --- src/tsd/GraphHandler.java | 6 +++--- src/tsd/client/QueryUi.java | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/tsd/GraphHandler.java b/src/tsd/GraphHandler.java index 3df3086a90..2669d6b8c6 100644 --- a/src/tsd/GraphHandler.java +++ b/src/tsd/GraphHandler.java @@ -230,14 +230,14 @@ private void doGraph(final TSDB tsdb, final HttpQuery query) class ErrorCB implements Callback { public Object call(final Exception e) throws Exception { - LOG.info("Failed to retrieve global annotations: ", e); + LOG.warn("Failed to retrieve global annotations: ", e); throw e; } } class GlobalCB implements Callback> { - public Object call(final List globalAnnotations) throws Exception { - rungnuplot.plot.setGlobals(globalAnnotations); + public Object call(final List global_annotations) throws Exception { + rungnuplot.plot.setGlobals(global_annotations); execGnuplot(rungnuplot, query); return null; diff --git a/src/tsd/client/QueryUi.java b/src/tsd/client/QueryUi.java index cf78c6feca..e81a50ac70 100644 --- a/src/tsd/client/QueryUi.java +++ b/src/tsd/client/QueryUi.java @@ -148,6 +148,10 @@ public class QueryUi implements EntryPoint, HistoryListener { private final CheckBox smooth = new CheckBox(); private final ListBox styles = new ListBox(); private String timezone = ""; + + // Annotations handling flags. + private boolean hide_annotations = false; + private boolean show_global_annotations = false; /** * Handles every change to the query form and gets a new graph. @@ -788,6 +792,9 @@ private void refreshFromQueryString() { autoreload.setValue(qs.containsKey("autoreload"), true); maybeSetTextbox(qs, "autoreload", autoreoload_interval); + show_global_annotations = qs.containsKey("global_annotations") ? true : false; + hide_annotations = qs.containsKey("no_annotations") ? true : false; + //get the tz param value final ArrayList tzvalues = qs.get("tz"); if (tzvalues == null) @@ -931,6 +938,14 @@ private void refreshGraph() { url.append("&smooth=csplines"); } url.append("&style=").append(styles.getValue(styles.getSelectedIndex())); + + if (hide_annotations) { + url.append("&no_annotations=true"); + } + if (show_global_annotations) { + url.append("&global_annotations=true"); + } + final String unencodedUri = url.toString(); final String uri = URL.encode(unencodedUri); if (uri.equals(lastgraphuri)) { From 27bb483a8e6b8aff912a5c70fa5a2186c6aa4967 Mon Sep 17 00:00:00 2001 From: Kevin Bowling Date: Mon, 25 Apr 2016 00:06:37 -0700 Subject: [PATCH 42/68] Fix comment for behavior of tsd.network.tcp_no_delay --- src/opentsdb.conf | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/opentsdb.conf b/src/opentsdb.conf index aea14cf9c0..a06420e9f2 100644 --- a/src/opentsdb.conf +++ b/src/opentsdb.conf @@ -6,15 +6,14 @@ tsd.network.port = # The IPv4 network address to bind to, defaults to all addresses # tsd.network.bind = 0.0.0.0 -# Enables Nagel's algorithm to reduce the number of packets sent over the -# network, default is True +# Disable Nagel's algorithm, default is True #tsd.network.tcpnodelay = true -# Determines whether or not to send keepalive packets to peers, default +# Determines whether or not to send keepalive packets to peers, default # is True #tsd.network.keep_alive = true -# Determines if the same socket should be used for new connections, default +# Determines if the same socket should be used for new connections, default # is True #tsd.network.reuseaddress = true @@ -42,7 +41,7 @@ tsd.http.cachedir = # Whether or not to enable data compaction in HBase, default is True #tsd.storage.enable_compaction = true -# How often, in milliseconds, to flush the data point queue to storage, +# How often, in milliseconds, to flush the data point queue to storage, # default is 1,000 # tsd.storage.flush_interval = 1000 @@ -55,7 +54,7 @@ tsd.http.cachedir = # Path under which the znode for the -ROOT- region is located, default is "/hbase" #tsd.storage.hbase.zk_basedir = /hbase -# A comma separated list of Zookeeper hosts to connect to, with or without +# A comma separated list of Zookeeper hosts to connect to, with or without # port specifiers, default is "localhost" #tsd.storage.hbase.zk_quorum = localhost From 4c98026764077b26567eb1c3beed9cb578377de7 Mon Sep 17 00:00:00 2001 From: Kevin Bowling Date: Mon, 25 Apr 2016 00:07:20 -0700 Subject: [PATCH 43/68] A few fixes for deb and rpm opentsdb.conf options --- build-aux/deb/opentsdb.conf | 7 +++---- build-aux/rpm/opentsdb.conf | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/build-aux/deb/opentsdb.conf b/build-aux/deb/opentsdb.conf index 3d7db5bfa3..70afee8737 100644 --- a/build-aux/deb/opentsdb.conf +++ b/build-aux/deb/opentsdb.conf @@ -6,9 +6,8 @@ tsd.network.port = 4242 # The IPv4 network address to bind to, defaults to all addresses # tsd.network.bind = 0.0.0.0 -# Enables Nagel's algorithm to reduce the number of packets sent over the -# network, default is True -#tsd.network.tcpnodelay = true +# Disable Nagel's algorithm. Default is True +#tsd.network.tcp_no_delay = true # Determines whether or not to send keepalive packets to peers, default # is True @@ -16,7 +15,7 @@ tsd.network.port = 4242 # Determines if the same socket should be used for new connections, default # is True -#tsd.network.reuseaddress = true +#tsd.network.reuse_address = true # Number of worker threads dedicated to Netty, defaults to # of CPUs * 2 #tsd.network.worker_threads = 8 diff --git a/build-aux/rpm/opentsdb.conf b/build-aux/rpm/opentsdb.conf index caf4599acc..052936b962 100644 --- a/build-aux/rpm/opentsdb.conf +++ b/build-aux/rpm/opentsdb.conf @@ -6,9 +6,8 @@ tsd.network.port = 4242 # The IPv4 network address to bind to, defaults to all addresses # tsd.network.bind = 0.0.0.0 -# Enables Nagel's algorithm to reduce the number of packets sent over the -# network, default is True -#tsd.network.tcpnodelay = true +# Disable Nagel's algorithm, default is True +#tsd.network.tcp_no_delay = true # Determines whether or not to send keepalive packets to peers, default # is True @@ -16,7 +15,7 @@ tsd.network.port = 4242 # Determines if the same socket should be used for new connections, default # is True -#tsd.network.reuseaddress = true +#tsd.network.reuse_address = true # Number of worker threads dedicated to Netty, defaults to # of CPUs * 2 #tsd.network.worker_threads = 8 From a4cc4fc9efaeeb53c9b6b5019a86bbb3901ede11 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 1 May 2016 12:19:36 -0700 Subject: [PATCH 44/68] Fix issue #778 by allowing the creating of a TSMeta object without a TSUID. Also fix up some UTs in the UIDRPC class where the JSON order can change. --- src/tsd/UniqueIdRpc.java | 9 ++- test/tsd/TestUniqueIdRpc.java | 142 ++++++++++++++++++++-------------- 2 files changed, 91 insertions(+), 60 deletions(-) diff --git a/src/tsd/UniqueIdRpc.java b/src/tsd/UniqueIdRpc.java index b1c2ddaf67..60ff461341 100644 --- a/src/tsd/UniqueIdRpc.java +++ b/src/tsd/UniqueIdRpc.java @@ -486,8 +486,13 @@ private UIDMeta parseUIDMetaQS(final HttpQuery query) { * be parsed */ private TSMeta parseTSMetaQS(final HttpQuery query) { - final String tsuid = query.getRequiredQueryStringParam("tsuid"); - final TSMeta meta = new TSMeta(tsuid); + final String tsuid = query.getQueryStringParam("tsuid"); + final TSMeta meta; + if (tsuid != null && !tsuid.isEmpty()) { + meta = new TSMeta(tsuid); + } else { + meta = new TSMeta(); + } final String display_name = query.getQueryStringParam("display_name"); if (display_name != null) { diff --git a/test/tsd/TestUniqueIdRpc.java b/test/tsd/TestUniqueIdRpc.java index 1e7c5240b6..41836ae928 100644 --- a/test/tsd/TestUniqueIdRpc.java +++ b/test/tsd/TestUniqueIdRpc.java @@ -107,9 +107,10 @@ public void assignQsMetricDouble() throws Exception { "/api/uid/assign?metric=sys.cpu.0,sys.cpu.2"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals( - "{\"metric\":{\"sys.cpu.0\":\"000001\",\"sys.cpu.2\":\"000003\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"sys.cpu.0\":\"000001\"")); + assertTrue(json.contains("\"sys.cpu.2\":\"000003\"")); } @Test @@ -119,9 +120,10 @@ public void assignQsMetricSingleBad() throws Exception { "/api/uid/assign?metric=sys.cpu.1"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); - assertEquals("{\"metric_errors\":{\"sys.cpu.1\":\"Name already exists with " - + "UID: 000002\"},\"metric\":{}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"sys.cpu.1\":\"Name already exists with " + + "UID: 000002\"}")); } @Test @@ -131,10 +133,12 @@ public void assignQsMetric2Good1Bad() throws Exception { "/api/uid/assign?metric=sys.cpu.0,sys.cpu.1,sys.cpu.2"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); - assertEquals("{\"metric_errors\":{\"sys.cpu.1\":\"Name already exists with " - + "UID: 000002\"},\"metric\":{\"sys.cpu.0\":\"000001\",\"sys.cpu.2\":" - + "\"000003\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"sys.cpu.1\":\"Name already exists with " + + "UID: 000002\"}")); + assertTrue(json.contains("{\"sys.cpu.0\":\"000001\",\"sys.cpu.2\":" + + "\"000003\"}")); } @Test @@ -155,9 +159,10 @@ public void assignQsTagkDouble() throws Exception { "/api/uid/assign?tagk=host,fqdn"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals( - "{\"tagk\":{\"fqdn\":\"000003\",\"host\":\"000001\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"fqdn\":\"000003\"")); + assertTrue(json.contains("\"host\":\"000001\"")); } @Test @@ -167,9 +172,10 @@ public void assignQsTagkSingleBad() throws Exception { "/api/uid/assign?tagk=datacenter"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); - assertEquals("{\"tagk_errors\":{\"datacenter\":\"Name already exists with " - + "UID: 000002\"},\"tagk\":{}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"tagk_errors\":{\"datacenter\":" + + "\"Name already exists with UID: 000002\"}")); } @Test @@ -179,9 +185,12 @@ public void assignQsTagk2Good1Bad() throws Exception { "/api/uid/assign?tagk=host,datacenter,fqdn"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); - assertEquals("{\"tagk_errors\":{\"datacenter\":\"Name already exists with " - + "UID: 000002\"},\"tagk\":{\"fqdn\":\"000003\",\"host\":\"000001\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"datacenter\":\"Name already exists with " + + "UID: 000002\"}")); + assertTrue(json.contains("\"fqdn\":\"000003\"")); + assertTrue(json.contains("\"host\":\"000001\"")); } @Test @@ -202,9 +211,10 @@ public void assignQsTagvDouble() throws Exception { "/api/uid/assign?tagv=localhost,foo"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals( - "{\"tagv\":{\"foo\":\"000003\",\"localhost\":\"000001\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"foo\":\"000003\"")); + assertTrue(json.contains("\"localhost\":\"000001\"")); } @Test @@ -214,9 +224,10 @@ public void assignQsTagvSingleBad() throws Exception { "/api/uid/assign?tagv=myserver"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); - assertEquals("{\"tagv\":{},\"tagv_errors\":{\"myserver\":\"Name already " - + "exists with UID: 000002\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"tagv_errors\":{\"myserver\":\"Name already " + + "exists with UID: 000002\"}")); } @Test @@ -226,10 +237,12 @@ public void assignQsTagv2Good1Bad() throws Exception { "/api/uid/assign?tagv=localhost,myserver,foo"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); - assertEquals("{\"tagv\":{\"foo\":\"000003\",\"localhost\":\"000001\"}," - + "\"tagv_errors\":{\"myserver\":\"Name already exists with " - + "UID: 000002\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"foo\":\"000003\"")); + assertTrue(json.contains("\"localhost\":\"000001\"")); + assertTrue(json.contains("{\"myserver\":\"Name already exists with " + + "UID: 000002\"}")); } @Test @@ -297,9 +310,10 @@ public void assignPostMetricDouble() throws Exception { "{\"metric\":[\"sys.cpu.0\",\"sys.cpu.2\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals( - "{\"metric\":{\"sys.cpu.0\":\"000001\",\"sys.cpu.2\":\"000003\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"sys.cpu.0\":\"000001\"")); + assertTrue(json.contains("\"sys.cpu.2\":\"000003\"")); } public void assignPostMetricSingleBad() throws Exception { @@ -308,9 +322,10 @@ public void assignPostMetricSingleBad() throws Exception { "{\"metric\":[\"sys.cpu.2\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals("{\"metric_errors\":{\"sys.cpu.1\":\"Name already exists with " - + "UID: 000002\"},\"metric\":{}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"sys.cpu.1\":\"Name already exists with " + + "UID: 000002\"}")); } public void assignPostMetric2Good1Bad() throws Exception { @@ -319,10 +334,12 @@ public void assignPostMetric2Good1Bad() throws Exception { "{\"metric\":[\"sys.cpu.0\",\"sys.cpu.1\",\"sys.cpu.2\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals("{\"metric_errors\":{\"sys.cpu.1\":\"Name already exists with " - + "UID: 000002\"},\"metric\":{\"sys.cpu.0\":\"000001\",\"sys.cpu.2\":" - + "\"000003\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"sys.cpu.1\":\"Name already exists with " + + "UID: 000002\"}")); + assertTrue(json.contains("\"sys.cpu.0\":\"000001\"")); + assertTrue(json.contains("\"sys.cpu.2\":\"000003\"")); } @Test @@ -342,9 +359,10 @@ public void assignPostTagkDouble() throws Exception { "{\"tagk\":[\"host\",\"fqdn\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals( - "{\"tagk\":{\"fqdn\":\"000003\",\"host\":\"000001\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"fqdn\":\"000003\"")); + assertTrue(json.contains("\"host\":\"000001\"")); } public void assignPostTagkSingleBad() throws Exception { @@ -353,9 +371,10 @@ public void assignPostTagkSingleBad() throws Exception { "{\"tagk\":[\"datacenter\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals("{\"tagk_errors\":{\"datacenter\":\"Name already exists with " - + "UID: 000002\"},\"tagk\":{}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"datacenter\":\"Name already exists with " + + "UID: 000002\"")); } public void assignPostTagk2Good1Bad() throws Exception { @@ -364,9 +383,12 @@ public void assignPostTagk2Good1Bad() throws Exception { "{\"tagk\":[\"host\",\"datacenter\",\"fqdn\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals("{\"tagk_errors\":{\"datacenter\":\"Name already exists with " - + "UID: 000002\"},\"tagk\":{\"fqdn\":\"000003\",\"host\":\"000001\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("{\"datacenter\":\"Name already exists with " + + "UID: 000002\"}")); + assertTrue(json.contains("\"fqdn\":\"000003\"")); + assertTrue(json.contains("\"host\":\"000001\"")); } @Test @@ -386,9 +408,10 @@ public void assignPostTagvDouble() throws Exception { "{\"tagv\":[\"localhost\",\"foo\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals( - "{\"tagv\":{\"foo\":\"000003\",\"localhost\":\"000001\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"foo\":\"000003\"")); + assertTrue(json.contains("\"localhost\":\"000001\"")); } public void assignPostTagvSingleBad() throws Exception { @@ -397,9 +420,10 @@ public void assignPostTagvSingleBad() throws Exception { "{\"tagv\":[\"myserver\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals("{\"tagv\":{},\"tagv_errors\":{\"myserver\":\"Name already " - + "exists with UID: 000002\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"tagv_errors\":{\"myserver\":\"Name already " + + "exists with UID: 000002\"}")); } public void assignPostTagv2Good1Bad() throws Exception { @@ -408,10 +432,12 @@ public void assignPostTagv2Good1Bad() throws Exception { "{\"tagv\":[\"localhost\",\"myserver\",\"foo\"]}"); this.rpc.execute(tsdb, query); assertEquals(HttpResponseStatus.OK, query.response().getStatus()); - assertEquals("{\"tagv\":{\"foo\":\"000003\",\"localhost\":\"000001\"}," - + "\"tagv_errors\":{\"myserver\":\"Name already exists with " - + "UID: 000002\"}}", - query.response().getContent().toString(Charset.forName("UTF-8"))); + final String json = query.response().getContent() + .toString(Charset.forName("UTF-8")); + assertTrue(json.contains("\"foo\":\"000003\"")); + assertTrue(json.contains("\"localhost\":\"000001\"")); + assertTrue(json.contains("\"tagv_errors\":{\"myserver\":\"Name already exists with " + + "UID: 000002\"}")); } @Test From 050d507cb8db95dee794e6f92d317fc19ab43b71 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 1 May 2016 13:09:19 -0700 Subject: [PATCH 45/68] Fix issue #784 by adding an estimate for the number of data points from storage. Also fix the average calculation when salting is not enabled. --- src/core/SaltScanner.java | 73 ++++++++++++++++++++++++++++++++++++-- src/core/TsdbQuery.java | 74 +++++++++++++++++++++++++++++++++++++-- src/stats/QueryStats.java | 11 +++--- 3 files changed, 149 insertions(+), 9 deletions(-) diff --git a/src/core/SaltScanner.java b/src/core/SaltScanner.java index 9498284e00..3a929cd2af 100644 --- a/src/core/SaltScanner.java +++ b/src/core/SaltScanner.java @@ -316,6 +316,8 @@ final class ScannerCB implements Callback> rows) final List> lookups = filters != null && !filters.isEmpty() ? new ArrayList>(rows.size()) : null; - + + rows_pre_filter += rows.size(); for (final ArrayList row : rows) { final byte[] key = row.get(0).key(); if (RowKey.rowKeyContainsMetric(metric, key) != 0) { @@ -393,6 +396,36 @@ public Object call(final ArrayList> rows) return null; } + // calculate estimated data point count. We don't want to deserialize + // the byte arrays so we'll just get a rough estimate of compacted + // columns. + for (final KeyValue kv : row) { + if (kv.qualifier().length % 2 == 0) { + if (kv.qualifier().length == 2 || kv.qualifier().length == 4) { + ++dps_pre_filter; + } else { + // for now we'll assume that all compacted columns are of the + // same precision. This is likely incorrect. + if (Internal.inMilliseconds(kv.qualifier())) { + dps_pre_filter += (kv.qualifier().length / 4); + } else { + dps_pre_filter += (kv.qualifier().length / 2); + } + } + } else if (kv.qualifier()[0] == AppendDataPoints.APPEND_COLUMN_PREFIX) { + // with appends we don't have a good rough estimate as the length + // can vary widely with the value length variability. Therefore we + // have to iterate. + int idx = 0; + int qlength = 0; + while (idx < kv.value().length) { + qlength = Internal.getQualifierLength(kv.value(), idx); + idx += qlength + Internal.getValueLengthFromQualifier(kv.value(), idx); + ++dps_pre_filter; + } + } + } + // If any filters have made it this far then we need to resolve // the row key UIDs to their names for string comparison. We'll // try to avoid the resolution with some sets but we may dupe @@ -486,6 +519,7 @@ public Object call(final ArrayList group) throws Exception { * @param row The row to add */ void processRow(final byte[] key, final ArrayList row) { + ++rows_post_filter; if (delete) { final DeleteRequest del = new DeleteRequest(tsdb.dataTable(), key); tsdb.getClient().delete(del); @@ -496,6 +530,36 @@ void processRow(final byte[] key, final ArrayList row) { notes = new ArrayList(); annotations.put(key, notes); } + + // calculate estimated data point count. We don't want to deserialize + // the byte arrays so we'll just get a rough estimate of compacted + // columns. + for (final KeyValue kv : row) { + if (kv.qualifier().length % 2 == 0) { + if (kv.qualifier().length == 2 || kv.qualifier().length == 4) { + ++dps_post_filter; + } else { + // for now we'll assume that all compacted columns are of the + // same precision. This is likely incorrect. + if (Internal.inMilliseconds(kv.qualifier())) { + dps_post_filter += (kv.qualifier().length / 4); + } else { + dps_post_filter += (kv.qualifier().length / 2); + } + } + } else if (kv.qualifier()[0] == AppendDataPoints.APPEND_COLUMN_PREFIX) { + // with appends we don't have a good rough estimate as the length + // can vary widely with the value length variability. Therefore we + // have to iterate. + int idx = 0; + int qlength = 0; + while (idx < kv.value().length) { + qlength = Internal.getQualifierLength(kv.value(), idx); + idx += qlength + Internal.getValueLengthFromQualifier(kv.value(), idx); + ++dps_post_filter; + } + } + } final KeyValue compacted; // let IllegalDataExceptions bubble up so the handler above can close @@ -542,11 +606,14 @@ void close(final boolean ok) { QueryStat.SUCCESSFUL_SCAN, ok ? 1 : 0); // Post Scan stats - /* TODO - fix up/add these counters + query_stats.addScannerStat(query_index, index, + QueryStat.ROWS_PRE_FILTER, rows_pre_filter); + query_stats.addScannerStat(query_index, index, + QueryStat.DPS_PRE_FILTER, dps_pre_filter); query_stats.addScannerStat(query_index, index, QueryStat.ROWS_POST_FILTER, rows_post_filter); query_stats.addScannerStat(query_index, index, - QueryStat.DPS_POST_FILTER, dps_post_filter); */ + QueryStat.DPS_POST_FILTER, dps_post_filter); query_stats.addScannerStat(query_index, index, QueryStat.SCANNER_UID_TO_STRING_TIME, uid_resolve_time); query_stats.addScannerStat(query_index, index, diff --git a/src/core/TsdbQuery.java b/src/core/TsdbQuery.java index 257d3cf018..0b64ebe5c6 100644 --- a/src/core/TsdbQuery.java +++ b/src/core/TsdbQuery.java @@ -583,6 +583,10 @@ final class ScannerCB implements Callback> rows) throw new InterruptedException("Query timeout exceeded!"); } + rows_pre_filter += rows.size(); + // used for UID resolution if a filter is involved final List> lookups = filters != null && !filters.isEmpty() ? @@ -645,6 +651,36 @@ public Object call(final ArrayList> rows) + " with " + Arrays.toString(metric)); } + // calculate estimated data point count. We don't want to deserialize + // the byte arrays so we'll just get a rough estimate of compacted + // columns. + for (final KeyValue kv : row) { + if (kv.qualifier().length % 2 == 0) { + if (kv.qualifier().length == 2 || kv.qualifier().length == 4) { + ++dps_pre_filter; + } else { + // for now we'll assume that all compacted columns are of the + // same precision. This is likely incorrect. + if (Internal.inMilliseconds(kv.qualifier())) { + dps_pre_filter += (kv.qualifier().length / 4); + } else { + dps_pre_filter += (kv.qualifier().length / 2); + } + } + } else if (kv.qualifier()[0] == AppendDataPoints.APPEND_COLUMN_PREFIX) { + // with appends we don't have a good rough estimate as the length + // can vary widely with the value length variability. Therefore we + // have to iterate. + int idx = 0; + int qlength = 0; + while (idx < kv.value().length) { + qlength = Internal.getQualifierLength(kv.value(), idx); + idx += qlength + Internal.getValueLengthFromQualifier(kv.value(), idx); + ++dps_pre_filter; + } + } + } + // If any filters have made it this far then we need to resolve // the row key UIDs to their names for string comparison. We'll // try to avoid the resolution with some sets but we may dupe @@ -735,11 +771,42 @@ public Object call(final ArrayList group) throws Exception { * @param row The row to add */ void processRow(final byte[] key, final ArrayList row) { + ++rows_post_filter; if (delete) { final DeleteRequest del = new DeleteRequest(tsdb.dataTable(), key); tsdb.getClient().delete(del); } + // calculate estimated data point count. We don't want to deserialize + // the byte arrays so we'll just get a rough estimate of compacted + // columns. + for (final KeyValue kv : row) { + if (kv.qualifier().length % 2 == 0) { + if (kv.qualifier().length == 2 || kv.qualifier().length == 4) { + ++dps_post_filter; + } else { + // for now we'll assume that all compacted columns are of the + // same precision. This is likely incorrect. + if (Internal.inMilliseconds(kv.qualifier())) { + dps_post_filter += (kv.qualifier().length / 4); + } else { + dps_post_filter += (kv.qualifier().length / 2); + } + } + } else if (kv.qualifier()[0] == AppendDataPoints.APPEND_COLUMN_PREFIX) { + // with appends we don't have a good rough estimate as the length + // can vary widely with the value length variability. Therefore we + // have to iterate. + int idx = 0; + int qlength = 0; + while (idx < kv.value().length) { + qlength = Internal.getQualifierLength(kv.value(), idx); + idx += qlength + Internal.getValueLengthFromQualifier(kv.value(), idx); + ++dps_post_filter; + } + } + } + Span datapoints = spans.get(key); if (datapoints == null) { datapoints = new Span(tsdb); @@ -777,11 +844,14 @@ void close(final Exception e) { QueryStat.SUCCESSFUL_SCAN, e == null ? 1 : 0); // Post Scan stats - /* TODO - fix up/add these counters + query_stats.addScannerStat(query_index, index, + QueryStat.ROWS_PRE_FILTER, rows_pre_filter); + query_stats.addScannerStat(query_index, index, + QueryStat.DPS_PRE_FILTER, dps_pre_filter); query_stats.addScannerStat(query_index, index, QueryStat.ROWS_POST_FILTER, rows_post_filter); query_stats.addScannerStat(query_index, index, - QueryStat.DPS_POST_FILTER, dps_post_filter); */ + QueryStat.DPS_POST_FILTER, dps_post_filter); query_stats.addScannerStat(query_index, index, QueryStat.SCANNER_UID_TO_STRING_TIME, uid_resolve_time); query_stats.addScannerStat(query_index, index, diff --git a/src/stats/QueryStats.java b/src/stats/QueryStats.java index 757a9bdca9..1bf23c5d4e 100644 --- a/src/stats/QueryStats.java +++ b/src/stats/QueryStats.java @@ -140,6 +140,8 @@ public enum QueryStat { SUCCESSFUL_SCAN ("successfulScan", false), // Single Scanner stats + DPS_PRE_FILTER ("dpsPreFilter", false), + ROWS_PRE_FILTER ("rowsPreFilter", false), DPS_POST_FILTER ("dpsPostFilter", false), ROWS_POST_FILTER ("rowsPostFilter", false), SCANNER_UID_TO_STRING_TIME ("scannerUidToStringTime", true), @@ -535,7 +537,8 @@ public void aggQueryStats() { final Pair names = AGG_MAP.get(cumulation.getKey()); addStat(names.getKey(), (cumulation.getValue().getKey() / - (scanner_stats.size() * Const.SALT_BUCKETS()))); + (scanner_stats.size() * + Const.SALT_WIDTH() > 0 ? Const.SALT_BUCKETS() : 1))); addStat(names.getValue(), cumulation.getValue().getValue()); } overall_cumulations.clear(); @@ -610,8 +613,8 @@ public void addScannerStat(final int query_index, final int id, final QueryStat name, final long value) { Map> qs = scanner_stats.get(query_index); if (qs == null) { - qs = new ConcurrentHashMap>(Const.SALT_BUCKETS()); + qs = new ConcurrentHashMap>( + Const.SALT_WIDTH() > 0 ? Const.SALT_BUCKETS() : 1); scanner_stats.put(query_index, qs); } Map scanner_stat_map = qs.get(id); @@ -633,7 +636,7 @@ public void addScannerServers(final int query_index, final int id, Map> query_servers = scanner_servers.get(query_index); if (query_servers == null) { query_servers = new ConcurrentHashMap>( - Const.SALT_BUCKETS()); + Const.SALT_WIDTH() > 0 ? Const.SALT_BUCKETS() : 1); scanner_servers.put(query_index, query_servers); } query_servers.put(id, servers); From 6d011d526ffac598273f5544fa14411d9ae23f3c Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 1 May 2016 13:36:44 -0700 Subject: [PATCH 46/68] f --- test/tsd/TestUniqueIdRpc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tsd/TestUniqueIdRpc.java b/test/tsd/TestUniqueIdRpc.java index 41836ae928..266e11dd0d 100644 --- a/test/tsd/TestUniqueIdRpc.java +++ b/test/tsd/TestUniqueIdRpc.java @@ -226,7 +226,7 @@ public void assignQsTagvSingleBad() throws Exception { assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus()); final String json = query.response().getContent() .toString(Charset.forName("UTF-8")); - assertTrue(json.contains("{\"tagv_errors\":{\"myserver\":\"Name already " + assertTrue(json.contains("\"tagv_errors\":{\"myserver\":\"Name already " + "exists with UID: 000002\"}")); } From 7d2f7a996b53c483d5ee9c31a71eb94337195902 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 17 Sep 2016 15:19:26 -0700 Subject: [PATCH 47/68] Move to AsyncHBase 1.7.2 --- third_party/hbase/asynchbase-1.7.0.jar.md5 | 1 - third_party/hbase/asynchbase-1.7.2.jar.md5 | 1 + third_party/hbase/include.mk | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 third_party/hbase/asynchbase-1.7.0.jar.md5 create mode 100644 third_party/hbase/asynchbase-1.7.2.jar.md5 diff --git a/third_party/hbase/asynchbase-1.7.0.jar.md5 b/third_party/hbase/asynchbase-1.7.0.jar.md5 deleted file mode 100644 index 5cb6466209..0000000000 --- a/third_party/hbase/asynchbase-1.7.0.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -f1aed41b7f16345d2f58797ffa77f36a \ No newline at end of file diff --git a/third_party/hbase/asynchbase-1.7.2.jar.md5 b/third_party/hbase/asynchbase-1.7.2.jar.md5 new file mode 100644 index 0000000000..df83960284 --- /dev/null +++ b/third_party/hbase/asynchbase-1.7.2.jar.md5 @@ -0,0 +1 @@ +35fdde5a8e6009553e6aab5357ed8896 \ No newline at end of file diff --git a/third_party/hbase/include.mk b/third_party/hbase/include.mk index bb25220311..7cb693bb8a 100644 --- a/third_party/hbase/include.mk +++ b/third_party/hbase/include.mk @@ -13,7 +13,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see . -ASYNCHBASE_VERSION := 1.7.1 +ASYNCHBASE_VERSION := 1.7.2 ASYNCHBASE := third_party/hbase/asynchbase-$(ASYNCHBASE_VERSION).jar ASYNCHBASE_BASE_URL := http://central.maven.org/maven2/org/hbase/asynchbase/$(ASYNCHBASE_VERSION) From ef35ae24e205e868beaf64edc446051a835898ea Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 31 Dec 2016 13:28:57 -0800 Subject: [PATCH 48/68] Modify the config loads for the RPCManager class to use the config's getBoolean function to parse various types of boolean setting flags. --- src/tsd/RpcManager.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/tsd/RpcManager.java b/src/tsd/RpcManager.java index 86638c1a12..0e4046c993 100644 --- a/src/tsd/RpcManager.java +++ b/src/tsd/RpcManager.java @@ -252,11 +252,12 @@ private void initializeBuiltinRpcs(final String mode, final ImmutableMap.Builder telnet, final ImmutableMap.Builder http) { - final Boolean enableApi = tsdb.getConfig().getString("tsd.core.enable_api").equals("true"); - final Boolean enableUi = tsdb.getConfig().getString("tsd.core.enable_ui").equals("true"); - final Boolean enableDieDieDie = tsdb.getConfig().getString("tsd.no_diediedie").equals("false"); + final boolean enableApi = tsdb.getConfig().getBoolean("tsd.core.enable_api"); + final boolean enableUi = tsdb.getConfig().getBoolean("tsd.core.enable_ui"); + final boolean enableDieDieDie = tsdb.getConfig().getBoolean("tsd.no_diediedie"); - LOG.info("Mode: {}, HTTP UI Enabled: {}, HTTP API Enabled: {}", mode, enableUi, enableApi); + LOG.info("Mode: {}, HTTP UI Enabled: {}, HTTP API Enabled: {}", mode, + enableUi, enableApi); if (mode.equals("rw") || mode.equals("wo")) { final PutDataPointRpc put = new PutDataPointRpc(); From 4728986e6ea3066cf5879fc1b25d0cec43b2aa82 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 28 Jan 2017 15:05:31 -0800 Subject: [PATCH 49/68] Avoid double computing the expressions for the /query/exp endpoint. Also make sure both versions of next() handle booleans. --- src/query/expression/ExpressionIterator.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/query/expression/ExpressionIterator.java b/src/query/expression/ExpressionIterator.java index d090382bfb..d7dcfececd 100644 --- a/src/query/expression/ExpressionIterator.java +++ b/src/query/expression/ExpressionIterator.java @@ -338,9 +338,9 @@ public ExpressionDataPoint[] next(final long timestamp) { } final Object output = expression.execute(context); if (output instanceof Double) { - result = (Double) expression.execute(context); + result = (Double) output; } else if (output instanceof Boolean) { - result = (((Boolean) expression.execute(context)) ? 1 : 0); + result = (((Boolean) output) ? 1 : 0); } else { throw new IllegalStateException("Expression returned a result of type: " + output.getClass().getName() + " for " + this); @@ -465,7 +465,15 @@ public void next(final int i) { } } } - result = (Double)expression.execute(context); + final Object output = expression.execute(context); + if (output instanceof Double) { + result = (Double) output; + } else if (output instanceof Boolean) { + result = (((Boolean) output) ? 1 : 0); + } else { + throw new IllegalStateException("Expression returned a result of type: " + + output.getClass().getName() + " for " + this); + } dps[i].reset(ts, result); } From cedd46ea7a0571f9c8780cfc2188cdd3d14edd18 Mon Sep 17 00:00:00 2001 From: goll Date: Wed, 1 Feb 2017 20:21:17 +0100 Subject: [PATCH 50/68] Bump javassist to 3.21.0-GA --- third_party/javassist/include.mk | 2 +- third_party/javassist/javassist-3.21.0-GA.jar.md5 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 third_party/javassist/javassist-3.21.0-GA.jar.md5 diff --git a/third_party/javassist/include.mk b/third_party/javassist/include.mk index 2df2cf6063..e639914f79 100644 --- a/third_party/javassist/include.mk +++ b/third_party/javassist/include.mk @@ -23,7 +23,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -JAVASSIST_VERSION := 3.18.1-GA +JAVASSIST_VERSION := 3.21.0-GA JAVASSIST := third_party/javassist/javassist-$(JAVASSIST_VERSION).jar JAVASSIST_BASE_URL := http://central.maven.org/maven2/org/javassist/javassist/$(JAVASSIST_VERSION) diff --git a/third_party/javassist/javassist-3.21.0-GA.jar.md5 b/third_party/javassist/javassist-3.21.0-GA.jar.md5 new file mode 100644 index 0000000000..fba94d1657 --- /dev/null +++ b/third_party/javassist/javassist-3.21.0-GA.jar.md5 @@ -0,0 +1 @@ +3dba2305f842c2891df0a0926e18bcfa \ No newline at end of file From 97bdd3c3c004f50ebfaeae37f9b781f3b80bcb1c Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sat, 11 Mar 2017 10:56:48 -0800 Subject: [PATCH 51/68] Fix #915 by simply copying the entire tools directory into the destination directory. Thanks @shyamraj242. --- Makefile.am | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index ab2a3fa505..1ae0a175e7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -601,11 +601,13 @@ install-data-tools: $(tsdb_DEPS) $(jar) destdatatoolsdir="$(DESTDIR)$(pkgdatadir)/tools" ; \ echo " $(mkdir_p) $$destdatatoolsdir"; \ $(mkdir_p) "$$destdatatoolsdir" || exit 1; \ - tools="$$tools $(top_srcdir)/tools/*" ; \ tools="$$tools $(top_srcdir)/src/create_table.sh" ; \ tools="$$tools $(top_srcdir)/src/upgrade_1to2.sh" ; \ echo " $(INSTALL_SCRIPT)" $$tools "$$destdatatoolsdir" ; \ - $(INSTALL_SCRIPT) $$tools "$$destdatatoolsdir" || exit 1; + $(INSTALL_SCRIPT) $$tools "$$destdatatoolsdir" || exit 1; \ + tools="-r $(top_srcdir)/tools/*" ; \ + echo " cp" $$tools "$$destdatatoolsdir" ; \ + cp $$tools "$$destdatatoolsdir" || exit 1; uninstall-data-tools: @$(NORMAL_UNINSTALL) From be2b995a1247f84f50d4664d529eb932ffa6216f Mon Sep 17 00:00:00 2001 From: Jagmeet Singh bali Date: Tue, 2 May 2017 18:13:52 +0530 Subject: [PATCH 52/68] Log query stats in case of Closed Channel also Signed-off-by: Chris Larsen --- src/tsd/AbstractHttpQuery.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/tsd/AbstractHttpQuery.java b/src/tsd/AbstractHttpQuery.java index 035404a5ad..b09bc8b8d7 100644 --- a/src/tsd/AbstractHttpQuery.java +++ b/src/tsd/AbstractHttpQuery.java @@ -386,6 +386,9 @@ public void notFound() { */ public void sendStatusOnly(final HttpResponseStatus status) { if (!chan.isConnected()) { + if(stats != null) { + stats.markSendFailed(); + } done(); return; } @@ -414,6 +417,9 @@ public void sendBuffer(final HttpResponseStatus status, final ChannelBuffer buf, final String contentType) { if (!chan.isConnected()) { + if(stats != null) { + stats.markSendFailed(); + } done(); return; } @@ -442,7 +448,10 @@ public void sendBuffer(final HttpResponseStatus status, private class SendSuccess implements ChannelFutureListener { @Override public void operationComplete(final ChannelFuture future) throws Exception { - stats.markSent(); + if(future.isSuccess()) { + stats.markSent();} + else + stats.markSendFailed(); } } From e7ccbfde04cd9a5daffa8d076e31055ef5395418 Mon Sep 17 00:00:00 2001 From: Ioan Date: Wed, 12 Apr 2017 13:09:21 +0200 Subject: [PATCH 53/68] added java-8-oracle PATH in JDK_DIRS added /usr/lib/jvm/java-8-oracle in JDK_DIRS ...if JAVA_HOME is not defined in $DEFAULT Signed-off-by: Chris Larsen --- build-aux/deb/init.d/opentsdb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-aux/deb/init.d/opentsdb b/build-aux/deb/init.d/opentsdb index 4eb8ee3847..9e46e9037e 100644 --- a/build-aux/deb/init.d/opentsdb +++ b/build-aux/deb/init.d/opentsdb @@ -29,7 +29,8 @@ MAX_OPEN_FILES=65535 # The first existing directory is used for JAVA_HOME # (if JAVA_HOME is not defined in $DEFAULT) -JDK_DIRS="/usr/lib/jvm/java-7-oracle /usr/lib/jvm/java-7-openjdk \ +JDK_DIRS="/usr/lib/jvm/java-8-oracle \ + /usr/lib/jvm/java-7-oracle /usr/lib/jvm/java-7-openjdk \ /usr/lib/jvm/java-7-openjdk-amd64/ /usr/lib/jvm/java-7-openjdk-i386/ \ /usr/lib/jvm/java-6-sun /usr/lib/jvm/java-6-openjdk \ /usr/lib/jvm/java-6-openjdk-amd64 /usr/lib/jvm/java-6-openjdk-i386 \ From f31916ebeeca169d46dc3d3c19997357f3acfc48 Mon Sep 17 00:00:00 2001 From: Jason Harvey Date: Fri, 10 Mar 2017 23:18:40 -0900 Subject: [PATCH 54/68] Pass FAMILY to get(). Signed-off-by: Chris Larsen --- src/core/TSDB.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/TSDB.java b/src/core/TSDB.java index 57e591ef07..1a5683386f 100644 --- a/src/core/TSDB.java +++ b/src/core/TSDB.java @@ -1586,7 +1586,7 @@ final void scheduleForCompaction(final byte[] row, final int base_time) { /** Gets the entire given row from the data table. */ final Deferred> get(final byte[] key) { - return client.get(new GetRequest(table, key)); + return client.get(new GetRequest(table, key, FAMILY)); } /** Puts the given value into the data table. */ From 58e6c72f3721cf8f14d5c2aa9c07c4a1fd7ce0b9 Mon Sep 17 00:00:00 2001 From: Marcin Januszkiewicz Date: Mon, 23 Jan 2017 09:55:51 +0100 Subject: [PATCH 55/68] Fix spotted incorrect conditional. Fix #708 by using better comparisons The implemented Comparable interface was very inconsistent, leading to randomly sorted lists. --- src/tsd/client/MetricForm.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/tsd/client/MetricForm.java b/src/tsd/client/MetricForm.java index 0c2665416f..e273b51d97 100644 --- a/src/tsd/client/MetricForm.java +++ b/src/tsd/client/MetricForm.java @@ -475,7 +475,7 @@ private List getFilters(final boolean group_by) { if (filter.tagk.isEmpty() || filter.tagv.isEmpty()) { continue; } - if (filter.is_groupby = group_by) { + if (filter.is_groupby == group_by) { filters.add(filter); } } @@ -752,8 +752,12 @@ public int compareTo(final Filter filter) { if (filter == this) { return 0; } - return tagk.compareTo(filter.tagk) + - tagv.compareTo(filter.tagv); + int tagkv_order = tagk.compareTo(filter.tagk) == 0 ? + tagv.compareTo(filter.tagv) : tagk.compareTo(filter.tagk); + + int groupby_order = is_groupby == filter.is_groupby ? 0 : (is_groupby ? 1 : -1); + + return tagkv_order == 0 ? groupby_order : tagkv_order; } } From e918e8f69224f5f08a1265ac9812779a0319bcbb Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 11 Jun 2017 16:34:10 -0700 Subject: [PATCH 56/68] Fix #994 by filtering on the annotation start time when serializing so that we skip any that do not start within the query timespan. --- src/tsd/HttpJsonSerializer.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/tsd/HttpJsonSerializer.java b/src/tsd/HttpJsonSerializer.java index b29a834bab..675e66cb5c 100644 --- a/src/tsd/HttpJsonSerializer.java +++ b/src/tsd/HttpJsonSerializer.java @@ -32,6 +32,7 @@ import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; +import net.opentsdb.core.Const; import net.opentsdb.core.DataPoint; import net.opentsdb.core.DataPoints; import net.opentsdb.core.FillPolicy; @@ -733,6 +734,13 @@ public Object call(final ArrayList deferreds) throws Exception { Collections.sort(annotations); json.writeArrayFieldStart("annotations"); for (Annotation note : annotations) { + long ts = note.getStartTime(); + if (!((ts & Const.SECOND_MASK) != 0)) { + ts *= 1000; + } + if (ts < data_query.startTime() || ts > data_query.endTime()) { + continue; + } json.writeObject(note); } json.writeEndArray(); @@ -742,6 +750,13 @@ public Object call(final ArrayList deferreds) throws Exception { Collections.sort(globals); json.writeArrayFieldStart("globalAnnotations"); for (Annotation note : globals) { + long ts = note.getStartTime(); + if (!((ts & Const.SECOND_MASK) != 0)) { + ts *= 1000; + } + if (ts < data_query.startTime() || ts > data_query.endTime()) { + continue; + } json.writeObject(note); } json.writeEndArray(); From e7ff2fb49490ec1a3429a3edbcad728fe40e9854 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 11 Jun 2017 17:15:10 -0700 Subject: [PATCH 57/68] Fix #967 by changing the conversions to UTF-8 for the static byte methods in the UID class. This should properly decode the UTF strings now. Also remove the CHARSET from the UID class, use Const instead. --- src/tools/CliUtils.java | 21 ++++++++------------- src/tools/UidManager.java | 3 ++- src/uid/UniqueId.java | 15 ++++++--------- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/tools/CliUtils.java b/src/tools/CliUtils.java index f67c966dd4..17b74b0292 100644 --- a/src/tools/CliUtils.java +++ b/src/tools/CliUtils.java @@ -41,8 +41,6 @@ final class CliUtils { static final Method toBytes; /** Function used to convert a byte[] to a String. */ static final Method fromBytes; - /** Charset used to convert Strings to byte arrays and back. */ - static final Charset CHARSET; /** The single column family used by this class. */ static final byte[] ID_FAMILY; /** The single column family used by this class. */ @@ -58,9 +56,6 @@ final class CliUtils { // "THIS IS INTERNAL DO NOT USE". If only Java had C++'s "friend" or // a less stupid notion of a package. Field f; - f = uidclass.getDeclaredField("CHARSET"); - f.setAccessible(true); - CHARSET = (Charset) f.get(null); f = uidclass.getDeclaredField("ID_FAMILY"); f.setAccessible(true); ID_FAMILY = (byte[]) f.get(null); @@ -79,17 +74,17 @@ final class CliUtils { } } /** Qualifier for metrics meta data */ - static final byte[] METRICS_META = "metric_meta".getBytes(CHARSET); + static final byte[] METRICS_META = "metric_meta".getBytes(Const.ASCII_CHARSET); /** Qualifier for tagk meta data */ - static final byte[] TAGK_META = "tagk_meta".getBytes(CHARSET); + static final byte[] TAGK_META = "tagk_meta".getBytes(Const.ASCII_CHARSET); /** Qualifier for tagv meta data */ - static final byte[] TAGV_META = "tagv_meta".getBytes(CHARSET); + static final byte[] TAGV_META = "tagv_meta".getBytes(Const.ASCII_CHARSET); /** Qualifier for metrics UIDs */ - static final byte[] METRICS = "metrics".getBytes(CHARSET); + static final byte[] METRICS = "metrics".getBytes(Const.ASCII_CHARSET); /** Qualifier for tagk UIDs */ - static final byte[] TAGK = "tagk".getBytes(CHARSET); + static final byte[] TAGK = "tagk".getBytes(Const.ASCII_CHARSET); /** Qualifier for tagv UIDs */ - static final byte[] TAGV = "tagv".getBytes(CHARSET); + static final byte[] TAGV = "tagv".getBytes(Const.ASCII_CHARSET); /** * Returns the max metric ID from the UID table @@ -103,8 +98,8 @@ static long getMaxMetricID(final TSDB tsdb) { // first up, we need the max metric ID so we can split up the data table // amongst threads. final GetRequest get = new GetRequest(tsdb.uidTable(), new byte[] { 0 }); - get.family("id".getBytes(CHARSET)); - get.qualifier("metrics".getBytes(CHARSET)); + get.family("id".getBytes(Const.ASCII_CHARSET)); + get.qualifier("metrics".getBytes(Const.ASCII_CHARSET)); ArrayList row; try { row = tsdb.getClient().get(get).joinUninterruptibly(); diff --git a/src/tools/UidManager.java b/src/tools/UidManager.java index 5e4185d4f4..bcd01453f6 100644 --- a/src/tools/UidManager.java +++ b/src/tools/UidManager.java @@ -35,6 +35,7 @@ import org.hbase.async.PutRequest; import org.hbase.async.Scanner; +import net.opentsdb.core.Const; import net.opentsdb.core.TSDB; import net.opentsdb.meta.TSMeta; import net.opentsdb.uid.NoSuchUniqueId; @@ -300,7 +301,7 @@ private static int grep(final HBaseClient client, if (ignorecase) { regexp = "(?i)" + regexp; } - scanner.setKeyRegexp(regexp, CliUtils.CHARSET); + scanner.setKeyRegexp(regexp, Const.ASCII_CHARSET); boolean found = false; try { ArrayList> rows; diff --git a/src/uid/UniqueId.java b/src/uid/UniqueId.java index 22a384fc82..274ed04dee 100644 --- a/src/uid/UniqueId.java +++ b/src/uid/UniqueId.java @@ -12,7 +12,6 @@ // see . package net.opentsdb.uid; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -65,8 +64,6 @@ public enum UniqueIdType { TAGV } - /** Charset used to convert Strings to byte arrays and back. */ - private static final Charset CHARSET = Charset.forName("ISO-8859-1"); /** The single column family used by this class. */ private static final byte[] ID_FAMILY = toBytes("id"); /** The single column family used by this class. */ @@ -1260,11 +1257,11 @@ private void hbasePutWithRetry(final PutRequest put, short attempts, short wait) } private static byte[] toBytes(final String s) { - return s.getBytes(CHARSET); + return s.getBytes(Const.UTF8_CHARSET); } private static String fromBytes(final byte[] b) { - return new String(b, CHARSET); + return new String(b, Const.UTF8_CHARSET); } /** Returns a human readable string representation of the object. */ @@ -1588,21 +1585,21 @@ public Map call(final ArrayList row) // and the user hasn't put any metrics in, so log and return 0s LOG.info("Could not find the UID assignment row"); for (final byte[] kind : kinds) { - results.put(new String(kind, CHARSET), 0L); + results.put(new String(kind, Const.ASCII_CHARSET), 0L); } return results; } for (final KeyValue column : row) { - results.put(new String(column.qualifier(), CHARSET), + results.put(new String(column.qualifier(), Const.ASCII_CHARSET), Bytes.getLong(column.value())); } // if the user is starting with a fresh UID table, we need to account // for missing columns for (final byte[] kind : kinds) { - if (results.get(new String(kind, CHARSET)) == null) { - results.put(new String(kind, CHARSET), 0L); + if (results.get(new String(kind, Const.ASCII_CHARSET)) == null) { + results.put(new String(kind, Const.ASCII_CHARSET), 0L); } } return results; From c5379e3a4a09aad081c36c41ce9f2333493fe057 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 11 Jun 2017 21:02:03 -0700 Subject: [PATCH 58/68] Take a stab at fixing #953 by at least hunting for back-ticks before passing parameters to Gnuplot. Metrics and tags are already handled by the char list. Thanks @gsocgsoc --- src/tsd/GraphHandler.java | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/tsd/GraphHandler.java b/src/tsd/GraphHandler.java index 2669d6b8c6..206b5b1857 100644 --- a/src/tsd/GraphHandler.java +++ b/src/tsd/GraphHandler.java @@ -179,6 +179,15 @@ private void doGraph(final TSDB tsdb, final HttpQuery query) } else if (options.size() != tsdbqueries.length) { throw new BadRequestException(options.size() + " `o' parameters, but " + tsdbqueries.length + " `m' parameters."); + } else { + for (final String option : options) { + // TODO - far from perfect, should help a little. + if (option.contains("`") || option.contains("%60") || + option.contains("`")) { + throw new BadRequestException("Option contained a back-tick. " + + "That's a no-no."); + } + } } for (final Query tsdbquery : tsdbqueries) { try { @@ -627,6 +636,12 @@ private HashMap loadCachedJson(final HttpQuery query, static void setPlotDimensions(final HttpQuery query, final Plot plot) { final String wxh = query.getQueryStringParam("wxh"); if (wxh != null && !wxh.isEmpty()) { + // TODO - far from perfect, should help a little. + if (wxh.contains("`") || wxh.contains("%60") || + wxh.contains("`")) { + throw new BadRequestException("WXH contained a back-tick. " + + "That's a no-no."); + } final int wxhlength = wxh.length(); if (wxhlength < 7) { // 100x100 minimum. throw new BadRequestException("Parameter wxh too short: " + wxh); @@ -677,7 +692,14 @@ private static String popParam(final Map> querystring, if (params == null) { return null; } - return params.get(params.size() - 1); + final String given = params.get(params.size() - 1); + // TODO - far from perfect, should help a little. + if (given.contains("`") || given.contains("%60") || + given.contains("`")) { + throw new BadRequestException("Parameter " + param + " contained a " + + "back-tick. That's a no-no."); + } + return given; } /** From 11958abd8e976bd8c8160c07d42085c89fce2a24 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 11 Jun 2017 21:45:20 -0700 Subject: [PATCH 59/68] Bump AsyncHBase client to 1.8.0 for HBase 1.3.x compat. --- third_party/hbase/asynchbase-1.7.1-20151004.015637-1.jar.md5 | 1 - third_party/hbase/asynchbase-1.7.1.jar.md5 | 1 - third_party/hbase/asynchbase-1.8.0.jar.md5 | 1 + third_party/hbase/include.mk | 4 ++-- 4 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 third_party/hbase/asynchbase-1.7.1-20151004.015637-1.jar.md5 delete mode 100644 third_party/hbase/asynchbase-1.7.1.jar.md5 create mode 100644 third_party/hbase/asynchbase-1.8.0.jar.md5 diff --git a/third_party/hbase/asynchbase-1.7.1-20151004.015637-1.jar.md5 b/third_party/hbase/asynchbase-1.7.1-20151004.015637-1.jar.md5 deleted file mode 100644 index 75abc13db6..0000000000 --- a/third_party/hbase/asynchbase-1.7.1-20151004.015637-1.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -898d34a463b52e570addf0f0160add48 \ No newline at end of file diff --git a/third_party/hbase/asynchbase-1.7.1.jar.md5 b/third_party/hbase/asynchbase-1.7.1.jar.md5 deleted file mode 100644 index 45ad0e9669..0000000000 --- a/third_party/hbase/asynchbase-1.7.1.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -f236854721eac6d40b6710ec7d59f4a8 diff --git a/third_party/hbase/asynchbase-1.8.0.jar.md5 b/third_party/hbase/asynchbase-1.8.0.jar.md5 new file mode 100644 index 0000000000..daf07749fd --- /dev/null +++ b/third_party/hbase/asynchbase-1.8.0.jar.md5 @@ -0,0 +1 @@ +84ce0ce7f048a80755105f001352e14e diff --git a/third_party/hbase/include.mk b/third_party/hbase/include.mk index 7cb693bb8a..cb2b4f58c9 100644 --- a/third_party/hbase/include.mk +++ b/third_party/hbase/include.mk @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 The OpenTSDB Authors. +# Copyright (C) 2011-2017 The OpenTSDB Authors. # # This library is free software: you can redistribute it and/or modify it # under the terms of the GNU Lesser General Public License as published @@ -13,7 +13,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see . -ASYNCHBASE_VERSION := 1.7.2 +ASYNCHBASE_VERSION := 1.8.0 ASYNCHBASE := third_party/hbase/asynchbase-$(ASYNCHBASE_VERSION).jar ASYNCHBASE_BASE_URL := http://central.maven.org/maven2/org/hbase/asynchbase/$(ASYNCHBASE_VERSION) From 8e0ebf04c7624754125ab913bb4b27b447c08b0d Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Sun, 16 Jul 2017 12:16:38 -0700 Subject: [PATCH 60/68] Remove the "final" modifier from the meta classes (TSMeta, UIDMeta and Annotations) so that plugins can extend them as needed. This can be useful in the search plugin. --- src/meta/Annotation.java | 2 +- src/meta/TSMeta.java | 2 +- src/meta/UIDMeta.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/meta/Annotation.java b/src/meta/Annotation.java index b595ed4057..9fa255a976 100644 --- a/src/meta/Annotation.java +++ b/src/meta/Annotation.java @@ -75,7 +75,7 @@ @JsonAutoDetect(fieldVisibility = Visibility.PUBLIC_ONLY) @JsonInclude(Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) -public final class Annotation implements Comparable { +public class Annotation implements Comparable { private static final Logger LOG = LoggerFactory.getLogger(Annotation.class); /** Charset used to convert Strings to byte arrays and back. */ diff --git a/src/meta/TSMeta.java b/src/meta/TSMeta.java index 65e0030dd0..cefd86c02b 100644 --- a/src/meta/TSMeta.java +++ b/src/meta/TSMeta.java @@ -70,7 +70,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(Include.NON_NULL) @JsonAutoDetect(fieldVisibility = Visibility.PUBLIC_ONLY) -public final class TSMeta { +public class TSMeta { private static final Logger LOG = LoggerFactory.getLogger(TSMeta.class); /** Charset used to convert Strings to byte arrays and back. */ diff --git a/src/meta/UIDMeta.java b/src/meta/UIDMeta.java index 21f0e5470b..e96eb25aec 100644 --- a/src/meta/UIDMeta.java +++ b/src/meta/UIDMeta.java @@ -67,7 +67,7 @@ */ @JsonIgnoreProperties(ignoreUnknown = true) @JsonAutoDetect(fieldVisibility = Visibility.PUBLIC_ONLY) -public final class UIDMeta { +public class UIDMeta { private static final Logger LOG = LoggerFactory.getLogger(UIDMeta.class); /** Charset used to convert Strings to byte arrays and back. */ From 212d4f8e067272ec14be53da15dceb76eee9363e Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Thu, 8 Feb 2018 13:14:59 -0800 Subject: [PATCH 61/68] Try screwdriver. --- .travis.yml => .travis.yml_BKP | 0 screwdriver.yaml | 7 +++++++ 2 files changed, 7 insertions(+) rename .travis.yml => .travis.yml_BKP (100%) create mode 100644 screwdriver.yaml diff --git a/.travis.yml b/.travis.yml_BKP similarity index 100% rename from .travis.yml rename to .travis.yml_BKP diff --git a/screwdriver.yaml b/screwdriver.yaml new file mode 100644 index 0000000000..6218b8597d --- /dev/null +++ b/screwdriver.yaml @@ -0,0 +1,7 @@ +shared: + image: maven + +jobs: + main: + steps: + - run_arbitrary_script: mvn clean test From 6f380a56c3c989b9541e0af9d93b7ab59ad4936e Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Thu, 8 Feb 2018 13:16:55 -0800 Subject: [PATCH 62/68] Fix SD --- screwdriver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/screwdriver.yaml b/screwdriver.yaml index 6218b8597d..7b8befb526 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -4,4 +4,4 @@ shared: jobs: main: steps: - - run_arbitrary_script: mvn clean test + - run_arbitrary_script: /build.sh pom.xml && mvn clean test --quiet From 804ea588ed1c90512d3f1216bf2dbf5e8486c51f Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Thu, 8 Feb 2018 13:18:57 -0800 Subject: [PATCH 63/68] Doh, missed period --- screwdriver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/screwdriver.yaml b/screwdriver.yaml index 7b8befb526..5531f34326 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -4,4 +4,4 @@ shared: jobs: main: steps: - - run_arbitrary_script: /build.sh pom.xml && mvn clean test --quiet + - run_arbitrary_script: ./build.sh pom.xml && mvn clean test --quiet From 3ae0816579e383f39b87aaedf3513afac989d8da Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Thu, 8 Feb 2018 13:26:29 -0800 Subject: [PATCH 64/68] Install autoconf? --- screwdriver.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/screwdriver.yaml b/screwdriver.yaml index 5531f34326..ef222038b0 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -4,4 +4,5 @@ shared: jobs: main: steps: + - init: apt-get install autoconf - run_arbitrary_script: ./build.sh pom.xml && mvn clean test --quiet From 9d3cc9a3edab36495c55f5a14e76438ad7e5720f Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Thu, 8 Feb 2018 13:29:52 -0800 Subject: [PATCH 65/68] SD didn't like init --- screwdriver.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/screwdriver.yaml b/screwdriver.yaml index ef222038b0..d25f8c83d6 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -4,5 +4,4 @@ shared: jobs: main: steps: - - init: apt-get install autoconf - - run_arbitrary_script: ./build.sh pom.xml && mvn clean test --quiet + - run_arbitrary_script: apt-get install autoconf && ./build.sh pom.xml && mvn clean test --quiet From 6b174599f1ee0d41cc6fd955d54101754d0c1f46 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Thu, 8 Feb 2018 14:44:05 -0800 Subject: [PATCH 66/68] Now it sets up the docker --- screwdriver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/screwdriver.yaml b/screwdriver.yaml index d25f8c83d6..11a4352285 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -4,4 +4,4 @@ shared: jobs: main: steps: - - run_arbitrary_script: apt-get install autoconf && ./build.sh pom.xml && mvn clean test --quiet + - run_arbitrary_script: apt-get update && apt-get install autoconf make -y && ./build.sh pom.xml && mvn clean test --quiet From a19db84cbb2d93456f4625ec4c2e0a730c40729e Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Thu, 8 Feb 2018 16:52:16 -0800 Subject: [PATCH 67/68] Fail a ut --- test/core/TestSpan.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/core/TestSpan.java b/test/core/TestSpan.java index b9c1b668f3..717201982d 100644 --- a/test/core/TestSpan.java +++ b/test/core/TestSpan.java @@ -73,7 +73,12 @@ public void before() throws Exception { when(RowKey.metricNameAsync(tsdb, HOUR1)) .thenReturn(Deferred.fromResult("sys.cpu.user")); } - + +@Test +public void failme() { + assertTrue(false); +} + @Test public void addRow() { final byte[] qual1 = { 0x00, 0x07 }; From a7fd85c1767f3e9a29f49c639e264d25cfa4bc55 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Thu, 8 Feb 2018 18:53:19 -0800 Subject: [PATCH 68/68] Test a pr. --- .travis.yml_BKP | 9 --------- src/core/TSDB.java | 1 + 2 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 .travis.yml_BKP diff --git a/.travis.yml_BKP b/.travis.yml_BKP deleted file mode 100644 index 9354fb4ea2..0000000000 --- a/.travis.yml_BKP +++ /dev/null @@ -1,9 +0,0 @@ -language: java -before_script: ./build.sh pom.xml -script: export MAVEN_OPTS="-Xmx1024m" && mvn test --quiet -addons: - hostname: short-hostname -jdk: - - oraclejdk8 -notifications: - email: false diff --git a/src/core/TSDB.java b/src/core/TSDB.java index 1a5683386f..9b3a872ba2 100644 --- a/src/core/TSDB.java +++ b/src/core/TSDB.java @@ -782,6 +782,7 @@ public Histogram getScanLatencyHistogram() { */ private static void collectUidStats(final UniqueId uid, final StatsCollector collector) { +System.out.println("Just testing prs."); collector.record("uid.cache-hit", uid.cacheHits(), "kind=" + uid.kind()); collector.record("uid.cache-miss", uid.cacheMisses(), "kind=" + uid.kind()); collector.record("uid.cache-size", uid.cacheSize(), "kind=" + uid.kind());