From b06012d24dcb329a9d35bd9c629c0c1516f63cb1 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 4 Jun 2014 12:31:01 +0200 Subject: [PATCH 01/61] updates sbt-assembly to a version that does exist --- build.sbt | 2 +- project/plugins.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 426af60..02cb65d 100644 --- a/build.sbt +++ b/build.sbt @@ -14,6 +14,6 @@ scalaVersion := "2.9.1" assemblySettings -jarName in assembly <<= (version) { v => "sqltap_" + v + ".jar" } +jarName in assembly := { s"${name.value.toLowerCase}-${version.value}.jar" } fork in run := true diff --git a/project/plugins.sbt b/project/plugins.sbt index 9814732..9685838 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns) -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.8.5") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2") From e0fa875a55165f88ffd652c7500db8dc2bbc0930 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 16 Sep 2014 10:05:41 +0200 Subject: [PATCH 02/61] .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e08c794..311cc1e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ target/ .DS_Store +/bin From 6d2b92cd5822a246ae0ceea79beb802d4303bc51 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 16 Sep 2014 10:24:42 +0200 Subject: [PATCH 03/61] porting sqltap to Scala 2.10 --- build.sbt | 2 +- src/com/paulasmuth/sqltap/ExpirationJob.scala | 2 +- src/com/paulasmuth/sqltap/ReplicationFeed.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 02cb65d..90656dd 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") scalaSource in Compile <<= baseDirectory(_ / "src") -scalaVersion := "2.9.1" +scalaVersion := "2.10.4" assemblySettings diff --git a/src/com/paulasmuth/sqltap/ExpirationJob.scala b/src/com/paulasmuth/sqltap/ExpirationJob.scala index 1d8f04b..354ec50 100644 --- a/src/com/paulasmuth/sqltap/ExpirationJob.scala +++ b/src/com/paulasmuth/sqltap/ExpirationJob.scala @@ -45,7 +45,7 @@ class ExpirationJob(worker: Worker, ctree: CTree) extends ReadyCallback[Record] if (tuple._1 == primary_id) { val key = ctree.key(tuple._1, record_id.toString, tuple._2) handler.execute(worker, key) - keys = keys - tuple + keys = keys.filter(_ != tuple) } } } diff --git a/src/com/paulasmuth/sqltap/ReplicationFeed.scala b/src/com/paulasmuth/sqltap/ReplicationFeed.scala index 6f76440..52c988d 100644 --- a/src/com/paulasmuth/sqltap/ReplicationFeed.scala +++ b/src/com/paulasmuth/sqltap/ReplicationFeed.scala @@ -42,7 +42,7 @@ object ReplicationFeed extends Worker with AbstractSQLConnectionPool { } else { val row = query.rows.last val position = row.last.toInt - val filename = row.first + val filename = row.head conn.start_binlog(filename, position) } From b1612602d2a58d7ee8ddbf872e5aee3190e3e517 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Thu, 27 Aug 2015 13:21:02 +0200 Subject: [PATCH 04/61] mysql: adds TIMESTAMP2 table column type to TableMapBinlogEvent --- src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala b/src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala index 4c1f0c0..214bb45 100644 --- a/src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala +++ b/src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala @@ -46,6 +46,7 @@ class TableMapBinlogEvent(data: Array[Byte], ts: Long, fmt: FormatDescriptionBin case 0x0c => 0 // 0x0c DATETIME case 0x0d => 0 // 0x0d YEAR case 0x0f => read_int(2) // 0x0f VARCHAR + case 0x11 => 0 // 0x11 TIMESTAMP2 case 0x12 => 0 // 0x12 DATETIME2 case 0xf6 => read_int(2) // 0xf6 NEWDECIMAL case 0xfc => read_int(1) // 0xfc BLOB From 9328b3fb2efb8ea0f00911250d86ce4ed83f4abc Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 16 Sep 2014 10:24:42 +0200 Subject: [PATCH 05/61] porting sqltap to Scala 2.10 --- build.sbt | 2 +- src/com/paulasmuth/sqltap/ExpirationJob.scala | 2 +- src/com/paulasmuth/sqltap/ReplicationFeed.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index cad9983..1223be8 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") scalaSource in Compile <<= baseDirectory(_ / "src") -scalaVersion := "2.9.1" +scalaVersion := "2.10.4" assemblySettings diff --git a/src/com/paulasmuth/sqltap/ExpirationJob.scala b/src/com/paulasmuth/sqltap/ExpirationJob.scala index 1d8f04b..354ec50 100644 --- a/src/com/paulasmuth/sqltap/ExpirationJob.scala +++ b/src/com/paulasmuth/sqltap/ExpirationJob.scala @@ -45,7 +45,7 @@ class ExpirationJob(worker: Worker, ctree: CTree) extends ReadyCallback[Record] if (tuple._1 == primary_id) { val key = ctree.key(tuple._1, record_id.toString, tuple._2) handler.execute(worker, key) - keys = keys - tuple + keys = keys.filter(_ != tuple) } } } diff --git a/src/com/paulasmuth/sqltap/ReplicationFeed.scala b/src/com/paulasmuth/sqltap/ReplicationFeed.scala index 6f76440..52c988d 100644 --- a/src/com/paulasmuth/sqltap/ReplicationFeed.scala +++ b/src/com/paulasmuth/sqltap/ReplicationFeed.scala @@ -42,7 +42,7 @@ object ReplicationFeed extends Worker with AbstractSQLConnectionPool { } else { val row = query.rows.last val position = row.last.toInt - val filename = row.first + val filename = row.head conn.start_binlog(filename, position) } From 3b9b93bfbe95b3003727a27c185cc96ca962ba9b Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 28 Oct 2014 15:44:42 +0100 Subject: [PATCH 06/61] some patches --- patches/sqltap-encoding-fix.diff | 62 +++++++++++++++++ patches/sqltap-log-slow-queries.diff | 100 +++++++++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 patches/sqltap-encoding-fix.diff create mode 100644 patches/sqltap-log-slow-queries.diff diff --git a/patches/sqltap-encoding-fix.diff b/patches/sqltap-encoding-fix.diff new file mode 100644 index 0000000..5bb509c --- /dev/null +++ b/patches/sqltap-encoding-fix.diff @@ -0,0 +1,62 @@ +diff --git a/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala b/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala +index 2145f56..cf17011 100644 +--- a/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala ++++ b/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala +@@ -1,5 +1,5 @@ + // This file is part of the "SQLTap" project +-// (c) 2011-2013 Paul Asmuth ++// (c) 2014 Paul Asmuth, Google Inc. + // + // Licensed under the MIT License (the "License"); you may not use this + // file except in compliance with the License. You may obtain a copy of +@@ -30,8 +30,10 @@ object ExpirationHandlerFactory { + case "noop" => + handler = new NoopExpirationHandler() + +- case "purge" => ++ case "purge" => { + handler = new PurgeExpirationHandler() ++ ReplicationFeed.start() ++ } + + case _ => + throw new ParseException("unknown expiration handler: " + name) +diff --git a/src/com/paulasmuth/sqltap/HTTPParser.scala b/src/com/paulasmuth/sqltap/HTTPParser.scala +index 22b27be..33cd26d 100644 +--- a/src/com/paulasmuth/sqltap/HTTPParser.scala ++++ b/src/com/paulasmuth/sqltap/HTTPParser.scala +@@ -1,5 +1,5 @@ + // This file is part of the "SQLTap" project +-// (c) 2011-2013 Paul Asmuth ++// (c) 2014 Paul Asmuth, Google Inc. + // + // Licensed under the MIT License (the "License"); you may not use this + // file except in compliance with the License. You may obtain a copy of +@@ -130,7 +130,7 @@ class HTTPParser { + } + + def uri_parts() : List[String] = { +- val uri = URLDecoder.decode(http_uri, "UTF-8") ++ val uri = URLDecoder.decode(http_uri) + var pos = uri.length + var cur = pos - 1 + var ret = new ListBuffer[String]() +diff --git a/src/com/paulasmuth/sqltap/SQLTap.scala b/src/com/paulasmuth/sqltap/SQLTap.scala +index 357fc20..05c8921 100644 +--- a/src/com/paulasmuth/sqltap/SQLTap.scala ++++ b/src/com/paulasmuth/sqltap/SQLTap.scala +@@ -1,5 +1,5 @@ + // This file is part of the "SQLTap" project +-// (c) 2011-2013 Paul Asmuth ++// (c) 2014 Paul Asmuth, Google Inc. + // + // Licensed under the MIT License (the "License"); you may not use this + // file except in compliance with the License. You may obtain a copy of +@@ -108,7 +108,6 @@ object SQLTap{ + Manifest.load(new File(Config.get('config_base))) + RelationTrace.load(Manifest.resources) + ExpirationHandlerFactory.configure(Config.get('expiration_handler)) +- ReplicationFeed.start() + + val server = new Server(Config.get('threads).toInt) + server.run(Config.get('http_port).toInt) diff --git a/patches/sqltap-log-slow-queries.diff b/patches/sqltap-log-slow-queries.diff new file mode 100644 index 0000000..959f46b --- /dev/null +++ b/patches/sqltap-log-slow-queries.diff @@ -0,0 +1,100 @@ +diff -r -u sqltap/src/com/paulasmuth/sqltap/HTTPConnection.scala sqltap-0.7.21/src/com/paulasmuth/sqltap/HTTPConnection.scala +--- sqltap/src/com/paulasmuth/sqltap/HTTPConnection.scala 2014-02-12 16:13:05.009387526 +0100 ++++ sqltap-0.7.21/src/com/paulasmuth/sqltap/HTTPConnection.scala 2014-09-04 14:31:28.000000000 +0200 +@@ -1,10 +1,9 @@ + // This file is part of the "SQLTap" project +-// (c) 2011-2013 Paul Asmuth ++// (c) 2014 Paul Asmuth, Google Inc. + // + // Licensed under the MIT License (the "License"); you may not use this + // file except in compliance with the License. You may obtain a copy of + // the License at: http://opensource.org/licenses/MIT +- + package com.paulasmuth.sqltap + + import java.nio.channels.{SocketChannel,SelectionKey} +@@ -23,6 +22,7 @@ + private val parser = new HTTPParser() + private var state = HTTP_STATE_INIT + private var last_event : SelectionKey = null ++ private var last_uri : String = "" // for debugging only + private var keepalive : Boolean = false + private var resp_buf : ByteBuffer = null + +@@ -190,6 +190,7 @@ + + idle_timer.cancel() + stime = System.nanoTime ++ last_uri = parser.http_uri + seq += 1 + + if (parser.http_version == "1.1") +@@ -302,8 +303,13 @@ + } + + def finish() : Unit = { +- Statistics.incr('http_request_time_mean, +- (System.nanoTime - stime) / 1000000.0) ++ val runtime_millis = (System.nanoTime - stime) / 1000000.0 ++ Statistics.incr('http_request_time_mean, runtime_millis) ++ ++ if (Config.has_key('log_slow_queries) && ++ runtime_millis >= Config.get('log_slow_queries).toInt) { ++ Logger.log("[HTTP] [Slow Query] (" + runtime_millis + "ms): " + last_uri) ++ } + + if (!keepalive) + return close() +diff -r -u sqltap/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala sqltap-0.7.21/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala +--- sqltap/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala 2014-02-12 16:13:05.013389526 +0100 ++++ sqltap-0.7.21/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala 2014-09-04 14:31:38.000000000 +0200 +@@ -1,10 +1,9 @@ + // This file is part of the "SQLTap" project +-// (c) 2011-2013 Paul Asmuth ++// (c) 2014 Paul Asmuth, Google Inc. + // + // Licensed under the MIT License (the "License"); you may not use this + // file except in compliance with the License. You may obtain a copy of + // the License at: http://opensource.org/licenses/MIT +- + package com.paulasmuth.sqltap.mysql + + import com.paulasmuth.sqltap._ +@@ -44,8 +43,14 @@ + tok = System.nanoTime + qtime = tok - tik + +- Statistics.incr('sql_request_time_mean, qtime / 1000000.0) +- Logger.debug("Finished (" + (qtime / 1000000.0) + "ms): " + query) ++ val runtime_millis = qtime / 1000000.0 ++ Statistics.incr('sql_request_time_mean, runtime_millis) ++ Logger.debug("Finished (" + runtime_millis + "ms): " + query) ++ ++ if (Config.has_key('log_slow_queries) && ++ runtime_millis >= Config.get('log_slow_queries).toInt) { ++ Logger.log("[SQL] [Slow Query] (" + runtime_millis + "ms): " + query) ++ } + } + + def error(err: Throwable) : Unit = { +diff -r -u sqltap/src/com/paulasmuth/sqltap/SQLTap.scala sqltap-0.7.21/src/com/paulasmuth/sqltap/SQLTap.scala +--- sqltap/src/com/paulasmuth/sqltap/SQLTap.scala 2014-02-25 17:15:47.049982142 +0100 ++++ sqltap-0.7.21/src/com/paulasmuth/sqltap/SQLTap.scala 2014-09-04 14:35:51.000000000 +0200 +@@ -72,6 +71,9 @@ + else if (args(n) == "--disable-keepalive") + { Config.set('http_keepalive, "false"); n += 1 } + ++ else if (args(n) == "--log-slow-queries") ++ { Config.set('log_slow_queries, args(n+1)); n += 2 } ++ + else if ((args(n) == "-t") || (args(n) == "--threads")) + { Config.set('threads, args(n+1)); n += 2 } + +@@ -138,6 +139,7 @@ + println(" --memcache-queuelen max mysql queue size per worker ") + println(" --memcache-numconns max number of mysql connections per worker ") + println(" --memcache-mode replication mode (copy, shard) ") ++ println(" --log-slow-queries log all queries with a runtime > val in ms ") + println(" -h, --help you're reading it... ") + println(" -d, --debug debug mode ") + } From 4f292d789fc16e70c1c3035ba33cbc67f3b254b4 Mon Sep 17 00:00:00 2001 From: Cagdas Senol Date: Fri, 21 Nov 2014 17:59:23 +0100 Subject: [PATCH 07/61] Update to Scala 2.11.4 --- build.sbt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 1223be8..9250384 100644 --- a/build.sbt +++ b/build.sbt @@ -10,10 +10,12 @@ mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") scalaSource in Compile <<= baseDirectory(_ / "src") -scalaVersion := "2.10.4" +scalaVersion := "2.11.4" assemblySettings jarName in assembly := "sqltap_0.7.21.jar" fork in run := true + +libraryDependencies += "org.scala-lang.modules" %% "scala-xml" % "1.0.2" From b134c968f48b69df8d4433d44b58c3b627953884 Mon Sep 17 00:00:00 2001 From: Cagdas Senol Date: Fri, 21 Nov 2014 16:17:34 +0100 Subject: [PATCH 08/61] Parametrize memcache connection setting --- src/com/paulasmuth/sqltap/Config.scala | 2 ++ src/com/paulasmuth/sqltap/MemcacheConnection.scala | 5 +---- .../paulasmuth/sqltap/MemcacheConnectionPool.scala | 4 +++- src/com/paulasmuth/sqltap/SQLTap.scala | 12 ++++++++---- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/com/paulasmuth/sqltap/Config.scala b/src/com/paulasmuth/sqltap/Config.scala index 94bab39..3cc2b13 100644 --- a/src/com/paulasmuth/sqltap/Config.scala +++ b/src/com/paulasmuth/sqltap/Config.scala @@ -24,6 +24,8 @@ object Config { 'memcache_mode -> "copy", 'memcache_queue_max_len -> "4096", 'memcache_max_connections -> "10", + 'memcache_host -> "127.0.0.1", + 'memcache_port -> "11211", 'threads -> "4", 'expiration_handler -> "purge", 'cache_backend -> "memcache" diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 8c3c5f4..bc3b01a 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -12,10 +12,7 @@ import java.nio.channels.{SocketChannel,SelectionKey} import java.nio.{ByteBuffer,ByteOrder} import java.net.{InetSocketAddress,ConnectException} -class MemcacheConnection(pool: MemcacheConnectionPool) extends TimeoutCallback { - - var hostname : String = "127.0.0.1" - var port : Int = 11211 +class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : Int) extends TimeoutCallback { private val MC_STATE_INIT = 0 private val MC_STATE_CONN = 1 diff --git a/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala b/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala index 6c45c8f..12885f4 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala @@ -108,7 +108,9 @@ class MemcacheConnectionPool extends CacheBackend { } private def connect() : Unit = { - val conn = new MemcacheConnection(this) + val port: Int = Config.get('memcache_port).toInt + val host: String = Config.get('memcache_host) + val conn = new MemcacheConnection(this, host, port) conn.connect() connections += conn diff --git a/src/com/paulasmuth/sqltap/SQLTap.scala b/src/com/paulasmuth/sqltap/SQLTap.scala index 8f6477b..67aadf0 100644 --- a/src/com/paulasmuth/sqltap/SQLTap.scala +++ b/src/com/paulasmuth/sqltap/SQLTap.scala @@ -56,9 +56,6 @@ object SQLTap{ else if (args(n) == "--cache-backend") { Config.set('cache_backend, args(n+1)); n += 2 } - else if (args(n) == "--memcache-hosts") - { Config.set('memcache_hosts, args(n+1)); n += 2 } - else if (args(n) == "--memcache-mode") { Config.set('memcache_mode, args(n+1)); n += 2 } @@ -74,6 +71,12 @@ object SQLTap{ else if (args(n) == "--log-slow-queries") { Config.set('log_slow_queries, args(n+1)); n += 2 } + else if (args(n) == "--memcache-host") + { Config.set('memcache_host, args(n+1)); n += 2 } + + else if (args(n) == "--memcache-port") + { Config.set('memcache_port, args(n+1)); n += 2 } + else if ((args(n) == "-t") || (args(n) == "--threads")) { Config.set('threads, args(n+1)); n += 2 } @@ -135,7 +138,8 @@ object SQLTap{ println(" --mysql-numconns max number of mysql connections per worker ") println(" --expiration-handler expiration handler (noop, purge, refresh) ") println(" --cache-backend cache backend (memcache) ") - println(" --memcache-hosts comma-seperated memcache servers (host:port) ") + println(" --memcache-host memcache server host ") + println(" --memcache-port memcache server port ") println(" --memcache-queuelen max mysql queue size per worker ") println(" --memcache-numconns max number of mysql connections per worker ") println(" --memcache-mode replication mode (copy, shard) ") From 9f9fa5cfb4dad7b067fd1e3434ba30a77fe65c07 Mon Sep 17 00:00:00 2001 From: Cagdas Senol Date: Mon, 2 Feb 2015 17:33:30 +0100 Subject: [PATCH 09/61] Add TTL support per ctree --- src/com/paulasmuth/sqltap/CTree.scala | 1 + src/com/paulasmuth/sqltap/CTreeCache.scala | 2 +- src/com/paulasmuth/sqltap/CacheStoreRequest.scala | 2 +- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/com/paulasmuth/sqltap/CTree.scala b/src/com/paulasmuth/sqltap/CTree.scala index 2d2f941..8d4a626 100644 --- a/src/com/paulasmuth/sqltap/CTree.scala +++ b/src/com/paulasmuth/sqltap/CTree.scala @@ -12,6 +12,7 @@ class CTree(doc: xml.Node) { val name : String = elem.attr("name", true) val query : String = elem.attr("query", true) + val expire : Int = elem.attr("expire", false, "300").toInt val allow_conditions : Boolean = elem.attr("allow_conditions", false, "true").equals("true") diff --git a/src/com/paulasmuth/sqltap/CTreeCache.scala b/src/com/paulasmuth/sqltap/CTreeCache.scala index 4287492..234f58a 100644 --- a/src/com/paulasmuth/sqltap/CTreeCache.scala +++ b/src/com/paulasmuth/sqltap/CTreeCache.scala @@ -17,7 +17,7 @@ object CTreeCache { CTreeMarshal.serialize(ctree_buf, ctree.stack.head, ins) - val request = new CacheStoreRequest(key, buf) + val request = new CacheStoreRequest(key, buf, ctree.expire) request.worker = worker worker.cache.enqueue(request) diff --git a/src/com/paulasmuth/sqltap/CacheStoreRequest.scala b/src/com/paulasmuth/sqltap/CacheStoreRequest.scala index e39c009..578300c 100644 --- a/src/com/paulasmuth/sqltap/CacheStoreRequest.scala +++ b/src/com/paulasmuth/sqltap/CacheStoreRequest.scala @@ -9,7 +9,7 @@ package com.paulasmuth.sqltap import scala.collection.mutable.{ListBuffer} -class CacheStoreRequest(_key: String, _buf: ElasticBuffer) extends CacheRequest { +class CacheStoreRequest(_key: String, _buf: ElasticBuffer, val expire: Int) extends CacheRequest { val key : String = _key buffer = _buf diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index bc3b01a..8ab4ab9 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -98,7 +98,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.put(32.toByte) write_buf.put(48.toByte) write_buf.put(32.toByte) - write_buf.put(48.toByte) + write_buf.put(request.expire.toString.getBytes("UTF-8")) write_buf.put(32.toByte) write_buf.put(len.toString.getBytes("UTF-8")) write_buf.put(13.toByte) From bf870f58e34467d1481fe09190a9d16856d83e04 Mon Sep 17 00:00:00 2001 From: Tadas Sce Date: Mon, 4 May 2015 15:47:15 +0300 Subject: [PATCH 10/61] Escape backslashes when writing json --- src/com/paulasmuth/sqltap/JSONWriter.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/com/paulasmuth/sqltap/JSONWriter.scala b/src/com/paulasmuth/sqltap/JSONWriter.scala index 94b4502..2902650 100644 --- a/src/com/paulasmuth/sqltap/JSONWriter.scala +++ b/src/com/paulasmuth/sqltap/JSONWriter.scala @@ -81,10 +81,12 @@ class JSONWriter(buf: WrappedBuffer) { val b = byte & 0x000000ff if (b == 0xA) { - buf.write(Array(0x5C.toByte, 0x6E.toByte)) + buf.write(Array(0x5C.toByte, 0x6E.toByte)) // \n } else if (b == 0x22) { - buf.write(Array(0x5C.toByte, 0x22.toByte)) - } else if ((b == 0) || ((b >= 0x20) && (b != 0x5C))) { + buf.write(Array(0x5C.toByte, 0x22.toByte)) // \" + } else if (b == 0x5C) { + buf.write(Array(0x5C.toByte, 0x5C.toByte)) // \\ + } else if ((b == 0) || ((b >= 0x20))) { buf.write(byte) } } From 2ff2a4c2f64707292479e9a561ff25365346ed28 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 16 Sep 2014 10:05:41 +0200 Subject: [PATCH 11/61] .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e08c794..311cc1e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ target/ .DS_Store +/bin From 890163e61b3eaa28aff3a400beee850984897c54 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 4 Jun 2014 12:31:01 +0200 Subject: [PATCH 12/61] updates sbt-assembly to a version that does exist --- build.sbt | 2 +- project/plugins.sbt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 project/plugins.sbt diff --git a/build.sbt b/build.sbt index 9250384..fcd6c62 100644 --- a/build.sbt +++ b/build.sbt @@ -14,7 +14,7 @@ scalaVersion := "2.11.4" assemblySettings -jarName in assembly := "sqltap_0.7.21.jar" +jarName in assembly := { s"${name.value.toLowerCase}-${version.value}.jar" } fork in run := true diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 0000000..9685838 --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,3 @@ +resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns) + +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2") From a00e4a6b1803455054b5f45c458e386ccab29c96 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Thu, 1 Oct 2015 17:03:55 +0200 Subject: [PATCH 13/61] fixes memcache_connections_open / sql_connections_open statistics bug --- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 8ab4ab9..407615d 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -226,7 +226,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : pool.close(this) sock.close() - Statistics.decr('sql_connections_open) + Statistics.decr('memcache_connections_open) } def timeout() : Unit = { From 7a00108c881a9e71491e4a2de09e40bd05371d00 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Thu, 1 Oct 2015 17:04:37 +0200 Subject: [PATCH 14/61] symbolize literals to make the source code a little more readable --- .../sqltap/MemcacheConnection.scala | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 407615d..ef78fa9 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -14,6 +14,10 @@ import java.net.{InetSocketAddress,ConnectException} class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : Int) extends TimeoutCallback { + private val CR = 13 + private val LF = 10 + private val SP = 32 + private val MC_STATE_INIT = 0 private val MC_STATE_CONN = 1 private val MC_STATE_IDLE = 2 @@ -66,12 +70,12 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.put("get".getBytes) for (key <- keys) { - write_buf.put(32.toByte) + write_buf.put(SP.toByte) write_buf.put(key.getBytes("UTF-8")) } - write_buf.put(13.toByte) - write_buf.put(10.toByte) + write_buf.put(CR.toByte) + write_buf.put(LF.toByte) write_buf.flip state = MC_STATE_CMD_MGET @@ -93,19 +97,19 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.clear write_buf.put("set".getBytes) - write_buf.put(32.toByte) + write_buf.put(SP.toByte) write_buf.put(key.getBytes("UTF-8")) - write_buf.put(32.toByte) + write_buf.put(SP.toByte) write_buf.put(48.toByte) - write_buf.put(32.toByte) + write_buf.put(SP.toByte) write_buf.put(request.expire.toString.getBytes("UTF-8")) - write_buf.put(32.toByte) + write_buf.put(SP.toByte) write_buf.put(len.toString.getBytes("UTF-8")) - write_buf.put(13.toByte) - write_buf.put(10.toByte) + write_buf.put(CR.toByte) + write_buf.put(LF.toByte) write_buf.put(buf) - write_buf.put(13.toByte) - write_buf.put(10.toByte) + write_buf.put(CR.toByte) + write_buf.put(LF.toByte) write_buf.flip state = MC_STATE_CMD_SET @@ -120,10 +124,10 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.clear write_buf.put("delete".getBytes) - write_buf.put(32.toByte) + write_buf.put(SP.toByte) write_buf.put(key.getBytes("UTF-8")) - write_buf.put(13.toByte) - write_buf.put(10.toByte) + write_buf.put(CR.toByte) + write_buf.put(LF.toByte) write_buf.flip state = MC_STATE_CMD_DELETE @@ -171,7 +175,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : pos = cur } else { - if (read_buf.get(cur) == 10) { + if (read_buf.get(cur) == LF) { next(new String(read_buf.array, pos, cur - 1 - pos, "UTF-8")) pos = cur + 1 } From 36dc9c3b92110a662fd1aa5db6f30d25c892018f Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Thu, 1 Oct 2015 17:07:00 +0200 Subject: [PATCH 15/61] helps 48.toByte to be more readable, it's a '0' literal ;-) --- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index ef78fa9..7c05184 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -100,7 +100,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.put(SP.toByte) write_buf.put(key.getBytes("UTF-8")) write_buf.put(SP.toByte) - write_buf.put(48.toByte) + write_buf.put('0'.toByte) write_buf.put(SP.toByte) write_buf.put(request.expire.toString.getBytes("UTF-8")) write_buf.put(SP.toByte) From 610f3a18b6f4f45d30356936edca093ff831b301 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Fri, 2 Oct 2015 17:16:56 +0200 Subject: [PATCH 16/61] Fixes bug with too large memcached responses and its follow-up resource leakage --- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 7c05184..08f1643 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -27,8 +27,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : private val MC_STATE_READ = 7 private val MC_STATE_CLOSE = 8 - private val MC_WRITE_BUF_LEN = 65535 - private val MC_READ_BUF_LEN = (65535 * 8) + private val MC_WRITE_BUF_LEN = (65535 * 3) + private val MC_READ_BUF_LEN = (MC_WRITE_BUF_LEN * 8) private var state = MC_STATE_INIT private var last_event : SelectionKey = null @@ -87,6 +87,13 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : val len = buf.position request.ready() + + if (len > MC_WRITE_BUF_LEN - 512) { + // some hacky safe margin + // minus 512 because of the first command line + return; + } + buf.position(0) buf.limit(len) From b5ba51fd542f2874ffdd8e0b4e7e8729a500c902 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Fri, 2 Oct 2015 18:16:04 +0200 Subject: [PATCH 17/61] some personally helpful documentation --- .../sqltap/MemcacheConnection.scala | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 08f1643..d0a6974 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -40,8 +40,13 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : sock.configureBlocking(false) private var timer = TimeoutScheduler.schedule(1000, this) + private var requests : List[CacheRequest] = null + + // buffer for the current value to be received private var cur_buf : ElasticBuffer = null + + // length in bytes of the currently received value private var cur_len = 0 def connect() : Unit = { @@ -102,6 +107,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : timer.start() + // "set $key 0 $expiry $len\r\n$buf\r\n" write_buf.clear write_buf.put("set".getBytes) write_buf.put(SP.toByte) @@ -155,6 +161,12 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : idle(event) } + /** + * @brief Callback, invoked when underlyingsocket is non-blocking readable. + * + * Processes any incoming data, i.e. the response from the underlying + * memcached server. + */ def read(event: SelectionKey) : Unit = { val chunk = sock.read(read_buf) @@ -201,6 +213,13 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } } + /** + * @brief Callback, invoked when underlying connection is non-blocking + * writable. + * + * When all data has been flushed out to the memcached server, + * we will stop watching for WRITE events and switch back to READ. + */ def write(event: SelectionKey) : Unit = { try { sock.write(write_buf) @@ -287,6 +306,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : return idle(last_event) } + // expect ["VALUE", key, 0, dataLength] if (parts.length != 4) { throw new ExecutionException("[Memcache] protocol error: " + cmd) } From 60c309d72545ed5bd2520fd46aff5b01f576315c Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 10:20:58 +0200 Subject: [PATCH 18/61] documentation updates, def rearrangements, and minor code cleanups No logic has been changed. --- .../sqltap/MemcacheConnection.scala | 198 +++++++++--------- 1 file changed, 103 insertions(+), 95 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index d0a6974..3809442 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -12,20 +12,27 @@ import java.nio.channels.{SocketChannel,SelectionKey} import java.nio.{ByteBuffer,ByteOrder} import java.net.{InetSocketAddress,ConnectException} +/** + * @param pool pool that owns this connection + * @param hostname memcached-server's hostname + * @param port memcached-server's port number + */ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : Int) extends TimeoutCallback { - private val CR = 13 - private val LF = 10 - private val SP = 32 - - private val MC_STATE_INIT = 0 - private val MC_STATE_CONN = 1 - private val MC_STATE_IDLE = 2 - private val MC_STATE_CMD_DELETE = 4 - private val MC_STATE_CMD_SET = 5 - private val MC_STATE_CMD_MGET = 6 - private val MC_STATE_READ = 7 - private val MC_STATE_CLOSE = 8 + private val CR = 13.toByte + private val LF = 10.toByte + private val SP = 32.toByte + + // states: + // Uninitialized, Connecting, Idle, Cmd{Delete,Set,MGet}, Reading, Closed. + private val MC_STATE_INIT = 0 // just created, no connect() invoked yet (idle) + private val MC_STATE_CONN = 1 // just connected (idle) + private val MC_STATE_IDLE = 2 // idle (after a command) + private val MC_STATE_CMD_DELETE = 4 // + private val MC_STATE_CMD_SET = 5 // + private val MC_STATE_CMD_MGET = 6 // executing a multi-key GET + private val MC_STATE_READ = 7 // + private val MC_STATE_CLOSE = 8 // private val MC_WRITE_BUF_LEN = (65535 * 3) private val MC_READ_BUF_LEN = (MC_WRITE_BUF_LEN * 8) @@ -49,6 +56,11 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : // length in bytes of the currently received value private var cur_len = 0 + /** + * @brief Asynchronously establishes connection to the Memcached server. + * + * @see ready(event: SelectionKey) + */ def connect() : Unit = { Statistics.incr('memcache_connections_open) @@ -63,6 +75,42 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : .attach(this) } + /** + * @brief Callback, invoked upon non-blocking connect() completion. + * + * Moves MemcacheConnection state from MC_STATE_CONN into MC_STATE_IDLE state. + */ + def ready(event: SelectionKey) : Unit = { + try { + sock.finishConnect + } catch { + case e: ConnectException => { + Logger.error("[Memcache] connection failed: " + e.toString, false) + return close(e) + } + } + + idle(event) + } + + /** + * @brief Puts connection into ready state and then back into the idle pool. + */ + private def idle(event: SelectionKey) : Unit = { + timer.cancel() + state = MC_STATE_IDLE + event.interestOps(0) + last_event = event + requests = null + pool.ready(this) + } + + /** + * @brief Retrieves values for multiple keys. + * + * @param keys list of keys to retrieve + * @param _requests list of CacheRequest objects to store the values to + */ def execute_mget(keys: List[String], _requests: List[CacheRequest]) : Unit = { requests = _requests @@ -75,12 +123,12 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.put("get".getBytes) for (key <- keys) { - write_buf.put(SP.toByte) + write_buf.put(SP) write_buf.put(key.getBytes("UTF-8")) } - write_buf.put(CR.toByte) - write_buf.put(LF.toByte) + write_buf.put(CR) + write_buf.put(LF) write_buf.flip state = MC_STATE_CMD_MGET @@ -110,19 +158,19 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : // "set $key 0 $expiry $len\r\n$buf\r\n" write_buf.clear write_buf.put("set".getBytes) - write_buf.put(SP.toByte) + write_buf.put(SP) write_buf.put(key.getBytes("UTF-8")) - write_buf.put(SP.toByte) + write_buf.put(SP) write_buf.put('0'.toByte) - write_buf.put(SP.toByte) + write_buf.put(SP) write_buf.put(request.expire.toString.getBytes("UTF-8")) - write_buf.put(SP.toByte) + write_buf.put(SP) write_buf.put(len.toString.getBytes("UTF-8")) - write_buf.put(CR.toByte) - write_buf.put(LF.toByte) + write_buf.put(CR) + write_buf.put(LF) write_buf.put(buf) - write_buf.put(CR.toByte) - write_buf.put(LF.toByte) + write_buf.put(CR) + write_buf.put(LF) write_buf.flip state = MC_STATE_CMD_SET @@ -137,32 +185,18 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : write_buf.clear write_buf.put("delete".getBytes) - write_buf.put(SP.toByte) + write_buf.put(SP) write_buf.put(key.getBytes("UTF-8")) - write_buf.put(CR.toByte) - write_buf.put(LF.toByte) + write_buf.put(CR) + write_buf.put(LF) write_buf.flip state = MC_STATE_CMD_DELETE last_event.interestOps(SelectionKey.OP_WRITE) } - - def ready(event: SelectionKey) : Unit = { - try { - sock.finishConnect - } catch { - case e: ConnectException => { - Logger.error("[Memcache] connection failed: " + e.toString, false) - return close(e) - } - } - - idle(event) - } - /** - * @brief Callback, invoked when underlyingsocket is non-blocking readable. + * @brief Callback, invoked when underlying socket is non-blocking readable. * * Processes any incoming data, i.e. the response from the underlying * memcached server. @@ -195,7 +229,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : pos = cur } else { if (read_buf.get(cur) == LF) { - next(new String(read_buf.array, pos, cur - 1 - pos, "UTF-8")) + val headline = new String(read_buf.array, pos, cur - 1 - pos, "UTF-8") + next(headline) pos = cur + 1 } @@ -264,37 +299,29 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : close() } + /** @brief Retrieves the corresponding CacheRequest to the given @p key. + * + * @return never null but the CacheRequest object. + */ + private def get_request_by_key(key: String) : CacheRequest = { + for (req <- requests) { + if (req.buffer == null && req.key == key) { + return req + } + } + throw new ExecutionException("[Memcache] invalid response key: " + key) + } + private def next(cmd: String) : Unit = { state match { - - case MC_STATE_CMD_DELETE => { - cmd match { - - case "DELETED" => { - idle(last_event) - } - - case "NOT_FOUND" => { - idle(last_event) - } - - } + case MC_STATE_CMD_DELETE => cmd match { + case "DELETED" => idle(last_event) + case "NOT_FOUND" => idle(last_event) } - - case MC_STATE_CMD_SET => { - cmd match { - - case "STORED" => { - idle(last_event) - } - - case "NOT_STORED" => { - idle(last_event) - } - - } + case MC_STATE_CMD_SET => cmd match { + case "STORED" => idle(last_event) + case "NOT_STORED" => idle(last_event) } - case MC_STATE_CMD_MGET => { val parts = cmd.split(" ") @@ -311,36 +338,17 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : throw new ExecutionException("[Memcache] protocol error: " + cmd) } - for (req <- requests) { - if (req.buffer == null && req.key == parts(1)) { - val buf = new ElasticBuffer(65535 * 8) - req.buffer = buf + val req = get_request_by_key(parts(1)) + cur_buf = new ElasticBuffer(65535 * 8) + cur_len = parts(3).toInt + req.buffer = cur_buf - cur_buf = buf - cur_len = parts(3).toInt - - state = MC_STATE_READ - return - } - } + state = MC_STATE_READ } - case _ => { - throw new ExecutionException( - "unexpected token " + cmd + " (" + state.toString + ")") + throw new ExecutionException("unexpected token " + cmd + + " (" + state.toString + ")") } - } } - - private def idle(event: SelectionKey) : Unit = { - timer.cancel() - state = MC_STATE_IDLE - event.interestOps(0) - last_event = event - requests = null - pool.ready(this) - } - - } From e2e323b9249cd3cc668142c85e6c5d1ec2c5f0a1 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 10:34:43 +0200 Subject: [PATCH 19/61] code readability improvements --- .../sqltap/CacheBackendFactory.scala | 15 +++------------ .../paulasmuth/sqltap/MemcacheConnection.scala | 6 ++++++ .../sqltap/MemcacheConnectionPool.scala | 18 +++--------------- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/src/com/paulasmuth/sqltap/CacheBackendFactory.scala b/src/com/paulasmuth/sqltap/CacheBackendFactory.scala index ec883b6..c5fadc0 100644 --- a/src/com/paulasmuth/sqltap/CacheBackendFactory.scala +++ b/src/com/paulasmuth/sqltap/CacheBackendFactory.scala @@ -8,25 +8,16 @@ package com.paulasmuth.sqltap object CacheBackendFactory { - def get(worker: Worker) : CacheBackend = { val name = Config.get('cache_backend) val backend = name match { - - case "memcache" => - new MemcacheConnectionPool() - - case "noop" => - new NoopCacheBackend() - - case _ => - throw new ParseException("unknown cache backend: " + name) - + case "memcache" => new MemcacheConnectionPool() + case "noop" => new NoopCacheBackend() + case _ => throw new ParseException("unknown cache backend: " + name) } backend.loop = worker.loop backend } - } diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 3809442..4684bcb 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -112,6 +112,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : * @param _requests list of CacheRequest objects to store the values to */ def execute_mget(keys: List[String], _requests: List[CacheRequest]) : Unit = { + Logger.debug("[Memcache] mget: " + keys.mkString(", ")) + requests = _requests if (state != MC_STATE_IDLE) @@ -136,6 +138,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } def execute_set(key: String, request: CacheStoreRequest) : Unit = { + Logger.debug("[Memcache] store: " + key) + val buf = request.buffer.buffer val len = buf.position @@ -178,6 +182,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } def execute_delete(key: String) : Unit = { + Logger.debug("[Memcache] delete: " + key) + if (state != MC_STATE_IDLE) throw new ExecutionException("memcache connection busy") diff --git a/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala b/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala index 12885f4..0f42f4e 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala @@ -28,8 +28,7 @@ class MemcacheConnectionPool extends CacheBackend { if (queue.length >= max_queue_len) { requests.foreach(_.ready()) - Logger.exception( - new TemporaryException("memcache queue is full"), false) + Logger.exception(new TemporaryException("memcache queue is full"), false) return } @@ -91,7 +90,6 @@ class MemcacheConnectionPool extends CacheBackend { batch += req } - Logger.debug("[Memcache] mget: " + keys.mkString(", ")) conn.execute_mget(keys.toList, batch.toList) execute_next() @@ -118,18 +116,8 @@ class MemcacheConnectionPool extends CacheBackend { private def execute(connection: MemcacheConnection, req: CacheRequest) = { req match { - - case set: CacheStoreRequest => { - Logger.debug("[Memcache] store: " + req.key) - connection.execute_set(req.key, set) - } - - case purge: CachePurgeRequest => { - Logger.debug("[Memcache] delete: " + req.key) - connection.execute_delete(purge.key) - } - + case set: CacheStoreRequest => connection.execute_set(req.key, set) + case purge: CachePurgeRequest => connection.execute_delete(purge.key) } } - } From ce21b89ac6d7708f529dd4e65774396b636d63ac Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 13:23:43 +0200 Subject: [PATCH 20/61] documentation --- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 4684bcb..2324ea6 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -31,8 +31,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : private val MC_STATE_CMD_DELETE = 4 // private val MC_STATE_CMD_SET = 5 // private val MC_STATE_CMD_MGET = 6 // executing a multi-key GET - private val MC_STATE_READ = 7 // - private val MC_STATE_CLOSE = 8 // + private val MC_STATE_READ = 7 // reading an execute_mget value chunk + private val MC_STATE_CLOSE = 8 // connection closed private val MC_WRITE_BUF_LEN = (65535 * 3) private val MC_READ_BUF_LEN = (MC_WRITE_BUF_LEN * 8) @@ -318,6 +318,16 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : throw new ExecutionException("[Memcache] invalid response key: " + key) } + /** + * @brief Processes a command response. + * + * @param the first line of the response + * + * Usually commands have a response of only one command, thus, they'll + * directly transition the connection to the idle-state. + * + * Other commands (such as GET) may require reading more data. + */ private def next(cmd: String) : Unit = { state match { case MC_STATE_CMD_DELETE => cmd match { From 6f765c8bb1b6bca72b8f012dde6cb1bc1b56c87a Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 13:57:38 +0200 Subject: [PATCH 21/61] code cleanup: keep single-statement for-loops more consistent with what has been found in the code base already --- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 2324ea6..866edfd 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -283,9 +283,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : try { if (requests != null) { - for (req <- requests) { - req.ready() - } + requests.foreach(_.ready()) } } catch { case e: Exception => { @@ -342,10 +340,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : val parts = cmd.split(" ") if (parts.length == 1 && parts.head == "END") { - for (req <- requests) { - req.ready() - } - + requests.foreach(_.ready()) return idle(last_event) } From fad16b6672e885979197c7c808ca6071df0f189e Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 19:23:33 +0200 Subject: [PATCH 22/61] more documentationary --- .../sqltap/MemcacheConnection.scala | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 866edfd..1ec8c1b 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -18,7 +18,6 @@ import java.net.{InetSocketAddress,ConnectException} * @param port memcached-server's port number */ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : Int) extends TimeoutCallback { - private val CR = 13.toByte private val LF = 10.toByte private val SP = 32.toByte @@ -53,7 +52,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : // buffer for the current value to be received private var cur_buf : ElasticBuffer = null - // length in bytes of the currently received value + // length in bytes of the currently received value still to be processed. private var cur_len = 0 /** @@ -216,18 +215,20 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : return } - var cur = 0 - var pos = 0 + var cur = 0 //!< the offset in read_buf currently being parsed + var pos = 0 //!< offset of the first byte of the currently processed chunk (message line or value payload) while (cur < read_buf.position) { if (state == MC_STATE_READ) { + // process response body chunk, from pos to min(maxpos, pos + cur_len) cur = math.min(read_buf.position, pos + cur_len) cur_len -= cur - pos cur_buf.write(read_buf.array, pos, cur - pos) + // GET-value response chunk fully consumed? if (cur_len == 0) { - cur += 2 + cur += 2 // skip "\r\n" cur_buf.buffer.flip() state = MC_STATE_CMD_MGET } @@ -277,6 +278,11 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } } + /** + * @brief Closes this connection and notifies the owning pool about the close. + * + * @param err The exception that potentially caused the close. + */ def close(err: Throwable = null) : Unit = { if (state == MC_STATE_CLOSE) return @@ -298,6 +304,10 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : Statistics.decr('memcache_connections_open) } + /** @brief Callback, invoked upon I/O completion timeout. + * + * Closes the memcache connection. + */ def timeout() : Unit = { Logger.error("[Memcache] connection timed out...", false) close() @@ -350,8 +360,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } val req = get_request_by_key(parts(1)) - cur_buf = new ElasticBuffer(65535 * 8) cur_len = parts(3).toInt + cur_buf = new ElasticBuffer(65535 * 8) // FIXME why not of size cur_len? req.buffer = cur_buf state = MC_STATE_READ From c4a5c83974b8ddae249aaddd0720127173c9a31f Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 19:25:16 +0200 Subject: [PATCH 23/61] minor code-refactor --- src/com/paulasmuth/sqltap/MemcacheConnection.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 1ec8c1b..81d744b 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -318,12 +318,10 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : * @return never null but the CacheRequest object. */ private def get_request_by_key(key: String) : CacheRequest = { - for (req <- requests) { - if (req.buffer == null && req.key == key) { - return req - } + requests.find(r => r.key == key && r.buffer == null) match { + case Some(r) => r + case None => throw new ExecutionException("[Memcache] invalid response key: " + key) } - throw new ExecutionException("[Memcache] invalid response key: " + key) } /** From 793037ff4f1c3ea3ccb4059635a41f917ceef8cc Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 19:57:26 +0200 Subject: [PATCH 24/61] cleanup: remove unnecessary SelectionKey passing, as it is always the same. --- .../sqltap/MemcacheConnection.scala | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 81d744b..964a27d 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -69,9 +69,8 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : timer.start() - sock - .register(pool.loop, SelectionKey.OP_CONNECT) - .attach(this) + last_event = sock.register(pool.loop, SelectionKey.OP_CONNECT) + last_event.attach(this) } /** @@ -89,17 +88,16 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } } - idle(event) + idle() } /** * @brief Puts connection into ready state and then back into the idle pool. */ - private def idle(event: SelectionKey) : Unit = { + private def idle() : Unit = { timer.cancel() state = MC_STATE_IDLE - event.interestOps(0) - last_event = event + last_event.interestOps(0) requests = null pool.ready(this) } @@ -274,7 +272,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : if (write_buf.remaining == 0) { write_buf.clear - event.interestOps(SelectionKey.OP_READ) + last_event.interestOps(SelectionKey.OP_READ) } } @@ -337,19 +335,19 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : private def next(cmd: String) : Unit = { state match { case MC_STATE_CMD_DELETE => cmd match { - case "DELETED" => idle(last_event) - case "NOT_FOUND" => idle(last_event) + case "DELETED" => idle() + case "NOT_FOUND" => idle() } case MC_STATE_CMD_SET => cmd match { - case "STORED" => idle(last_event) - case "NOT_STORED" => idle(last_event) + case "STORED" => idle() + case "NOT_STORED" => idle() } case MC_STATE_CMD_MGET => { val parts = cmd.split(" ") if (parts.length == 1 && parts.head == "END") { requests.foreach(_.ready()) - return idle(last_event) + return idle() } // expect ["VALUE", key, 0, dataLength] From cbec21b845f4a58c52cb583e9d17d8daf5162a7e Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 21:34:03 +0200 Subject: [PATCH 25/61] bugfix for the infamous Memcached protocol error --- .../sqltap/MemcacheConnection.scala | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/com/paulasmuth/sqltap/MemcacheConnection.scala index 964a27d..c43829b 100644 --- a/src/com/paulasmuth/sqltap/MemcacheConnection.scala +++ b/src/com/paulasmuth/sqltap/MemcacheConnection.scala @@ -213,44 +213,45 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : return } - var cur = 0 //!< the offset in read_buf currently being parsed - var pos = 0 //!< offset of the first byte of the currently processed chunk (message line or value payload) - - while (cur < read_buf.position) { + while (read_buf.position > 0) { if (state == MC_STATE_READ) { // process response body chunk, from pos to min(maxpos, pos + cur_len) - cur = math.min(read_buf.position, pos + cur_len) + val cur_chunk_len = math.min(read_buf.position, cur_len) - cur_len -= cur - pos - cur_buf.write(read_buf.array, pos, cur - pos) + cur_len -= cur_chunk_len + cur_buf.write(read_buf.array, 0, cur_chunk_len) // GET-value response chunk fully consumed? if (cur_len == 0) { - cur += 2 // skip "\r\n" + cur_buf.retrieve.limit(cur_buf.retrieve.limit() - 2) cur_buf.buffer.flip() state = MC_STATE_CMD_MGET } - pos = cur + read_buf.limit(read_buf.position) + read_buf.position(cur_chunk_len) + read_buf.compact() } else { - if (read_buf.get(cur) == LF) { - val headline = new String(read_buf.array, pos, cur - 1 - pos, "UTF-8") - next(headline) - pos = cur + 1 + var found = false; + var i = 0; + + while (!found && i < read_buf.position) { + if (read_buf.get(i) == LF) { + val headline = new String(read_buf.array, 0, i - 1, "UTF-8") + read_buf.limit(read_buf.position) + read_buf.position(i + 1) + read_buf.compact() + next(headline) + found = true; + } + i = i + 1; } - cur += 1 + if (!found) { + return; + } } } - - if (cur < read_buf.position) { - println("READ REMAINING") - read_buf.limit(read_buf.position) - read_buf.position(cur) - read_buf.compact() - } else { - read_buf.clear() - } } /** @@ -356,7 +357,7 @@ class MemcacheConnection(pool: MemcacheConnectionPool, hostname : String, port : } val req = get_request_by_key(parts(1)) - cur_len = parts(3).toInt + cur_len = parts(3).toInt + 2 cur_buf = new ElasticBuffer(65535 * 8) // FIXME why not of size cur_len? req.buffer = cur_buf From e77af946cc5e8b3c70235aff1c5867b110a489c8 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 23:28:21 +0200 Subject: [PATCH 26/61] mute record-not-found log messages --- src/com/paulasmuth/sqltap/Worker.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/com/paulasmuth/sqltap/Worker.scala b/src/com/paulasmuth/sqltap/Worker.scala index 3b7a97b..40ae809 100644 --- a/src/com/paulasmuth/sqltap/Worker.scala +++ b/src/com/paulasmuth/sqltap/Worker.scala @@ -80,6 +80,11 @@ class Worker() extends Thread { conn.write(event) } catch { + case e: NotFoundException => { + // but do not log it, as it's a client side error we do not care + // about in the server side log + conn.close(e) + } case e: Exception => { Logger.error("[SQL] exception: " + e.toString, false) Logger.exception(e, false) From a45101a5a38829926d2cdb0ba0d93d14c610e2b3 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sun, 4 Oct 2015 23:28:40 +0200 Subject: [PATCH 27/61] version bump 0.8.0 --- build.sbt | 2 +- src/com/paulasmuth/sqltap/SQLTap.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index fcd6c62..bb30c99 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ name := "SQLTap" organization := "com.paulasmuth" -version := "0.7.21" +version := "0.8.0" mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") diff --git a/src/com/paulasmuth/sqltap/SQLTap.scala b/src/com/paulasmuth/sqltap/SQLTap.scala index 67aadf0..e6152b2 100644 --- a/src/com/paulasmuth/sqltap/SQLTap.scala +++ b/src/com/paulasmuth/sqltap/SQLTap.scala @@ -19,7 +19,7 @@ import java.io.File object SQLTap{ - val VERSION = "v0.7.21" + val VERSION = "v0.8.0" def main(args: Array[String]) : Unit = { var n = 0 From 80d6887f5f2e38e113b1c15c12b1ce455a4a3580 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 02:07:53 +0200 Subject: [PATCH 28/61] dead code cleanup --- src/com/paulasmuth/sqltap/StubCache.scala | 45 ----------------------- 1 file changed, 45 deletions(-) delete mode 100644 src/com/paulasmuth/sqltap/StubCache.scala diff --git a/src/com/paulasmuth/sqltap/StubCache.scala b/src/com/paulasmuth/sqltap/StubCache.scala deleted file mode 100644 index 7fdb2cc..0000000 --- a/src/com/paulasmuth/sqltap/StubCache.scala +++ /dev/null @@ -1,45 +0,0 @@ -// This file is part of the "SQLTap" project -// (c) 2011-2013 Paul Asmuth -// -// Licensed under the MIT License (the "License"); you may not use this -// file except in compliance with the License. You may obtain a copy of -// the License at: http://opensource.org/licenses/MIT - -package com.paulasmuth.sqltap - -import scala.collection.mutable.{HashMap} - -// STUB! -class StubCache extends CacheBackend { - - val stubcache = new HashMap[String,ElasticBuffer]() - - def connect() : Unit = () - - def execute(requests: List[CacheRequest]) = { - for (req <- requests) { - req match { - case get: CacheGetRequest => { - Logger.debug("[CACHE] retrieve: " + req.key) - stubcache.get(req.key) match { - case Some(buf: ElasticBuffer) => { - get.buffer = buf.clone() - } - case None => () - } - } - case set: CacheStoreRequest => { - Logger.debug("[CACHE] store: " + req.key) - stubcache.put(req.key, set.buffer) - } - case purge: CachePurgeRequest => { - Logger.debug("[CACHE] purge: " + req.key) - stubcache.remove(req.key) - } - } - - req.ready() - } - } - -} From 504f1ae80ffdcd54cec890ad7f5b857a7997df84 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 23:21:41 +0200 Subject: [PATCH 29/61] Improve logger api --- src/com/paulasmuth/sqltap/Logger.scala | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/com/paulasmuth/sqltap/Logger.scala b/src/com/paulasmuth/sqltap/Logger.scala index 0cf4469..6b003a6 100644 --- a/src/com/paulasmuth/sqltap/Logger.scala +++ b/src/com/paulasmuth/sqltap/Logger.scala @@ -19,6 +19,11 @@ object Logger { println("[" + df.format(new Date()) + "] " + msg) } + def fatal(msg: String) : Unit = { + log("[FATAL] " + msg) + System.exit(1) + } + def error(msg: String, fatal: Boolean) : Unit = { log("[ERROR] " + msg) @@ -26,6 +31,14 @@ object Logger { System.exit(1) } + def notice(msg: String) : Unit = { + log("[NOTICE] " + msg) + } + + def info(msg: String) : Unit = { + log("[INFO] " + msg) + } + def debug(msg: String) : Unit = { if (Config.debug) log("[DEBUG] " + msg) From f049da75a3c90cbe8e7d5f331a759a054617e887 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 23:21:52 +0200 Subject: [PATCH 30/61] fixes compiler warning --- src/com/paulasmuth/sqltap/GZIPTranscoder.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/com/paulasmuth/sqltap/GZIPTranscoder.scala b/src/com/paulasmuth/sqltap/GZIPTranscoder.scala index 0be3aa3..50a56a9 100644 --- a/src/com/paulasmuth/sqltap/GZIPTranscoder.scala +++ b/src/com/paulasmuth/sqltap/GZIPTranscoder.scala @@ -28,7 +28,7 @@ class GZIPTranscoder(buffer: ElasticBuffer) { buf.put(target.toByteArray()) } - def decode() : Unit = try { + def decode() : Unit = { val buf = buffer.retrieve() val source = new ByteArrayInputStream(buf.array.clone()) val gzip = new GZIPInputStream(source) @@ -50,7 +50,6 @@ class GZIPTranscoder(buffer: ElasticBuffer) { gzip.close source.close } - } From d234cfc4e2c12e6ef4ae8a1550716bc5d7bb66d3 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 23:22:29 +0200 Subject: [PATCH 31/61] code cleanup --- src/com/paulasmuth/sqltap/CTreeIndex.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/com/paulasmuth/sqltap/CTreeIndex.scala b/src/com/paulasmuth/sqltap/CTreeIndex.scala index e528950..1017313 100644 --- a/src/com/paulasmuth/sqltap/CTreeIndex.scala +++ b/src/com/paulasmuth/sqltap/CTreeIndex.scala @@ -36,11 +36,12 @@ object CTreeIndex { score += ctree.base_score Logger.debug("CTree: evaluating candidate: '" + ctree.name + - "' (score: " + score + ", cost: " + cost + ") for: " + root.resource_name) + "' (score: " + score + ", cost: " + cost + ") for: " + + root.resource_name) - var matches = (cost == 0 && winner_cost > 0) - matches ||= (score > top_score && winner_cost != 0) - matches ||= (score == top_score && cost > winner_cost) + val matches = (cost == 0 && winner_cost > 0) || + (score > top_score && winner_cost != 0) || + (score == top_score && cost > winner_cost) if (matches) { winner = ctree From 22869f1a6ae2310ee17761bccb81a8baaf8b3aa8 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 23:39:37 +0200 Subject: [PATCH 32/61] generalize sending simple text HTTP responses --- src/com/paulasmuth/sqltap/HTTPConnection.scala | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/com/paulasmuth/sqltap/HTTPConnection.scala b/src/com/paulasmuth/sqltap/HTTPConnection.scala index b146876..85ab50d 100644 --- a/src/com/paulasmuth/sqltap/HTTPConnection.scala +++ b/src/com/paulasmuth/sqltap/HTTPConnection.scala @@ -216,22 +216,28 @@ class HTTPConnection(sock: SocketChannel, worker: Worker) extends ReadyCallback[ Statistics.incr('http_requests_per_second) } - private def execute_ping() : Unit = { + private def execute_text(code: Integer, text: String) : Unit = { val http_buf = new HTTPWriter(buf) buf.clear - http_buf.write_status(200) - http_buf.write_content_length(6) + val body = text.getBytes + + http_buf.write_status(code) + http_buf.write_content_length(body.length) http_buf.write_default_headers() http_buf.finish_headers() - buf.put("pong\r\n".getBytes) + buf.put(body) buf.flip worker.requests_success.incrementAndGet() flush() } + private def execute_ping() : Unit = { + execute_text(200, "pong\r\n") + } + private def execute_request(params: List[String]) : Unit = { if (seq > 1) worker.requests_queued.incrementAndGet() From 57dedd08b3a85cfbbd5263ff099ec4935814954d Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 23:40:36 +0200 Subject: [PATCH 33/61] add the ability to optionally log Sqltap queries. CLI- and runtime configurable. --- src/com/paulasmuth/sqltap/Config.scala | 1 + src/com/paulasmuth/sqltap/HTTPConnection.scala | 10 ++++++++++ src/com/paulasmuth/sqltap/QueryParser.scala | 5 +++++ src/com/paulasmuth/sqltap/SQLTap.scala | 3 +++ 4 files changed, 19 insertions(+) diff --git a/src/com/paulasmuth/sqltap/Config.scala b/src/com/paulasmuth/sqltap/Config.scala index 3cc2b13..bb5f093 100644 --- a/src/com/paulasmuth/sqltap/Config.scala +++ b/src/com/paulasmuth/sqltap/Config.scala @@ -27,6 +27,7 @@ object Config { 'memcache_host -> "127.0.0.1", 'memcache_port -> "11211", 'threads -> "4", + 'log_queries -> "false", 'expiration_handler -> "purge", 'cache_backend -> "memcache" ) diff --git a/src/com/paulasmuth/sqltap/HTTPConnection.scala b/src/com/paulasmuth/sqltap/HTTPConnection.scala index 85ab50d..cdb282c 100644 --- a/src/com/paulasmuth/sqltap/HTTPConnection.scala +++ b/src/com/paulasmuth/sqltap/HTTPConnection.scala @@ -209,6 +209,16 @@ class HTTPConnection(sock: SocketChannel, worker: Worker) extends ReadyCallback[ else if (route.length > 1 && route.head == "expire") execute_expire(route.tail) + else if (route.length == 1 && route.head == "log_queries") { + Config.set('log_queries, "true") + execute_text(200, "sqltap query logging enabled\r\n") + } + + else if (route.length == 1 && route.head == "no_log_queries") { + Config.set('log_queries, "false") + execute_text(200, "sqltap query logging disabled\r\n") + } + else http_error(404, "not found") diff --git a/src/com/paulasmuth/sqltap/QueryParser.scala b/src/com/paulasmuth/sqltap/QueryParser.scala index 2fe6edc..b56459e 100644 --- a/src/com/paulasmuth/sqltap/QueryParser.scala +++ b/src/com/paulasmuth/sqltap/QueryParser.scala @@ -18,6 +18,11 @@ object QueryParser { private val PARSER_STATE_BODY = 5 def parse(stack: InstructionStack, qry: String) : Unit = { + + if (Config.get('log_queries).equals("true")) { + Logger.info("[QueryParser] parse: " + qry) + } + var args = new ListBuffer[String]() var state = PARSER_STATE_NEXT diff --git a/src/com/paulasmuth/sqltap/SQLTap.scala b/src/com/paulasmuth/sqltap/SQLTap.scala index e6152b2..b575d07 100644 --- a/src/com/paulasmuth/sqltap/SQLTap.scala +++ b/src/com/paulasmuth/sqltap/SQLTap.scala @@ -80,6 +80,9 @@ object SQLTap{ else if ((args(n) == "-t") || (args(n) == "--threads")) { Config.set('threads, args(n+1)); n += 2 } + else if (args(n) == "--log-queries") + { Config.set('log_queries, "true"); n += 1 } + else if ((args(n) == "-c") || (args(n) == "--config")) { Config.set('config_base, args(n+1)); n += 2 } From 12d6f47af0b0747b22079be4d0c61ae3286b32a8 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 5 Oct 2015 23:42:01 +0200 Subject: [PATCH 34/61] bump scala version to 2.11.7 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index bb30c99..d6783d9 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") scalaSource in Compile <<= baseDirectory(_ / "src") -scalaVersion := "2.11.4" +scalaVersion := "2.11.7" assemblySettings From bc7a0e427be3dfefdce6205d1d614ba8bc879b4e Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Thu, 8 Oct 2015 23:31:11 +0200 Subject: [PATCH 35/61] adds SQL slow query statistics --- src/com/paulasmuth/sqltap/Statistics.scala | 3 +++ src/com/paulasmuth/sqltap/mysql/SQLQuery.scala | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/com/paulasmuth/sqltap/Statistics.scala b/src/com/paulasmuth/sqltap/Statistics.scala index f943ef8..f795129 100644 --- a/src/com/paulasmuth/sqltap/Statistics.scala +++ b/src/com/paulasmuth/sqltap/Statistics.scala @@ -21,6 +21,9 @@ object Statistics { 'sql_requests_total -> new IntegralStatistic, 'sql_requests_per_second -> new DeltaStatistic, 'sql_request_time_mean -> new MeanStatistic, + 'sql_slow_queries_total -> new IntegralStatistic, + 'sql_slow_queries_per_second -> new DeltaStatistic, + 'sql_slow_queries_time_mean -> new MeanStatistic, 'memcache_requests_total -> new IntegralStatistic, 'memcache_requests_per_second -> new DeltaStatistic, 'memcache_connections_open -> new IntegralStatistic diff --git a/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala b/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala index deeac0e..7c896b3 100644 --- a/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala +++ b/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala @@ -49,6 +49,10 @@ class SQLQuery(query_str: String) extends TimeoutCallback { if (Config.has_key('log_slow_queries) && runtime_millis >= Config.get('log_slow_queries).toInt) { + Statistics.incr('sql_slow_queries_total) + Statistics.incr('sql_slow_queries_per_second) + Statistics.incr('sql_slow_queries_time_mean, runtime_millis) + Logger.log("[SQL] [Slow Query] (" + runtime_millis + "ms): " + query) } } From adc1fbbc249622889c38d820a44fd6ab00785232 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Fri, 23 Oct 2015 11:43:41 +0200 Subject: [PATCH 36/61] DeltaStatistic: code readability improvements --- src/com/paulasmuth/sqltap/DeltaStatistic.scala | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/com/paulasmuth/sqltap/DeltaStatistic.scala b/src/com/paulasmuth/sqltap/DeltaStatistic.scala index 8a8b45b..c1b4da9 100644 --- a/src/com/paulasmuth/sqltap/DeltaStatistic.scala +++ b/src/com/paulasmuth/sqltap/DeltaStatistic.scala @@ -11,25 +11,23 @@ import java.util.concurrent.atomic.{AtomicInteger} import java.text.{DecimalFormat} class DeltaStatistic extends Statistic { - - private val bucket = new AtomicInteger() - private var value : Double = 0.0 + private val current = new AtomicInteger() + private var last : Double = 0.0 private val format = new DecimalFormat("0.00") def incr(delta: Double) : Unit= { - bucket.getAndAdd(delta.toInt) + current.getAndAdd(delta.toInt) } def decr(delta: Double) : Unit = { - bucket.getAndAdd(delta.toInt * -1) + current.getAndAdd(delta.toInt * -1) } def get() : String = { - format.format(value) + format.format(last) } def flush(f: Double) : Unit = { - value = bucket.getAndSet(0) / f + last = current.getAndSet(0) / f } - } From d66b3a204305ab5668c8d0695de7e51a3478df26 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 23 Oct 2015 12:18:33 +0200 Subject: [PATCH 37/61] move src/* to convention src/main/scala/* --- build.sbt | 2 -- .../scala}/com/paulasmuth/sqltap/AbstractWrappedBuffer.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CTree.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CTreeBuffer.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CTreeCache.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CTreeIndex.scala | 0 .../scala}/com/paulasmuth/sqltap/CTreeInstruction.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CTreeMarshal.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CacheAdapter.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CacheBackend.scala | 0 .../scala}/com/paulasmuth/sqltap/CacheBackendFactory.scala | 0 .../scala}/com/paulasmuth/sqltap/CacheGetRequest.scala | 0 .../scala}/com/paulasmuth/sqltap/CachePurgeRequest.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/CacheRequest.scala | 0 .../scala}/com/paulasmuth/sqltap/CacheStoreRequest.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Config.scala | 0 .../scala}/com/paulasmuth/sqltap/CountInstruction.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/DeltaStatistic.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/ElasticBuffer.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Exceptions.scala | 0 .../scala}/com/paulasmuth/sqltap/ExpirationHandler.scala | 0 .../scala}/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/ExpirationJob.scala | 0 .../scala}/com/paulasmuth/sqltap/FindMultiInstruction.scala | 0 .../scala}/com/paulasmuth/sqltap/FindSingleInstruction.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/GZIPTranscoder.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/HTTPConnection.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/HTTPParser.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/HTTPWriter.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Instruction.scala | 0 .../scala}/com/paulasmuth/sqltap/InstructionFactory.scala | 0 .../scala}/com/paulasmuth/sqltap/InstructionStack.scala | 0 .../scala}/com/paulasmuth/sqltap/IntegralStatistic.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/JSONWriter.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Logger.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Manifest.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/MeanStatistic.scala | 0 .../scala}/com/paulasmuth/sqltap/MemcacheConnection.scala | 0 .../scala}/com/paulasmuth/sqltap/MemcacheConnectionPool.scala | 0 .../scala}/com/paulasmuth/sqltap/NoopCacheBackend.scala | 0 .../scala}/com/paulasmuth/sqltap/NoopExpirationHandler.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/PhiInstruction.scala | 0 .../scala}/com/paulasmuth/sqltap/PurgeExpirationHandler.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Query.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/QueryParser.scala | 0 .../scala}/com/paulasmuth/sqltap/RawSQLInstruction.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/ReadyCallback.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Record.scala | 0 .../scala}/com/paulasmuth/sqltap/RecordLookupJob.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/RelationTrace.scala | 0 .../scala}/com/paulasmuth/sqltap/ReplicationFeed.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Request.scala | 0 .../scala}/com/paulasmuth/sqltap/ResourceManifest.scala | 0 .../scala}/com/paulasmuth/sqltap/ResourceRelation.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/SQLBuilder.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/SQLHelper.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/SQLInstruction.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/SQLTap.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Server.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Statistic.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Statistics.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Timeout.scala | 0 .../scala}/com/paulasmuth/sqltap/TimeoutCallback.scala | 0 .../scala}/com/paulasmuth/sqltap/TimeoutScheduler.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Watchdog.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/Worker.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/WrappedBuffer.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/XMLHelper.scala | 0 .../com/paulasmuth/sqltap/mysql/AbstractSQLConnectionPool.scala | 0 .../com/paulasmuth/sqltap/mysql/AuthSwitchResponsePacket.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/BinaryInteger.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/BinaryString.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/BinlogDumpPacket.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/BinlogEvent.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/BinlogEventPacket.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/ColumnDefinition.scala | 0 .../paulasmuth/sqltap/mysql/FormatDescriptionBinlogEvent.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/HandshakePacket.scala | 0 .../com/paulasmuth/sqltap/mysql/HandshakeResponsePacket.scala | 0 .../com/paulasmuth/sqltap/mysql/LengthEncodedInteger.scala | 0 .../com/paulasmuth/sqltap/mysql/LengthEncodedString.scala | 0 .../com/paulasmuth/sqltap/mysql/OldPasswordAuthentication.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/PingPacket.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/SQLConnection.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/SQLConnectionPool.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/SQLPacket.scala | 0 src/{ => main/scala}/com/paulasmuth/sqltap/mysql/SQLQuery.scala | 0 .../paulasmuth/sqltap/mysql/SecurePasswordAuthentication.scala | 0 .../com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala | 0 .../scala}/com/paulasmuth/sqltap/mysql/UnknownBinlogEvent.scala | 0 .../com/paulasmuth/sqltap/mysql/UpdateRowsBinlogEvent.scala | 0 92 files changed, 2 deletions(-) rename src/{ => main/scala}/com/paulasmuth/sqltap/AbstractWrappedBuffer.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CTree.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CTreeBuffer.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CTreeCache.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CTreeIndex.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CTreeInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CTreeMarshal.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CacheAdapter.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CacheBackend.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CacheBackendFactory.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CacheGetRequest.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CachePurgeRequest.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CacheRequest.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CacheStoreRequest.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Config.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/CountInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/DeltaStatistic.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ElasticBuffer.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Exceptions.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ExpirationHandler.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ExpirationJob.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/FindMultiInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/FindSingleInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/GZIPTranscoder.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/HTTPConnection.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/HTTPParser.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/HTTPWriter.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Instruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/InstructionFactory.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/InstructionStack.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/IntegralStatistic.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/JSONWriter.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Logger.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Manifest.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/MeanStatistic.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/MemcacheConnection.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/MemcacheConnectionPool.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/NoopCacheBackend.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/NoopExpirationHandler.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/PhiInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/PurgeExpirationHandler.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Query.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/QueryParser.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/RawSQLInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ReadyCallback.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Record.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/RecordLookupJob.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/RelationTrace.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ReplicationFeed.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Request.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ResourceManifest.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/ResourceRelation.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/SQLBuilder.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/SQLHelper.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/SQLInstruction.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/SQLTap.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Server.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Statistic.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Statistics.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Timeout.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/TimeoutCallback.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/TimeoutScheduler.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Watchdog.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/Worker.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/WrappedBuffer.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/XMLHelper.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/AbstractSQLConnectionPool.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/AuthSwitchResponsePacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/BinaryInteger.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/BinaryString.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/BinlogDumpPacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/BinlogEvent.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/BinlogEventPacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/ColumnDefinition.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/FormatDescriptionBinlogEvent.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/HandshakePacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/HandshakeResponsePacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/LengthEncodedInteger.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/OldPasswordAuthentication.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/PingPacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/SQLConnection.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/SQLConnectionPool.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/SQLPacket.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/SQLQuery.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/SecurePasswordAuthentication.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/UnknownBinlogEvent.scala (100%) rename src/{ => main/scala}/com/paulasmuth/sqltap/mysql/UpdateRowsBinlogEvent.scala (100%) diff --git a/build.sbt b/build.sbt index d6783d9..c48479e 100644 --- a/build.sbt +++ b/build.sbt @@ -8,8 +8,6 @@ version := "0.8.0" mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") -scalaSource in Compile <<= baseDirectory(_ / "src") - scalaVersion := "2.11.7" assemblySettings diff --git a/src/com/paulasmuth/sqltap/AbstractWrappedBuffer.scala b/src/main/scala/com/paulasmuth/sqltap/AbstractWrappedBuffer.scala similarity index 100% rename from src/com/paulasmuth/sqltap/AbstractWrappedBuffer.scala rename to src/main/scala/com/paulasmuth/sqltap/AbstractWrappedBuffer.scala diff --git a/src/com/paulasmuth/sqltap/CTree.scala b/src/main/scala/com/paulasmuth/sqltap/CTree.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CTree.scala rename to src/main/scala/com/paulasmuth/sqltap/CTree.scala diff --git a/src/com/paulasmuth/sqltap/CTreeBuffer.scala b/src/main/scala/com/paulasmuth/sqltap/CTreeBuffer.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CTreeBuffer.scala rename to src/main/scala/com/paulasmuth/sqltap/CTreeBuffer.scala diff --git a/src/com/paulasmuth/sqltap/CTreeCache.scala b/src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CTreeCache.scala rename to src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala diff --git a/src/com/paulasmuth/sqltap/CTreeIndex.scala b/src/main/scala/com/paulasmuth/sqltap/CTreeIndex.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CTreeIndex.scala rename to src/main/scala/com/paulasmuth/sqltap/CTreeIndex.scala diff --git a/src/com/paulasmuth/sqltap/CTreeInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/CTreeInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CTreeInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/CTreeInstruction.scala diff --git a/src/com/paulasmuth/sqltap/CTreeMarshal.scala b/src/main/scala/com/paulasmuth/sqltap/CTreeMarshal.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CTreeMarshal.scala rename to src/main/scala/com/paulasmuth/sqltap/CTreeMarshal.scala diff --git a/src/com/paulasmuth/sqltap/CacheAdapter.scala b/src/main/scala/com/paulasmuth/sqltap/CacheAdapter.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CacheAdapter.scala rename to src/main/scala/com/paulasmuth/sqltap/CacheAdapter.scala diff --git a/src/com/paulasmuth/sqltap/CacheBackend.scala b/src/main/scala/com/paulasmuth/sqltap/CacheBackend.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CacheBackend.scala rename to src/main/scala/com/paulasmuth/sqltap/CacheBackend.scala diff --git a/src/com/paulasmuth/sqltap/CacheBackendFactory.scala b/src/main/scala/com/paulasmuth/sqltap/CacheBackendFactory.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CacheBackendFactory.scala rename to src/main/scala/com/paulasmuth/sqltap/CacheBackendFactory.scala diff --git a/src/com/paulasmuth/sqltap/CacheGetRequest.scala b/src/main/scala/com/paulasmuth/sqltap/CacheGetRequest.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CacheGetRequest.scala rename to src/main/scala/com/paulasmuth/sqltap/CacheGetRequest.scala diff --git a/src/com/paulasmuth/sqltap/CachePurgeRequest.scala b/src/main/scala/com/paulasmuth/sqltap/CachePurgeRequest.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CachePurgeRequest.scala rename to src/main/scala/com/paulasmuth/sqltap/CachePurgeRequest.scala diff --git a/src/com/paulasmuth/sqltap/CacheRequest.scala b/src/main/scala/com/paulasmuth/sqltap/CacheRequest.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CacheRequest.scala rename to src/main/scala/com/paulasmuth/sqltap/CacheRequest.scala diff --git a/src/com/paulasmuth/sqltap/CacheStoreRequest.scala b/src/main/scala/com/paulasmuth/sqltap/CacheStoreRequest.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CacheStoreRequest.scala rename to src/main/scala/com/paulasmuth/sqltap/CacheStoreRequest.scala diff --git a/src/com/paulasmuth/sqltap/Config.scala b/src/main/scala/com/paulasmuth/sqltap/Config.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Config.scala rename to src/main/scala/com/paulasmuth/sqltap/Config.scala diff --git a/src/com/paulasmuth/sqltap/CountInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/CountInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/CountInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/CountInstruction.scala diff --git a/src/com/paulasmuth/sqltap/DeltaStatistic.scala b/src/main/scala/com/paulasmuth/sqltap/DeltaStatistic.scala similarity index 100% rename from src/com/paulasmuth/sqltap/DeltaStatistic.scala rename to src/main/scala/com/paulasmuth/sqltap/DeltaStatistic.scala diff --git a/src/com/paulasmuth/sqltap/ElasticBuffer.scala b/src/main/scala/com/paulasmuth/sqltap/ElasticBuffer.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ElasticBuffer.scala rename to src/main/scala/com/paulasmuth/sqltap/ElasticBuffer.scala diff --git a/src/com/paulasmuth/sqltap/Exceptions.scala b/src/main/scala/com/paulasmuth/sqltap/Exceptions.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Exceptions.scala rename to src/main/scala/com/paulasmuth/sqltap/Exceptions.scala diff --git a/src/com/paulasmuth/sqltap/ExpirationHandler.scala b/src/main/scala/com/paulasmuth/sqltap/ExpirationHandler.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ExpirationHandler.scala rename to src/main/scala/com/paulasmuth/sqltap/ExpirationHandler.scala diff --git a/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala b/src/main/scala/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala rename to src/main/scala/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala diff --git a/src/com/paulasmuth/sqltap/ExpirationJob.scala b/src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ExpirationJob.scala rename to src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala diff --git a/src/com/paulasmuth/sqltap/FindMultiInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/FindMultiInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/FindMultiInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/FindMultiInstruction.scala diff --git a/src/com/paulasmuth/sqltap/FindSingleInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/FindSingleInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/FindSingleInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/FindSingleInstruction.scala diff --git a/src/com/paulasmuth/sqltap/GZIPTranscoder.scala b/src/main/scala/com/paulasmuth/sqltap/GZIPTranscoder.scala similarity index 100% rename from src/com/paulasmuth/sqltap/GZIPTranscoder.scala rename to src/main/scala/com/paulasmuth/sqltap/GZIPTranscoder.scala diff --git a/src/com/paulasmuth/sqltap/HTTPConnection.scala b/src/main/scala/com/paulasmuth/sqltap/HTTPConnection.scala similarity index 100% rename from src/com/paulasmuth/sqltap/HTTPConnection.scala rename to src/main/scala/com/paulasmuth/sqltap/HTTPConnection.scala diff --git a/src/com/paulasmuth/sqltap/HTTPParser.scala b/src/main/scala/com/paulasmuth/sqltap/HTTPParser.scala similarity index 100% rename from src/com/paulasmuth/sqltap/HTTPParser.scala rename to src/main/scala/com/paulasmuth/sqltap/HTTPParser.scala diff --git a/src/com/paulasmuth/sqltap/HTTPWriter.scala b/src/main/scala/com/paulasmuth/sqltap/HTTPWriter.scala similarity index 100% rename from src/com/paulasmuth/sqltap/HTTPWriter.scala rename to src/main/scala/com/paulasmuth/sqltap/HTTPWriter.scala diff --git a/src/com/paulasmuth/sqltap/Instruction.scala b/src/main/scala/com/paulasmuth/sqltap/Instruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Instruction.scala rename to src/main/scala/com/paulasmuth/sqltap/Instruction.scala diff --git a/src/com/paulasmuth/sqltap/InstructionFactory.scala b/src/main/scala/com/paulasmuth/sqltap/InstructionFactory.scala similarity index 100% rename from src/com/paulasmuth/sqltap/InstructionFactory.scala rename to src/main/scala/com/paulasmuth/sqltap/InstructionFactory.scala diff --git a/src/com/paulasmuth/sqltap/InstructionStack.scala b/src/main/scala/com/paulasmuth/sqltap/InstructionStack.scala similarity index 100% rename from src/com/paulasmuth/sqltap/InstructionStack.scala rename to src/main/scala/com/paulasmuth/sqltap/InstructionStack.scala diff --git a/src/com/paulasmuth/sqltap/IntegralStatistic.scala b/src/main/scala/com/paulasmuth/sqltap/IntegralStatistic.scala similarity index 100% rename from src/com/paulasmuth/sqltap/IntegralStatistic.scala rename to src/main/scala/com/paulasmuth/sqltap/IntegralStatistic.scala diff --git a/src/com/paulasmuth/sqltap/JSONWriter.scala b/src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala similarity index 100% rename from src/com/paulasmuth/sqltap/JSONWriter.scala rename to src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala diff --git a/src/com/paulasmuth/sqltap/Logger.scala b/src/main/scala/com/paulasmuth/sqltap/Logger.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Logger.scala rename to src/main/scala/com/paulasmuth/sqltap/Logger.scala diff --git a/src/com/paulasmuth/sqltap/Manifest.scala b/src/main/scala/com/paulasmuth/sqltap/Manifest.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Manifest.scala rename to src/main/scala/com/paulasmuth/sqltap/Manifest.scala diff --git a/src/com/paulasmuth/sqltap/MeanStatistic.scala b/src/main/scala/com/paulasmuth/sqltap/MeanStatistic.scala similarity index 100% rename from src/com/paulasmuth/sqltap/MeanStatistic.scala rename to src/main/scala/com/paulasmuth/sqltap/MeanStatistic.scala diff --git a/src/com/paulasmuth/sqltap/MemcacheConnection.scala b/src/main/scala/com/paulasmuth/sqltap/MemcacheConnection.scala similarity index 100% rename from src/com/paulasmuth/sqltap/MemcacheConnection.scala rename to src/main/scala/com/paulasmuth/sqltap/MemcacheConnection.scala diff --git a/src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala b/src/main/scala/com/paulasmuth/sqltap/MemcacheConnectionPool.scala similarity index 100% rename from src/com/paulasmuth/sqltap/MemcacheConnectionPool.scala rename to src/main/scala/com/paulasmuth/sqltap/MemcacheConnectionPool.scala diff --git a/src/com/paulasmuth/sqltap/NoopCacheBackend.scala b/src/main/scala/com/paulasmuth/sqltap/NoopCacheBackend.scala similarity index 100% rename from src/com/paulasmuth/sqltap/NoopCacheBackend.scala rename to src/main/scala/com/paulasmuth/sqltap/NoopCacheBackend.scala diff --git a/src/com/paulasmuth/sqltap/NoopExpirationHandler.scala b/src/main/scala/com/paulasmuth/sqltap/NoopExpirationHandler.scala similarity index 100% rename from src/com/paulasmuth/sqltap/NoopExpirationHandler.scala rename to src/main/scala/com/paulasmuth/sqltap/NoopExpirationHandler.scala diff --git a/src/com/paulasmuth/sqltap/PhiInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/PhiInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/PhiInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/PhiInstruction.scala diff --git a/src/com/paulasmuth/sqltap/PurgeExpirationHandler.scala b/src/main/scala/com/paulasmuth/sqltap/PurgeExpirationHandler.scala similarity index 100% rename from src/com/paulasmuth/sqltap/PurgeExpirationHandler.scala rename to src/main/scala/com/paulasmuth/sqltap/PurgeExpirationHandler.scala diff --git a/src/com/paulasmuth/sqltap/Query.scala b/src/main/scala/com/paulasmuth/sqltap/Query.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Query.scala rename to src/main/scala/com/paulasmuth/sqltap/Query.scala diff --git a/src/com/paulasmuth/sqltap/QueryParser.scala b/src/main/scala/com/paulasmuth/sqltap/QueryParser.scala similarity index 100% rename from src/com/paulasmuth/sqltap/QueryParser.scala rename to src/main/scala/com/paulasmuth/sqltap/QueryParser.scala diff --git a/src/com/paulasmuth/sqltap/RawSQLInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/RawSQLInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/RawSQLInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/RawSQLInstruction.scala diff --git a/src/com/paulasmuth/sqltap/ReadyCallback.scala b/src/main/scala/com/paulasmuth/sqltap/ReadyCallback.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ReadyCallback.scala rename to src/main/scala/com/paulasmuth/sqltap/ReadyCallback.scala diff --git a/src/com/paulasmuth/sqltap/Record.scala b/src/main/scala/com/paulasmuth/sqltap/Record.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Record.scala rename to src/main/scala/com/paulasmuth/sqltap/Record.scala diff --git a/src/com/paulasmuth/sqltap/RecordLookupJob.scala b/src/main/scala/com/paulasmuth/sqltap/RecordLookupJob.scala similarity index 100% rename from src/com/paulasmuth/sqltap/RecordLookupJob.scala rename to src/main/scala/com/paulasmuth/sqltap/RecordLookupJob.scala diff --git a/src/com/paulasmuth/sqltap/RelationTrace.scala b/src/main/scala/com/paulasmuth/sqltap/RelationTrace.scala similarity index 100% rename from src/com/paulasmuth/sqltap/RelationTrace.scala rename to src/main/scala/com/paulasmuth/sqltap/RelationTrace.scala diff --git a/src/com/paulasmuth/sqltap/ReplicationFeed.scala b/src/main/scala/com/paulasmuth/sqltap/ReplicationFeed.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ReplicationFeed.scala rename to src/main/scala/com/paulasmuth/sqltap/ReplicationFeed.scala diff --git a/src/com/paulasmuth/sqltap/Request.scala b/src/main/scala/com/paulasmuth/sqltap/Request.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Request.scala rename to src/main/scala/com/paulasmuth/sqltap/Request.scala diff --git a/src/com/paulasmuth/sqltap/ResourceManifest.scala b/src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ResourceManifest.scala rename to src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala diff --git a/src/com/paulasmuth/sqltap/ResourceRelation.scala b/src/main/scala/com/paulasmuth/sqltap/ResourceRelation.scala similarity index 100% rename from src/com/paulasmuth/sqltap/ResourceRelation.scala rename to src/main/scala/com/paulasmuth/sqltap/ResourceRelation.scala diff --git a/src/com/paulasmuth/sqltap/SQLBuilder.scala b/src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala similarity index 100% rename from src/com/paulasmuth/sqltap/SQLBuilder.scala rename to src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala diff --git a/src/com/paulasmuth/sqltap/SQLHelper.scala b/src/main/scala/com/paulasmuth/sqltap/SQLHelper.scala similarity index 100% rename from src/com/paulasmuth/sqltap/SQLHelper.scala rename to src/main/scala/com/paulasmuth/sqltap/SQLHelper.scala diff --git a/src/com/paulasmuth/sqltap/SQLInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/SQLInstruction.scala similarity index 100% rename from src/com/paulasmuth/sqltap/SQLInstruction.scala rename to src/main/scala/com/paulasmuth/sqltap/SQLInstruction.scala diff --git a/src/com/paulasmuth/sqltap/SQLTap.scala b/src/main/scala/com/paulasmuth/sqltap/SQLTap.scala similarity index 100% rename from src/com/paulasmuth/sqltap/SQLTap.scala rename to src/main/scala/com/paulasmuth/sqltap/SQLTap.scala diff --git a/src/com/paulasmuth/sqltap/Server.scala b/src/main/scala/com/paulasmuth/sqltap/Server.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Server.scala rename to src/main/scala/com/paulasmuth/sqltap/Server.scala diff --git a/src/com/paulasmuth/sqltap/Statistic.scala b/src/main/scala/com/paulasmuth/sqltap/Statistic.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Statistic.scala rename to src/main/scala/com/paulasmuth/sqltap/Statistic.scala diff --git a/src/com/paulasmuth/sqltap/Statistics.scala b/src/main/scala/com/paulasmuth/sqltap/Statistics.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Statistics.scala rename to src/main/scala/com/paulasmuth/sqltap/Statistics.scala diff --git a/src/com/paulasmuth/sqltap/Timeout.scala b/src/main/scala/com/paulasmuth/sqltap/Timeout.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Timeout.scala rename to src/main/scala/com/paulasmuth/sqltap/Timeout.scala diff --git a/src/com/paulasmuth/sqltap/TimeoutCallback.scala b/src/main/scala/com/paulasmuth/sqltap/TimeoutCallback.scala similarity index 100% rename from src/com/paulasmuth/sqltap/TimeoutCallback.scala rename to src/main/scala/com/paulasmuth/sqltap/TimeoutCallback.scala diff --git a/src/com/paulasmuth/sqltap/TimeoutScheduler.scala b/src/main/scala/com/paulasmuth/sqltap/TimeoutScheduler.scala similarity index 100% rename from src/com/paulasmuth/sqltap/TimeoutScheduler.scala rename to src/main/scala/com/paulasmuth/sqltap/TimeoutScheduler.scala diff --git a/src/com/paulasmuth/sqltap/Watchdog.scala b/src/main/scala/com/paulasmuth/sqltap/Watchdog.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Watchdog.scala rename to src/main/scala/com/paulasmuth/sqltap/Watchdog.scala diff --git a/src/com/paulasmuth/sqltap/Worker.scala b/src/main/scala/com/paulasmuth/sqltap/Worker.scala similarity index 100% rename from src/com/paulasmuth/sqltap/Worker.scala rename to src/main/scala/com/paulasmuth/sqltap/Worker.scala diff --git a/src/com/paulasmuth/sqltap/WrappedBuffer.scala b/src/main/scala/com/paulasmuth/sqltap/WrappedBuffer.scala similarity index 100% rename from src/com/paulasmuth/sqltap/WrappedBuffer.scala rename to src/main/scala/com/paulasmuth/sqltap/WrappedBuffer.scala diff --git a/src/com/paulasmuth/sqltap/XMLHelper.scala b/src/main/scala/com/paulasmuth/sqltap/XMLHelper.scala similarity index 100% rename from src/com/paulasmuth/sqltap/XMLHelper.scala rename to src/main/scala/com/paulasmuth/sqltap/XMLHelper.scala diff --git a/src/com/paulasmuth/sqltap/mysql/AbstractSQLConnectionPool.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/AbstractSQLConnectionPool.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/AbstractSQLConnectionPool.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/AbstractSQLConnectionPool.scala diff --git a/src/com/paulasmuth/sqltap/mysql/AuthSwitchResponsePacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/AuthSwitchResponsePacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/AuthSwitchResponsePacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/AuthSwitchResponsePacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/BinaryInteger.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/BinaryInteger.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/BinaryInteger.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/BinaryInteger.scala diff --git a/src/com/paulasmuth/sqltap/mysql/BinaryString.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/BinaryString.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/BinaryString.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/BinaryString.scala diff --git a/src/com/paulasmuth/sqltap/mysql/BinlogDumpPacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/BinlogDumpPacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/BinlogDumpPacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/BinlogDumpPacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/BinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/BinlogEvent.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/BinlogEvent.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/BinlogEvent.scala diff --git a/src/com/paulasmuth/sqltap/mysql/BinlogEventPacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/BinlogEventPacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/BinlogEventPacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/BinlogEventPacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/ColumnDefinition.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/ColumnDefinition.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/ColumnDefinition.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/ColumnDefinition.scala diff --git a/src/com/paulasmuth/sqltap/mysql/FormatDescriptionBinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/FormatDescriptionBinlogEvent.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/FormatDescriptionBinlogEvent.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/FormatDescriptionBinlogEvent.scala diff --git a/src/com/paulasmuth/sqltap/mysql/HandshakePacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/HandshakePacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/HandshakePacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/HandshakePacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/HandshakeResponsePacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/HandshakeResponsePacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/HandshakeResponsePacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/HandshakeResponsePacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/LengthEncodedInteger.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedInteger.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/LengthEncodedInteger.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedInteger.scala diff --git a/src/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/LengthEncodedString.scala diff --git a/src/com/paulasmuth/sqltap/mysql/OldPasswordAuthentication.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/OldPasswordAuthentication.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/OldPasswordAuthentication.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/OldPasswordAuthentication.scala diff --git a/src/com/paulasmuth/sqltap/mysql/PingPacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/PingPacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/PingPacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/PingPacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala diff --git a/src/com/paulasmuth/sqltap/mysql/SQLConnection.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnection.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/SQLConnection.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnection.scala diff --git a/src/com/paulasmuth/sqltap/mysql/SQLConnectionPool.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnectionPool.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/SQLConnectionPool.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/SQLConnectionPool.scala diff --git a/src/com/paulasmuth/sqltap/mysql/SQLPacket.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/SQLPacket.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/SQLPacket.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/SQLPacket.scala diff --git a/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/SQLQuery.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/SQLQuery.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/SQLQuery.scala diff --git a/src/com/paulasmuth/sqltap/mysql/SecurePasswordAuthentication.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/SecurePasswordAuthentication.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/SecurePasswordAuthentication.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/SecurePasswordAuthentication.scala diff --git a/src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/TableMapBinlogEvent.scala diff --git a/src/com/paulasmuth/sqltap/mysql/UnknownBinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/UnknownBinlogEvent.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/UnknownBinlogEvent.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/UnknownBinlogEvent.scala diff --git a/src/com/paulasmuth/sqltap/mysql/UpdateRowsBinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/UpdateRowsBinlogEvent.scala similarity index 100% rename from src/com/paulasmuth/sqltap/mysql/UpdateRowsBinlogEvent.scala rename to src/main/scala/com/paulasmuth/sqltap/mysql/UpdateRowsBinlogEvent.scala From 657cd1524fd18333b10336d3af9c26567ea09b4d Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 23 Oct 2015 13:16:23 +0200 Subject: [PATCH 38/61] add scalatest as test dependency --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index c48479e..aa2e091 100644 --- a/build.sbt +++ b/build.sbt @@ -17,3 +17,4 @@ jarName in assembly := { s"${name.value.toLowerCase}-${version.value}.jar" } fork in run := true libraryDependencies += "org.scala-lang.modules" %% "scala-xml" % "1.0.2" +libraryDependencies += "org.scalatest" % "scalatest_2.11" % "2.2.4" % "test" From 860f2d645e634e332594b6d4cfe9f9552a21b21e Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 23 Oct 2015 13:18:41 +0200 Subject: [PATCH 39/61] add specs for existing JSONWriter behaviour --- .../paulasmuth/sqltap/JSONWriterSpec.scala | 43 +++++++++++++++++++ .../com/paulasmuth/sqltap/UnitSpec.scala | 7 +++ 2 files changed, 50 insertions(+) create mode 100644 src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala create mode 100644 src/test/scala/com/paulasmuth/sqltap/UnitSpec.scala diff --git a/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala b/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala new file mode 100644 index 0000000..3201c7d --- /dev/null +++ b/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala @@ -0,0 +1,43 @@ +package com.paulasmuth.sqltap + +import java.nio.ByteBuffer + +class JSONWriterSpec extends UnitSpec { + + describe("write_escaped") { + it("0x20 -> ' '") { + val str = write_escaped(Array(0x20.toByte)) + assert(str == " ") + } + + it("0xA -> '\\n'") { + val str = write_escaped(Array(0xA.toByte)) + assert(str == "\\n") + } + + it("0x22 -> '\\\"'") { + val str = write_escaped(Array(0x22.toByte)) + assert(str == "\\\"") + } + + it("0x5C -> '\\\\'") { + val str = write_escaped(Array(0x5C.toByte)) + assert(str == "\\\\") + } + } + + def buffer_to_string(buf: ByteBuffer) : String = { + val bytes = new Array[Byte](buf.position()) + buf.rewind() + buf.get(bytes) + new String(bytes, "UTF-8") + } + + def write_escaped(in_bytes: Array[Byte]) : String = { + val buf = ByteBuffer.allocate(10) + val writer = new JSONWriter(new WrappedBuffer(buf)) + writer.write_escaped(new String(in_bytes, "UTF-8")) + buffer_to_string(buf) + } + +} diff --git a/src/test/scala/com/paulasmuth/sqltap/UnitSpec.scala b/src/test/scala/com/paulasmuth/sqltap/UnitSpec.scala new file mode 100644 index 0000000..e2bc883 --- /dev/null +++ b/src/test/scala/com/paulasmuth/sqltap/UnitSpec.scala @@ -0,0 +1,7 @@ +package com.paulasmuth.sqltap + +import org.scalatest.FunSpec +import org.scalatest.BeforeAndAfter +import org.scalatest.Assertions._ + +abstract class UnitSpec extends FunSpec with BeforeAndAfter From 841b35250712d42ca10a44a0102a00d9ade71863 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 23 Oct 2015 13:44:29 +0200 Subject: [PATCH 40/61] add json string escaping for bytes < 0x20 --- src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala | 5 ++++- src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala b/src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala index 2902650..576ce3b 100644 --- a/src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala +++ b/src/main/scala/com/paulasmuth/sqltap/JSONWriter.scala @@ -86,7 +86,10 @@ class JSONWriter(buf: WrappedBuffer) { buf.write(Array(0x5C.toByte, 0x22.toByte)) // \" } else if (b == 0x5C) { buf.write(Array(0x5C.toByte, 0x5C.toByte)) // \\ - } else if ((b == 0) || ((b >= 0x20))) { + } else if (b < 0x20) { + buf.write(Array(0x5C.toByte, 0x75.toByte)) + buf.write("%04x".format(b).getBytes) // \u000b + } else { buf.write(byte) } } diff --git a/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala b/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala index 3201c7d..0e2866b 100644 --- a/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala +++ b/src/test/scala/com/paulasmuth/sqltap/JSONWriterSpec.scala @@ -24,6 +24,11 @@ class JSONWriterSpec extends UnitSpec { val str = write_escaped(Array(0x5C.toByte)) assert(str == "\\\\") } + + it("0xB -> '\\u000b'") { + val str = write_escaped(Array(0xB.toByte)) + assert(str == "\\u000b") + } } def buffer_to_string(buf: ByteBuffer) : String = { From 3522f1fa9fedb7939559a1560a5f87a62669a2d0 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Mon, 2 Nov 2015 14:59:55 +0100 Subject: [PATCH 41/61] add escaping for default_order field --- src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala b/src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala index 6471c98..af2df00 100644 --- a/src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala +++ b/src/main/scala/com/paulasmuth/sqltap/ResourceManifest.scala @@ -31,7 +31,7 @@ class ResourceManifest(doc: xml.Node) { elem.attr("id_field", false, "id") val default_order : String = - elem.attr("default_order", false, id_field + " DESC") + elem.attr("default_order", false, "`" + id_field + "` DESC") val relations = ((List[ResourceRelation]() /: (doc \ "relation")) From cebe5cd5f1402a1df00e1a7ba9d8242a7aad4396 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 3 Nov 2015 12:22:25 +0100 Subject: [PATCH 42/61] build.sbt: compile fix --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index aa2e091..8b8d79d 100644 --- a/build.sbt +++ b/build.sbt @@ -17,4 +17,5 @@ jarName in assembly := { s"${name.value.toLowerCase}-${version.value}.jar" } fork in run := true libraryDependencies += "org.scala-lang.modules" %% "scala-xml" % "1.0.2" + libraryDependencies += "org.scalatest" % "scalatest_2.11" % "2.2.4" % "test" From 8cd2bdd6633bcbd6c308b3872434afe7662a2340 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 3 Nov 2015 12:22:36 +0100 Subject: [PATCH 43/61] version bump to 0.8.1 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 8b8d79d..6041778 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ name := "SQLTap" organization := "com.paulasmuth" -version := "0.8.0" +version := "0.8.1" mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap") From 5cac6f5630ce037d3c496ece4449ace47f2dcc9b Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Fri, 22 Jan 2016 15:47:18 +0100 Subject: [PATCH 44/61] Dockerize --- .dockerignore | 3 +++ Dockerfile | 47 +++++++++++++++++++++++++++++++++++++++++++++++ sqltap.sh | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100755 sqltap.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..35706e6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +.gitignore +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e0cee51 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +FROM java:8 + +MAINTAINER DaWanda + +ENV SCALA_VERSION="2.11.7" \ + SBT_VERSION="0.13.8" \ + SQLTAP_VERSION="0.8.1" + +ADD http://downloads.typesafe.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.tgz /tmp/scala.tgz +RUN tar -C /opt -xzf /tmp/scala.tgz && rm -f /tmp/scala.tgz + +ENV PATH="/opt/scala-$SCALA_VERSION/bin:$PATH" + +ADD https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb /tmp/sbt.deb +RUN dpkg -i /tmp/sbt.deb && rm -f /tmp/sbt.deb +RUN apt-get update && apt-get install sbt + +RUN sbt + +ADD . /opt/sqltap +WORKDIR /opt/sqltap +RUN sbt assembly + +ENV SQLTAP_HTTP_PORT="3000" \ + SQLTAP_THREADS="16" \ + SQLTAP_JARFILE="/opt/sqltap/target/scala-2.11/sqltap-${SQLTAP_VERSION}.jar" \ + SQLTAP_SCHEMA="/var/schema.xml" \ + SQLTAP_OPTS="" \ + MYSQL_HOST="127.0.0.1" \ + MYSQL_PORT="3306" \ + MYSQL_USER="fetch" \ + MYSQL_DATABASE="test" \ + MYSQL_NUMCONNS="6" \ + MYSQL_QUEUELEN="2500" \ + JMX_PORT="9191" \ + RMI_BIND="127.0.0.1" \ + JAVA_XMX="16384M" \ + HOSTNAME="sqltap1" \ + CACHE_BACKEND="memcache" \ + MEMCACHE_HOST="127.0.0.1" \ + MEMCACHE_PORT="11211" \ + MEMCACHE_QUEUELEN="8192" \ + MEMCACHE_NUMCONNS="20" + +EXPOSE 3000 + +CMD ["/opt/sqltap/sqltap.sh"] diff --git a/sqltap.sh b/sqltap.sh new file mode 100755 index 0000000..fdde1c8 --- /dev/null +++ b/sqltap.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +if [[ "$SCHEMA_URL" != "" ]]; then + curl -sL "$SCHEMA_URL" -o /var/schema.xml +fi + +opts="" +opts="$opts --config '${SQLTAP_SCHEMA}'" +opts="$opts --http ${SQLTAP_HTTP_PORT}" +opts="$opts --disable-keepalive" +opts="$opts -t ${SQLTAP_THREADS}" +opts="$opts --mysql-host ${MYSQL_HOST}" +opts="$opts --mysql-port ${MYSQL_PORT}" +opts="$opts --mysql-user ${MYSQL_USER}" +opts="$opts --mysql-database ${MYSQL_DATABASE}" +opts="$opts --mysql-numconns ${MYSQL_NUMCONNS}" +opts="$opts --mysql-queuelen ${MYSQL_QUEUELEN}" +opts="$opts --cache-backend ${CACHE_BACKEND}" +if [[ "${CACHE_BACKEND}" == "memcache" ]]; then + opts="$opts --memcache-host ${MEMCACHE_HOST}" + opts="$opts --memcache-port ${MEMCACHE_PORT}" + opts="$opts --memcache-queuelen ${MEMCACHE_QUEUELEN}" + opts="$opts --memcache-numconns ${MEMCACHE_NUMCONNS}" +fi +opts="$opts ${SQLTAP_OPTS}" + +exec java \ + -Djava.rmi.server.hostname="${RMI_BIND}" \ + -Dcom.sun.management.jmxremote \ + -Dcom.sun.management.jmxremote.port="${JMX_PORT}" \ + -Dcom.sun.management.jmxremote.ssl=false \ + -Dcom.sun.management.jmxremote.authenticate=false \ + -Xmx"${JAVA_XMX}" -XX:GCTimeRatio=99 -XX:+UseConcMarkSweepGC \ + -jar "${SQLTAP_JARFILE}" \ + $opts From 22c6318d46397ef739ec4efff0adf91a0d24b825 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Fri, 22 Jan 2016 17:31:43 +0100 Subject: [PATCH 45/61] Dockerfile: adds missing ENV --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index e0cee51..8764ea0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ ENV SQLTAP_HTTP_PORT="3000" \ SQLTAP_THREADS="16" \ SQLTAP_JARFILE="/opt/sqltap/target/scala-2.11/sqltap-${SQLTAP_VERSION}.jar" \ SQLTAP_SCHEMA="/var/schema.xml" \ + SCHEMA_URL="" \ SQLTAP_OPTS="" \ MYSQL_HOST="127.0.0.1" \ MYSQL_PORT="3306" \ From cf76c52ee22d477e735580a7eda4981af3dc93b9 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 02:13:58 +0100 Subject: [PATCH 46/61] Dockerfile refactoring, killing unnecessary statements and do some cleanup --- Dockerfile | 36 ++++++++++++++---------------------- build.sbt | 2 +- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8764ea0..bf29d3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,12 @@ FROM java:8 - MAINTAINER DaWanda -ENV SCALA_VERSION="2.11.7" \ - SBT_VERSION="0.13.8" \ - SQLTAP_VERSION="0.8.1" - -ADD http://downloads.typesafe.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.tgz /tmp/scala.tgz -RUN tar -C /opt -xzf /tmp/scala.tgz && rm -f /tmp/scala.tgz - -ENV PATH="/opt/scala-$SCALA_VERSION/bin:$PATH" - -ADD https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb /tmp/sbt.deb -RUN dpkg -i /tmp/sbt.deb && rm -f /tmp/sbt.deb -RUN apt-get update && apt-get install sbt - -RUN sbt - -ADD . /opt/sqltap -WORKDIR /opt/sqltap -RUN sbt assembly +ARG SBT_VERSION="0.13.8" +ARG SQLTAP_JARFILE="/usr/lib/sqltap.jar" ENV SQLTAP_HTTP_PORT="3000" \ SQLTAP_THREADS="16" \ - SQLTAP_JARFILE="/opt/sqltap/target/scala-2.11/sqltap-${SQLTAP_VERSION}.jar" \ - SQLTAP_SCHEMA="/var/schema.xml" \ + SQLTAP_SCHEMA="/etc/sqltap-schema.xml" \ SCHEMA_URL="" \ SQLTAP_OPTS="" \ MYSQL_HOST="127.0.0.1" \ @@ -43,6 +25,16 @@ ENV SQLTAP_HTTP_PORT="3000" \ MEMCACHE_QUEUELEN="8192" \ MEMCACHE_NUMCONNS="20" -EXPOSE 3000 +ADD https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb /tmp/sbt.deb +RUN dpkg -i /tmp/sbt.deb && rm -f /tmp/sbt.deb +RUN apt-get update && apt-get install sbt + +ADD . /opt/sqltap +RUN cd /opt/sqltap && \ + sbt assembly && \ + cp -vpi /opt/sqltap/target/scala-*/sqltap.jar $SQLTAP_JARFILE && \ + rm -rf /opt/sqltap + +EXPOSE $SQLTAP_HTTP_PORT CMD ["/opt/sqltap/sqltap.sh"] diff --git a/build.sbt b/build.sbt index 6041778..9ce4fdd 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ scalaVersion := "2.11.7" assemblySettings -jarName in assembly := { s"${name.value.toLowerCase}-${version.value}.jar" } +jarName in assembly := { s"${name.value.toLowerCase}.jar" } fork in run := true From f35d059964dc0474ad9c891eb05e070513bcccb3 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:09:49 +0100 Subject: [PATCH 47/61] docker: improved bootup script to be a little more user-friendly --- sqltap.sh => bootup.sh | 47 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) rename sqltap.sh => bootup.sh (55%) diff --git a/sqltap.sh b/bootup.sh similarity index 55% rename from sqltap.sh rename to bootup.sh index fdde1c8..ace9dfc 100755 --- a/sqltap.sh +++ b/bootup.sh @@ -1,9 +1,47 @@ #!/bin/bash +set -e if [[ "$SCHEMA_URL" != "" ]]; then - curl -sL "$SCHEMA_URL" -o /var/schema.xml + curl -sL "$SCHEMA_URL" -o "${SQLTAP_SCHEMA}" fi +require_arg() { + local name="$1" + local val=`eval "echo \\\$${name}"` + + echo "${name}: ${val}" + + if [[ "${val}" == "" ]]; then + echo "Error. Required argument ${name} missing." 1>&2 + exit 1 + fi +} + +require_args() { + while [[ $# -ne 0 ]]; do + require_arg $1 + shift + done +} + +require_args SQLTAP_SCHEMA \ + MYSQL_PORT \ + SQLTAP_HTTP_PORT \ + SQLTAP_THREADS \ + SQLTAP_SCHEMA \ + SCHEMA_URL \ + SQLTAP_OPTS \ + MYSQL_HOST \ + MYSQL_PORT \ + MYSQL_USER \ + MYSQL_DATABASE \ + MYSQL_NUMCONNS \ + MYSQL_QUEUELEN \ + JMX_PORT \ + RMI_BIND \ + JAVA_XMX \ + CACHE_BACKEND + opts="" opts="$opts --config '${SQLTAP_SCHEMA}'" opts="$opts --http ${SQLTAP_HTTP_PORT}" @@ -16,12 +54,19 @@ opts="$opts --mysql-database ${MYSQL_DATABASE}" opts="$opts --mysql-numconns ${MYSQL_NUMCONNS}" opts="$opts --mysql-queuelen ${MYSQL_QUEUELEN}" opts="$opts --cache-backend ${CACHE_BACKEND}" + if [[ "${CACHE_BACKEND}" == "memcache" ]]; then + require_args MEMCACHE_HOST \ + MEMCACHE_PORT \ + MEMCACHE_QUEUELEN \ + MEMCACHE_NUMCONNS + opts="$opts --memcache-host ${MEMCACHE_HOST}" opts="$opts --memcache-port ${MEMCACHE_PORT}" opts="$opts --memcache-queuelen ${MEMCACHE_QUEUELEN}" opts="$opts --memcache-numconns ${MEMCACHE_NUMCONNS}" fi + opts="$opts ${SQLTAP_OPTS}" exec java \ From c55d041f45ad2e026ab7173f4446c8d10c7ca54f Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:10:40 +0100 Subject: [PATCH 48/61] .dockerignore update --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index 35706e6..44c9325 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ .git .gitignore .DS_Store +/target +/bin From 24bcb8ffc932feef0585277264e1e67d94da51ea Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:33:05 +0100 Subject: [PATCH 49/61] Dockerfile: drop unnecessary HOSTNAME env var --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bf29d3e..4836d00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,6 @@ ENV SQLTAP_HTTP_PORT="3000" \ JMX_PORT="9191" \ RMI_BIND="127.0.0.1" \ JAVA_XMX="16384M" \ - HOSTNAME="sqltap1" \ CACHE_BACKEND="memcache" \ MEMCACHE_HOST="127.0.0.1" \ MEMCACHE_PORT="11211" \ From 7d833de61c5fe6bbd8bd521b1ded3975e9bf4bea Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:33:47 +0100 Subject: [PATCH 50/61] Dockerfile: remove default arg for MEMCACHE_HOST as it must be set explicitely when cache-backend is memcache --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4836d00..48409d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ ENV SQLTAP_HTTP_PORT="3000" \ RMI_BIND="127.0.0.1" \ JAVA_XMX="16384M" \ CACHE_BACKEND="memcache" \ - MEMCACHE_HOST="127.0.0.1" \ + MEMCACHE_HOST="" \ MEMCACHE_PORT="11211" \ MEMCACHE_QUEUELEN="8192" \ MEMCACHE_NUMCONNS="20" From 14d6523ffc4aaa989766a54a177d240ea1e6875d Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:34:22 +0100 Subject: [PATCH 51/61] Dockerfile: use new startup location --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 48409d0..1492ff8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,4 +36,4 @@ RUN cd /opt/sqltap && \ EXPOSE $SQLTAP_HTTP_PORT -CMD ["/opt/sqltap/sqltap.sh"] +CMD ["/bootup.sh"] From 3d3787dca24136a6f71c80db8f10cbcb7e7e0d16 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:34:37 +0100 Subject: [PATCH 52/61] Dockerfile: improved build process --- Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1492ff8..4cb43ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,11 +28,15 @@ ADD https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb /tmp/sbt.deb RUN dpkg -i /tmp/sbt.deb && rm -f /tmp/sbt.deb RUN apt-get update && apt-get install sbt -ADD . /opt/sqltap -RUN cd /opt/sqltap && \ +ADD project /usr/src/project/ +ADD src /usr/src/src/ +ADD build.sbt /usr/src/ +ADD bootup.sh /bootup.sh + +RUN cd /usr/src && \ sbt assembly && \ - cp -vpi /opt/sqltap/target/scala-*/sqltap.jar $SQLTAP_JARFILE && \ - rm -rf /opt/sqltap + cp -vpi /usr/src/target/scala-*/sqltap.jar $SQLTAP_JARFILE && \ + rm -rf /usr/src/* EXPOSE $SQLTAP_HTTP_PORT From eda87f942bc7b6f499c3801263efd31616f009f6 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 03:36:57 +0100 Subject: [PATCH 53/61] adds Makefile for convinience --- Makefile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..51e40f7 --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +# This file is part of the "sqltap" project, http://github.com/dawanda/sqltap> +# (c) 2016 Christian Parpart +# +# Licensed under the MIT License (the "License"); you may not use this +# file except in compliance with the License. You may obtain a copy of +# the License at: http://opensource.org/licenses/MIT + +IMAGE = sqltap +VERSION = $(shell grep ^version build.sbt | cut -d\" -f2) + +image: + docker build -t ${IMAGE}:${VERSION} . + +push: image + docker push ${IMAGE}:${VERSION} + +clean: + rm -rf target + +.PHONY: image clean + +# vim:ts=8:noet From 8089a8a1a6cd3d83e15107ff6815dd0baa30fdcc Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 04:18:21 +0100 Subject: [PATCH 54/61] bootup.sh: drop requirement of SQLTAP_OPTS env var --- bootup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/bootup.sh b/bootup.sh index ace9dfc..b5e70c3 100755 --- a/bootup.sh +++ b/bootup.sh @@ -30,7 +30,6 @@ require_args SQLTAP_SCHEMA \ SQLTAP_THREADS \ SQLTAP_SCHEMA \ SCHEMA_URL \ - SQLTAP_OPTS \ MYSQL_HOST \ MYSQL_PORT \ MYSQL_USER \ From ddbf11904abd80c5c8499a7ab743a6de5f245c6b Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 04:47:41 +0100 Subject: [PATCH 55/61] docker fixes --- Dockerfile | 2 +- bootup.sh | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4cb43ad..8a839b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,4 +40,4 @@ RUN cd /usr/src && \ EXPOSE $SQLTAP_HTTP_PORT -CMD ["/bootup.sh"] +CMD ["/bootup.sh", "$SQLTAP_JARFILE"] diff --git a/bootup.sh b/bootup.sh index b5e70c3..059f2fa 100755 --- a/bootup.sh +++ b/bootup.sh @@ -1,5 +1,7 @@ #!/bin/bash -set -e +set -ex + +SQLTAP_JARFILE="${1}" if [[ "$SCHEMA_URL" != "" ]]; then curl -sL "$SCHEMA_URL" -o "${SQLTAP_SCHEMA}" @@ -28,7 +30,6 @@ require_args SQLTAP_SCHEMA \ MYSQL_PORT \ SQLTAP_HTTP_PORT \ SQLTAP_THREADS \ - SQLTAP_SCHEMA \ SCHEMA_URL \ MYSQL_HOST \ MYSQL_PORT \ @@ -42,7 +43,7 @@ require_args SQLTAP_SCHEMA \ CACHE_BACKEND opts="" -opts="$opts --config '${SQLTAP_SCHEMA}'" +opts="$opts --config ${SQLTAP_SCHEMA}" opts="$opts --http ${SQLTAP_HTTP_PORT}" opts="$opts --disable-keepalive" opts="$opts -t ${SQLTAP_THREADS}" From ba78cc0ca6fe3eac9e00407b6138bed46414be03 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 04:55:18 +0100 Subject: [PATCH 56/61] docker: fixing SQLTAP_JARFILE --- Dockerfile | 5 ++--- bootup.sh | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8a839b7..b2fc0e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,6 @@ FROM java:8 MAINTAINER DaWanda ARG SBT_VERSION="0.13.8" -ARG SQLTAP_JARFILE="/usr/lib/sqltap.jar" ENV SQLTAP_HTTP_PORT="3000" \ SQLTAP_THREADS="16" \ @@ -35,9 +34,9 @@ ADD bootup.sh /bootup.sh RUN cd /usr/src && \ sbt assembly && \ - cp -vpi /usr/src/target/scala-*/sqltap.jar $SQLTAP_JARFILE && \ + cp -vpi /usr/src/target/scala-*/sqltap.jar /usr/lib/sqltap.jar && \ rm -rf /usr/src/* EXPOSE $SQLTAP_HTTP_PORT -CMD ["/bootup.sh", "$SQLTAP_JARFILE"] +CMD ["/bootup.sh"] diff --git a/bootup.sh b/bootup.sh index 059f2fa..c77ae1c 100755 --- a/bootup.sh +++ b/bootup.sh @@ -1,7 +1,7 @@ #!/bin/bash set -ex -SQLTAP_JARFILE="${1}" +SQLTAP_JARFILE="/usr/lib/sqltap.jar" if [[ "$SCHEMA_URL" != "" ]]; then curl -sL "$SCHEMA_URL" -o "${SQLTAP_SCHEMA}" From 040c0b87c454530c7bdb6381831bf457d7d36e08 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 23 Jan 2016 04:56:12 +0100 Subject: [PATCH 57/61] docker: move ADD bootup.sh at the most bottom place as possible --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b2fc0e7..8a8d65e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,6 @@ RUN apt-get update && apt-get install sbt ADD project /usr/src/project/ ADD src /usr/src/src/ ADD build.sbt /usr/src/ -ADD bootup.sh /bootup.sh RUN cd /usr/src && \ sbt assembly && \ @@ -39,4 +38,5 @@ RUN cd /usr/src && \ EXPOSE $SQLTAP_HTTP_PORT +ADD bootup.sh /bootup.sh CMD ["/bootup.sh"] From 344415bd4101dc2af933c187d7cfb9ffb0401dcf Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 25 Jan 2016 11:14:50 +0100 Subject: [PATCH 58/61] Makefile: adds a p[h]ony --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 51e40f7..9c7b446 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,6 @@ push: image clean: rm -rf target -.PHONY: image clean +.PHONY: image clean push # vim:ts=8:noet From 393243bca72f8575491c0094e4d257fdbdec8d39 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 25 Jan 2016 11:15:15 +0100 Subject: [PATCH 59/61] bootup.sh: env var cleanup --- bootup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootup.sh b/bootup.sh index c77ae1c..9a04783 100755 --- a/bootup.sh +++ b/bootup.sh @@ -9,7 +9,7 @@ fi require_arg() { local name="$1" - local val=`eval "echo \\\$${name}"` + local val="${!name}" echo "${name}: ${val}" From 1975517e5ecdb8fd2b98abe0ee671154d9af1514 Mon Sep 17 00:00:00 2001 From: Paul Asmuth Date: Wed, 2 Mar 2016 17:21:07 +0100 Subject: [PATCH 60/61] handle >31bit integer field ids --- src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala | 2 +- src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala | 2 +- .../com/paulasmuth/sqltap/FindMultiInstruction.scala | 6 +++--- .../com/paulasmuth/sqltap/FindSingleInstruction.scala | 4 ++-- src/main/scala/com/paulasmuth/sqltap/Record.scala | 8 ++++---- .../scala/com/paulasmuth/sqltap/RecordLookupJob.scala | 2 +- .../scala/com/paulasmuth/sqltap/ReplicationFeed.scala | 2 +- src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala | 4 ++-- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala b/src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala index 234f58a..bf98e3d 100644 --- a/src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala +++ b/src/main/scala/com/paulasmuth/sqltap/CTreeCache.scala @@ -46,7 +46,7 @@ object CTreeCache { * @param record_id the primary id of the resource/record to be expired * @param resource_name the name of the resource to be expired */ - def expire(worker: Worker, resource_name: String, record_id: Int) : Unit = { + def expire(worker: Worker, resource_name: String, record_id: Long) : Unit = { if (!Manifest.has_resource(resource_name)) throw new ParseException("unknown resource: " + resource_name) diff --git a/src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala b/src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala index 354ec50..58716c2 100644 --- a/src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala +++ b/src/main/scala/com/paulasmuth/sqltap/ExpirationJob.scala @@ -34,7 +34,7 @@ class ExpirationJob(worker: Worker, ctree: CTree) extends ReadyCallback[Record] * * @param record the primary record id */ - def execute(record_id: Int) : Unit = { + def execute(record_id: Long) : Unit = { val primary_id = ctree.resource.id_field Logger.debug( diff --git a/src/main/scala/com/paulasmuth/sqltap/FindMultiInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/FindMultiInstruction.scala index 9d7a62c..f597796 100644 --- a/src/main/scala/com/paulasmuth/sqltap/FindMultiInstruction.scala +++ b/src/main/scala/com/paulasmuth/sqltap/FindMultiInstruction.scala @@ -29,7 +29,7 @@ class FindMultiInstruction extends SQLInstruction with CTreeInstruction { var offset : String = null var expanded : Boolean = false - var join_id : Int = 0 + var join_id : Long = 0 var join_conditions : String = null def execute(_worker: Worker) : Unit = { @@ -75,7 +75,7 @@ class FindMultiInstruction extends SQLInstruction with CTreeInstruction { else if (relation.join_field_local != null && prev.is_finished) { state = INS_STATE_READY - join_id = prev.record.get(relation.join_field_local).toInt + join_id = prev.record.get(relation.join_field_local).toLong } } @@ -186,7 +186,7 @@ class FindMultiInstruction extends SQLInstruction with CTreeInstruction { var found = false val this_id = row( - query.columns.indexOf(relation.resource.id_field)).toInt + query.columns.indexOf(relation.resource.id_field)).toLong while (!found && n > 0) { val this_ins = next(n - 1) diff --git a/src/main/scala/com/paulasmuth/sqltap/FindSingleInstruction.scala b/src/main/scala/com/paulasmuth/sqltap/FindSingleInstruction.scala index e53ca26..a5765f5 100644 --- a/src/main/scala/com/paulasmuth/sqltap/FindSingleInstruction.scala +++ b/src/main/scala/com/paulasmuth/sqltap/FindSingleInstruction.scala @@ -26,7 +26,7 @@ class FindSingleInstruction extends SQLInstruction with CTreeInstruction { var order : String = null var join_field : String = null - var join_id : Int = 0 + var join_id : Long = 0 var allow_empty : Boolean = false def execute(_worker: Worker) : Unit = { @@ -66,7 +66,7 @@ class FindSingleInstruction extends SQLInstruction with CTreeInstruction { } state = INS_STATE_READY - join_id = join_id_str.toInt + join_id = join_id_str.toLong if (join_id == 0) { state = INS_STATE_DONE diff --git a/src/main/scala/com/paulasmuth/sqltap/Record.scala b/src/main/scala/com/paulasmuth/sqltap/Record.scala index b655062..6f094c4 100644 --- a/src/main/scala/com/paulasmuth/sqltap/Record.scala +++ b/src/main/scala/com/paulasmuth/sqltap/Record.scala @@ -15,16 +15,16 @@ class Record(_resource: ResourceManifest) { var fields = ListBuffer[String]() var data = ListBuffer[String]() - def id() : Int = { - get(resource.id_field).toInt + def id() : Long = { + get(resource.id_field).toLong } - def set_id(id: Int) : Unit = { + def set_id(id: Long) : Unit = { set(resource.id_field, id.toString) } def set_id(id: String) : Unit = { - set_id(id.toInt) + set_id(id.toLong) } def has_id() : Boolean = { diff --git a/src/main/scala/com/paulasmuth/sqltap/RecordLookupJob.scala b/src/main/scala/com/paulasmuth/sqltap/RecordLookupJob.scala index ca4030a..900cfa1 100644 --- a/src/main/scala/com/paulasmuth/sqltap/RecordLookupJob.scala +++ b/src/main/scala/com/paulasmuth/sqltap/RecordLookupJob.scala @@ -14,7 +14,7 @@ class RecordLookupJob(worker: Worker, resource: ResourceManifest) extends ReadyC private val callbacks = new ListBuffer[ReadyCallback[Record]]() - def execute(record_id: Int) : Unit = { + def execute(record_id: Long) : Unit = { if (callbacks.length == 0) { return // RecordLookupJob is a noop without callbacks } diff --git a/src/main/scala/com/paulasmuth/sqltap/ReplicationFeed.scala b/src/main/scala/com/paulasmuth/sqltap/ReplicationFeed.scala index 52c988d..0ac2ac5 100644 --- a/src/main/scala/com/paulasmuth/sqltap/ReplicationFeed.scala +++ b/src/main/scala/com/paulasmuth/sqltap/ReplicationFeed.scala @@ -29,7 +29,7 @@ object ReplicationFeed extends Worker with AbstractSQLConnectionPool { CTreeCache.expire(this, Manifest.resource_name_for_table(evt.table_name), - evt.primary_key.toInt) + evt.primary_key.toLong) } } diff --git a/src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala b/src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala index c074a3f..7ff6da7 100644 --- a/src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala +++ b/src/main/scala/com/paulasmuth/sqltap/SQLBuilder.scala @@ -12,7 +12,7 @@ object SQLBuilder { def select( res: ResourceManifest, id_field: String, - id: Int, + id: Long, fields: List[String], cond: String, order: String, @@ -45,7 +45,7 @@ object SQLBuilder { def count( res: ResourceManifest, id_field: String, - id: Int, + id: Long, cond: String ) : String = ( From 7f0d6551577f0865c5870c321657bc512f3462f7 Mon Sep 17 00:00:00 2001 From: Paul Asmuth Date: Wed, 2 Mar 2016 17:38:04 +0100 Subject: [PATCH 61/61] also support 64bit integers in RowsBinlogEvent --- .../com/paulasmuth/sqltap/mysql/BinaryInteger.scala | 9 +++++++++ .../com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/scala/com/paulasmuth/sqltap/mysql/BinaryInteger.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/BinaryInteger.scala index c3a9108..d67c571 100644 --- a/src/main/scala/com/paulasmuth/sqltap/mysql/BinaryInteger.scala +++ b/src/main/scala/com/paulasmuth/sqltap/mysql/BinaryInteger.scala @@ -18,4 +18,13 @@ object BinaryInteger { return value } + def readLong(data: Array[Byte], pos: Int, len: Int) : Long = { + var value : Long = 0 + + for (n <- (0 until len)) + value += (data(pos + n) & 0x000000ff) << (8*n) + + return value + } + } diff --git a/src/main/scala/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala b/src/main/scala/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala index f4a954b..e9e6cc5 100644 --- a/src/main/scala/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala +++ b/src/main/scala/com/paulasmuth/sqltap/mysql/RowsBinlogEvent.scala @@ -65,8 +65,8 @@ trait RowsBinlogEvent extends BinlogEvent { num._1 } - def read_int(bytes: Int) : Int = { - val num = BinaryInteger.read(data, cur, bytes) + def read_int(bytes: Int) : Long = { + val num = BinaryInteger.readLong(data, cur, bytes) cur += bytes num }