diff --git a/Commands/Test.php b/Commands/Test.php index 92df691..f78de26 100644 --- a/Commands/Test.php +++ b/Commands/Test.php @@ -75,7 +75,7 @@ protected function doExecute(): int $output->writeln('Timeout: ' . $settings->redisTimeout->getValue()); $output->writeln('Password: ' . $settings->redisPassword->getValue()); $output->writeln('Database: ' . $settings->redisDatabase->getValue()); - $output->writeln('UseSentinelBackend: ' . (int) $settings->useSentinelBackend->getValue()); + $output->writeln('RedisBackendType: ' . $settings->getRedisType()); $output->writeln('SentinelMasterName: ' . $settings->sentinelMasterName->getValue()); $output->writeln(''); @@ -222,9 +222,14 @@ protected function doExecute(): int */ private function getRedisConfig($redis, $configName) { - $config = $redis->config('GET', $configName); - $value = strtolower(array_shift($config)); + if ($redis instanceof \RedisCluster) { + $config = $redis->config('CONFIG', 'GET', $configName); + unset($config[0]); + } else { + $config = $redis->config('GET', $configName); + } + $value = strtolower(array_shift($config)); return $value; } diff --git a/Queue/Backend/RedisCluster.php b/Queue/Backend/RedisCluster.php new file mode 100644 index 0000000..7d5319c --- /dev/null +++ b/Queue/Backend/RedisCluster.php @@ -0,0 +1,348 @@ +connectIfNeeded(); + return 'TEST' === $this->redis->echo('TEST_ECHO', 'TEST'); + + } catch (\Exception $e) { + Log::debug($e->getMessage()); + } + + return false; + } + + public function getServerVersion() + { + $this->connectIfNeeded(); + + $server = $this->redis->info('server'); + + if (empty($server)) { + return ''; + } + + $version = $server['redis_version']; + + return $version; + } + + public function getLastError() + { + $this->connectIfNeeded(); + + return $this->redis->getLastError(); + } + + /** + * Returns converted bytes to B,K,M,G,T. + * @param int|float|double $bytes byte number. + * @param int $precision decimal round. + * * @return string + */ + private function formatBytes($bytes, $precision = 2) { + $units = array('B', 'K', 'M', 'G', 'T'); + + $bytes = max($bytes, 0); + $pow = floor(($bytes ? log($bytes) : 0) / log(1024)); + $pow = min($pow, count($units) - 1); + $bytes /= (1 << (10 * $pow)); + + return round($bytes, $precision) . $units[$pow]; + } + + public function getMemoryStats() + { + $this->connectIfNeeded(); + + $hosts = explode(',', $this->host); + $ports = explode(',', $this->port); + + $memory = array ( + 'used_memory_human' => 0, + 'used_memory_peak_human' => 0 + ); + + foreach ($hosts as $idx=>$host) { + $info = $this->redis->info(array($host, (int)$ports[$idx]), 'memory'); + $memory['used_memory_human'] += $info['used_memory'] ?? 0; + $memory['used_memory_peak_human'] += $info['used_memory_peak'] ?? 0; + } + + $memory['used_memory_human'] = $this->formatBytes($memory['used_memory_human']); + $memory['used_memory_peak_human'] = $this->formatBytes($memory['used_memory_peak_human']); + + return $memory; + } + + /** + * Returns the time to live of a key that can expire in ms. + * @param $key + * @return int + */ + public function getTimeToLive($key) + { + $this->connectIfNeeded(); + + $ttl = $this->redis->pttl($key); + + if ($ttl == -1) { + // key exists but has no associated expire + return 99999999; + } + + if ($ttl == -2) { + // key does not exist + return 0; + } + + return $ttl; + } + + public function appendValuesToList($key, $values) + { + $this->connectIfNeeded(); + + foreach ($values as $value) { + $this->redis->rPush($key, gzcompress($value)); + } + + // usually we would simply do call_user_func_array(array($redis, 'rPush'), $values); as rpush supports multiple values + // at once but it seems to be not implemented yet see https://github.com/nicolasff/phpredis/issues/366 + // doing it in one command should be much faster as it requires less tcp communication. Anyway, we currently do + // not write multiple values at once ... so it is ok! + } + + public function getFirstXValuesFromList($key, $numValues) + { + if ($numValues <= 0) { + return array(); + } + + $this->connectIfNeeded(); + $values = $this->redis->lRange($key, 0, $numValues - 1); + foreach($values as $key => $value) { + $tmpValue = @gzuncompress($value); // Avoid warning if not compressed + + // if empty, string is not compressed. Use original value + if(empty($tmpValue)) { + $values[$key] = $value; + } else { + $values[$key] = $tmpValue; + } + } + + return $values; + } + + public function removeFirstXValuesFromList($key, $numValues) + { + if ($numValues <= 0) { + return; + } + + $this->connectIfNeeded(); + $this->redis->ltrim($key, $numValues, -1); + } + + public function hasAtLeastXRequestsQueued($key, $numValuesRequired) + { + if ($numValuesRequired <= 0) { + return true; + } + + $numActual = $this->getNumValuesInList($key); + + return $numActual >= $numValuesRequired; + } + + public function getNumValuesInList($key) + { + $this->connectIfNeeded(); + + return $this->redis->lLen($key); + } + + public function setIfNotExists($key, $value, $ttlInSeconds) + { + $this->connectIfNeeded(); + $wasSet = $this->redis->set($key, $value, array('nx', 'ex' => $ttlInSeconds)); + + return $wasSet; + } + + /** + * @internal for tests only + * @return \RedisCluster + */ + public function getConnection() + { + return $this->redis; + } + + /** + * @internal for tests only + */ + public function delete($key) + { + $this->connectIfNeeded(); + + return $this->redis->del($key) > 0; + } + + public function deleteIfKeyHasValue($key, $value) + { + if (empty($value)) { + return false; + } + + $this->connectIfNeeded(); + + // see http://redis.io/topics/distlock + $script = 'if redis.call("GET",KEYS[1]) == ARGV[1] then + return redis.call("DEL",KEYS[1]) +else + return 0 +end'; + + // ideally we would use evalSha to reduce bandwidth! + return (bool) $this->evalScript($script, array($key), array($value)); + } + + protected function evalScript($script, $keys, $args) + { + return $this->redis->eval($script, array_merge($keys, $args), count($keys)); + } + + public function getKeysMatchingPattern($pattern) + { + $this->connectIfNeeded(); + + return $this->redis->keys($pattern); + } + + public function expireIfKeyHasValue($key, $value, $ttlInSeconds) + { + if (empty($value)) { + return false; + } + + $this->connectIfNeeded(); + + $script = 'if redis.call("GET",KEYS[1]) == ARGV[1] then + return redis.call("EXPIRE",KEYS[1], ARGV[2]) +else + return 0 +end'; + // ideally we would use evalSha to reduce bandwidth! + return (bool) $this->evalScript($script, array($key), array($value, (int) $ttlInSeconds)); + } + + public function get($key) + { + $this->connectIfNeeded(); + + return $this->redis->get($key); + } + + /** + * @internal + */ + public function flushAll() + { + $this->connectIfNeeded(); + + $hosts = explode(',', $this->host); + $ports = explode(',', $this->port); + + foreach ($hosts as $idx=>$host) { + $this->redis->flushDB(array($host, (int)$ports[$idx])); + } + } + + private function connectIfNeeded() + { + if (!$this->isConnected()) { + $this->connect(); + } + } + + protected function connect() + { + $hosts = explode(',', $this->host); + $ports = explode(',', $this->port); + + if (count($hosts) !== count($ports)) { + throw new Exception(Piwik::translate('QueuedTracking_NumHostsNotMatchNumPorts')); + } + + $hostsPorts = array_map(fn($host, $port): string => "$host:$port", $hosts, $ports); + + try { + $this->redis = new \RedisCluster(NULL, $hostsPorts, $this->timeout, $this->timeout, true, $this->password); + return true; + } catch (Exception $e) { + throw new Exception('Could not connect to redis cluster: ' . $e->getMessage()); + } + } + + public function setConfig($host, $port, $timeout, $password) + { + $this->disconnect(); + + $this->host = $host; + $this->port = $port; + $this->timeout = $timeout; + + if (!empty($password)) { + $this->password = $password; + } + } + + private function disconnect() + { + if ($this->isConnected()) { + $this->redis->close(); + } + + $this->redis = null; + } + + private function isConnected() + { + return isset($this->redis); + } + + public function setDatabase($database) + { + $this->database = $database; + } +} diff --git a/Queue/Factory.php b/Queue/Factory.php index 6db34cd..1caaefc 100644 --- a/Queue/Factory.php +++ b/Queue/Factory.php @@ -70,14 +70,17 @@ public static function makeBackendFromSettings(SystemSettings $settings) } else { $redis = new Queue\Backend\Sentinel(); $redis->setSentinelMasterName($masterName); + $redis->setDatabase($database); } + } + elseif($settings->isUsingClusterBackend()) { + $redis = new Queue\Backend\RedisCluster(); } else { $redis = new Queue\Backend\Redis(); + $redis->setDatabase($database); } $redis->setConfig($host, $port, $timeout, $password); - $redis->setDatabase($database); - return $redis; } diff --git a/SystemSettings.php b/SystemSettings.php index 381fee9..329b897 100644 --- a/SystemSettings.php +++ b/SystemSettings.php @@ -9,6 +9,7 @@ namespace Piwik\Plugins\QueuedTracking; use Piwik\Plugins\QueuedTracking\Settings\NumWorkers; +use Piwik\Settings\Plugin\SystemSetting; use Piwik\Settings\Setting; use Piwik\Settings\FieldConfig; use Piwik\Plugins\QueuedTracking\Queue\Factory; @@ -53,11 +54,20 @@ class SystemSettings extends \Piwik\Settings\Plugin\SystemSettings public $numRequestsToProcess; /** @var Setting */ - public $useSentinelBackend; + public $useWhatRedisBackendType; /** @var Setting */ public $sentinelMasterName; + public function getAvailableRedisBackendTypes() + { + return array( + 1=>Piwik::translate('QueuedTracking_AvailableRedisBackendTypeStandAlone'), + 2=>Piwik::translate('QueuedTracking_AvailableRedisBackendTypeSentinel'), + 3=>Piwik::translate('QueuedTracking_AvailableRedisBackendTypeCluster') + ); + } + protected function assignValueIsIntValidator (FieldConfig $field) { $field->validate = function ($value) { if ((is_string($value) && !ctype_digit($value)) || (!is_string($value) && !is_int($value))) { @@ -69,7 +79,7 @@ protected function assignValueIsIntValidator (FieldConfig $field) { protected function init() { $this->backend = $this->createBackendSetting(); - $this->useSentinelBackend = $this->createUseSentinelBackend(); + $this->useWhatRedisBackendType = $this->createUseWhatRedisBackendType(); $this->sentinelMasterName = $this->createSetSentinelMasterName(); $this->redisHost = $this->createRedisHostSetting(); $this->redisPort = $this->createRedisPortSetting(); @@ -84,7 +94,17 @@ protected function init() public function isUsingSentinelBackend() { - return $this->useSentinelBackend->getValue(); + return $this->useWhatRedisBackendType->getValue() === 2; + } + + public function isUsingClusterBackend() + { + return $this->useWhatRedisBackendType->getValue() === 3; + } + + public function getRedisType() + { + return $this->useWhatRedisBackendType->getValue(); } public function getSentinelMasterName() @@ -109,7 +129,7 @@ private function createRedisHostSetting() $field->inlineHelp = Piwik::translate('QueuedTracking_RedisHostFieldHelp') . '

' . Piwik::translate('QueuedTracking_RedisHostFieldHelpExtended') . '
'; - if ($self->isUsingSentinelBackend()) { + if ($self->isUsingSentinelBackend() || $self->isUsingClusterBackend()) { $field->inlineHelp .= '
' . Piwik::translate('QueuedTracking_RedisHostFieldHelpExtendedSentinel') . '
'; } @@ -143,20 +163,20 @@ private function createRedisPortSetting() $field->uiControlAttributes = array('size' => 100); $field->inlineHelp = Piwik::translate('QueuedTracking_RedisPortFieldHelp') . '
'; - if ($self->isUsingSentinelBackend()) { + if ($self->isUsingSentinelBackend() || $self->isUsingClusterBackend()) { $field->inlineHelp .= '
' . Piwik::translate('QueuedTracking_RedisHostFieldHelpExtendedSentinel') . '
'; } $field->validate = function ($value) use ($self) { $self->checkMultipleServersOnlyConfiguredWhenSentinelIsEnabled($value); - if (!$self->isUsingSentinelBackend()) { - (new NumberRange(0, 65535))->validate($value); - } else { + if ($self->isUsingSentinelBackend() || $self->isUsingClusterBackend()) { $ports = explode(',', $value); foreach ($ports as $port) { (new NumberRange(0, 65535))->validate(trim($port)); } + } else { + (new NumberRange(0, 65535))->validate($value); } }; @@ -279,7 +299,7 @@ private function createProcessInTrackingRequestSetting() public function checkMultipleServersOnlyConfiguredWhenSentinelIsEnabled($value) { - if ($this->isUsingSentinelBackend()) { + if ($this->isUsingSentinelBackend() || $this->isUsingClusterBackend()) { return; } @@ -322,14 +342,14 @@ private function createBackendSetting() }); } - private function createUseSentinelBackend() + private function createUseWhatRedisBackendType() { - return $this->makeSetting('useSentinelBackend', $default = false, FieldConfig::TYPE_BOOL, function (FieldConfig $field) { - $field->title = Piwik::translate('QueuedTracking_UseSentinelFieldTitle'); - $field->uiControl = FieldConfig::UI_CONTROL_CHECKBOX; - $field->uiControlAttributes = array('size' => 3); + return $this->makeSetting('useWhatRedisBackendType', $default = 1, FieldConfig::TYPE_INT, function (FieldConfig $field) { + $field->title = 'Redis type'; + $field->uiControl = FieldConfig::UI_CONTROL_RADIO; + $field->availableValues = $this->getAvailableRedisBackendTypes(); $field->condition = 'backend=="redis"'; - $field->inlineHelp = Piwik::translate('QueuedTracking_UseSentinelFieldHelp') . '
'; + $field->inlineHelp = Piwik::translate('QueuedTracking_WhatRedisBackEndType') . '
'; }); } diff --git a/Updates/5.0.8.php b/Updates/5.0.8.php new file mode 100644 index 0000000..4dd583e --- /dev/null +++ b/Updates/5.0.8.php @@ -0,0 +1,34 @@ +getValue() == 0) { + $tmp_useWhatRedisBackendType->setValue($old_useSentinelBackend->getValue() == true ? 2 : 1); + $tmp_useWhatRedisBackendType->save(); + } + } +} diff --git a/docs/faq.md b/docs/faq.md index b624c55..d8f2c1b 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -90,6 +90,8 @@ There should be only one Redis server to make sure the data will be replayed in If you want to configure Redis HA (High Availability) it is possible to use Redis Sentinel see further down. We currently write into the Redis default database by default but you can configure to use a different one. +You can also use a "Redis Cluster" to distribute all tracking requests data across multiple Redis masters/shards, complete with the HA feature. + __Why do some tests fail on my local Piwik instance?__ Make sure the requirements mentioned above are met and Redis needs to run on 127.0.0.1:6379 with no password for the diff --git a/lang/en.json b/lang/en.json index 5b74327..2e2f6fe 100644 --- a/lang/en.json +++ b/lang/en.json @@ -12,6 +12,10 @@ "NumberOfQueueWorkersFieldTitle": "Number of queue workers", "NumberOfQueueWorkersFieldHelp": "Number of allowed maximum queue workers. Accepts a number between 1 and 16. Best practice is to set the number of CPUs you want to make available for queue processing. Be aware you need to make sure to start the workers manually. We recommend to not use 9-15 workers, rather use 8 or 16 as the queue might not be distributed evenly into different queues.", "NumberOfQueueWorkersFieldHelpNew": "Number of allowed maximum queue workers. Accepts a number between 1 and 4096. Best practice is to set the number of CPUs you want to make available for queue processing. Be aware you need to make sure to start the workers manually. We recommend to not use 9-15 workers, rather use 8 or 16 as the queue might not be distributed evenly into different queues.", + "WhatRedisBackEndType": "Select which type of redis to use. Make sure to update host and port if needed. Once you have selected and saved the change, you will be able to specify multiple hosts and ports using comma separated lists for \"Sentinel\" and \"Cluster\" type only.", + "AvailableRedisBackendTypeStandAlone": "Stand-alone", + "AvailableRedisBackendTypeSentinel": "Sentinel", + "AvailableRedisBackendTypeCluster": "Cluster", "RedisPasswordFieldTitle": "Redis password", "RedisPasswordFieldHelp": "Password set on the Redis server, if any. Redis can be instructed to require a password before allowing clients to execute commands.", "RedisDatabaseFieldTitle": "Redis database", diff --git a/plugin.json b/plugin.json index 7d89bb2..84ac0c8 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "name": "QueuedTracking", - "version": "5.0.7", + "version": "5.0.8", "description": "Scale your large traffic Matomo service by queuing tracking requests in Redis or MySQL for better performance and reliability when experiencing peaks.", "theme": false, "keywords": ["tracker", "tracking", "queue", "redis"], diff --git a/tests/Framework/TestCase/IntegrationTestCase.php b/tests/Framework/TestCase/IntegrationTestCase.php index 0d440de..0abf734 100644 --- a/tests/Framework/TestCase/IntegrationTestCase.php +++ b/tests/Framework/TestCase/IntegrationTestCase.php @@ -35,7 +35,7 @@ public function setUp(): void protected function enableRedisSentinel($master = 'mymaster') { - Config::getInstance()->QueuedTracking = array('useSentinelBackend' => '1', 'sentinelMasterName' => $master); + Config::getInstance()->QueuedTracking = array('useWhatRedisBackendType' => '2', 'sentinelMasterName' => $master); } protected function disableRedisSentinel() diff --git a/tests/Integration/Queue/FactoryTest.php b/tests/Integration/Queue/FactoryTest.php index 15eeef3..2bf653d 100644 --- a/tests/Integration/Queue/FactoryTest.php +++ b/tests/Integration/Queue/FactoryTest.php @@ -63,13 +63,13 @@ public function test_makeBackend_shouldFailToCreateASentinelInstance_IfNotFullyC $this->expectException(\Exception::class); $this->expectExceptionMessage('You must configure a sentinel master name'); - Config::getInstance()->QueuedTracking = array('useSentinelBackend' => '1', 'sentinelMasterName' => ''); + Config::getInstance()->QueuedTracking = array('useWhatRedisBackendType' => '2', 'sentinelMasterName' => ''); Factory::makeBackend(); } public function test_makeBackend_shouldReturnASentinelInstanceIfConfigured() { - Config::getInstance()->QueuedTracking = array('useSentinelBackend' => '1', 'sentinelMasterName' => 'mymaster'); + Config::getInstance()->QueuedTracking = array('useWhatRedisBackendType' => '2', 'sentinelMasterName' => 'mymaster'); $backend = Factory::makeBackend(); Config::getInstance()->QueuedTracking = array(); $this->assertTrue($backend instanceof Queue\Backend\Sentinel); diff --git a/tests/Integration/SettingsTest.php b/tests/Integration/SettingsTest.php index 8f86397..7c1abc5 100644 --- a/tests/Integration/SettingsTest.php +++ b/tests/Integration/SettingsTest.php @@ -236,13 +236,27 @@ public function test_sentinelMasterName_ShouldTrimTheGivenValue_IfNotEmpty() $this->assertSame('test', $this->settings->sentinelMasterName->getValue()); } - public function test_useSentinelBackend() + public function test_useWhatRedisBackendType() { - $this->settings->useSentinelBackend->setValue('0'); - $this->assertFalse($this->settings->useSentinelBackend->getValue()); + $this->settings->useWhatRedisBackendType->setValue(1); + $this->assertFalse($this->settings->isUsingSentinelBackend()); + + $this->settings->useWhatRedisBackendType->setValue(3); + $this->assertFalse($this->settings->isUsingSentinelBackend()); + + $this->settings->useWhatRedisBackendType->setValue(2); + $this->assertTrue($this->settings->isUsingSentinelBackend()); + } + + public function testIsUsingClusterBackend() + { + $this->settings->useWhatRedisBackendType->setValue(1); + $this->assertFalse($this->settings->isUsingClusterBackend()); + $this->settings->useWhatRedisBackendType->setValue(2); + $this->assertFalse($this->settings->isUsingClusterBackend()); - $this->settings->useSentinelBackend->setValue('1'); - $this->assertTrue($this->settings->useSentinelBackend->getValue()); + $this->settings->useWhatRedisBackendType->setValue(3); + $this->assertTrue($this->settings->isUsingClusterBackend()); } public function test_redisPort_ShouldFailWhenMultipleValuesGiven_IfSentinelNotEnabled() @@ -290,9 +304,9 @@ public function test_queueEnabled_ShouldBeDisabledByDefault() $this->assertFalse($this->settings->queueEnabled->getValue()); } - public function test_useSentinelBackend_ShouldBeDisabledByDefault() + public function test_useWhatRedisBackendType_ShouldBe1Default() { - $this->assertFalse($this->settings->useSentinelBackend->getValue()); + $this->assertSame(1, $this->settings->useWhatRedisBackendType->getValue()); } public function test_sentinelMasterName_shouldHaveValueByDefault() diff --git a/tests/UI/QueuedTrackingSettings_spec.js b/tests/UI/QueuedTrackingSettings_spec.js index 487296b..b0ed9af 100644 --- a/tests/UI/QueuedTrackingSettings_spec.js +++ b/tests/UI/QueuedTrackingSettings_spec.js @@ -52,7 +52,7 @@ describe("QueuedTrackingSettings", function () { it("should display the settings page with sentinel enabled", async function () { testEnvironment.overrideConfig('QueuedTracking', { - useSentinelBackend: '1' + useWhatRedisBackendType: '2' }); testEnvironment.save(); diff --git a/tests/UI/expected-ui-screenshots/QueuedTrackingSettings_settings_page.png b/tests/UI/expected-ui-screenshots/QueuedTrackingSettings_settings_page.png index 95606e6..5db2f67 100644 Binary files a/tests/UI/expected-ui-screenshots/QueuedTrackingSettings_settings_page.png and b/tests/UI/expected-ui-screenshots/QueuedTrackingSettings_settings_page.png differ diff --git a/tests/UI/expected-ui-screenshots/QueuedTrackingSettings_settings_save_error.png b/tests/UI/expected-ui-screenshots/QueuedTrackingSettings_settings_save_error.png index 39ed0c1..a118399 100644 Binary files a/tests/UI/expected-ui-screenshots/QueuedTrackingSettings_settings_save_error.png and b/tests/UI/expected-ui-screenshots/QueuedTrackingSettings_settings_save_error.png differ