From fd3cfe5cdf50aaf82585a46a1b642c9d4bda7c36 Mon Sep 17 00:00:00 2001 From: Stefan Scheu Date: Thu, 25 May 2023 12:08:56 +0200 Subject: [PATCH 1/3] feat: #47835 added redis garbage collector --- .../Cache/Adapter/AbstractTagAwareAdapter.php | 2 +- .../Cache/Adapter/RedisTagAwareAdapter.php | 192 +++++++++++++++++- 2 files changed, 192 insertions(+), 2 deletions(-) diff --git a/src/Symfony/Component/Cache/Adapter/AbstractTagAwareAdapter.php b/src/Symfony/Component/Cache/Adapter/AbstractTagAwareAdapter.php index ef62b4fb21c7f..9daf03d387452 100644 --- a/src/Symfony/Component/Cache/Adapter/AbstractTagAwareAdapter.php +++ b/src/Symfony/Component/Cache/Adapter/AbstractTagAwareAdapter.php @@ -35,7 +35,7 @@ abstract class AbstractTagAwareAdapter implements TagAwareAdapterInterface, TagA use AbstractAdapterTrait; use ContractsTrait; - private const TAGS_PREFIX = "\1tags\1"; + protected const TAGS_PREFIX = "\1tags\1"; protected function __construct(string $namespace = '', int $defaultLifetime = 0) { diff --git a/src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php b/src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php index a3ef9f10960b6..16c3782fafadb 100644 --- a/src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php +++ b/src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php @@ -23,6 +23,7 @@ use Symfony\Component\Cache\Marshaller\DeflateMarshaller; use Symfony\Component\Cache\Marshaller\MarshallerInterface; use Symfony\Component\Cache\Marshaller\TagAwareMarshaller; +use Symfony\Component\Cache\PruneableInterface; use Symfony\Component\Cache\Traits\RedisTrait; /** @@ -44,7 +45,7 @@ * @author Nicolas Grekas * @author André Rømcke */ -class RedisTagAwareAdapter extends AbstractTagAwareAdapter +class RedisTagAwareAdapter extends AbstractTagAwareAdapter implements PruneableInterface { use RedisTrait; @@ -305,4 +306,193 @@ private function getRedisEvictionPolicy(): string return $this->redisEvictionPolicy = ''; } + + + private function getPrefix(): string + { + if ($this->redis instanceof \Predis\ClientInterface) { + $prefix = $this->redis->getOptions()->prefix ? $this->redis->getOptions()->prefix->getPrefix() : ''; + } elseif (\is_array($prefix = $this->redis->getOption(\Redis::OPT_PREFIX) ?? '')) { + $prefix = current($prefix); + } + return $prefix; + } + + /** + * Returns all existing tag keys from the cache. + * + * @TODO Verify the LUA scripts are redis-cluster safe. + * + * @return array + */ + protected function getAllTagKeys(): array + { + $tagKeys = []; + $prefix = $this->getPrefix(); + // need to trim the \0 for lua script + $tagsPrefix = trim(self::TAGS_PREFIX); + + // get all SET entries which are tagged + $getTagsLua = <<<'EOLUA' + redis.replicate_commands() + local cursor = ARGV[1] + local prefix = ARGV[2] + local tagPrefix = string.gsub(KEYS[1], prefix, "") + return redis.call('SCAN', cursor, 'COUNT', 5000, 'MATCH', '*' .. tagPrefix .. '*', 'TYPE', 'set') + EOLUA; + $cursor = null; + do { + $results = $this->pipeline(function () use ($getTagsLua, $cursor, $prefix, $tagsPrefix) { + yield 'eval' => [$getTagsLua, [$tagsPrefix, $cursor, $prefix], 1]; + }); + + $setKeys = $results->valid() ? iterator_to_array($results) : []; + [$cursor, $ids] = $setKeys[$tagsPrefix] ?? [null, null]; + // merge the fetched ids together + $tagKeys = array_merge($tagKeys, $ids); + } while ($cursor = (int) $cursor); + + return $tagKeys; + } + + + /** + * Checks all tags in the cache for orphaned items and creates a "report" array. + * + * By default, only completely orphaned tag keys are reported. If + * compressMode is enabled the report will include all tag keys + * that have any orphaned references to cache items + * + * @TODO Verify the LUA scripts are redis-cluster safe. + * @TODO Is there anything that can be done to reduce memory footprint? + * + * @param bool $compressMode + * @return array{tagKeys: string[], orphanedTagKeys: string[], orphanedTagReferenceKeys?: array} + * tagKeys: List of all tags in the cache. + * orphanedTagKeys: List of tags that only reference orphaned cache items. + * orphanedTagReferenceKeys: List of all orphaned cache item references per tag. + * Keyed by tag, value is the list of orphaned cache item keys. + */ + private function getOrphanedTagsStats(bool $compressMode = false): array + { + $prefix = $this->getPrefix(); + $tagKeys = $this->getAllTagKeys(); + + // lua for fetching all entries/content from a SET + $getSetContentLua = <<<'EOLUA' + redis.replicate_commands() + local cursor = ARGV[1] + return redis.call('SSCAN', KEYS[1], cursor, 'COUNT', 5000) + EOLUA; + + $orphanedTagReferenceKeys = []; + $orphanedTagKeys = []; + // Iterate over each tag and check if its entries reference orphaned + // cache items. + foreach ($tagKeys as $tagKey) { + $tagKey = substr($tagKey, strlen($prefix)); + $cursor = null; + $hasExistingKeys = false; + do { + // Fetch all referenced cache keys from the tag entry. + $results = $this->pipeline(function () use ($getSetContentLua, $tagKey, $cursor) { + yield 'eval' => [$getSetContentLua, [$tagKey, $cursor], 1]; + }); + [$cursor, $referencedCacheKeys] = $results->valid() ? $results->current() : [null, null]; + + if (!empty($referencedCacheKeys)) { + // Counts how many of the referenced cache items exist. + $existingCacheKeysResult = $this->pipeline(function () use ($referencedCacheKeys) { + yield 'exists' => $referencedCacheKeys; + }); + $existingCacheKeysCount = $existingCacheKeysResult->valid() ? $existingCacheKeysResult->current() : 0; + $hasExistingKeys = $hasExistingKeys || ($existingCacheKeysCount > 0 ?? false); + + // If compression mode is enabled and the count between + // referenced and existing cache keys differs collect the + // missing references. + if ($compressMode && count($referencedCacheKeys) > $existingCacheKeysCount) { + // In order to create the delta each single reference + // has to be checked. + foreach ($referencedCacheKeys as $cacheKey) { + $existingCacheKeyResult = $this->pipeline(function () use ($cacheKey) { + yield 'exists' => [$cacheKey]; + }); + if ($existingCacheKeyResult->valid() && !$existingCacheKeyResult->current()) { + $orphanedTagReferenceKeys[$tagKey][] = $cacheKey; + } + } + } + // Stop processing cursors in case compression mode is + // disabled and the tag references existing keys. + if (!$compressMode && $hasExistingKeys) { + break; + } + } + } while ($cursor = (int) $cursor); + if (!$hasExistingKeys) { + $orphanedTagKeys[] = $tagKey; + } + } + + $stats = ['orphanedTagKeys' => $orphanedTagKeys, 'tagKeys' => $tagKeys]; + if ($compressMode) { + $stats['orphanedTagReferenceKeys'] = $orphanedTagReferenceKeys; + } + return $stats; + } + + /** + * + * @TODO Verify the LUA scripts are redis-cluster safe. + * + * @param bool $compressMode + * @return bool + */ + private function pruneOrphanedTags(bool $compressMode = false): bool + { + $success = true; + $orphanedTagsStats = $this->getOrphanedTagsStats($compressMode); + + // Delete all tags that don't reference any existing cache item. + foreach ($orphanedTagsStats['orphanedTagKeys'] as $orphanedTagKey) { + $result = $this->pipeline(function () use ($orphanedTagKey) { + yield 'del' => [$orphanedTagKey]; + }); + if (!$result->valid() || $result->current() !== 1) { + $success = false; + } + } + // If orphaned cache key references are provided prune them too. + if (!empty($orphanedTagsStats['orphanedTagReferenceKeys'])) { + // lua for deleting member from a SET + $removeSetMemberLua = <<<'EOLUA' + redis.replicate_commands() + return redis.call('SREM', KEYS[1], KEYS[2]) + EOLUA; + // Loop through all tags with orphaned cache item references. + foreach ($orphanedTagsStats['orphanedTagReferenceKeys'] as $tagKey => $orphanedCacheKeys) { + // Remove each cache item reference from the tag set. + foreach ($orphanedCacheKeys as $orphanedCacheKey) { + $result = $this->pipeline(function () use ($removeSetMemberLua, $tagKey, $orphanedCacheKey) { + yield 'srem' => [$tagKey, $orphanedCacheKey]; + }); + if (!$result->valid() || $result->current() !== 1) { + $success = false; + } + } + } + } + return $success; + } + + /** + * @TODO Make compression mode flag configurable. + * + * @return bool + */ + public function prune(): bool + { + return $this->pruneOrphanedTags(true); + } } From 0102578f0218a814aa51405cfcde1dc47e6bc8b8 Mon Sep 17 00:00:00 2001 From: Stefan Scheu Date: Wed, 7 Jun 2023 11:19:18 +0200 Subject: [PATCH 2/3] fix: apply patch for coding standards --- .../Cache/Adapter/RedisTagAwareAdapter.php | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php b/src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php index 16c3782fafadb..447c30799e235 100644 --- a/src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php +++ b/src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php @@ -307,7 +307,6 @@ private function getRedisEvictionPolicy(): string return $this->redisEvictionPolicy = ''; } - private function getPrefix(): string { if ($this->redis instanceof \Predis\ClientInterface) { @@ -315,6 +314,7 @@ private function getPrefix(): string } elseif (\is_array($prefix = $this->redis->getOption(\Redis::OPT_PREFIX) ?? '')) { $prefix = current($prefix); } + return $prefix; } @@ -322,8 +322,6 @@ private function getPrefix(): string * Returns all existing tag keys from the cache. * * @TODO Verify the LUA scripts are redis-cluster safe. - * - * @return array */ protected function getAllTagKeys(): array { @@ -355,7 +353,6 @@ protected function getAllTagKeys(): array return $tagKeys; } - /** * Checks all tags in the cache for orphaned items and creates a "report" array. * @@ -366,12 +363,11 @@ protected function getAllTagKeys(): array * @TODO Verify the LUA scripts are redis-cluster safe. * @TODO Is there anything that can be done to reduce memory footprint? * - * @param bool $compressMode * @return array{tagKeys: string[], orphanedTagKeys: string[], orphanedTagReferenceKeys?: array} - * tagKeys: List of all tags in the cache. - * orphanedTagKeys: List of tags that only reference orphaned cache items. - * orphanedTagReferenceKeys: List of all orphaned cache item references per tag. - * Keyed by tag, value is the list of orphaned cache item keys. + * tagKeys: List of all tags in the cache. + * orphanedTagKeys: List of tags that only reference orphaned cache items. + * orphanedTagReferenceKeys: List of all orphaned cache item references per tag. + * Keyed by tag, value is the list of orphaned cache item keys. */ private function getOrphanedTagsStats(bool $compressMode = false): array { @@ -390,7 +386,7 @@ private function getOrphanedTagsStats(bool $compressMode = false): array // Iterate over each tag and check if its entries reference orphaned // cache items. foreach ($tagKeys as $tagKey) { - $tagKey = substr($tagKey, strlen($prefix)); + $tagKey = substr($tagKey, \strlen($prefix)); $cursor = null; $hasExistingKeys = false; do { @@ -411,7 +407,7 @@ private function getOrphanedTagsStats(bool $compressMode = false): array // If compression mode is enabled and the count between // referenced and existing cache keys differs collect the // missing references. - if ($compressMode && count($referencedCacheKeys) > $existingCacheKeysCount) { + if ($compressMode && \count($referencedCacheKeys) > $existingCacheKeysCount) { // In order to create the delta each single reference // has to be checked. foreach ($referencedCacheKeys as $cacheKey) { @@ -426,7 +422,7 @@ private function getOrphanedTagsStats(bool $compressMode = false): array // Stop processing cursors in case compression mode is // disabled and the tag references existing keys. if (!$compressMode && $hasExistingKeys) { - break; + break; } } } while ($cursor = (int) $cursor); @@ -439,15 +435,12 @@ private function getOrphanedTagsStats(bool $compressMode = false): array if ($compressMode) { $stats['orphanedTagReferenceKeys'] = $orphanedTagReferenceKeys; } + return $stats; } /** - * * @TODO Verify the LUA scripts are redis-cluster safe. - * - * @param bool $compressMode - * @return bool */ private function pruneOrphanedTags(bool $compressMode = false): bool { @@ -459,7 +452,7 @@ private function pruneOrphanedTags(bool $compressMode = false): bool $result = $this->pipeline(function () use ($orphanedTagKey) { yield 'del' => [$orphanedTagKey]; }); - if (!$result->valid() || $result->current() !== 1) { + if (!$result->valid() || 1 !== $result->current()) { $success = false; } } @@ -474,22 +467,21 @@ private function pruneOrphanedTags(bool $compressMode = false): bool foreach ($orphanedTagsStats['orphanedTagReferenceKeys'] as $tagKey => $orphanedCacheKeys) { // Remove each cache item reference from the tag set. foreach ($orphanedCacheKeys as $orphanedCacheKey) { - $result = $this->pipeline(function () use ($removeSetMemberLua, $tagKey, $orphanedCacheKey) { + $result = $this->pipeline(function () use ($tagKey, $orphanedCacheKey) { yield 'srem' => [$tagKey, $orphanedCacheKey]; }); - if (!$result->valid() || $result->current() !== 1) { + if (!$result->valid() || 1 !== $result->current()) { $success = false; } } } } + return $success; } /** * @TODO Make compression mode flag configurable. - * - * @return bool */ public function prune(): bool { From 70b183978093c1243f28c1e1e4cf28e7ba1a6656 Mon Sep 17 00:00:00 2001 From: zoidbergx Date: Thu, 8 Jun 2023 10:29:09 +0200 Subject: [PATCH 3/3] Update src/Symfony/Component/Cache/Adapter/AbstractTagAwareAdapter.php apply suggestion Co-authored-by: Alexander M. Turek --- src/Symfony/Component/Cache/Adapter/AbstractTagAwareAdapter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Symfony/Component/Cache/Adapter/AbstractTagAwareAdapter.php b/src/Symfony/Component/Cache/Adapter/AbstractTagAwareAdapter.php index 9daf03d387452..96d59a13f0bf4 100644 --- a/src/Symfony/Component/Cache/Adapter/AbstractTagAwareAdapter.php +++ b/src/Symfony/Component/Cache/Adapter/AbstractTagAwareAdapter.php @@ -35,7 +35,7 @@ abstract class AbstractTagAwareAdapter implements TagAwareAdapterInterface, TagA use AbstractAdapterTrait; use ContractsTrait; - protected const TAGS_PREFIX = "\1tags\1"; + final protected const TAGS_PREFIX = "\1tags\1"; protected function __construct(string $namespace = '', int $defaultLifetime = 0) {