Add cache fallback into DB library

Dockerfile

@@ -34,6 +34,8 @@ RUN \
     git \
     brotli-dev \
     linux-headers \
+    docker-cli \
+    docker-cli-compose \
   && docker-php-ext-install opcache pgsql pdo_mysql pdo_pgsql \
   && apk del postgresql-dev \
   && rm -rf /var/cache/apk/*

docker-compose.yml

@@ -15,6 +15,8 @@ services:
       - ./dev:/usr/src/code/dev
       - ./phpunit.xml:/usr/src/code/phpunit.xml
       - ./dev/xdebug.ini:/usr/local/etc/php/conf.d/xdebug.ini
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./docker-compose.yml:/usr/src/code/docker-compose.yml
 
   adminer:
     image: adminer

src/Database/Database.php

@@ -4,6 +4,7 @@
 
 use Exception;
 use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
 use Utopia\Database\Exception as DatabaseException;
 use Utopia\Database\Exception\Authorization as AuthorizationException;
 use Utopia\Database\Exception\Conflict as ConflictException;
@@ -2957,7 +2958,14 @@ public function getDocument(string $collection, string $id, array $queries = [],
             $documentCacheHash .= ':' . \md5(\implode($selections));
         }
 
-        if ($cache = $this->cache->load($documentCacheKey, self::TTL, $documentCacheHash)) {
+        try {
+            $cache = $this->cache->load($documentCacheKey, self::TTL, $documentCacheHash);
+        } catch (Exception $e) {
+            Console::warning('Warning: Failed to get document from cache: ' . $e->getMessage());
+            $cache = null;
+        }
+
+        if ($cache) {
             $document = new Document($cache);
 
             if ($collection->getId() !== self::METADATA) {
@@ -3020,21 +3028,35 @@ public function getDocument(string $collection, string $id, array $queries = [],
         foreach ($this->map as $key => $value) {
             [$k, $v] = \explode('=>', $key);
             $ck = $this->cacheName . '-cache-' . $this->getNamespace() . ':' . $this->adapter->getTenant() . ':map:' . $k;
-            $cache = $this->cache->load($ck, self::TTL, $ck);
+
+            try {
+                $cache = $this->cache->load($ck, self::TTL, $ck);
+            } catch (Exception $e) {
+                Console::warning('Failed to load document from cache: ' . $e->getMessage());
+                $cache = [];
+            }
             if (empty($cache)) {
                 $cache = [];
             }
             if (!\in_array($v, $cache)) {
                 $cache[] = $v;
-                $this->cache->save($ck, $cache, $ck);
+                try {
+                    $this->cache->save($ck, $cache, $ck);
+                } catch (Exception $e) {
+                    Console::warning('Failed to save document to cache: ' . $e->getMessage());
+                }
             }
         }
 
         // Don't save to cache if it's part of a relationship
         if (!$hasTwoWayRelationship && empty($relationships)) {
-            $this->cache->save($documentCacheKey, $document->getArrayCopy(), $documentCacheHash);
-            // Add document reference to the collection key
-            $this->cache->save($collectionCacheKey, 'empty', $documentCacheKey);
+            try {
+                $this->cache->save($documentCacheKey, $document->getArrayCopy(), $documentCacheHash);
+                // Add document reference to the collection key
+                $this->cache->save($collectionCacheKey, 'empty', $documentCacheKey);
+            } catch (Exception $e) {
+                Console::warning('Failed to save document to cache: ' . $e->getMessage());
+            }
         }
 
         // Remove internal attributes if not queried for select query
@@ -3952,6 +3974,9 @@ public function updateDocument(string $collection, string $id, Document $documen
 
             $this->adapter->updateDocument($collection->getId(), $id, $document);
 
+            $this->purgeRelatedDocuments($collection, $id);
+            $this->purgeCachedDocument($collection->getId(), $id);
+
             return $document;
         });
 
@@ -3961,8 +3986,6 @@ public function updateDocument(string $collection, string $id, Document $documen
 
         $document = $this->decode($collection, $document);
 
-        $this->purgeRelatedDocuments($collection, $id);
-        $this->purgeCachedDocument($collection->getId(), $id);
         $this->trigger(self::EVENT_DOCUMENT_UPDATE, $document);
 
         return $document;
@@ -4762,11 +4785,13 @@ public function deleteDocument(string $collection, string $id): bool
                 $document = $this->silent(fn () => $this->deleteDocumentRelationships($collection, $document));
             }
 
-            return $this->adapter->deleteDocument($collection->getId(), $id);
-        });
+            $result = $this->adapter->deleteDocument($collection->getId(), $id);
 
-        $this->purgeRelatedDocuments($collection, $id);
-        $this->purgeCachedDocument($collection->getId(), $id);
+            $this->purgeRelatedDocuments($collection, $id);
+            $this->purgeCachedDocument($collection->getId(), $id);
+
+            return $result;
+        });
 
         $this->trigger(self::EVENT_DOCUMENT_DELETE, $document);
 
@@ -5300,6 +5325,7 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba
     public function purgeCachedCollection(string $collectionId): bool
     {
         $collectionKey = $this->cacheName . '-cache-' . $this->getNamespace() . ':' . $this->adapter->getTenant() . ':collection:' . $collectionId;
+
         $documentKeys = $this->cache->list($collectionKey);
         foreach ($documentKeys as $documentKey) {
             $this->cache->purge($documentKey);
@@ -6029,7 +6055,9 @@ private function purgeRelatedDocuments(Document $collection, string $id): void
         }
 
         $key = $this->cacheName . '-cache-' . $this->getNamespace() . ':map:' . $collection->getId() . ':' . $id;
+
         $cache = $this->cache->load($key, self::TTL, $key);
+
         if (!empty($cache)) {
             foreach ($cache as $v) {
                 list($collectionId, $documentId) = explode(':', $v);

tests/e2e/Adapter/Base.php

@@ -5,6 +5,7 @@
 use Exception;
 use PHPUnit\Framework\TestCase;
 use Throwable;
+use Utopia\CLI\Console;
 use Utopia\Database\Adapter\SQL;
 use Utopia\Database\Database;
 use Utopia\Database\DateTime;
@@ -40,7 +41,7 @@ abstract class Base extends TestCase
     /**
      * @return Database
      */
-    abstract protected static function getDatabase(): Database;
+    abstract protected static function getDatabase(bool $fresh = false): Database;
 
     /**
      * @param string $collection
@@ -17314,4 +17315,66 @@ public function testEvents(): void
             $database->delete('hellodb');
         });
     }
+
+    public function testCacheFallback(): void
+    {
+        Authorization::cleanRoles();
+        Authorization::setRole(Role::any()->toString());
+        $database = static::getDatabase(true);
+
+        // Write mock data
+        $database->createCollection('testRedisFallback', attributes: [
+            new Document([
+                '$id' => ID::custom('string'),
+                'type' => Database::VAR_STRING,
+                'size' => 767,
+                'required' => true,
+            ])
+        ], permissions: [
+            Permission::read(Role::any()),
+            Permission::create(Role::any()),
+            Permission::update(Role::any()),
+            Permission::delete(Role::any())
+        ]);
+
+        $database->createDocument('testRedisFallback', new Document([
+            '$id' => 'doc1',
+            'string' => 'text📝',
+        ]));
+
+        $database->createIndex('testRedisFallback', 'index1', Database::INDEX_KEY, ['string']);
+        $this->assertCount(1, $database->find('testRedisFallback', [Query::equal('string', ['text📝'])]));
+
+        // Bring down Redis
+        $stdout = '';
+        $stderr = '';
+        Console::execute('docker ps -a --filter "name=utopia-redis" --format "{{.Names}}" | xargs -r docker stop', "", $stdout, $stderr);
+
+        // Check we can read data still
+        $this->assertCount(1, $database->find('testRedisFallback', [Query::equal('string', ['text📝'])]));
+        $this->assertFalse(($database->getDocument('testRedisFallback', 'doc1'))->isEmpty());
+
+        // Check we cannot modify data
+        try {
+            $database->updateDocument('testRedisFallback', 'doc1', new Document([
+                'string' => 'text📝 updated',
+            ]));
+            $this->fail('Failed to throw exception');
+        } catch (\Throwable $e) {
+            $this->assertEquals('Redis server redis:6379 went away', $e->getMessage());
+        }
+
+        try {
+            $database->deleteDocument('testRedisFallback', 'doc1');
+            $this->fail('Failed to throw exception');
+        } catch (\Throwable $e) {
+            $this->assertEquals('Redis server redis:6379 went away', $e->getMessage());
+        }
+
+        // Bring backup Redis
+        Console::execute('docker ps -a --filter "name=utopia-redis" --format "{{.Names}}" | xargs -r docker start', "", $stdout, $stderr);
+        sleep(5);
+
+        $this->assertCount(1, $database->find('testRedisFallback', [Query::equal('string', ['text📝'])]));
+    }
 }

tests/e2e/Adapter/MariaDBTest.php

@@ -41,6 +41,7 @@ public static function getDatabase(bool $fresh = false): Database
         $dbPass = 'password';
 
         $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MariaDB::getPDOAttributes());
+
         $redis = new Redis();
         $redis->connect('redis', 6379);
         $redis->flushAll();

tests/e2e/Adapter/MirrorTest.php

@@ -4,6 +4,7 @@
 
 use PDO;
 use Redis;
+use Utopia\Cache\Adapter\None;
 use Utopia\Cache\Adapter\Redis as RedisAdapter;
 use Utopia\Cache\Cache;
 use Utopia\Database\Adapter\MariaDB;
@@ -45,10 +46,15 @@ protected static function getDatabase(bool $fresh = false): Mirror
         $dbPass = 'password';
 
         $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MariaDB::getPDOAttributes());
-        $redis = new Redis();
-        $redis->connect('redis');
-        $redis->flushAll();
-        $cache = new Cache(new RedisAdapter($redis));
+
+        try {
+            $redis = new Redis();
+            $redis->connect('redis', 6379);
+            $redis->flushAll();
+            $cache = new Cache(new RedisAdapter($redis));
+        } catch (\Exception $e) {
+            $cache = new Cache(new None());
+        }
 
         self::$sourcePdo = $pdo;
         self::$source = new Database(new MariaDB($pdo), $cache);
@@ -59,10 +65,15 @@ protected static function getDatabase(bool $fresh = false): Mirror
         $mirrorPass = 'password';
 
         $mirrorPdo = new PDO("mysql:host={$mirrorHost};port={$mirrorPort};charset=utf8mb4", $mirrorUser, $mirrorPass, MariaDB::getPDOAttributes());
-        $mirrorRedis = new Redis();
-        $mirrorRedis->connect('redis-mirror');
-        $mirrorRedis->flushAll();
-        $mirrorCache = new Cache(new RedisAdapter($mirrorRedis));
+
+        try {
+            $mirrorRedis = new Redis();
+            $mirrorRedis->connect('redis-mirror');
+            $mirrorRedis->flushAll();
+            $mirrorCache = new Cache(new RedisAdapter($mirrorRedis));
+        } catch (\Exception $e) {
+            $mirrorCache = new Cache(new None());
+        }
 
         self::$destinationPdo = $mirrorPdo;
         self::$destination = new Database(new MariaDB($mirrorPdo), $mirrorCache);

tests/e2e/Adapter/MongoDBTest.php

@@ -4,6 +4,7 @@
 
 use Exception;
 use Redis;
+use Utopia\Cache\Adapter\None;
 use Utopia\Cache\Adapter\Redis as RedisAdapter;
 use Utopia\Cache\Cache;
 use Utopia\Database\Adapter\Mongo;
@@ -29,16 +30,20 @@ public static function getAdapterName(): string
      * @return Database
      * @throws Exception
      */
-    public static function getDatabase(): Database
+    public static function getDatabase(bool $fresh = false): Database
     {
-        if (!is_null(self::$database)) {
+        if (!is_null(self::$database) && !$fresh) {
             return self::$database;
         }
 
-        $redis = new Redis();
-        $redis->connect('redis', 6379);
-        $redis->flushAll();
-        $cache = new Cache(new RedisAdapter($redis));
+        try {
+            $redis = new Redis();
+            $redis->connect('redis', 6379);
+            $redis->flushAll();
+            $cache = new Cache(new RedisAdapter($redis));
+        } catch (\Exception $e) {
+            $cache = new Cache(new None());
+        }
 
         $schema = 'utopiaTests'; // same as $this->testDatabase
         $client = new Client(

tests/e2e/Adapter/MySQLTest.php

@@ -4,6 +4,7 @@
 
 use PDO;
 use Redis;
+use Utopia\Cache\Adapter\None;
 use Utopia\Cache\Adapter\Redis as RedisAdapter;
 use Utopia\Cache\Cache;
 use Utopia\Database\Adapter\MySQL;
@@ -29,9 +30,9 @@ public static function getAdapterName(): string
     /**
      * @return Database
      */
-    public static function getDatabase(): Database
+    public static function getDatabase(bool $fresh = false): Database
     {
-        if (!is_null(self::$database)) {
+        if (!is_null(self::$database) && !$fresh) {
             return self::$database;
         }
 
@@ -42,11 +43,14 @@ public static function getDatabase(): Database
 
         $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MySQL::getPDOAttributes());
 
-        $redis = new Redis();
-        $redis->connect('redis', 6379);
-        $redis->flushAll();
-
-        $cache = new Cache(new RedisAdapter($redis));
+        try {
+            $redis = new Redis();
+            $redis->connect('redis', 6379);
+            $redis->flushAll();
+            $cache = new Cache(new RedisAdapter($redis));
+        } catch (\Exception $e) {
+            $cache = new Cache(new None());
+        }
 
         $database = new Database(new MySQL($pdo), $cache);
         $database

tests/e2e/Adapter/PostgresTest.php

@@ -4,6 +4,7 @@
 
 use PDO;
 use Redis;
+use Utopia\Cache\Adapter\None;
 use Utopia\Cache\Adapter\Redis as RedisAdapter;
 use Utopia\Cache\Cache;
 use Utopia\Database\Adapter\Postgres;
@@ -28,9 +29,9 @@ public static function getAdapterName(): string
     /**
      * @reture Adapter
      */
-    public static function getDatabase(): Database
+    public static function getDatabase(bool $fresh = false): Database
     {
-        if (!is_null(self::$database)) {
+        if (!is_null(self::$database) && !$fresh) {
             return self::$database;
         }
 
@@ -40,10 +41,14 @@ public static function getDatabase(): Database
         $dbPass = 'password';
 
         $pdo = new PDO("pgsql:host={$dbHost};port={$dbPort};", $dbUser, $dbPass, Postgres::getPDOAttributes());
-        $redis = new Redis();
-        $redis->connect('redis', 6379);
-        $redis->flushAll();
-        $cache = new Cache(new RedisAdapter($redis));
+        try {
+            $redis = new Redis();
+            $redis->connect('redis', 6379);
+            $redis->flushAll();
+            $cache = new Cache(new RedisAdapter($redis));
+        } catch (\Exception $e) {
+            $cache = new Cache(new None());
+        }
 
         $database = new Database(new Postgres($pdo), $cache);
         $database