diff --git a/src/Database/Database.php b/src/Database/Database.php index 1fe59e723..4b20de9c2 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -8565,159 +8565,111 @@ public function find(string $collection, array $queries = [], string $forPermiss } /** - * Find documents using an Appwrite list-cache compatible key and field. + * Execute a callback behind a cache-aside lookup. * - * @param string $collection - * @param array $queries - * @param int $ttl Cache TTL in seconds. Values above TTL are clamped. Set to 0 to disable caching. - * @param string|null $cacheCollection - * @param string|null $namespace - * @param int|string|null $tenant - * @param array $roles - * @param string $field - * @param string $payloadKey - * @param string $forPermission - * @return array - * @throws DatabaseException - * @throws QueryException - * @throws TimeoutException - * @throws Exception + * The callback runs on cache miss and its value is returned to the caller. + * A literal false value is treated as a cache miss and is not cacheable. + * + * @template T + * @param string $key + * @param callable(): T $callback + * @param string $hash + * @return T */ - public function findCached( - string $collection, - array $queries = [], - int $ttl = self::TTL, - ?string $cacheCollection = null, - ?string $namespace = null, - int|string|null $tenant = null, - array $roles = [], - string $field = 'documents', - string $payloadKey = 'documents', - string $forPermission = Database::PERMISSION_READ, - ): array { - $this->checkQueryTypes($queries); + public function withCache( + string $key, + callable $callback, + string $hash = '', + ): mixed { + return $this->withCachedPayload( + key: $key, + callback: $callback, + hash: $hash, + fromCache: fn (mixed $cached): mixed => \is_array($cached) && \array_key_exists('value', $cached) ? $cached['value'] : false, + toCache: fn (mixed $value): array => ['value' => $value], + ); + } + /** + * Execute a callback behind a cache-aside lookup with cache payload hooks. + * + * @template T + * @param string $key + * @param callable(): T $callback + * @param int $ttl + * @param string $hash + * @param (callable(mixed): (T|false))|null $fromCache + * @param (callable(T): (array|string))|null $toCache + * @param (callable(T, mixed): void)|null $onCacheHit + * @param bool $touchOnHit + * @return T + */ + private function withCachedPayload( + string $key, + callable $callback, + int $ttl = self::TTL, + string $hash = '', + ?callable $fromCache = null, + ?callable $toCache = null, + ?callable $onCacheHit = null, + bool $touchOnHit = false, + ): mixed { if ($ttl <= 0) { - return $this->find($collection, $queries, $forPermission); - } - - if ($this->authorization->getStatus()) { - return $this->find($collection, $queries, $forPermission); - } - - foreach ($queries as $query) { - if ($query instanceof Query && $query->getMethod() === Query::TYPE_ORDER_RANDOM) { - return $this->find($collection, $queries, $forPermission); - } + return $callback(); } $ttl = \min($ttl, self::TTL); - - $collectionDocument = $this->silent(fn () => $this->getCollection($collection)); - - if ($collectionDocument->isEmpty()) { - throw new NotFoundException('Collection not found'); - } - - if ($this->validate) { - $validator = new DocumentsValidator( - $collectionDocument->getAttribute('attributes', []), - $collectionDocument->getAttribute('indexes', []), - $this->adapter->getIdAttributeType(), - $this->maxQueryValues, - $this->adapter->getMaxUIDLength(), - $this->adapter->getMinDateTime(), - $this->adapter->getMaxDateTime(), - $this->adapter->getSupportForAttributes(), - $this->adapter->getSupportForUnsignedBigInt() - ); - if (!$validator->isValid($queries)) { - throw new QueryException($validator->getDescription()); - } - } - - $cacheKey = $this->getFindCacheKey($cacheCollection ?? $collectionDocument->getId(), $namespace, $tenant); - $cacheField = $this->getFindCacheField($collectionDocument, $queries, $roles, $field, $payloadKey); + $shouldRefreshCache = false; try { - $cached = $this->cache->load($cacheKey, $ttl, $cacheField); - } catch (Exception $e) { - Console::warning('Warning: Failed to get list result from cache: ' . $e->getMessage()); - $cached = null; + $cached = $this->cache->load($key, $ttl, $hash); + } catch (Throwable $e) { + Console::warning('Warning: Failed to load cache value: ' . $e->getMessage()); + $cached = false; } - if (\is_array($cached) && isset($cached[$payloadKey]) && \is_array($cached[$payloadKey])) { - [$documents, $shouldRefreshCache] = $this->decodeFindCachePayload($collectionDocument, $cached[$payloadKey]); + if ($cached !== false && $cached !== null) { + $value = $fromCache === null ? $cached : $fromCache($cached); - if ($shouldRefreshCache) { - try { - $this->cache->purge($cacheKey, $cacheField); - } catch (Exception $e) { - Console::warning('Warning: Failed to purge expired list result cache: ' . $e->getMessage()); + if ($value !== false) { + if ($touchOnHit) { + try { + $this->cache->touch($key, $hash); + } catch (Throwable $e) { + Console::warning('Warning: Failed to touch cache value: ' . $e->getMessage()); + } } - $documents = $this->find($collectionDocument->getId(), $queries, $forPermission); - try { - $this->cache->save($cacheKey, [ - $payloadKey => \array_map( - static fn (Document $document): array => $document->getArrayCopy(), - $documents, - ), - ], $cacheField); - } catch (Exception $e) { - Console::warning('Failed to save list result to cache: ' . $e->getMessage()); + if ($onCacheHit !== null) { + $onCacheHit($value, $cached); } - return $documents; + return $value; } - $this->trigger(self::EVENT_DOCUMENT_FIND, $documents); - - return $documents; - } - - $documents = $this->find($collectionDocument->getId(), $queries, $forPermission); - - try { - $this->cache->save($cacheKey, [ - $payloadKey => \array_map( - static fn (Document $document): array => $document->getArrayCopy(), - $documents, - ), - ], $cacheField); - } catch (Exception $e) { - Console::warning('Failed to save list result to cache: ' . $e->getMessage()); + $shouldRefreshCache = true; } - return $documents; - } - - /** - * @param array $payload - * @return array{0: array, 1: bool} - */ - private function decodeFindCachePayload(Document $collection, array $payload): array - { - $results = []; - $shouldRefreshCache = false; - - foreach ($payload as $document) { - if (!\is_array($document)) { - $shouldRefreshCache = true; - continue; + if ($shouldRefreshCache) { + try { + $this->cache->purge($key, $hash); + } catch (Throwable $e) { + Console::warning('Warning: Failed to purge rejected cache value: ' . $e->getMessage()); } + } - $document = $this->createDocumentInstance($collection->getId(), $document); + $value = $callback(); + $payload = $toCache === null ? $value : $toCache($value); - if ($this->isTtlExpired($collection, $document)) { - $shouldRefreshCache = true; - continue; + if ($value !== false && (\is_array($payload) || \is_string($payload))) { + try { + $this->cache->save($key, $payload, $hash); + } catch (Throwable $e) { + Console::warning('Warning: Failed to save cache value: ' . $e->getMessage()); } - - $results[] = $document; } - return [$results, $shouldRefreshCache]; + return $value; } /** diff --git a/tests/unit/ListCacheTest.php b/tests/unit/ListCacheTest.php index 5f51f4137..f5bb2b439 100644 --- a/tests/unit/ListCacheTest.php +++ b/tests/unit/ListCacheTest.php @@ -4,236 +4,183 @@ use PHPUnit\Framework\TestCase; use Utopia\Cache\Adapter; -use Utopia\Cache\Adapter\Memory as CacheMemory; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Memory as DatabaseMemory; use Utopia\Database\Database; -use Utopia\Database\Document; -use Utopia\Database\Exception\Query as QueryException; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; -use Utopia\Database\Query; class ListCacheTest extends TestCase { - private Database $database; - - protected function setUp(): void + private function createDatabase(Adapter $cache): Database { - $this->database = $this->createDatabase(new CacheMemory()); - } - - private function createDatabase(Adapter $cache, ?DatabaseMemory $adapter = null): Database - { - $database = new Database($adapter ?? new DatabaseMemory(), new Cache($cache)); + $database = new Database(new DatabaseMemory(), new Cache($cache)); $database ->setDatabase('utopiaTests') ->setNamespace('list_cache_' . \uniqid()); $database->create(); - $database->createCollection('projects'); - $database->createAttribute('projects', 'name', Database::VAR_STRING, 255, true); return $database; } - private function seedProject(Database $database, string $id, string $name): void - { - $database->createDocument('projects', new Document([ - '$id' => $id, - '$permissions' => [Permission::read(Role::any())], - 'name' => $name, - ])); - } - - public function testFindCachedReturnsStaleResultUntilListKeyIsPurged(): void + public function testWithCacheUsesCallbackOnMissAndCachesResult(): void { $cache = new HashMemoryCache(); $database = $this->createDatabase($cache); - $this->seedProject($database, 'first', 'First'); - - $documents = $database->getAuthorization()->skip(fn () => $database->findCached( - 'projects', - [Query::orderAsc('name')], - ttl: 3600, - cacheCollection: 'wafrules', - namespace: '_39', - roles: ['waf'], - payloadKey: 'rules', - )); - $this->assertCount(1, $documents); - $this->assertSame('first', $documents[0]->getId()); - - $this->seedProject($database, 'second', 'Second'); - - $documents = $database->getAuthorization()->skip(fn () => $database->findCached( - 'projects', - [Query::orderAsc('name')], - ttl: 3600, - cacheCollection: 'wafrules', - namespace: '_39', - roles: ['waf'], - payloadKey: 'rules', - )); - $this->assertCount(1, $documents); - $this->assertSame('first', $documents[0]->getId()); - - $cache->purge($database->getFindCacheKey('wafrules', '_39')); - - $documents = $database->getAuthorization()->skip(fn () => $database->findCached( - 'projects', - [Query::orderAsc('name')], - ttl: 3600, - cacheCollection: 'wafrules', - namespace: '_39', - roles: ['waf'], - payloadKey: 'rules', - )); - $this->assertCount(2, $documents); + + $callbackCalls = 0; + + $value = $database->withCache( + 'key', + function () use (&$callbackCalls): array { + $callbackCalls++; + return ['value' => 'fresh']; + }, + ); + + $this->assertSame(['value' => 'fresh'], $value); + $this->assertSame(1, $callbackCalls); + + $value = $database->withCache( + 'key', + function () use (&$callbackCalls): array { + $callbackCalls++; + return ['value' => 'new']; + }, + ); + + $this->assertSame(['value' => 'fresh'], $value); + $this->assertSame(1, $callbackCalls); } - public function testFindCachedUsesListCacheKeyAndField(): void + public function testWithCacheCachesEmptyValues(): void { $cache = new HashMemoryCache(); $database = $this->createDatabase($cache); - $this->seedProject($database, 'first', 'First'); - - $database->getAuthorization()->skip(fn () => $database->findCached( - 'projects', - [Query::orderAsc('name')], - ttl: 3600, - cacheCollection: 'wafrules', - namespace: '_39', - roles: ['waf'], - payloadKey: 'rules', - )); - - $fields = $cache->list($database->getFindCacheKey('wafrules', '_39')); - - $this->assertCount(1, $fields); - $this->assertStringEndsWith(':documents:rules', $fields[0]); - $this->assertSame(4, \substr_count($fields[0], ':')); + + $callbackCalls = 0; + + $value = $database->withCache( + 'key', + function () use (&$callbackCalls): array { + $callbackCalls++; + return []; + }, + ); + + $this->assertSame([], $value); + + $value = $database->withCache( + 'key', + function () use (&$callbackCalls): array { + $callbackCalls++; + return ['value' => 'miss']; + }, + ); + + $this->assertSame([], $value); + $this->assertSame(1, $callbackCalls); } - public function testFindCachedRefetchesExpiredCachedDocuments(): void + public function testWithCacheCachesNullValues(): void { $cache = new HashMemoryCache(); - $database = $this->createDatabase($cache, new TtlMemoryAdapter()); - $database->createAttribute('projects', 'expiresAt', Database::VAR_DATETIME, 0, false); - $database->createIndex('projects', 'expiresAtTtl', Database::INDEX_TTL, ['expiresAt'], ttl: 1); - - $database->createDocument('projects', new Document([ - '$id' => 'first', - '$permissions' => [Permission::read(Role::any())], - 'name' => 'First', - 'expiresAt' => '2999-01-01T00:00:00.000+00:00', - ])); - $this->seedProject($database, 'second', 'Second'); - - $queries = [Query::orderAsc('name'), Query::limit(1)]; - $database->getAuthorization()->skip(fn () => $database->findCached( - 'projects', - $queries, - ttl: 3600, - cacheCollection: 'wafrules', - namespace: '_39', - roles: ['waf'], - payloadKey: 'rules', - )); - - $collection = $database->getCollection('projects'); - $cache->setCachedPayloadDocumentAttribute( - $database->getFindCacheKey('wafrules', '_39'), - $database->getFindCacheField($collection, $queries, ['waf'], 'documents', 'rules'), - 'rules', - 'first', - 'expiresAt', - '2000-01-01T00:00:00.000+00:00', + $database = $this->createDatabase($cache); + + $callbackCalls = 0; + + $value = $database->withCache( + 'key', + function () use (&$callbackCalls): mixed { + $callbackCalls++; + return null; + }, + ); + + $this->assertNull($value); + + $value = $database->withCache( + 'key', + function () use (&$callbackCalls): string { + $callbackCalls++; + return 'miss'; + }, ); - $database->getAuthorization()->skip(fn () => $database->updateDocument('projects', 'first', new Document(['name' => 'Zulu']))); - - $documents = $database->getAuthorization()->skip(fn () => $database->findCached( - 'projects', - $queries, - ttl: 3600, - cacheCollection: 'wafrules', - namespace: '_39', - roles: ['waf'], - payloadKey: 'rules', - )); - $this->assertCount(1, $documents); - $this->assertSame('second', $documents[0]->getId()); + + $this->assertNull($value); + $this->assertSame(1, $callbackCalls); } - public function testFindCachedRefetchesInvalidCachedPayload(): void + public function testWithCacheSeparatesPayloadsByHashField(): void { $cache = new HashMemoryCache(); $database = $this->createDatabase($cache); - $this->seedProject($database, 'first', 'First'); - - $queries = [Query::orderAsc('name')]; - $database->getAuthorization()->skip(fn () => $database->findCached( - 'projects', - $queries, - ttl: 3600, - cacheCollection: 'wafrules', - namespace: '_39', - roles: ['waf'], - payloadKey: 'rules', - )); - - $this->seedProject($database, 'second', 'Second'); - $cache->setCachedPayload( - $database->getFindCacheKey('wafrules', '_39'), - $database->getFindCacheField($database->getCollection('projects'), $queries, ['waf'], 'documents', 'rules'), - 'rules', - ['invalid'], + + $firstCalls = 0; + $secondCalls = 0; + + $first = $database->withCache( + 'key', + function () use (&$firstCalls): array { + $firstCalls++; + return ['value' => 'first']; + }, + 'first-field', + ); + + $second = $database->withCache( + 'key', + function () use (&$secondCalls): array { + $secondCalls++; + return ['value' => 'second']; + }, + 'second-field', ); - $documents = $database->getAuthorization()->skip(fn () => $database->findCached( - 'projects', - $queries, - ttl: 3600, - cacheCollection: 'wafrules', - namespace: '_39', - roles: ['waf'], - payloadKey: 'rules', - )); - - $this->assertCount(2, $documents); + $cachedFirst = $database->withCache( + 'key', + function () use (&$firstCalls): array { + $firstCalls++; + return ['value' => 'miss']; + }, + 'first-field', + ); + + $this->assertSame(['value' => 'first'], $first); + $this->assertSame(['value' => 'second'], $second); + $this->assertSame(['value' => 'first'], $cachedFirst); + $this->assertSame(1, $firstCalls); + $this->assertSame(1, $secondCalls); + $this->assertSame(['first-field', 'second-field'], $cache->list('key')); } - public function testFindCachedValidatesQuerySemanticsBeforeReadingCache(): void + public function testWithCacheDoesNotCacheFalseValues(): void { - $this->expectException(QueryException::class); - $cache = new HashMemoryCache(); $database = $this->createDatabase($cache); - $this->seedProject($database, 'first', 'First'); - - $database->getAuthorization()->skip(fn () => $database->findCached( - 'projects', - [Query::equal('missing', ['value'])], - ttl: 3600, - cacheCollection: 'wafrules', - namespace: '_39', - )); - } - public function testFindCachedValidatesQueryTypesBeforeCaching(): void - { - $this->expectException(QueryException::class); - - $queries = ['invalid']; - - $this->database->getAuthorization()->skip(fn () => $this->database->findCached( - 'projects', - /** @phpstan-ignore-next-line intentionally passing invalid query type */ - $queries, - ttl: 3600, - cacheCollection: 'wafrules', - namespace: '_39', - )); + $callbackCalls = 0; + + $value = $database->withCache( + 'key', + function () use (&$callbackCalls): bool { + $callbackCalls++; + return false; + }, + ); + + $this->assertFalse($value); + $this->assertSame([], $cache->list('key')); + + $value = $database->withCache( + 'key', + function () use (&$callbackCalls): string { + $callbackCalls++; + return 'fresh'; + }, + ); + + $this->assertSame('fresh', $value); + $this->assertSame(2, $callbackCalls); } } @@ -282,44 +229,6 @@ public function touch(string $key, string $hash = ''): bool return true; } - public function setCachedPayloadDocumentAttribute(string $key, string $hash, string $payload, string $documentId, string $attribute, mixed $value): void - { - $data = $this->store[$key][$hash]['data'] ?? []; - if (!\is_array($data)) { - return; - } - - $documents = $data[$payload] ?? []; - if (!\is_array($documents)) { - return; - } - - foreach ($documents as $index => $document) { - if (!\is_array($document) || ($document['$id'] ?? '') !== $documentId) { - continue; - } - - $documents[$index][$attribute] = $value; - $data[$payload] = $documents; - $this->store[$key][$hash]['data'] = $data; - return; - } - } - - /** - * @param array $documents - */ - public function setCachedPayload(string $key, string $hash, string $payload, array $documents): void - { - $data = $this->store[$key][$hash]['data'] ?? []; - if (!\is_array($data)) { - return; - } - - $data[$payload] = $documents; - $this->store[$key][$hash]['data'] = $data; - } - /** * @return array */ @@ -362,11 +271,3 @@ public function getName(?string $key = null): string return 'hash-memory'; } } - -class TtlMemoryAdapter extends DatabaseMemory -{ - public function getSupportForTTLIndexes(): bool - { - return true; - } -}