Skip to content

Commit fbcfa23

Browse files
ArtificialOwlArtificialOwl
committed
using etag to manage metadata
Signed-off-by: Maxence Lange <maxence@artificial-owl.com>
1 parent 414a4ca commit fbcfa23

7 files changed

Lines changed: 171 additions & 40 deletions

File tree

‎lib/private/Blurhash/Listener/GenerateBlurhashMetadata.php‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ public function handle(Event $event): void {
7272
return;
7373
}
7474

75+
$metadata = $event->getMetadata();
76+
if ($metadata->isUpToDate('blurhash')) {
77+
return;
78+
}
79+
7580
// too heavy to run on the live thread, request a rerun as a background job
7681
if ($event instanceof MetadataLiveEvent) {
7782
$event->requestBackgroundJob();
@@ -95,7 +100,6 @@ public function handle(Event $event): void {
95100
return;
96101
}
97102

98-
$metadata = $event->getMetadata();
99103
$metadata->setString('blurhash', $this->generateBlurHash($image));
100104
}
101105

‎lib/private/FilesMetadata/FilesMetadataManager.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,9 +96,9 @@ public function refreshMetadata(
9696
string $namedEvent = ''
9797
): IFilesMetadata {
9898
try {
99-
$metadata = $this->metadataRequestService->getMetadataFromFileId($node->getId());
99+
$metadata = $this->metadataRequestService->getMetadataFromFileId($node->getId(), $node->getEtag());
100100
} catch (FilesMetadataNotFoundException) {
101-
$metadata = new FilesMetadata($node->getId());
101+
$metadata = new FilesMetadata($node->getId(), $node->getEtag());
102102
}
103103

104104
// if $process is LIVE, we enforce LIVE

‎lib/private/FilesMetadata/Model/FilesMetadata.php‎

Lines changed: 61 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ class FilesMetadata implements IFilesMetadata {
4646
private string $syncToken = '';
4747

4848
public function __construct(
49-
private int $fileId = 0
49+
private int $fileId = 0,
50+
private string $fileEtag = '',
5051
) {
5152
}
5253

@@ -59,6 +60,15 @@ public function getFileId(): int {
5960
return $this->fileId;
6061
}
6162

63+
/**
64+
* @inheritDoc
65+
* @return string related file etag
66+
* @since 29.0.0
67+
*/
68+
public function getFileEtag(): string {
69+
return $this->fileEtag;
70+
}
71+
6272
/**
6373
* @inheritDoc
6474
* @return int timestamp
@@ -156,6 +166,35 @@ public function setEditPermission(string $key, int $permission): void {
156166
$this->metadata[$key]->setEditPermission($permission);
157167
}
158168

169+
170+
public function getEtag(string $key): string {
171+
if (!array_key_exists($key, $this->metadata)) {
172+
throw new FilesMetadataNotFoundException();
173+
}
174+
175+
return $this->metadata[$key]->getEtag();
176+
}
177+
178+
public function setEtag(string $key, string $etag): void {
179+
if (!array_key_exists($key, $this->metadata)) {
180+
throw new FilesMetadataNotFoundException();
181+
}
182+
183+
$this->metadata[$key]->setEtag($etag);
184+
}
185+
186+
/**
187+
* @param string $key metadata key
188+
*
189+
* @inheritDoc
190+
* @return bool
191+
* @since 29.0.0
192+
*/
193+
public function isUpToDate(string $key): bool {
194+
return ($this->fileEtag !== '' && $this->hasKey($key) && $this->metadata[$key]->getEtag() === $this->getFileEtag());
195+
}
196+
197+
159198
/**
160199
* @param string $key metadata key
161200
*
@@ -311,16 +350,16 @@ public function getType(string $key): string {
311350
public function setString(string $key, string $value, bool $index = false): IFilesMetadata {
312351
$this->confirmKeyFormat($key);
313352
try {
314-
if ($this->getString($key) === $value && $index === $this->isIndex($key)) {
315-
return $this; // we ignore if value and index have not changed
353+
if ($this->getString($key) === $value && $index === $this->isIndex($key) && $this->isUpToDate($key)) {
354+
return $this; // we ignore if value and index have not changed and etag is still the same
316355
}
317356
} catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) {
318357
// if value does not exist, or type has changed, we keep on the writing
319358
}
320359

321360
$meta = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_STRING);
322361
$this->updated = true;
323-
$this->metadata[$key] = $meta->setValueString($value)->setIndexed($index);
362+
$this->metadata[$key] = $meta->setValueString($value)->setIndexed($index)->setEtag($this->getFileEtag());
324363

325364
return $this;
326365
}
@@ -338,15 +377,15 @@ public function setString(string $key, string $value, bool $index = false): IFil
338377
public function setInt(string $key, int $value, bool $index = false): IFilesMetadata {
339378
$this->confirmKeyFormat($key);
340379
try {
341-
if ($this->getInt($key) === $value && $index === $this->isIndex($key)) {
342-
return $this; // we ignore if value have not changed
380+
if ($this->getInt($key) === $value && $index === $this->isIndex($key) && $this->isUpToDate($key)) {
381+
return $this; // we ignore if value and index have not changed and etag is still the same
343382
}
344383
} catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) {
345384
// if value does not exist, or type has changed, we keep on the writing
346385
}
347386

348387
$meta = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_INT);
349-
$this->metadata[$key] = $meta->setValueInt($value)->setIndexed($index);
388+
$this->metadata[$key] = $meta->setValueInt($value)->setIndexed($index)->setEtag($this->getFileEtag());
350389
$this->updated = true;
351390

352391
return $this;
@@ -364,15 +403,15 @@ public function setInt(string $key, int $value, bool $index = false): IFilesMeta
364403
public function setFloat(string $key, float $value, bool $index = false): IFilesMetadata {
365404
$this->confirmKeyFormat($key);
366405
try {
367-
if ($this->getFloat($key) === $value && $index === $this->isIndex($key)) {
368-
return $this; // we ignore if value have not changed
406+
if ($this->getFloat($key) === $value && $index === $this->isIndex($key) && $this->isUpToDate($key)) {
407+
return $this; // we ignore if value and index have not changed and etag is still the same
369408
}
370409
} catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) {
371410
// if value does not exist, or type has changed, we keep on the writing
372411
}
373412

374413
$meta = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_FLOAT);
375-
$this->metadata[$key] = $meta->setValueFloat($value)->setIndexed($index);
414+
$this->metadata[$key] = $meta->setValueFloat($value)->setIndexed($index)->setEtag($this->getFileEtag());
376415
$this->updated = true;
377416

378417
return $this;
@@ -392,15 +431,15 @@ public function setFloat(string $key, float $value, bool $index = false): IFiles
392431
public function setBool(string $key, bool $value, bool $index = false): IFilesMetadata {
393432
$this->confirmKeyFormat($key);
394433
try {
395-
if ($this->getBool($key) === $value && $index === $this->isIndex($key)) {
396-
return $this; // we ignore if value have not changed
434+
if ($this->getBool($key) === $value && $index === $this->isIndex($key) && $this->isUpToDate($key)) {
435+
return $this; // we ignore if value and index have not changed and etag is still the same
397436
}
398437
} catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) {
399438
// if value does not exist, or type has changed, we keep on the writing
400439
}
401440

402441
$meta = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_BOOL);
403-
$this->metadata[$key] = $meta->setValueBool($value)->setIndexed($index);
442+
$this->metadata[$key] = $meta->setValueBool($value)->setIndexed($index)->setEtag($this->getFileEtag());
404443
$this->updated = true;
405444

406445
return $this;
@@ -419,15 +458,15 @@ public function setBool(string $key, bool $value, bool $index = false): IFilesMe
419458
public function setArray(string $key, array $value): IFilesMetadata {
420459
$this->confirmKeyFormat($key);
421460
try {
422-
if ($this->getArray($key) === $value) {
423-
return $this; // we ignore if value have not changed
461+
if ($this->getArray($key) === $value && $this->isUpToDate($key)) {
462+
return $this; // we ignore if value and index have not changed and etag is still the same
424463
}
425464
} catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) {
426465
// if value does not exist, or type has changed, we keep on the writing
427466
}
428467

429468
$meta = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_ARRAY);
430-
$this->metadata[$key] = $meta->setValueArray($value);
469+
$this->metadata[$key] = $meta->setValueArray($value)->setEtag($this->getFileEtag());
431470
$this->updated = true;
432471

433472
return $this;
@@ -446,15 +485,15 @@ public function setArray(string $key, array $value): IFilesMetadata {
446485
public function setStringList(string $key, array $value, bool $index = false): IFilesMetadata {
447486
$this->confirmKeyFormat($key);
448487
try {
449-
if ($this->getStringList($key) === $value) {
450-
return $this; // we ignore if value have not changed
488+
if ($this->getStringList($key) === $value && $this->isUpToDate($key)) {
489+
return $this; // we ignore if value and index have not changed and etag is still the same
451490
}
452491
} catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) {
453492
// if value does not exist, or type has changed, we keep on the writing
454493
}
455494

456495
$meta = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_STRING_LIST);
457-
$this->metadata[$key] = $meta->setValueStringList($value)->setIndexed($index);
496+
$this->metadata[$key] = $meta->setValueStringList($value)->setIndexed($index)->setEtag($this->getFileEtag());
458497
$this->updated = true;
459498

460499
return $this;
@@ -473,15 +512,15 @@ public function setStringList(string $key, array $value, bool $index = false): I
473512
public function setIntList(string $key, array $value, bool $index = false): IFilesMetadata {
474513
$this->confirmKeyFormat($key);
475514
try {
476-
if ($this->getIntList($key) === $value) {
477-
return $this; // we ignore if value have not changed
515+
if ($this->getIntList($key) === $value && $this->isUpToDate($key)) {
516+
return $this; // we ignore if value and index have not changed and etag is still the same
478517
}
479518
} catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) {
480519
// if value does not exist, or type has changed, we keep on the writing
481520
}
482521

483522
$valueWrapper = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_STRING_LIST);
484-
$this->metadata[$key] = $valueWrapper->setValueIntList($value)->setIndexed($index);
523+
$this->metadata[$key] = $valueWrapper->setValueIntList($value)->setIndexed($index)->setEtag($this->getFileEtag());
485524
$this->updated = true;
486525

487526
return $this;

‎lib/private/FilesMetadata/Model/MetadataValueWrapper.php‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ class MetadataValueWrapper implements IMetadataValueWrapper {
3838
private string $type;
3939
/** @var string|int|float|bool|array|string[]|int[] */
4040
private mixed $value = null;
41+
private string $etag = '';
4142
private bool $indexed = false;
4243
private int $editPermission = self::EDIT_FORBIDDEN;
4344

@@ -350,6 +351,27 @@ public function getValueAny(): mixed {
350351
return $this->value;
351352
}
352353

354+
/**
355+
* @inheritDoc
356+
* @return string stored etag
357+
* @since 29.0.0
358+
*/
359+
public function getEtag(): string {
360+
return $this->etag;
361+
}
362+
363+
/**
364+
* @param string $etag etag value
365+
*
366+
* @inheritDoc
367+
* @return self
368+
* @since 29.0.0
369+
*/
370+
public function setEtag(string $etag): self {
371+
$this->etag = $etag;
372+
return $this;
373+
}
374+
353375
/**
354376
* @param bool $indexed TRUE to set the stored value as an indexed value
355377
*
@@ -405,6 +427,7 @@ public function getEditPermission(): int {
405427
public function import(array $data): self {
406428
$this->value = $data['value'] ?? null;
407429
$this->type = $data['type'] ?? '';
430+
$this->setEtag($data['etag'] ?? '');
408431
$this->setIndexed($data['indexed'] ?? false);
409432
$this->setEditPermission($data['editPermission'] ?? self::EDIT_FORBIDDEN);
410433
return $this;
@@ -414,6 +437,7 @@ public function jsonSerialize(bool $emptyValues = false): array {
414437
return [
415438
'value' => ($emptyValues) ? null : $this->value,
416439
'type' => $this->getType(),
440+
'etag' => $this->getEtag(),
417441
'indexed' => $this->isIndexed(),
418442
'editPermission' => $this->getEditPermission()
419443
];

‎lib/private/FilesMetadata/Service/MetadataRequestService.php‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -70,33 +70,39 @@ public function store(IFilesMetadata $filesMetadata): void {
7070
* @return IFilesMetadata
7171
* @throws FilesMetadataNotFoundException if no metadata are found in database
7272
*/
73-
public function getMetadataFromFileId(int $fileId): IFilesMetadata {
73+
public function getMetadataFromFileId(int $fileId, string $etag = ''): IFilesMetadata {
7474
try {
7575
$qb = $this->dbConnection->getQueryBuilder();
76-
$qb->select('json', 'sync_token')->from(self::TABLE_METADATA);
77-
$qb->where(
78-
$qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))
79-
);
76+
$qb->select('m.json', 'm.sync_token')->from(self::TABLE_METADATA, 'm');
77+
if ($etag === '') {
78+
/**
79+
* in the rare case $etag is not known yet, we left-join filecache to
80+
* get the current etag linked to the file id.
81+
*/
82+
$qb->addSelect('f.etag')
83+
->leftJoin('m', 'filecache', 'f', $qb->expr()->eq('m.file_id', 'f.fileid'));
84+
}
85+
$qb->where($qb->expr()->eq('m.file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)));
8086
$result = $qb->executeQuery();
8187
$data = $result->fetch();
8288
$result->closeCursor();
8389
} catch (Exception $e) {
84-
$this->logger->warning(
85-
'exception while getMetadataFromDatabase()', ['exception' => $e, 'fileId' => $fileId]
86-
);
90+
$this->logger->warning('exception while getMetadataFromDatabase()', ['exception' => $e, 'fileId' => $fileId]);
8791
throw new FilesMetadataNotFoundException();
8892
}
8993

9094
if ($data === false) {
9195
throw new FilesMetadataNotFoundException();
9296
}
9397

94-
$metadata = new FilesMetadata($fileId);
98+
// we can assume $data['etag'] is null only if $etag is an empty string.
99+
$metadata = new FilesMetadata($fileId, $data['etag'] ?? $etag);
95100
$metadata->importFromDatabase($data);
96101

97102
return $metadata;
98103
}
99104

105+
100106
/**
101107
* returns metadata for multiple file ids
102108
*
@@ -109,16 +115,16 @@ public function getMetadataFromFileId(int $fileId): IFilesMetadata {
109115
*/
110116
public function getMetadataFromFileIds(array $fileIds): array {
111117
$qb = $this->dbConnection->getQueryBuilder();
112-
$qb->select('file_id', 'json', 'sync_token')->from(self::TABLE_METADATA);
113-
$qb->where(
114-
$qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))
115-
);
118+
$qb->select('m.file_id', 'm.json', 'm.sync_token')->from(self::TABLE_METADATA);
119+
$qb->addSelect('f.etag')
120+
->leftJoin('m', 'filecache', 'f', $qb->expr()->eq('m.file_id', 'f.fileid'));
121+
$qb->where($qb->expr()->in('m.file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)));
116122

117123
$list = [];
118124
$result = $qb->executeQuery();
119125
while ($data = $result->fetch()) {
120126
$fileId = (int) $data['file_id'];
121-
$metadata = new FilesMetadata($fileId);
127+
$metadata = new FilesMetadata($fileId, $data['etag'] ?? '');
122128
try {
123129
$metadata->importFromDatabase($data);
124130
} catch (FilesMetadataNotFoundException) {

0 commit comments

Comments
 (0)