Skip to content

Commit 0447f56

Browse files
Carl SchwanCarlSchwan
Carl Schwan
authored and
CarlSchwan
committed
perf(preview): Migrate previews to the new optimized table
Signed-off-by: Carl Schwan <[email protected]>
1 parent 7c9307e commit 0447f56

File tree

17 files changed

+377
-48
lines changed

17 files changed

+377
-48
lines changed

‎core/BackgroundJobs/MovePreviewJob.php‎

Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
6+
* SPDX-License-Identifier: AGPL-3.0-or-later
7+
*/
8+
9+
namespace OC\Core\BackgroundJobs;
10+
11+
use OC\Preview\Db\Preview;
12+
use OC\Preview\Db\PreviewMapper;
13+
use OC\Preview\Storage\StorageFactory;
14+
use OCP\AppFramework\Utility\ITimeFactory;
15+
use OCP\BackgroundJob\TimedJob;
16+
use OCP\DB\Exception;
17+
use OCP\Files\AppData\IAppDataFactory;
18+
use OCP\Files\IAppData;
19+
use OCP\Files\SimpleFS\ISimpleFile;
20+
use OCP\Files\SimpleFS\ISimpleFolder;
21+
use OCP\IAppConfig;
22+
use OCP\IDBConnection;
23+
use OCP\IPreview;
24+
25+
class MovePreviewJob extends TimedJob {
26+
private IAppData $appData;
27+
28+
public function __construct(
29+
ITimeFactory $time,
30+
private IAppConfig $appConfig,
31+
private PreviewMapper $previewMapper,
32+
private StorageFactory $storageFactory,
33+
private IDBConnection $connection,
34+
IAppDataFactory $appDataFactory,
35+
) {
36+
parent::__construct($time);
37+
38+
$this->appData = $appDataFactory->get('preview');
39+
$this->setTimeSensitivity(self::TIME_INSENSITIVE);
40+
$this->setInterval(24 * 60 * 60);
41+
}
42+
43+
protected function run(mixed $argument): void {
44+
try {
45+
$this->doRun($argument);
46+
} catch (\Throwable $exception) {
47+
echo $exception->getMessage();
48+
throw $exception;
49+
}
50+
}
51+
52+
private function doRun($argument): void {
53+
if ($this->appConfig->getValueBool('core', 'previewMovedDone')) {
54+
//return;
55+
}
56+
57+
$emptyHierarchicalPreviewFolders = false;
58+
59+
$startTime = time();
60+
while (true) {
61+
$previewFolders = [];
62+
63+
// Check new hierarchical preview folders first
64+
if (!$emptyHierarchicalPreviewFolders) {
65+
$qb = $this->connection->getQueryBuilder();
66+
$qb->select('*')
67+
->from('filecache')
68+
->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%/%/%/%/%/%/%/%')))
69+
->setMaxResults(100);
70+
71+
$result = $qb->executeQuery();
72+
while ($row = $result->fetch()) {
73+
$pathSplit = explode('/', $row['path']);
74+
assert(count($pathSplit) >= 2);
75+
$fileId = $pathSplit[count($pathSplit) - 2];
76+
$previewFolders[$fileId][] = $row['path'];
77+
}
78+
79+
if (!empty($previewFolders)) {
80+
$this->processPreviews($previewFolders, false);
81+
continue;
82+
}
83+
}
84+
85+
// And then the flat preview folder (legacy)
86+
$emptyHierarchicalPreviewFolders = true;
87+
$qb = $this->connection->getQueryBuilder();
88+
$qb->select('*')
89+
->from('filecache')
90+
->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.jpg')))
91+
->setMaxResults(100);
92+
93+
$result = $qb->executeQuery();
94+
while ($row = $result->fetch()) {
95+
$pathSplit = explode('/', $row['path']);
96+
assert(count($pathSplit) >= 2);
97+
$fileId = $pathSplit[count($pathSplit) - 2];
98+
array_pop($pathSplit);
99+
$path = implode('/', $pathSplit);
100+
if (!isset($previewFolders[$fileId])) {
101+
$previewFolders[$fileId] = [];
102+
}
103+
if (!in_array($path, $previewFolders[$fileId])) {
104+
$previewFolders[$fileId][] = $path;
105+
}
106+
}
107+
108+
if (empty($previewFolders)) {
109+
break;
110+
} else {
111+
$this->processPreviews($previewFolders, true);
112+
}
113+
114+
// Stop if execution time is more than one hour.
115+
if (time() - $startTime > 3600) {
116+
return;
117+
}
118+
}
119+
120+
// Delete any left over preview directory
121+
$this->appData->getFolder('.')->delete();
122+
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
123+
}
124+
125+
/**
126+
* @param array<string, string[]> $previewFolders
127+
*/
128+
private function processPreviews(array $previewFolders, bool $simplePaths): void {
129+
foreach ($previewFolders as $fileId => $previewFolder) {
130+
$internalPath = $this->getInternalFolder((string)$fileId, $simplePaths);
131+
$folder = $this->appData->getFolder($internalPath);
132+
133+
/**
134+
* @var list<array{file: ISimpleFile, width: int, height: int, crop: bool, max: bool, extension: string, mtime: int, size: int}> $previewFiles
135+
*/
136+
$previewFiles = [];
137+
138+
foreach ($folder->getDirectoryListing() as $previewFile) {
139+
[0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName());
140+
$nameSplit = explode('-', $baseName);
141+
142+
// TODO VERSION/PREFIX extraction
143+
144+
$width = $nameSplit[0];
145+
$height = $nameSplit[1];
146+
147+
if (isset($nameSplit[2])) {
148+
$crop = $nameSplit[2] === 'crop';
149+
$max = $nameSplit[2] === 'max';
150+
}
151+
152+
$previewFiles[] = [
153+
'file' => $previewFile,
154+
'width' => $width,
155+
'height' => $height,
156+
'crop' => $crop,
157+
'max' => $max,
158+
'extension' => $extension,
159+
'size' => $previewFile->getSize(),
160+
'mtime' => $previewFile->getMTime(),
161+
];
162+
}
163+
164+
$qb = $this->connection->getQueryBuilder();
165+
$qb->select('*')
166+
->from('filecache')
167+
->where($qb->expr()->like('fileid', $qb->createNamedParameter($fileId)));
168+
169+
$result = $qb->executeQuery();
170+
$result = $result->fetchAll();
171+
172+
if (count($result) > 0) {
173+
foreach ($previewFiles as $previewFile) {
174+
$preview = new Preview();
175+
$preview->setFileId((int)$fileId);
176+
$preview->setStorageId($result[0]['storage']);
177+
$preview->setEtag($result[0]['etag']);
178+
$preview->setMtime($previewFile['mtime']);
179+
$preview->setWidth($previewFile['width']);
180+
$preview->setHeight($previewFile['height']);
181+
$preview->setCrop($previewFile['crop']);
182+
$preview->setIsMax($previewFile['max']);
183+
$preview->setMimetype(match ($previewFile['extension']) {
184+
'png' => IPreview::MIMETYPE_PNG,
185+
'webp' => IPreview::MIMETYPE_WEBP,
186+
'gif' => IPreview::MIMETYPE_GIF,
187+
default => IPreview::MIMETYPE_JPEG,
188+
});
189+
$preview->setSize($previewFile['size']);
190+
try {
191+
$preview = $this->previewMapper->insert($preview);
192+
} catch (Exception $e) {
193+
// We already have this preview in the preview table, skip
194+
continue;
195+
}
196+
197+
try {
198+
$this->storageFactory->migratePreview($preview, $previewFile['file']);
199+
$previewFile['file']->delete();
200+
} catch (Exception $e) {
201+
$this->previewMapper->delete($preview);
202+
throw $e;
203+
}
204+
205+
}
206+
}
207+
208+
$this->deleteFolder($internalPath, $folder);
209+
}
210+
}
211+
212+
public static function getInternalFolder(string $name, bool $simplePaths): string {
213+
if ($simplePaths) {
214+
return '/' . $name;
215+
}
216+
return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name;
217+
}
218+
219+
private function deleteFolder(string $path, ISimpleFolder $folder): void {
220+
$folder->delete();
221+
222+
$current = $path;
223+
224+
while (true) {
225+
$current = dirname($current);
226+
if ($current === '/' || $current === '.' || $current === '') {
227+
break;
228+
}
229+
230+
231+
$folder = $this->appData->getFolder($current);
232+
if (count($folder->getDirectoryListing()) !== 0) {
233+
break;
234+
}
235+
$folder->delete();
236+
}
237+
}
238+
}

‎core/Migrations/Version33000Date20250819110529.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
3030
$table = $schema->createTable('previews');
3131
$table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]);
3232
$table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]);
33+
$table->addColumn('storage_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]);
3334
$table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]);
3435
$table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]);
3536
$table->addColumn('mimetype', Types::INTEGER, ['notnull' => true]);
@@ -38,7 +39,7 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
3839
$table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40]);
3940
$table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]);
4041
$table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]);
41-
$table->addColumn('version', Types::BIGINT, ['notnull' => false, 'unsigned' => true]);
42+
$table->addColumn('version', Types::BIGINT, ['notnull' => true, 'default' => -1]); // can not be null otherwise unique index doesn't work
4243
$table->setPrimaryKey(['id']);
4344
$table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'crop', 'version'], 'previews_file_uniq_idx');
4445
}

‎lib/composer/composer/autoload_classmap.php‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1248,6 +1248,7 @@
12481248
'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => $baseDir . '/core/BackgroundJobs/CleanupLoginFlowV2.php',
12491249
'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => $baseDir . '/core/BackgroundJobs/GenerateMetadataJob.php',
12501250
'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => $baseDir . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php',
1251+
'OC\\Core\\BackgroundJobs\\MovePreviewJob' => $baseDir . '/core/BackgroundJobs/MovePreviewJob.php',
12511252
'OC\\Core\\Command\\App\\Disable' => $baseDir . '/core/Command/App/Disable.php',
12521253
'OC\\Core\\Command\\App\\Enable' => $baseDir . '/core/Command/App/Enable.php',
12531254
'OC\\Core\\Command\\App\\GetPath' => $baseDir . '/core/Command/App/GetPath.php',
@@ -1522,7 +1523,7 @@
15221523
'OC\\Core\\Migrations\\Version32000Date20250620081925' => $baseDir . '/core/Migrations/Version32000Date20250620081925.php',
15231524
'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php',
15241525
'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php',
1525-
'OC\\Core\\Migrations\\Version33000Date20250819110523' => $baseDir . '/core/Migrations/Version33000Date20250819110523.php',
1526+
'OC\\Core\\Migrations\\Version33000Date20250819110529' => $baseDir . '/core/Migrations/Version33000Date20250819110529.php',
15261527
'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php',
15271528
'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php',
15281529
'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php',
@@ -1953,6 +1954,7 @@
19531954
'OC\\Repair\\AddCleanupDeletedUsersBackgroundJob' => $baseDir . '/lib/private/Repair/AddCleanupDeletedUsersBackgroundJob.php',
19541955
'OC\\Repair\\AddCleanupUpdaterBackupsJob' => $baseDir . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php',
19551956
'OC\\Repair\\AddMetadataGenerationJob' => $baseDir . '/lib/private/Repair/AddMetadataGenerationJob.php',
1957+
'OC\\Repair\\AddMovePreviewJob' => $baseDir . '/lib/private/Repair/AddMovePreviewJob.php',
19561958
'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => $baseDir . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php',
19571959
'OC\\Repair\\CleanTags' => $baseDir . '/lib/private/Repair/CleanTags.php',
19581960
'OC\\Repair\\CleanUpAbandonedApps' => $baseDir . '/lib/private/Repair/CleanUpAbandonedApps.php',

‎lib/composer/composer/autoload_static.php‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1289,6 +1289,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
12891289
'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupLoginFlowV2.php',
12901290
'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/GenerateMetadataJob.php',
12911291
'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php',
1292+
'OC\\Core\\BackgroundJobs\\MovePreviewJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/MovePreviewJob.php',
12921293
'OC\\Core\\Command\\App\\Disable' => __DIR__ . '/../../..' . '/core/Command/App/Disable.php',
12931294
'OC\\Core\\Command\\App\\Enable' => __DIR__ . '/../../..' . '/core/Command/App/Enable.php',
12941295
'OC\\Core\\Command\\App\\GetPath' => __DIR__ . '/../../..' . '/core/Command/App/GetPath.php',
@@ -1563,7 +1564,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
15631564
'OC\\Core\\Migrations\\Version32000Date20250620081925' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250620081925.php',
15641565
'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php',
15651566
'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php',
1566-
'OC\\Core\\Migrations\\Version33000Date20250819110523' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110523.php',
1567+
'OC\\Core\\Migrations\\Version33000Date20250819110529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110529.php',
15671568
'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php',
15681569
'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php',
15691570
'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php',
@@ -1994,6 +1995,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
19941995
'OC\\Repair\\AddCleanupDeletedUsersBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupDeletedUsersBackgroundJob.php',
19951996
'OC\\Repair\\AddCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php',
19961997
'OC\\Repair\\AddMetadataGenerationJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMetadataGenerationJob.php',
1998+
'OC\\Repair\\AddMovePreviewJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMovePreviewJob.php',
19971999
'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php',
19982000
'OC\\Repair\\CleanTags' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanTags.php',
19992001
'OC\\Repair\\CleanUpAbandonedApps' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanUpAbandonedApps.php',

‎lib/private/BackgroundJob/JobList.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,7 @@ private function buildJob(array $row): ?IJob {
321321
/** @var IJob $job */
322322
$job = \OCP\Server::get($row['class']);
323323
} catch (QueryException $e) {
324+
echo $e->getMessage();
324325
if (class_exists($row['class'])) {
325326
$class = $row['class'];
326327
$job = new $class();

‎lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
use OCP\IUser;
1515

1616
/**
17-
* @psalm-type ObjectStoreConfig array{class: class-string<IObjectStore>, arguments: array{multibucket: bool, objectPrefix: ?string, ...}}
17+
* @psalm-type ObjectStoreConfig array{class: class-string<IObjectStore>, arguments: array{multibucket: bool, objectPrefix?: string, ...}}
1818
*/
1919
class PrimaryObjectStoreConfig {
2020
public function __construct(
@@ -142,7 +142,7 @@ public function getObjectStoreConfigs(): ?array {
142142
}
143143

144144
/**
145-
* @param array|string $config
145+
* @param array{multibucket?: bool, objectPrefix?: string, ...}|string $config
146146
* @return string|ObjectStoreConfig
147147
*/
148148
private function validateObjectStoreConfig(array|string $config): array|string {

‎lib/private/Preview/Db/Preview.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public function __construct() {
7878
}
7979

8080
public function getName(): string {
81-
$path = ($this->getVersion() ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight();
81+
$path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight();
8282
if ($this->getCrop()) {
8383
$path .= '-crop';
8484
}
@@ -102,7 +102,7 @@ public function getMimetypeValue(): string {
102102

103103
public function getExtension(): string {
104104
return match ($this->mimetype) {
105-
IPreview::MIMETYPE_JPEG => 'jpeg',
105+
IPreview::MIMETYPE_JPEG => 'jpg',
106106
IPreview::MIMETYPE_PNG => 'png',
107107
IPreview::MIMETYPE_WEBP => 'webp',
108108
IPreview::MIMETYPE_GIF => 'gif',

0 commit comments

Comments
 (0)