[stable29] multi-instance object store tweaks #57141
+229
−114
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,140 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
| /** | ||
| * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors | ||
| * SPDX-License-Identifier: AGPL-3.0-or-later | ||
| */ | ||
|
|
||
| namespace OCA\Files\Command\Object\Multi; | ||
|
|
||
| use OC\Core\Command\Base; | ||
| use OC\Files\Mount\ObjectHomeMountProvider; | ||
| use OC\Files\ObjectStore\PrimaryObjectStoreConfig; | ||
| use OC\Files\ObjectStore\S3; | ||
| use OC\Files\Storage\StorageFactory; | ||
| use OCP\DB\QueryBuilder\IQueryBuilder; | ||
| use OCP\Files\FileInfo; | ||
| use OCP\Files\IMimeTypeLoader; | ||
| use OCP\IConfig; | ||
| use OCP\IDBConnection; | ||
| use OCP\IUser; | ||
| use OCP\IUserManager; | ||
| use Symfony\Component\Console\Input\InputInterface; | ||
| use Symfony\Component\Console\Input\InputOption; | ||
| use Symfony\Component\Console\Output\OutputInterface; | ||
|
|
||
| class Move extends Base { | ||
| public function __construct( | ||
| private PrimaryObjectStoreConfig $objectStoreConfig, | ||
| private IUserManager $userManager, | ||
| private IConfig $config, | ||
| private ObjectHomeMountProvider $mountProvider, | ||
| private IMimeTypeLoader $mimeTypeLoader, | ||
| private IDBConnection $connection, | ||
| ) { | ||
| parent::__construct(); | ||
| } | ||
|
|
||
| protected function configure(): void { | ||
| parent::configure(); | ||
| $this | ||
| ->setName('files:object:multi:move') | ||
| ->setDescription('Migrate user to the specified object store and bucket. The bucket must be created and known beforehand containing the same objects in the user\'s current bucket.') | ||
| ->addOption('object-store', 'o', InputOption::VALUE_REQUIRED, 'The name of the object store') | ||
| ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, 'The name of the bucket') | ||
| ->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'The user to migrate') | ||
| ->addOption('dry-run', null, InputOption::VALUE_NONE, 'Run command without commiting any changes'); | ||
| } | ||
|
|
||
| public function execute(InputInterface $input, OutputInterface $output): int { | ||
| $objectStore = $input->getOption('object-store'); | ||
| if (!$objectStore) { | ||
| $output->writeln('Please specify the object store'); | ||
| } | ||
| $bucket = $input->getOption('bucket'); | ||
| if (!$bucket) { | ||
| $output->writeln('Please specify the bucket'); | ||
| } | ||
|
|
||
| $configs = $this->objectStoreConfig->getObjectStoreConfigs(); | ||
| if (!isset($configs[$objectStore])) { | ||
| $output->writeln('<error>Unknown object store configuration: ' . $objectStore . '</error>'); | ||
| return 1; | ||
| } | ||
|
|
||
| if ($userId = $input->getOption('user')) { | ||
| $user = $this->userManager->get($userId); | ||
| if (!$user) { | ||
| $output->writeln('<error>User ' . $userId . ' not found</error>'); | ||
| return 1; | ||
| } | ||
| } else { | ||
| $output->writeln('<comment>Please specify a user id with --user</comment>'); | ||
| return 1; | ||
| } | ||
|
|
||
| $targetValid = $this->validateForUser($user, $objectStore, $bucket); | ||
| if ($targetValid) { | ||
| if (!$input->getOption('dry-run')) { | ||
| $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'objectstore', $objectStore); | ||
| $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $bucket); | ||
| } | ||
| $output->writeln('Moved <info>' . $user->getUID() . '</info> to object store <info>' . $objectStore . '</info> and bucket <info>' . $bucket . '</info>'); | ||
| } else { | ||
| $output->writeln('Object store <info>' . $objectStore . '</info> and bucket <info>' . $bucket . '</info> invalid for <info>' . $userId . '</info>. Bucket doesn\'t exist or contain expected user objects.'); | ||
| return 1; | ||
| } | ||
|
|
||
| return 0; | ||
| } | ||
|
|
||
| private function validateForUser(IUser $user, string $targetObjectStore, string $targetBucket): bool { | ||
| $currentObjectStore = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'objectstore', null); | ||
| if (!$currentObjectStore) { | ||
| return false; | ||
| } | ||
| if ($currentObjectStore === $targetObjectStore) { | ||
| return false; | ||
| } | ||
|
|
||
| $currentBucket = $this->objectStoreConfig->getSetBucketForUser($user); | ||
| if (!$currentBucket) { | ||
Check noticeCode scanning / Psalm RiskyTruthyFalsyComparison Note |
||
| return false; | ||
| } | ||
| if ($currentBucket === $targetBucket) { | ||
| return false; | ||
| } | ||
|
|
||
| $storageFactory = new StorageFactory(); | ||
| $homeMount = $this->mountProvider->getHomeMountForUser($user, $storageFactory); | ||
| if ($homeMount === null) { | ||
| return false; | ||
| } | ||
|
|
||
| $homeStorage = $homeMount->getStorage(); | ||
| $storageId = $homeStorage->getCache()->getNumericStorageId(); | ||
Check noticeCode scanning / Psalm PossiblyNullReference Note |
||
| $folderMimetype = $this->mimeTypeLoader->getId(FileInfo::MIMETYPE_FOLDER); | ||
|
|
||
| $query = $this->connection->getQueryBuilder(); | ||
| $query->select('fileid') | ||
| ->from('filecache') | ||
| ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) | ||
| ->andWhere($query->expr()->neq('mimetype', $query->createNamedParameter($folderMimetype, IQueryBuilder::PARAM_INT))); | ||
| $result = $query->execute(); | ||
Check noticeCode scanning / Psalm DeprecatedMethod Note
Comment on lines
+119
to
+124
Member
There was a problem hiding this comment. Since we already have the storage/cache, we can just do
Contributor
Author
There was a problem hiding this comment. Wouldn't that give the id of a user's root folder, which doesn't get saved as an object?
Member
There was a problem hiding this comment. Right, I forgot about that. It does probably make sense to add a limit to the number of fileids we check though. Checking ~5 files gives mostly the same safety as checking all of them without the potentially high cost of checking all files. |
||
| $fileIds = $result->fetchAll(\PDO::FETCH_COLUMN); | ||
Check noticeCode scanning / Psalm PossiblyInvalidMethodCall Note |
||
|
|
||
| // Use a new S3 client to 'peek' into the target bucket since it's not yet mounted | ||
| $targetConfig = $this->objectStoreConfig->getObjectStoreConfiguration($targetObjectStore); | ||
| $targetConfig['arguments']['bucket'] = $targetBucket; | ||
| $s3 = new S3($targetConfig['arguments']); | ||
|
|
||
| foreach ($fileIds as $fileId) { | ||
| if ($s3->objectExists('urn:oid:' . $fileId)) { | ||
| return true; | ||
| } | ||
| } | ||
|
|
||
| return false; | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
| /** | ||
| * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors | ||
| * SPDX-License-Identifier: AGPL-3.0-or-later | ||
| */ | ||
|
|
||
| namespace OCA\Files\Command\Object\Multi; | ||
|
|
||
| use OC\Core\Command\Base; | ||
| use OC\Files\ObjectStore\PrimaryObjectStoreConfig; | ||
| use OCP\IConfig; | ||
| use OCP\IUserManager; | ||
| use Symfony\Component\Console\Input\InputInterface; | ||
| use Symfony\Component\Console\Input\InputOption; | ||
| use Symfony\Component\Console\Output\OutputInterface; | ||
|
|
||
| class PreMigrate extends Base { | ||
| public function __construct( | ||
| private PrimaryObjectStoreConfig $objectStoreConfig, | ||
| private IUserManager $userManager, | ||
| private IConfig $config, | ||
| ) { | ||
| parent::__construct(); | ||
| } | ||
|
|
||
| protected function configure(): void { | ||
| parent::configure(); | ||
| $this | ||
| ->setName('files:object:multi:pre-migrate') | ||
| ->setDescription('Assign a configured object store to users who don\'t have one assigned yet.') | ||
| ->addOption('object-store', 'o', InputOption::VALUE_REQUIRED, 'The name of the configured object store') | ||
| ->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'The userId of the user to assign the object store') | ||
| ->addOption('all', 'a', InputOption::VALUE_NONE, 'Assign the object store to all users'); | ||
| } | ||
|
|
||
| public function execute(InputInterface $input, OutputInterface $output): int { | ||
| $objectStore = $input->getOption('object-store'); | ||
| if (!$objectStore) { | ||
| $output->writeln('Please specify the object store'); | ||
| return 1; | ||
| } | ||
|
|
||
| $configs = $this->objectStoreConfig->getObjectStoreConfigs(); | ||
| if (!isset($configs[$objectStore])) { | ||
| $output->writeln('<error>Unknown object store configuration: ' . $objectStore . '</error>'); | ||
| return 1; | ||
| } | ||
|
|
||
| if ($input->getOption('all')) { | ||
| $users = $this->userManager->getSeenUsers(); | ||
| } elseif ($userId = $input->getOption('user')) { | ||
| $user = $this->userManager->get($userId); | ||
| if (!$user) { | ||
| $output->writeln('<error>User ' . $userId . ' not found</error>'); | ||
| return 1; | ||
| } | ||
| $users = new \ArrayIterator([$user]); | ||
| } else { | ||
| $output->writeln('<comment>Please specify a user id with --user or --all for all users</comment>'); | ||
| return 1; | ||
| } | ||
|
|
||
| $count = 0; | ||
| foreach ($users as $user) { | ||
| if (!$this->config->getUserValue($user->getUID(), 'homeobjectstore', 'objectstore', null)) { | ||
| $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'objectstore', $objectStore); | ||
| $count++; | ||
| } | ||
| } | ||
| $output->writeln('Assigned object store <info>' . $objectStore . '</info> to <info>' . $count . '</info> users'); | ||
|
|
||
| return 0; | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.