diff --git a/appinfo/info.xml b/appinfo/info.xml index 912fa7830..abec28e9f 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -26,7 +26,7 @@ Share your tables and views with users and groups within your cloud. Have a good time and manage whatever you want. ]]> - 2.0.0-beta.1 + 2.0.0-beta.2 agpl Florian Steffens Tables @@ -57,6 +57,7 @@ Have a good time and manage whatever you want. OCA\Tables\Migration\NewDbStructureRepairStep OCA\Tables\Migration\DbRowSleeveSequence + OCA\Tables\Migration\CacheSleeveCells diff --git a/lib/Db/ColumnMapper.php b/lib/Db/ColumnMapper.php index ab38369bb..da036fec1 100644 --- a/lib/Db/ColumnMapper.php +++ b/lib/Db/ColumnMapper.php @@ -91,7 +91,7 @@ public function findAll(array $id): array { /** * @param integer $tableId - * @return array + * @return Column[] * @throws Exception */ public function findAllByTable(int $tableId): array { @@ -116,7 +116,7 @@ public function findAllByTable(int $tableId): array { /** * @param integer $tableID - * @return array + * @return int[] * @throws Exception */ public function findAllIdsByTable(int $tableID): array { diff --git a/lib/Db/Row2Mapper.php b/lib/Db/Row2Mapper.php index e69d6bb16..ad71e1dc9 100644 --- a/lib/Db/Row2Mapper.php +++ b/lib/Db/Row2Mapper.php @@ -10,6 +10,8 @@ use DateTime; use DateTimeImmutable; use OCA\Tables\Constants\UsergroupType; +use OCA\Tables\Db\RowLoader\CachedRowLoader; +use OCA\Tables\Db\RowLoader\NormalizedRowLoader; use OCA\Tables\Errors\InternalError; use OCA\Tables\Errors\NotFoundError; use OCA\Tables\Helper\ColumnsHelper; @@ -18,12 +20,8 @@ use OCP\AppFramework\Db\MultipleObjectsReturnedException; use OCP\AppFramework\Db\TTransactional; use OCP\DB\Exception; -use OCP\DB\IResult; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\IDBConnection; -use OCP\Server; -use Psr\Container\ContainerExceptionInterface; -use Psr\Container\NotFoundExceptionInterface; use Psr\Log\LoggerInterface; use Throwable; @@ -38,8 +36,15 @@ class Row2Mapper { protected ColumnMapper $columnMapper; private ColumnsHelper $columnsHelper; + /** + * @var array + */ + private array $rowLoaders; - public function __construct(?string $userId, IDBConnection $db, LoggerInterface $logger, UserHelper $userHelper, RowSleeveMapper $rowSleeveMapper, ColumnsHelper $columnsHelper, ColumnMapper $columnMapper) { + public function __construct(?string $userId, IDBConnection $db, LoggerInterface $logger, UserHelper $userHelper, RowSleeveMapper $rowSleeveMapper, ColumnsHelper $columnsHelper, ColumnMapper $columnMapper, + NormalizedRowLoader $normalizedRowLoader, + CachedRowLoader $cachedRowLoader, + ) { $this->rowSleeveMapper = $rowSleeveMapper; $this->userId = $userId; $this->db = $db; @@ -47,6 +52,10 @@ public function __construct(?string $userId, IDBConnection $db, LoggerInterface $this->userHelper = $userHelper; $this->columnsHelper = $columnsHelper; $this->columnMapper = $columnMapper; + $this->rowLoaders = [ + RowLoader\RowLoader::LOADER_NORMALIZED => $normalizedRowLoader, + RowLoader\RowLoader::LOADER_CACHED => $cachedRowLoader, + ]; } /** @@ -58,7 +67,7 @@ public function delete(Row2 $row): Row2 { $this->db->beginTransaction(); try { foreach ($this->columnsHelper->columns as $columnType) { - $this->getCellMapperFromType($columnType)->deleteAllForRow($row->getId()); + $this->columnsHelper->getCellMapperFromType($columnType)->deleteAllForRow($row->getId()); } $this->rowSleeveMapper->deleteById($row->getId()); $this->db->commit(); @@ -176,7 +185,7 @@ public function findAll(array $showColumnIds, int $tableId, ?int $limit = null, $wantedRowIdsArray = $this->getWantedRowIds($userId, $tableId, $filter, $sort, $limit, $offset); // Get rows without SQL sorting - $rows = $this->getRows($wantedRowIdsArray, $showColumnIds); + $rows = $this->getRows($wantedRowIdsArray, $showColumnIds, RowLoader\RowLoader::LOADER_CACHED); // Sort rows in PHP to preserve the order from getWantedRowIds return $this->sortRowsByIds($rows, $wantedRowIdsArray); @@ -189,61 +198,27 @@ public function findAll(array $showColumnIds, int $tableId, ?int $limit = null, /** * @param array $rowIds * @param array $columnIds + * @param RowLoader\RowLoader::LOADER_* $loader * @return Row2[] * @throws InternalError */ - private function getRows(array $rowIds, array $columnIds): array { - $qb = $this->db->getQueryBuilder(); - - $qbSqlForColumnTypes = null; - foreach ($this->columnsHelper->columns as $columnType) { - $qbTmp = $this->db->getQueryBuilder(); - $qbTmp->select('row_id', 'column_id', 'last_edit_at', 'last_edit_by') - ->selectAlias($qb->expr()->castColumn('value', IQueryBuilder::PARAM_STR), 'value'); - - // This is not ideal but I cannot think of a good way to abstract this away into the mapper right now - // Ideally we dynamically construct this query depending on what additional selects the column type requires - // however the union requires us to match the exact number of selects for each column type - if ($columnType === Column::TYPE_USERGROUP) { - $qbTmp->selectAlias($qb->expr()->castColumn('value_type', IQueryBuilder::PARAM_STR), 'value_type'); - } else { - $qbTmp->selectAlias($qbTmp->createFunction('NULL'), 'value_type'); - } - - $qbTmp - ->from('tables_row_cells_' . $columnType) - ->where($qb->expr()->in('column_id', $qb->createNamedParameter($columnIds, IQueryBuilder::PARAM_INT_ARRAY, ':columnIds'))) - ->andWhere($qb->expr()->in('row_id', $qb->createNamedParameter($rowIds, IQueryBuilder::PARAM_INT_ARRAY, ':rowsIds'))); - - if ($qbSqlForColumnTypes) { - $qbSqlForColumnTypes .= ' UNION ALL ' . $qbTmp->getSQL() . ' '; - } else { - $qbSqlForColumnTypes = '(' . $qbTmp->getSQL(); - } + private function getRows(array $rowIds, array $columnIds, string $loader = RowLoader\RowLoader::LOADER_NORMALIZED): array { + if (empty($rowIds)) { + return []; } - $qbSqlForColumnTypes .= ')'; - - $qb->select('row_id', 'column_id', 'created_by', 'created_at', 't1.last_edit_by', 't1.last_edit_at', 'value', 'table_id') - // Also should be more generic (see above) - ->addSelect('value_type') - ->from($qb->createFunction($qbSqlForColumnTypes), 't1') - ->innerJoin('t1', 'tables_row_sleeves', 'rs', 'rs.id = t1.row_id'); - try { - $result = $qb->executeQuery(); - } catch (Exception $e) { - $this->logger->error($e->getMessage(), ['exception' => $e]); - throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage(), ); + $columns = $mappers = []; + foreach ($columnIds as $columnId) { + $columns[$columnId] = $this->columnMapper->find($columnId); + $mappers[$columnId] = $this->columnsHelper->getCellMapperFromType($columns[$columnId]->getType()); } - try { - $sleeves = $this->rowSleeveMapper->findMultiple($rowIds); - } catch (Exception $e) { - $this->logger->error($e->getMessage(), ['exception' => $e]); - throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage()); + $rows = []; + foreach ($this->rowLoaders[$loader]->getRows($rowIds, $columns, $mappers) as $row) { + $rows[] = $row; } + return $rows; - return $this->parseEntities($result, $sleeves); } /** @@ -623,61 +598,6 @@ private function getSqlOperator(string $operator, IQueryBuilder $qb, string $col } } - /** - * @param IResult $result - * @param RowSleeve[] $sleeves - * @return Row2[] - * @throws InternalError - */ - private function parseEntities(IResult $result, array $sleeves): array { - $rows = []; - foreach ($sleeves as $sleeve) { - $rows[$sleeve->getId()] = new Row2(); - $rows[$sleeve->getId()]->setId($sleeve->getId()); - $rows[$sleeve->getId()]->setCreatedBy($sleeve->getCreatedBy()); - $rows[$sleeve->getId()]->setCreatedAt($sleeve->getCreatedAt()); - $rows[$sleeve->getId()]->setLastEditBy($sleeve->getLastEditBy()); - $rows[$sleeve->getId()]->setLastEditAt($sleeve->getLastEditAt()); - $rows[$sleeve->getId()]->setTableId($sleeve->getTableId()); - } - - $rowValues = []; - $keyToColumnId = []; - $keyToRowId = []; - $cellMapperCache = []; - - while ($rowData = $result->fetch()) { - if (!isset($rowData['row_id'], $rows[$rowData['row_id']])) { - break; - } - - $column = $this->columnMapper->find($rowData['column_id']); - $columnType = $column->getType(); - if (!isset($cellMapperCache[$columnType])) { - $cellMapperCache[$columnType] = $this->getCellMapperFromType($columnType); - } - $value = $cellMapperCache[$columnType]->formatRowData($column, $rowData); - $compositeKey = (string)$rowData['row_id'] . ',' . (string)$rowData['column_id']; - if ($cellMapperCache[$columnType]->hasMultipleValues()) { - if (array_key_exists($compositeKey, $rowValues)) { - $rowValues[$compositeKey][] = $value; - } else { - $rowValues[$compositeKey] = [$value]; - } - } else { - $rowValues[$compositeKey] = $value; - } - $keyToColumnId[$compositeKey] = $rowData['column_id']; - $keyToRowId[$compositeKey] = $rowData['row_id']; - } - - foreach ($rowValues as $compositeKey => $value) { - $rows[$keyToRowId[$compositeKey]]->addCell($keyToColumnId[$compositeKey], $value); - } - - return array_values($rows); - } - /** * @throws InternalError */ @@ -702,9 +622,12 @@ public function insert(Row2 $row): Row2 { } // write all cells to its db-table + $cachedCells = []; foreach ($row->getData() as $cell) { - $this->insertCell($rowSleeve->getId(), $cell['columnId'], $cell['value'], $rowSleeve->getLastEditAt(), $rowSleeve->getLastEditBy()); + $cachedCells[$cell['columnId']] = $this->insertCell($rowSleeve->getId(), $cell['columnId'], $cell['value'], $rowSleeve->getLastEditAt(), $rowSleeve->getLastEditBy()); } + $rowSleeve->setCachedCellsArray($cachedCells); + $this->rowSleeveMapper->update($rowSleeve); return $row; } @@ -721,9 +644,9 @@ public function update(Row2 $row): Row2 { // update meta data for sleeve try { - $sleeve = $this->rowSleeveMapper->find($row->getId()); - $this->updateMetaData($sleeve); - $this->rowSleeveMapper->update($sleeve); + $rowSleeve = $this->rowSleeveMapper->find($row->getId()); + $this->updateMetaData($rowSleeve); + $this->rowSleeveMapper->update($rowSleeve); } catch (DoesNotExistException|MultipleObjectsReturnedException|Exception $e) { $this->logger->error($e->getMessage(), ['exception' => $e]); throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage()); @@ -732,9 +655,12 @@ public function update(Row2 $row): Row2 { $this->columnMapper->preloadColumns(array_column($changedCells, 'columnId')); // write all changed cells to its db-table + $cachedCells = $rowSleeve->getCachedCellsArray(); foreach ($changedCells as $cell) { - $this->insertOrUpdateCell($sleeve->getId(), $cell['columnId'], $cell['value']); + $cachedCells[$cell['columnId']] = $this->insertOrUpdateCell($rowSleeve->getId(), $cell['columnId'], $cell['value']); } + $rowSleeve->setCachedCellsArray($cachedCells); + $this->rowSleeveMapper->update($rowSleeve); return $row; } @@ -786,9 +712,11 @@ private function updateMetaData($entity, bool $setCreate = false, ?string $lastE /** * Insert a cell to its specific db-table * + * @return array normalized cell data + * * @throws InternalError */ - private function insertCell(int $rowId, int $columnId, $value, ?string $lastEditAt = null, ?string $lastEditBy = null): void { + private function insertCell(int $rowId, int $columnId, $value, ?string $lastEditAt = null, ?string $lastEditBy = null): array { try { $column = $this->columnMapper->find($columnId); } catch (DoesNotExistException $e) { @@ -798,7 +726,7 @@ private function insertCell(int $rowId, int $columnId, $value, ?string $lastEdit // insert new cell $cellMapper = $this->getCellMapper($column); - + $cachedCell = []; try { $cellClassName = 'OCA\Tables\Db\RowCell' . ucfirst($column->getType()); if ($cellMapper->hasMultipleValues()) { @@ -810,6 +738,7 @@ private function insertCell(int $rowId, int $columnId, $value, ?string $lastEdit $this->updateMetaData($cell, false, $lastEditAt, $lastEditBy); $cellMapper->applyDataToEntity($column, $cell, $val); $cellMapper->insert($cell); + $cachedCell[] = $cellMapper->toArray($cell); } } else { /** @var RowCellSuper $cell */ @@ -819,11 +748,14 @@ private function insertCell(int $rowId, int $columnId, $value, ?string $lastEdit $this->updateMetaData($cell, false, $lastEditAt, $lastEditBy); $cellMapper->applyDataToEntity($column, $cell, $value); $cellMapper->insert($cell); + $cachedCell = $cellMapper->toArray($cell); } } catch (Exception $e) { $this->logger->error($e->getMessage(), ['exception' => $e]); throw new InternalError('Failed to insert column: ' . $e->getMessage(), 0, $e); } + + return $cachedCell; } /** @@ -831,53 +763,51 @@ private function insertCell(int $rowId, int $columnId, $value, ?string $lastEdit * @param RowCellMapperSuper $mapper * @param mixed $value the value should be parsed to the correct format within the row service * @param Column $column + * + * @return array normalized cell data * @throws InternalError */ - private function updateCell(RowCellSuper $cell, RowCellMapperSuper $mapper, $value, Column $column): void { - $this->getCellMapper($column)->applyDataToEntity($column, $cell, $value); + private function updateCell(RowCellSuper $cell, RowCellMapperSuper $mapper, $value, Column $column): array { + $cellMapper = $this->getCellMapper($column); + $cellMapper->applyDataToEntity($column, $cell, $value); $this->updateMetaData($cell); $mapper->updateWrapper($cell); + + return $cellMapper->toArray($cell); } /** + * @return array normalized cell data * @throws InternalError */ - private function insertOrUpdateCell(int $rowId, int $columnId, $value): void { + private function insertOrUpdateCell(int $rowId, int $columnId, $value): array { $column = $this->columnMapper->find($columnId); $cellMapper = $this->getCellMapper($column); + $cachedCell = []; try { if ($cellMapper->hasMultipleValues()) { - $this->atomic(function () use ($cellMapper, $rowId, $columnId, $value) { + $this->atomic(function () use ($cellMapper, $rowId, $columnId, $value, &$cachedCell) { // For a usergroup field with mutiple values, each is inserted as a new cell // we need to delete all previous cells for this row and column, otherwise we get duplicates $cellMapper->deleteAllForColumnAndRow($columnId, $rowId); - $this->insertCell($rowId, $columnId, $value); + $cachedCell = $this->insertCell($rowId, $columnId, $value); }, $this->db); } else { $cell = $cellMapper->findByRowAndColumn($rowId, $columnId); - $this->updateCell($cell, $cellMapper, $value, $column); + $cachedCell = $this->updateCell($cell, $cellMapper, $value, $column); } } catch (DoesNotExistException) { - $this->insertCell($rowId, $columnId, $value); + $cachedCell = $this->insertCell($rowId, $columnId, $value); } catch (MultipleObjectsReturnedException|Exception $e) { $this->logger->error($e->getMessage(), ['exception' => $e]); - throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage()); + throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage(), $e->getCode(), $e); } - } - private function getCellMapper(Column $column): RowCellMapperSuper { - return $this->getCellMapperFromType($column->getType()); + return $cachedCell; } - private function getCellMapperFromType(string $columnType): RowCellMapperSuper { - $cellMapperClassName = 'OCA\Tables\Db\RowCell' . ucfirst($columnType) . 'Mapper'; - /** @var RowCellMapperSuper $cellMapper */ - try { - return Server::get($cellMapperClassName); - } catch (NotFoundExceptionInterface|ContainerExceptionInterface $e) { - $this->logger->error($e->getMessage(), ['exception' => $e]); - throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage()); - } + private function getCellMapper(Column $column): RowCellMapperSuper { + return $this->columnsHelper->getCellMapperFromType($column->getType()); } /** diff --git a/lib/Db/RowCellMapperSuper.php b/lib/Db/RowCellMapperSuper.php index a3389b310..fa0cef35f 100644 --- a/lib/Db/RowCellMapperSuper.php +++ b/lib/Db/RowCellMapperSuper.php @@ -49,6 +49,10 @@ public function applyDataToEntity(Column $column, RowCellSuper $cell, $data): vo $cell->setValue($data); } + public function toArray(RowCellSuper $cell): array { + return ['value' => $cell->getValue()]; + } + public function getDbParamType() { return IQueryBuilder::PARAM_STR; } @@ -114,6 +118,21 @@ public function findByRowAndColumn(int $rowId, int $columnId): RowCellSuper { return $this->findEntity($qb); } + /** + * @throws MultipleObjectsReturnedException + * @throws DoesNotExistException + * @throws Exception + * @return RowCellSuper[] + */ + public function findManyByRowAndColumn(int $rowId, int $columnId): array { + $qb = $this->db->getQueryBuilder(); + $qb->select('*') + ->from($this->tableName) + ->where($qb->expr()->eq('row_id', $qb->createNamedParameter($rowId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('column_id', $qb->createNamedParameter($columnId, IQueryBuilder::PARAM_INT))); + return $this->findEntities($qb); + } + /** * @throws MultipleObjectsReturnedException * @throws DoesNotExistException diff --git a/lib/Db/RowCellUsergroupMapper.php b/lib/Db/RowCellUsergroupMapper.php index 41ed770c8..4f1934f83 100644 --- a/lib/Db/RowCellUsergroupMapper.php +++ b/lib/Db/RowCellUsergroupMapper.php @@ -58,6 +58,13 @@ public function formatRowData(Column $column, array $row) { ]; } + public function toArray(RowCellSuper $cell): array { + return [ + 'value' => $cell->getValue(), + 'value_type' => $cell->getValueType(), + ]; + } + public function hasMultipleValues(): bool { return true; } diff --git a/lib/Db/RowLoader/CachedRowLoader.php b/lib/Db/RowLoader/CachedRowLoader.php new file mode 100644 index 000000000..ab4b60cc9 --- /dev/null +++ b/lib/Db/RowLoader/CachedRowLoader.php @@ -0,0 +1,88 @@ +getRowsChunk($chunkedRowIds, $columns, $mappers); + } + } + + /** + * @param array $rowIds + * @param array $columns Column per columnId + * @param array $mappers Mapper per columnId + * @return Row2[] + * @throws InternalError + */ + private function getRowsChunk(array $rowIds, array $columns, array $mappers): array { + try { + $sleeves = $this->rowSleeveMapper->findMultiple($rowIds); + } catch (Exception $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage(), $e->getCode(), $e); + } + + return $this->parseResult($sleeves, $columns, $mappers); + } + + /** + * @param RowSleeve[] $sleeves + * @param array $columns Column per columnId + * @param array $mappers Mapper per columnId + * @return Row2[] + */ + private function parseResult(array $sleeves, array $columns, $mappers): array { + $rows = []; + foreach ($sleeves as $sleeve) { + $id = (int)$sleeve['id']; + $row = new Row2(); + $row->setId($id); + $row->setCreatedBy($sleeve['created_by']); + $row->setCreatedAt($sleeve['created_at']); + $row->setLastEditBy($sleeve['last_edit_by']); + $row->setLastEditAt($sleeve['last_edit_at']); + $row->setTableId((int)$sleeve['table_id']); + + $cachedCells = json_decode($sleeve['cached_cells'] ?? '{}', true); + foreach ($columns as $columnId => $column) { + if (empty($cachedCells[$columnId])) { + continue; + } + + $cellMapper = $mappers[$columnId]; + if ($cellMapper->hasMultipleValues()) { + $value = array_map(static fn ($rowData) => $cellMapper->formatRowData($column, $rowData), + $cachedCells[$columnId]); + } else { + $value = $cellMapper->formatRowData($column, $cachedCells[$columnId]); + } + + $row->addCell($columnId, $value); + } + + $rows[] = $row; + } + + return $rows; + } +} diff --git a/lib/Db/RowLoader/NormalizedRowLoader.php b/lib/Db/RowLoader/NormalizedRowLoader.php new file mode 100644 index 000000000..cea3d8b3a --- /dev/null +++ b/lib/Db/RowLoader/NormalizedRowLoader.php @@ -0,0 +1,149 @@ +columnsHelper->columns)); + $chunkSize = max(1, $maxParametersPerType - count($columns)); + + foreach (array_chunk($rowIds, $chunkSize) as $chunkedRowIds) { + yield from $this->getRowsChunk($chunkedRowIds, $columns, $mappers); + } + } + + /** + * Builds and executes the UNION ALL query for a specific chunk of rows. + * @param array $rowIds + * @param array $columns Column per columnId + * @param array $mappers Mapper per columnId + * @return Row2[] + * @throws InternalError + */ + private function getRowsChunk(array $rowIds, array $columns, array $mappers): array { + $qb = $this->db->getQueryBuilder(); + + $subqueries = []; + foreach ($this->columnsHelper->columns as $columnType) { + $qbValues = $this->db->getQueryBuilder(); + $qbValues->select('row_id', 'column_id', 'last_edit_at', 'last_edit_by') + ->selectAlias($qb->expr()->castColumn('value', IQueryBuilder::PARAM_STR), 'value'); + + // This is not ideal but I cannot think of a good way to abstract this away into the mapper right now + // Ideally we dynamically construct this query depending on what additional selects the column type requires + // however the union requires us to match the exact number of selects for each column type + if ($columnType === Column::TYPE_USERGROUP) { + $qbValues->selectAlias($qb->expr()->castColumn('value_type', IQueryBuilder::PARAM_STR), 'value_type'); + } else { + $qbValues->selectAlias($qbValues->createFunction('NULL'), 'value_type'); + } + + $qbValues + ->from('tables_row_cells_' . $columnType) + ->where($qb->expr()->in('column_id', $qb->createNamedParameter(array_keys($columns), IQueryBuilder::PARAM_INT_ARRAY, ':columnIds'))) + ->andWhere($qb->expr()->in('row_id', $qb->createNamedParameter($rowIds, IQueryBuilder::PARAM_INT_ARRAY, ':rowsIds'))); + + $subqueries[] = $qbValues->getSQL(); + } + + $qb->select('row_id', 'column_id', 'created_by', 'created_at', 't1.last_edit_by', 't1.last_edit_at', 'value', 'table_id') + // Also should be more generic (see above) + ->addSelect('value_type') + ->from($qb->createFunction('(' . implode(' UNION ALL ', $subqueries) . ')'), 't1') + ->innerJoin('t1', 'tables_row_sleeves', 'rs', 'rs.id = t1.row_id'); + + try { + $result = $qb->executeQuery(); + } catch (Exception $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage(), $e->getCode(), $e); + } + + try { + $sleeves = $this->rowSleeveMapper->findMultiple($rowIds); + } catch (Exception $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage(), $e->getCode(), $e); + } + + return $this->parseResult($result, $sleeves, $columns, $mappers); + } + + /** + * @param IResult $result + * @param RowSleeve[] $sleeves + * @param array $columns Column per columnId + * @param array $mappers Mapper per columnId + * @return Row2[] + * @throws InternalError + */ + private function parseResult(IResult $result, array $sleeves, array $columns, array $mappers): array { + $rows = []; + foreach ($sleeves as $sleeve) { + $id = (int)$sleeve['id']; + $rows[$id] = new Row2(); + $rows[$id]->setId($id); + $rows[$id]->setCreatedBy($sleeve['created_by']); + $rows[$id]->setCreatedAt($sleeve['created_at']); + $rows[$id]->setLastEditBy($sleeve['last_edit_by']); + $rows[$id]->setLastEditAt($sleeve['last_edit_at']); + $rows[$id]->setTableId((int)$sleeve['table_id']); + } + + $rowValues = []; + $keyToColumnId = []; + $keyToRowId = []; + + while ($rowData = $result->fetch()) { + if (!isset($rowData['row_id'], $rows[$rowData['row_id']])) { + break; + } + + $column = $columns[$rowData['column_id']]; + $cellMapper = $mappers[$rowData['column_id']]; + $value = $cellMapper->formatRowData($column, $rowData); + $compositeKey = (string)$rowData['row_id'] . ',' . (string)$rowData['column_id']; + if ($cellMapper->hasMultipleValues()) { + if (array_key_exists($compositeKey, $rowValues)) { + $rowValues[$compositeKey][] = $value; + } else { + $rowValues[$compositeKey] = [$value]; + } + } else { + $rowValues[$compositeKey] = $value; + } + $keyToColumnId[$compositeKey] = $rowData['column_id']; + $keyToRowId[$compositeKey] = $rowData['row_id']; + } + + foreach ($rowValues as $compositeKey => $value) { + $rows[$keyToRowId[$compositeKey]]->addCell($keyToColumnId[$compositeKey], $value); + } + + return array_values($rows); + } +} diff --git a/lib/Db/RowLoader/RowLoader.php b/lib/Db/RowLoader/RowLoader.php new file mode 100644 index 000000000..c2d149a02 --- /dev/null +++ b/lib/Db/RowLoader/RowLoader.php @@ -0,0 +1,24 @@ + $columns Column per columnId + * @param array $mappers Mapper per columnId + * @return iterable + * @throws InternalError + */ + public function getRows(array $rowIds, array $columns, array $mappers): iterable; +} diff --git a/lib/Db/RowSleeve.php b/lib/Db/RowSleeve.php index 3c765e817..5c417f30f 100644 --- a/lib/Db/RowSleeve.php +++ b/lib/Db/RowSleeve.php @@ -15,6 +15,8 @@ * @psalm-suppress PropertyNotSetInConstructor * @method getTableId(): ?int * @method setTableId(int $columnId) + * @method getCachedCells(): string + * @method setCachedCells(string $cachedCells) * @method getCreatedBy(): string * @method setCreatedBy(string $createdBy) * @method getCreatedAt(): string @@ -26,6 +28,7 @@ */ class RowSleeve extends Entity implements JsonSerializable { protected ?int $tableId = null; + protected ?string $cachedCells = null; protected ?string $createdBy = null; protected ?string $createdAt = null; protected ?string $lastEditBy = null; @@ -34,12 +37,25 @@ class RowSleeve extends Entity implements JsonSerializable { public function __construct() { $this->addType('id', 'integer'); $this->addType('tableId', 'integer'); + $this->addType('cachedCells', 'string'); + } + + /** + * @return array Indexed by column ID + */ + public function getCachedCellsArray(): array { + return json_decode($this->cachedCells, true) ?: []; + } + + public function setCachedCellsArray(array $array):void { + $this->setCachedCells(json_encode($array)); } public function jsonSerialize(): array { return [ 'id' => $this->id, 'tableId' => $this->tableId, + 'cachedCells' => $this->cachedCells, 'createdBy' => $this->createdBy, 'createdAt' => $this->createdAt, 'lastEditBy' => $this->lastEditBy, diff --git a/lib/Db/RowSleeveMapper.php b/lib/Db/RowSleeveMapper.php index d19b4a232..86bd26145 100644 --- a/lib/Db/RowSleeveMapper.php +++ b/lib/Db/RowSleeveMapper.php @@ -40,7 +40,7 @@ public function find(int $id): RowSleeve { /** * @param int[] $ids - * @return RowSleeve[] + * @return array Raw data from DB for the best performance * @throws Exception */ public function findMultiple(array $ids): array { @@ -49,6 +49,7 @@ public function findMultiple(array $ids): array { $qb->select( $sleeveAlias . '.id', $sleeveAlias . '.table_id', + $sleeveAlias . '.cached_cells', $sleeveAlias . '.created_by', $sleeveAlias . '.created_at', $sleeveAlias . '.last_edit_by', @@ -56,7 +57,8 @@ public function findMultiple(array $ids): array { ) ->from($this->table, $sleeveAlias) ->where($qb->expr()->in($sleeveAlias . '.id', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY))); - return $this->findEntities($qb); + + return $qb->executeQuery()->fetchAll(); } /** diff --git a/lib/Helper/ColumnsHelper.php b/lib/Helper/ColumnsHelper.php index 035dfcc1b..e06ece6bf 100644 --- a/lib/Helper/ColumnsHelper.php +++ b/lib/Helper/ColumnsHelper.php @@ -9,6 +9,12 @@ use OCA\Tables\Constants\UsergroupType; use OCA\Tables\Db\Column; +use OCA\Tables\Db\RowCellMapperSuper; +use OCA\Tables\Errors\InternalError; +use OCP\Server; +use Psr\Container\ContainerExceptionInterface; +use Psr\Container\NotFoundExceptionInterface; +use Psr\Log\LoggerInterface; class ColumnsHelper { @@ -23,6 +29,7 @@ class ColumnsHelper { public function __construct( private UserHelper $userHelper, private CircleHelper $circleHelper, + private LoggerInterface $logger, ) { } @@ -79,4 +86,15 @@ public function resolveSearchValue(string $placeholder, string $userId, ?Column default: return $placeholder; } } + + public function getCellMapperFromType(string $columnType): RowCellMapperSuper { + $cellMapperClassName = 'OCA\Tables\Db\RowCell' . ucfirst($columnType) . 'Mapper'; + /** @var RowCellMapperSuper $cellMapper */ + try { + return Server::get($cellMapperClassName); + } catch (NotFoundExceptionInterface|ContainerExceptionInterface $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage(), $e->getCode(), $e); + } + } } diff --git a/lib/Migration/CacheSleeveCells.php b/lib/Migration/CacheSleeveCells.php new file mode 100644 index 000000000..bedb14138 --- /dev/null +++ b/lib/Migration/CacheSleeveCells.php @@ -0,0 +1,105 @@ +config->getAppValue('tables', 'cachingSleeveCellsComplete', 'false') === 'true'; + if ($cachingSleeveCellsComplete) { + return; + } + + foreach ($this->getTableIds() as $tableId) { + $columns = $this->columnMapper->findAllByTable($tableId); + + while ($rowIds = $this->getPendingRowIds($tableId)) { + foreach ($rowIds as $rowId) { + $cachedCells = []; + foreach ($columns as $column) { + $cellMapper = $this->columnsHelper->getCellMapperFromType($column->getType()); + $cells = $cellMapper->findManyByRowAndColumn($rowId, $column->getId()); + foreach ($cells as $cell) { + if ($cellMapper->hasMultipleValues()) { + $cachedCells[$column->getId()][] = $cellMapper->toArray($cell); + } else { + $cachedCells[$column->getId()] = $cellMapper->toArray($cell); + } + } + } + + $sleeve = $this->rowSleeveMapper->find($rowId); + $sleeve->setCachedCellsArray($cachedCells); + $this->rowSleeveMapper->update($sleeve); + } + } + + $this->logger->info('Finished caching cells for table ' . $tableId); + } + + $this->config->setAppValue('tables', 'cachingSleeveCellsComplete', 'true'); + } + + /** + * @return int[] + */ + public function getTableIds(): array { + return $this->db->getQueryBuilder() + ->select('id') + ->from('tables_tables') + ->orderBy('id') + ->setMaxResults(PHP_INT_MAX) + ->executeQuery() + ->fetchAll(\PDO::FETCH_COLUMN); + + } + + /** + * @return int[] + */ + private function getPendingRowIds(int $tableId): array { + $qb = $this->db->getQueryBuilder(); + + return $qb->select('id') + ->from('tables_row_sleeves') + ->where($qb->expr()->isNull('cached_cells')) + ->andWhere($qb->expr()->eq('table_id', $qb->createNamedParameter($tableId, \PDO::PARAM_INT))) + ->orderBy('id') + ->setMaxResults(1000) + ->executeQuery() + ->fetchAll(\PDO::FETCH_COLUMN); + } +} diff --git a/lib/Migration/Version001010Date20251229000000.php b/lib/Migration/Version001010Date20251229000000.php new file mode 100644 index 000000000..8276d59e3 --- /dev/null +++ b/lib/Migration/Version001010Date20251229000000.php @@ -0,0 +1,40 @@ +getTable('tables_row_sleeves'); + if (!$table->hasColumn('tables_row_sleeves')) { + $table->addColumn('cached_cells', Types::JSON, [ + 'notnull' => false, + ]); + } + + return $schema; + } +}