Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use invalidations for performing check for intersecting periods being processed #22394

Open
wants to merge 5 commits into
base: 5.x-dev
Choose a base branch
from
Open

Use invalidations for performing check for intersecting periods being processed #22394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a0d73c0
Use database for performing check for intersecting periods being proc…
sgiehl Jul 12, 2024
b2c2a35
add some typehints
sgiehl Jul 16, 2024
9a9c553
add test for new method
sgiehl Jul 16, 2024
5978fd7
use period constants
sgiehl Jul 19, 2024
2d18aeb
Remove no longer used climulti request parser
sgiehl Aug 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 0 additions & 113 deletions core/CliMulti/RequestParser.php
View file
Open in desktop

This file was deleted.

8 changes: 0 additions & 8 deletions core/CronArchive.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
use Piwik\CronArchive\FixedSiteIds;
use Piwik\CronArchive\Performance\Logger;
use Piwik\Archive\ArchiveInvalidator;
use Piwik\CliMulti\RequestParser;
use Piwik\CronArchive\QueueConsumer;
use Piwik\CronArchive\SharedSiteIds;
use Piwik\CronArchive\StopArchiverException;
Expand Down Expand Up @@ -220,11 +219,6 @@ class CronArchive
*/
private $archiveFilter;

/**
* @var RequestParser
*/
private $cliMultiRequestParser;

/**
* @var bool|mixed
*/
Expand All @@ -249,7 +243,6 @@ public function __construct(?LoggerInterface $logger = null)
$this->periodIdsToLabels = array_flip(Piwik::$idPeriods);

$this->supportsAsync = $this->makeCliMulti()->supportsAsync();
$this->cliMultiRequestParser = new RequestParser($this->supportsAsync);

$this->archiveFilter = new ArchiveFilter();
}
Expand Down Expand Up @@ -379,7 +372,6 @@ public function run()
$this->model,
$this->segmentArchiving,
$this,
$this->cliMultiRequestParser,
$this->archiveFilter
);

Expand Down
78 changes: 35 additions & 43 deletions core/CronArchive/QueueConsumer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
use Piwik\ArchiveProcessor\Loader;
use Piwik\ArchiveProcessor\Parameters;
use Piwik\ArchiveProcessor\Rules;
use Piwik\CliMulti\RequestParser;
use Piwik\CronArchive;
use Piwik\DataAccess\ArchiveSelector;
use Piwik\DataAccess\Model;
Expand Down Expand Up @@ -78,11 +77,6 @@ class QueueConsumer
*/
private $periodIdsToLabels;

/**
* @var RequestParser
*/
private $cliMultiRequestParser;

/**
* @var int
*/
Expand Down Expand Up @@ -118,7 +112,6 @@ public function __construct(
Model $model,
SegmentArchiving $segmentArchiving,
CronArchive $cronArchive,
RequestParser $cliMultiRequestParser,
?ArchiveFilter $archiveFilter = null
) {
$this->logger = $logger;
Expand All @@ -128,7 +121,6 @@ public function __construct(
$this->model = $model;
$this->segmentArchiving = $segmentArchiving;
$this->cronArchive = $cronArchive;
$this->cliMultiRequestParser = $cliMultiRequestParser;
$this->archiveFilter = $archiveFilter;

// if we skip or can't process an idarchive, we want to ignore it the next time we look for an invalidated
Expand Down Expand Up @@ -291,7 +283,7 @@ public function getNextArchivesToProcess()
}

$reason = $this->shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress($invalidatedArchive);
if ($reason) {
if ($reason !== null) {
$this->logger->debug("Skipping invalidated archive, $reason: $invalidationDesc");
$invalidationsToExcludeInBatch[$invalidatedArchive['idinvalidation']] = true;
$this->addInvalidationToExclude($invalidatedArchive);
Expand Down Expand Up @@ -343,7 +335,7 @@ public function getNextArchivesToProcess()
return $archivesToProcess;
}

private function archiveArrayContainsArchive($archiveArray, $archive)
private function archiveArrayContainsArchive(array $archiveArray, array $archive): bool
{
foreach ($archiveArray as $entry) {
if (
Expand Down Expand Up @@ -411,7 +403,7 @@ private function shouldSkipArchive($archive)
}

// public for tests
public function canSkipArchiveBecauseNoPoint(array $invalidatedArchive)
public function canSkipArchiveBecauseNoPoint(array $invalidatedArchive): bool
{
$site = new Site($invalidatedArchive['idsite']);

Expand All @@ -430,22 +422,18 @@ public function canSkipArchiveBecauseNoPoint(array $invalidatedArchive)
return $loader->canSkipThisArchive(); // if no point in archiving, skip
}

public function shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress(array $archiveToProcess)
public function shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress(array $archiveToProcess): ?string
{
$inProgressArchives = $this->cliMultiRequestParser->getInProgressArchivingCommands();
$inProgressArchives = $this->model->getInvalidationsInProgress(
(int) $archiveToProcess['idsite']
);

$periods = array_flip(Piwik::$idPeriods);

foreach ($inProgressArchives as $archiveBeingProcessed) {
if (
empty($archiveBeingProcessed['period'])
|| empty($archiveBeingProcessed['date'])
) {
continue;
}
$this->findSegmentForArchive($archiveBeingProcessed);

if (
empty($archiveBeingProcessed['idSite'])
|| $archiveBeingProcessed['idSite'] != $archiveToProcess['idsite']
) {
if ($archiveBeingProcessed['idsite'] != $archiveToProcess['idsite']) {
continue; // different site
}

Expand All @@ -454,26 +442,31 @@ public function shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress(array $
!empty($archiveBeingProcessed['segment'])
&& !empty($archiveToProcess['segment'])
&& $archiveBeingProcessed['segment'] != $archiveToProcess['segment']
&& urldecode($archiveBeingProcessed['segment']) != $archiveToProcess['segment']
) {
continue;
}

$archiveBeingProcessed['periodObj'] = PeriodFactory::build($archiveBeingProcessed['period'], $archiveBeingProcessed['date']);
$archiveBeingProcessed['periodObj'] = PeriodFactory::build($periods[$archiveBeingProcessed['period']], $archiveBeingProcessed['date1']);

if ($this->isArchiveOfLowerPeriod($archiveToProcess, $archiveBeingProcessed)) {
return "lower or same period in progress in another local climulti process (period = {$archiveBeingProcessed['period']}, date = {$archiveBeingProcessed['date']})";
if (!$this->isArchiveOfLowerPeriod($archiveToProcess, $archiveBeingProcessed)) {
continue;
}

if ($this->isArchiveNonSegmentAndInProgressArchiveSegment($archiveToProcess, $archiveBeingProcessed)) {
return "segment archive in progress for same site/period ({$archiveBeingProcessed['segment']})";
if (empty($archiveToProcess['segment']) && !empty($archiveBeingProcessed['segment'])) {
return "segment archive in progress for same site with lower or same period ({$archiveBeingProcessed['segment']}, period = {$periods[$archiveBeingProcessed['period']]}, date = {$archiveBeingProcessed['date1']})";
}

if (!empty($archiveToProcess['segment']) && empty($archiveBeingProcessed['segment'])) {
return "all visits archive in progress for same site with lower or same period (period = {$periods[$archiveBeingProcessed['period']]}, date = {$archiveBeingProcessed['date1']})";
}

return "lower or same period in progress (period = {$periods[$archiveBeingProcessed['period']]}, date = {$archiveBeingProcessed['date1']})";
}

return false;
return null;
}

private function isArchiveOfLowerPeriod(array $archiveToProcess, $archiveBeingProcessed)
private function isArchiveOfLowerPeriod(array $archiveToProcess, array $archiveBeingProcessed): bool
{
/** @var Period $archiveToProcessPeriodObj */
$archiveToProcessPeriodObj = $archiveToProcess['periodObj'];
Expand All @@ -490,12 +483,11 @@ private function isArchiveOfLowerPeriod(array $archiveToProcess, $archiveBeingPr
return false;
}

private function isArchiveNonSegmentAndInProgressArchiveSegment(array $archiveToProcess, array $archiveBeingProcessed)
private function isArchiveNonSegmentAndInProgressArchiveSegment(array $archiveToProcess, array $archiveBeingProcessed): bool
{
// archive is for different site/period
if (
empty($archiveBeingProcessed['idSite'])
|| $archiveToProcess['idsite'] != $archiveBeingProcessed['idSite']
$archiveToProcess['idsite'] != $archiveBeingProcessed['idsite']
|| $archiveToProcess['periodObj']->getId() != $archiveBeingProcessed['periodObj']->getId()
|| $archiveToProcess['periodObj']->getDateStart()->toString() != $archiveBeingProcessed['periodObj']->getDateStart()->toString()
) {
Expand All @@ -505,13 +497,13 @@ private function isArchiveNonSegmentAndInProgressArchiveSegment(array $archiveTo
return empty($archiveToProcess['segment']) && !empty($archiveBeingProcessed['segment']);
}

private function detectPluginForArchive(&$archive)
private function detectPluginForArchive(&$archive): void
{
$archive['plugin'] = $this->getPluginNameForArchiveIfAny($archive);
}

// static so it can be unit tested
public static function hasIntersectingPeriod(array $archivesToProcess, $invalidatedArchive)
public static function hasIntersectingPeriod(array $archivesToProcess, $invalidatedArchive): bool
{
if (empty($archivesToProcess)) {
return false;
Expand Down Expand Up @@ -545,7 +537,7 @@ public static function hasIntersectingPeriod(array $archivesToProcess, $invalida
return false;
}

private function findSegmentForArchive(&$archive)
private function findSegmentForArchive(&$archive): bool
{
$flag = explode('.', $archive['name'])[0];
if ($flag == 'done') {
Expand All @@ -565,7 +557,7 @@ private function findSegmentForArchive(&$archive)
return $this->segmentArchiving->isAutoArchivingEnabledFor($storedSegment);
}

private function getPluginNameForArchiveIfAny($archive)
private function getPluginNameForArchiveIfAny(array $archive): ?string
{
$name = $archive['name'];
if (strpos($name, '.') === false) {
Expand All @@ -576,17 +568,17 @@ private function getPluginNameForArchiveIfAny($archive)
return $parts[1];
}

public function ignoreIdInvalidation($idinvalidation)
public function ignoreIdInvalidation($idinvalidation): void
{
$this->invalidationsToExclude[$idinvalidation] = $idinvalidation;
}

public function skipToNextSite()
public function skipToNextSite(): void
{
$this->idSite = null;
}

private function addInvalidationToExclude(array $invalidatedArchive)
private function addInvalidationToExclude(array $invalidatedArchive): void
{
$id = $invalidatedArchive['idinvalidation'];
if (empty($this->invalidationsToExclude[$id])) {
Expand All @@ -599,7 +591,7 @@ private function getNextIdSiteToArchive()
return $this->websiteIdArchiveList->getNextSiteId();
}

private function getInvalidationDescription(array $invalidatedArchive)
private function getInvalidationDescription(array $invalidatedArchive): string
{
return sprintf(
"[idinvalidation = %s, idsite = %s, period = %s(%s - %s), name = %s, segment = %s]",
Expand All @@ -614,7 +606,7 @@ private function getInvalidationDescription(array $invalidatedArchive)
}

// public for test
public function usableArchiveExists(array $invalidatedArchive)
public function usableArchiveExists(array $invalidatedArchive): array
{
$site = new Site($invalidatedArchive['idsite']);

Expand Down
Loading
Loading