diff --git a/core/CronArchive/ArchiveFilter.php b/core/CronArchive/ArchiveFilter.php index a723775864f..04fba3fc0a4 100644 --- a/core/CronArchive/ArchiveFilter.php +++ b/core/CronArchive/ArchiveFilter.php @@ -14,6 +14,7 @@ use Piwik\Date; use Piwik\Period\Factory; use Piwik\Period\Factory as PeriodFactory; +use Piwik\Period\Range; use Piwik\Piwik; use Piwik\Plugins\SegmentEditor\Model as SegmentEditorModel; use Piwik\Segment; @@ -89,7 +90,11 @@ public function filterArchive($archive) if (!empty($this->skipSegmentsForToday)) { $site = new Site($archive['idsite']); - $period = Factory::build($this->periodIdsToLabels[$archive['period']], $archive['date1']); + if ((int) $archive['period'] === Range::PERIOD_ID) { + $period = Factory::build($this->periodIdsToLabels[$archive['period']], "{$archive['date1']},{$archive['date2']}"); + } else { + $period = Factory::build($this->periodIdsToLabels[$archive['period']], $archive['date1']); + } $segment = new Segment($segment, [$archive['idsite']]); if (Archive::shouldSkipArchiveIfSkippingSegmentArchiveForToday($site, $period, $segment)) { return "skipping segment archives for today"; diff --git a/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php b/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php index 579c983e14d..7ae15d16572 100644 --- a/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php +++ b/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php @@ -13,7 +13,10 @@ use Piwik\CliMulti\RequestParser; use Piwik\Common; use Piwik\Config; +use Piwik\Period\Day; use Piwik\Period\Factory; +use Piwik\Period\Month; +use Piwik\Period\Range; use Piwik\Plugins\CustomDimensions; use Piwik\Container\StaticContainer; use Piwik\CronArchive; @@ -616,12 +619,15 @@ public function testSkipSegmentsToday() $segmentHash2 = (new Segment('browserCode==FF', [1]))->getHash(); $invalidations = [ - ['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null], - ['idarchive' => 1, 'name' => 'done' . $segmentHash2, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null], - ['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'report' => null], - ['idarchive' => 1, 'name' => 'done' . $segmentHash2 . '.ExamplePlugin', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null], - ['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], - ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null], + ['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => Day::PERIOD_ID, 'report' => null], + ['idarchive' => 2, 'name' => 'done' . $segmentHash2, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => Day::PERIOD_ID, 'report' => null], + ['idarchive' => 3, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => Day::PERIOD_ID, 'report' => null], + ['idarchive' => 4, 'name' => 'done' . $segmentHash2 . '.ExamplePlugin', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => Day::PERIOD_ID, 'report' => null], + ['idarchive' => 5, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => Month::PERIOD_ID, 'report' => null], + ['idarchive' => 6, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-31', 'period' => Range::PERIOD_ID, 'report' => null], + ['idarchive' => 7, 'name' => 'done' . $segmentHash2, 'idsite' => 1, 'date1' => '2018-02-02', 'date2' => '2018-03-04', 'period' => Range::PERIOD_ID, 'report' => null], + ['idarchive' => 8, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => Day::PERIOD_ID, 'report' => null], + ['idarchive' => 9, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-03', 'period' => Range::PERIOD_ID, 'report' => null], ]; shuffle($invalidations); @@ -646,60 +652,84 @@ public function testSkipSegmentsToday() } $expectedInvalidationsFound = [ - array ( - array ( - 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-04', - 'date2' => '2018-03-04', - 'period' => '1', - 'name' => 'done', - 'report' => null, - 'plugin' => null, - 'segment' => '', - 'ts_started' => null, - 'status' => '0', - ), - array ( - 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-03', - 'date2' => '2018-03-03', - 'period' => '1', - 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', - 'report' => null, - 'plugin' => null, - 'segment' => 'browserCode==IE', - 'ts_started' => null, - 'status' => '0', - ), - ), - array ( - 0 => - array ( - 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-01', - 'date2' => '2018-03-31', - 'period' => '3', - 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', - 'report' => null, - 'plugin' => null, - 'segment' => 'browserCode==IE', - 'ts_started' => null, - 'status' => '0', - ), - ), - array (// end of idsite=1 - ), + [ + [ + 'idarchive' => '8', + 'idsite' => '1', + 'date1' => '2018-03-04', + 'date2' => '2018-03-04', + 'period' => '1', + 'name' => 'done', + 'report' => null, + 'plugin' => null, + 'segment' => '', + 'ts_started' => null, + 'status' => '0', + ], + [ + 'idarchive' => '3', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', + 'report' => null, + 'plugin' => null, + 'segment' => 'browserCode==IE', + 'ts_started' => null, + 'status' => '0', + ], + ], + [ + [ + 'idarchive' => '5', + 'idsite' => '1', + 'date1' => '2018-03-01', + 'date2' => '2018-03-31', + 'period' => '3', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', + 'report' => null, + 'plugin' => null, + 'segment' => 'browserCode==IE', + 'ts_started' => null, + 'status' => '0', + ], + ], + [ + [ + 'idarchive' => '9', + 'name' => 'done', + 'idsite' => '1', + 'date1' => '2018-03-01', + 'date2' => '2018-03-03', + 'period' => '5', + 'ts_started' => null, + 'status' => '0', + 'report' => null, + 'plugin' => null, + 'segment' => '', + ], + ], + [ + [ + 'idarchive' => '7', + 'name' => 'done3736b708e4d20cfc10610e816a1b2341', + 'idsite' => '1', + 'date1' => '2018-02-02', + 'date2' => '2018-03-04', + 'period' => '5', + 'ts_started' => null, + 'status' => '0', + 'report' => null, + 'plugin' => null, + 'segment' => 'browserCode==FF', + ], + ], + [// end of idsite=1 + ], ]; - try { - $this->assertEquals($expectedInvalidationsFound, $iteratedInvalidations); - } catch (\Exception $ex) { - print "\nInvalidations inserted:\n" . var_export($invalidations, true) . "\n"; - throw $ex; - } + $this->assertEquals($expectedInvalidationsFound, $iteratedInvalidations); // automated check for no duplicates $invalidationDescs = []; @@ -714,6 +744,69 @@ public function testSkipSegmentsToday() $this->assertEquals($uniqueInvalidationDescs, $invalidationDescs, "Found duplicate archives being processed."); } + /** + * @dataProvider getAllPeriods + */ + public function testSkipSegmentsTodayShouldSkipAllSegmentInvalidationsIfPeriodBeginsToday(string $periodType) + { + if ($periodType === 'range') { + $period = Factory::build($periodType, '2018-03-04,2018-04-03'); + } else { + $period = Factory::build($periodType, '2018-03-04'); + } + + Date::$now = strtotime($period->getDateStart()->toString() . ' 01:00:00'); + + Fixture::createWebsite('2015-02-03'); + + Rules::setBrowserTriggerArchiving(false); + API::getInstance()->add('testegment', 'browserCode==IE', false, true); + Rules::setBrowserTriggerArchiving(true); + + // force archiving so we don't skip those without visits + Piwik::addAction('Archiving.getIdSitesToArchiveWhenNoVisits', function (&$idSites) { + $idSites[] = 1; + }); + + $cronArchive = new MockCronArchive(); + $cronArchive->init(); + + $archiveFilter = $this->makeTestArchiveFilter(null, null, null, false, true); + + $queueConsumer = new QueueConsumer( + StaticContainer::get(LoggerInterface::class), + new FixedSiteIds([1]), + 3, + 24, + new Model(), + new SegmentArchiving(), + $cronArchive, + new RequestParser(true), + $archiveFilter + ); + + $segmentHash1 = (new Segment('browserCode==IE', [1]))->getHash(); + + $invalidations = [ + ['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => $period->getDateStart()->toString(), 'date2' => $period->getDateEnd()->toString(), 'period' => $period::PERIOD_ID, 'report' => null], + ]; + + $this->insertInvalidations($invalidations); + + self::assertEmpty($queueConsumer->getNextArchivesToProcess()); + } + + public function getAllPeriods() + { + return [ + ['day'], + ['week'], + ['month'], + ['year'], + ['range'], + ]; + } + public function testMaxWebsitesToProcess() { Fixture::createWebsite('2021-11-16');