Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix archiving for range periods #22577

Merged
merged 2 commits into from
Sep 12, 2024
Merged

Fix archiving for range periods #22577

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
97e836b
Fix archiving for range periods
sgiehl Sep 11, 2024
ab0d8b0
adds another test
sgiehl Sep 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion core/CronArchive/ArchiveFilter.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
use Piwik\Date;
use Piwik\Period\Factory;
use Piwik\Period\Factory as PeriodFactory;
use Piwik\Period\Range;
use Piwik\Piwik;
use Piwik\Plugins\SegmentEditor\Model as SegmentEditorModel;
use Piwik\Segment;
Expand Down Expand Up @@ -89,7 +90,11 @@ public function filterArchive($archive)

if (!empty($this->skipSegmentsForToday)) {
$site = new Site($archive['idsite']);
$period = Factory::build($this->periodIdsToLabels[$archive['period']], $archive['date1']);
if ((int) $archive['period'] === Range::PERIOD_ID) {
$period = Factory::build($this->periodIdsToLabels[$archive['period']], "{$archive['date1']},{$archive['date2']}");
} else {
$period = Factory::build($this->periodIdsToLabels[$archive['period']], $archive['date1']);
}
$segment = new Segment($segment, [$archive['idsite']]);
if (Archive::shouldSkipArchiveIfSkippingSegmentArchiveForToday($site, $period, $segment)) {
return "skipping segment archives for today";
Expand Down
209 changes: 151 additions & 58 deletions tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@
use Piwik\CliMulti\RequestParser;
use Piwik\Common;
use Piwik\Config;
use Piwik\Period\Day;
use Piwik\Period\Factory;
use Piwik\Period\Month;
use Piwik\Period\Range;
use Piwik\Plugins\CustomDimensions;
use Piwik\Container\StaticContainer;
use Piwik\CronArchive;
Expand Down Expand Up @@ -616,12 +619,15 @@ public function testSkipSegmentsToday()
$segmentHash2 = (new Segment('browserCode==FF', [1]))->getHash();

$invalidations = [
['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null],
['idarchive' => 1, 'name' => 'done' . $segmentHash2, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null],
['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'report' => null],
['idarchive' => 1, 'name' => 'done' . $segmentHash2 . '.ExamplePlugin', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null],
['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null],
['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null],
['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => Day::PERIOD_ID, 'report' => null],
['idarchive' => 2, 'name' => 'done' . $segmentHash2, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => Day::PERIOD_ID, 'report' => null],
['idarchive' => 3, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => Day::PERIOD_ID, 'report' => null],
['idarchive' => 4, 'name' => 'done' . $segmentHash2 . '.ExamplePlugin', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => Day::PERIOD_ID, 'report' => null],
['idarchive' => 5, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => Month::PERIOD_ID, 'report' => null],
sgiehl marked this conversation as resolved.
Show resolved Hide resolved
['idarchive' => 6, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-31', 'period' => Range::PERIOD_ID, 'report' => null],
['idarchive' => 7, 'name' => 'done' . $segmentHash2, 'idsite' => 1, 'date1' => '2018-02-02', 'date2' => '2018-03-04', 'period' => Range::PERIOD_ID, 'report' => null],
['idarchive' => 8, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => Day::PERIOD_ID, 'report' => null],
['idarchive' => 9, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-03', 'period' => Range::PERIOD_ID, 'report' => null],
];
shuffle($invalidations);

Expand All @@ -646,60 +652,84 @@ public function testSkipSegmentsToday()
}

$expectedInvalidationsFound = [
array (
array (
'idarchive' => '1',
'idsite' => '1',
'date1' => '2018-03-04',
'date2' => '2018-03-04',
'period' => '1',
'name' => 'done',
'report' => null,
'plugin' => null,
'segment' => '',
'ts_started' => null,
'status' => '0',
),
array (
'idarchive' => '1',
'idsite' => '1',
'date1' => '2018-03-03',
'date2' => '2018-03-03',
'period' => '1',
'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc',
'report' => null,
'plugin' => null,
'segment' => 'browserCode==IE',
'ts_started' => null,
'status' => '0',
),
),
array (
0 =>
array (
'idarchive' => '1',
'idsite' => '1',
'date1' => '2018-03-01',
'date2' => '2018-03-31',
'period' => '3',
'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc',
'report' => null,
'plugin' => null,
'segment' => 'browserCode==IE',
'ts_started' => null,
'status' => '0',
),
),
array (// end of idsite=1
),
[
[
'idarchive' => '8',
'idsite' => '1',
'date1' => '2018-03-04',
'date2' => '2018-03-04',
'period' => '1',
'name' => 'done',
'report' => null,
'plugin' => null,
'segment' => '',
'ts_started' => null,
'status' => '0',
],
[
'idarchive' => '3',
'idsite' => '1',
'date1' => '2018-03-03',
'date2' => '2018-03-03',
'period' => '1',
'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc',
'report' => null,
'plugin' => null,
'segment' => 'browserCode==IE',
'ts_started' => null,
'status' => '0',
],
],
[
[
'idarchive' => '5',
'idsite' => '1',
'date1' => '2018-03-01',
'date2' => '2018-03-31',
'period' => '3',
'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc',
'report' => null,
'plugin' => null,
'segment' => 'browserCode==IE',
'ts_started' => null,
'status' => '0',
],
],
[
[
'idarchive' => '9',
'name' => 'done',
'idsite' => '1',
'date1' => '2018-03-01',
'date2' => '2018-03-03',
'period' => '5',
'ts_started' => null,
'status' => '0',
'report' => null,
'plugin' => null,
'segment' => '',
],
],
[
[
'idarchive' => '7',
'name' => 'done3736b708e4d20cfc10610e816a1b2341',
'idsite' => '1',
'date1' => '2018-02-02',
'date2' => '2018-03-04',
'period' => '5',
'ts_started' => null,
'status' => '0',
'report' => null,
'plugin' => null,
'segment' => 'browserCode==FF',
],
],
[// end of idsite=1
],
];

try {
$this->assertEquals($expectedInvalidationsFound, $iteratedInvalidations);
} catch (\Exception $ex) {
print "\nInvalidations inserted:\n" . var_export($invalidations, true) . "\n";
throw $ex;
}
$this->assertEquals($expectedInvalidationsFound, $iteratedInvalidations);

// automated check for no duplicates
$invalidationDescs = [];
Expand All @@ -714,6 +744,69 @@ public function testSkipSegmentsToday()
$this->assertEquals($uniqueInvalidationDescs, $invalidationDescs, "Found duplicate archives being processed.");
}

/**
* @dataProvider getAllPeriods
*/
public function testSkipSegmentsTodayShouldSkipAllSegmentInvalidationsIfPeriodBeginsToday(string $periodType)
{
if ($periodType === 'range') {
$period = Factory::build($periodType, '2018-03-04,2018-04-03');
} else {
$period = Factory::build($periodType, '2018-03-04');
}

Date::$now = strtotime($period->getDateStart()->toString() . ' 01:00:00');

Fixture::createWebsite('2015-02-03');

Rules::setBrowserTriggerArchiving(false);
API::getInstance()->add('testegment', 'browserCode==IE', false, true);
Rules::setBrowserTriggerArchiving(true);

// force archiving so we don't skip those without visits
Piwik::addAction('Archiving.getIdSitesToArchiveWhenNoVisits', function (&$idSites) {
$idSites[] = 1;
});

$cronArchive = new MockCronArchive();
$cronArchive->init();

$archiveFilter = $this->makeTestArchiveFilter(null, null, null, false, true);

$queueConsumer = new QueueConsumer(
StaticContainer::get(LoggerInterface::class),
new FixedSiteIds([1]),
3,
24,
new Model(),
new SegmentArchiving(),
$cronArchive,
new RequestParser(true),
$archiveFilter
);

$segmentHash1 = (new Segment('browserCode==IE', [1]))->getHash();

$invalidations = [
['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => $period->getDateStart()->toString(), 'date2' => $period->getDateEnd()->toString(), 'period' => $period::PERIOD_ID, 'report' => null],
];

$this->insertInvalidations($invalidations);

self::assertEmpty($queueConsumer->getNextArchivesToProcess());
}

public function getAllPeriods()
{
return [
['day'],
['week'],
['month'],
['year'],
['range'],
];
}

public function testMaxWebsitesToProcess()
{
Fixture::createWebsite('2021-11-16');
Expand Down
Loading