diff --git a/config/global.ini.php b/config/global.ini.php index c2e105930ee..f39b9ff8544 100644 --- a/config/global.ini.php +++ b/config/global.ini.php @@ -1154,6 +1154,10 @@ delete_reports_keep_range_reports = 0 delete_reports_keep_segment_reports = 0 +[ArchivingMetrics] +; retention_days - delete archiving metrics older than this many days. Set to 0 to disable cleanup. +retention_days = 180 + [mail] defaultHostnameIfEmpty = defaultHostnameIfEmpty.example.org ; default Email @hostname, if current host can't be read from system variables transport = ; smtp (using the configuration below) or empty (using built-in mail() function) diff --git a/core/Application/Kernel/PluginList.php b/core/Application/Kernel/PluginList.php index 410194c031e..ec86575917b 100644 --- a/core/Application/Kernel/PluginList.php +++ b/core/Application/Kernel/PluginList.php @@ -33,6 +33,7 @@ class PluginList * @var array */ private $corePluginsDisabledByDefault = array( + 'ArchivingMetrics', 'DBStats', 'ExamplePlugin', 'ExampleCommand', diff --git a/core/ArchiveProcessor/Loader.php b/core/ArchiveProcessor/Loader.php index 3045325ca36..776b583ab60 100644 --- a/core/ArchiveProcessor/Loader.php +++ b/core/ArchiveProcessor/Loader.php @@ -36,6 +36,13 @@ class Loader { private static $archivingDepth = 0; + /** + * Tracks whether the current prepareArchive run reused an existing archive instead of processing. + * + * @var boolean + */ + private $didReuseArchive = false; + /** * @var Parameters */ @@ -103,6 +110,11 @@ public function prepareArchive($pluginName) return Context::changeIdSite($this->params->getSite()->getId(), function () use ($pluginName) { try { ++self::$archivingDepth; + + if (self::$archivingDepth === 1) { + $this->didReuseArchive = false; + } + return $this->prepareArchiveImpl($pluginName); } finally { --self::$archivingDepth; @@ -135,6 +147,7 @@ private function prepareArchiveImpl($pluginName) // load existing data from archive $data = $this->loadArchiveData(); if (sizeof($data) == 2) { + $this->didReuseArchive = true; return $data; } [$idArchives, $visits, $visitsConverted, $foundRecords] = $data; @@ -153,6 +166,7 @@ private function prepareArchiveImpl($pluginName) $data = $this->loadArchiveData(); if (sizeof($data) == 2) { + $this->didReuseArchive = true; return $data; } @@ -542,6 +556,11 @@ public function canSkipThisArchiveWithReason(): array ]; } + public function didReuseArchive(): bool + { + return $this->didReuseArchive; + } + private function hasChildArchivesInPeriod($idSite, Period $period): bool { $cacheKey = CacheId::siteAware('Archiving.hasChildArchivesInPeriod.' . $period->getRangeString(), [$idSite]); diff --git a/core/Db/Schema/Mysql.php b/core/Db/Schema/Mysql.php index 545c47beaa6..137c191c407 100644 --- a/core/Db/Schema/Mysql.php +++ b/core/Db/Schema/Mysql.php @@ -334,6 +334,25 @@ public function getTablesCreateSql() ) $tableOptions ", + 'archiving_metrics' => "CREATE TABLE {$prefixTables}archiving_metrics ( + metadataid BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + idarchive BIGINT UNSIGNED NOT NULL, + idsite INTEGER UNSIGNED NOT NULL, + archive_name VARCHAR(255) NOT NULL, + date1 DATE NOT NULL, + date2 DATE NOT NULL, + period TINYINT UNSIGNED NOT NULL, + ts_started DATETIME NOT NULL, + ts_finished DATETIME NOT NULL, + total_time BIGINT UNSIGNED NOT NULL, + total_time_exclusive BIGINT UNSIGNED NOT NULL, + PRIMARY KEY(metadataid), + INDEX index_idarchive(idarchive), + INDEX index_idsite_archive_name(idsite, archive_name), + INDEX index_idsite_date1_period(idsite, date1, period) + ) $tableOptions + ", + 'archive_invalidations' => "CREATE TABLE `{$prefixTables}archive_invalidations` ( idinvalidation BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, idarchive INTEGER UNSIGNED NULL, diff --git a/core/Updates/5.7.0-b2.php b/core/Updates/5.7.0-b2.php new file mode 100644 index 00000000000..3213ad7d80c --- /dev/null +++ b/core/Updates/5.7.0-b2.php @@ -0,0 +1,62 @@ +migration = $factory; + } + + /** + * @param Updater $updater + * @return Migration[] + */ + public function getMigrations(Updater $updater) + { + return [ + $this->migration->db->createTable('archiving_metrics', [ + 'metadataid' => 'BIGINT UNSIGNED NOT NULL AUTO_INCREMENT', + 'idarchive' => 'BIGINT UNSIGNED NOT NULL', + 'idsite' => 'INTEGER UNSIGNED NOT NULL', + 'archive_name' => 'VARCHAR(255) NOT NULL', + 'date1' => 'DATE NOT NULL', + 'date2' => 'DATE NOT NULL', + 'period' => 'TINYINT UNSIGNED NOT NULL', + 'ts_started' => 'DATETIME NOT NULL', + 'ts_finished' => 'DATETIME NOT NULL', + 'total_time' => 'BIGINT UNSIGNED NOT NULL', + 'total_time_exclusive' => 'BIGINT UNSIGNED NOT NULL', + ], ['metadataid']), + $this->migration->db->addIndex('archiving_metrics', ['idarchive'], 'index_idarchive'), + $this->migration->db->addIndex('archiving_metrics', ['idsite', 'archive_name'], 'index_idsite_archive_name'), + $this->migration->db->addIndex('archiving_metrics', ['idsite', 'date1', 'period'], 'index_idsite_date1_period'), + ]; + } + + public function doUpdate(Updater $updater) + { + $updater->executeMigrations(__FILE__, $this->getMigrations($updater)); + } +} diff --git a/core/Version.php b/core/Version.php index 4d6958c8be7..8f5e4aac37d 100644 --- a/core/Version.php +++ b/core/Version.php @@ -22,7 +22,7 @@ final class Version * The current Matomo version. * @var string */ - public const VERSION = '5.7.0-b1'; + public const VERSION = '5.7.0-b2'; public const MAJOR_VERSION = 5; diff --git a/plugins/ArchivingMetrics/ArchivingMetrics.php b/plugins/ArchivingMetrics/ArchivingMetrics.php new file mode 100644 index 00000000000..529d3f1a326 --- /dev/null +++ b/plugins/ArchivingMetrics/ArchivingMetrics.php @@ -0,0 +1,64 @@ + 'onArchiveReportsStart', + 'CoreAdminHome.archiveReports.complete' => 'onArchiveReportsComplete', + ]; + } + + public function onArchiveReportsStart( + int $idSite, + Period $period, + Segment $segment, + string $plugin, + $report, + bool $isArchivePhpTriggered + ): void { + $timer = Timer::getInstance($isArchivePhpTriggered); + $context = $this->buildContext($idSite, $period, $segment, $plugin, $report); + + $timer->start($context); + } + + /** + * @param int[] $idArchives + */ + public function onArchiveReportsComplete( + int $idSite, + Period $period, + Segment $segment, + string $plugin, + $report, + bool $isArchivePhpTriggered, + array $idArchives, + bool $wasCached + ): void { + $timer = Timer::getInstance($isArchivePhpTriggered); + $context = $this->buildContext($idSite, $period, $segment, $plugin, $report); + + $timer->complete($context, $idArchives, $wasCached); + } + + private function buildContext(int $idSite, Period $period, Segment $segment, string $plugin, $report): Context + { + return new Context($idSite, $period, $segment, $plugin, $report); + } +} diff --git a/plugins/ArchivingMetrics/Clock/Clock.php b/plugins/ArchivingMetrics/Clock/Clock.php new file mode 100644 index 00000000000..89a32b645d5 --- /dev/null +++ b/plugins/ArchivingMetrics/Clock/Clock.php @@ -0,0 +1,20 @@ +idSite = $idSite; + $this->period = $period; + $this->segment = $segment; + $this->plugin = $plugin; + $this->report = $report; + } + + public function getKey(): string + { + return implode('|', [ + $this->idSite, + $this->period->getLabel(), + $this->segment->getString(), + $this->period->getDateTimeStart()->toString('Y-m-d'), + $this->period->getDateTimeEnd()->toString('Y-m-d'), + $this->plugin, + ]); + } +} diff --git a/plugins/ArchivingMetrics/Tasks.php b/plugins/ArchivingMetrics/Tasks.php new file mode 100644 index 00000000000..dc702be830e --- /dev/null +++ b/plugins/ArchivingMetrics/Tasks.php @@ -0,0 +1,58 @@ +weekly('purgeOldMetrics'); + $this->monthly('purgeMetricsForDeletedSites'); + } + + /** + * To test execute the following command: + * `./console core:run-scheduled-tasks --force "Piwik\Plugins\ArchivingMetrics\Tasks.purgeOldMetrics"` + */ + public function purgeOldMetrics() + { + $retentionDays = $this->getRetentionDays(); + if ($retentionDays <= 0) { + return; + } + + $cutoff = Date::now()->subDay($retentionDays)->getDatetime(); + $table = Common::prefixTable('archiving_metrics'); + Db::query("DELETE FROM {$table} WHERE ts_started < ?", [$cutoff]); + } + + public function purgeMetricsForDeletedSites() + { + $siteTable = Common::prefixTable('site'); + $table = Common::prefixTable('archiving_metrics'); + Db::query("DELETE a FROM {$table} a LEFT JOIN {$siteTable} s ON a.idsite = s.idsite WHERE s.idsite IS NULL"); + } + + private function getRetentionDays(): int + { + $config = Config::getInstance(); + $retentionDays = $config->ArchivingMetrics['retention_days'] ?? self::DEFAULT_RETENTION_DAYS; + return max(0, (int) $retentionDays); + } +} diff --git a/plugins/ArchivingMetrics/Timer.php b/plugins/ArchivingMetrics/Timer.php new file mode 100644 index 00000000000..830cb4a9d12 --- /dev/null +++ b/plugins/ArchivingMetrics/Timer.php @@ -0,0 +1,204 @@ +> + */ + private $runs = []; + + /** + * @var ?Timer + */ + private static $instance; + + public function __construct(bool $isArchivePhpTriggered, ClockInterface $clock, WriterInterface $writer) + { + $this->isArchivePhpTriggered = $isArchivePhpTriggered; + $this->clock = $clock; + $this->writer = $writer; + } + + public static function getInstance(bool $isArchivePhpTriggered, ?ClockInterface $clock = null, ?WriterInterface $writer = null): self + { + if (self::$instance !== null) { + return self::$instance; + } + + if ($clock === null) { + $clock = new Clock(); + } + + if ($writer === null) { + $writer = new DbWriter(); + } + + self::$instance = new self($isArchivePhpTriggered, $clock, $writer); + return self::$instance; + } + + /** + * @internal For tests only. + */ + public static function resetInstanceForTests(): void + { + self::$instance = null; + } + + public function start(Context $context): void + { + if (false === $this->isApplicableForTiming($context)) { + return; + } + + $this->runs[$context->getKey()] = [ + 'context' => $context, + 'timeStarted' => $this->clock->microtime(), + ]; + } + + public function complete(Context $context, array $idArchives, bool $wasCached): void + { + if (false === $this->isApplicableForTiming($context)) { + return; + } + + if (true === $wasCached) { + return; + } + + if (empty($idArchives) || count($idArchives) !== 1) { + return; + } + + $idArchive = reset($idArchives); + $key = $context->getKey(); + + if (!isset($this->runs[$key]['timeStarted'])) { + return; + } + + $finishedAt = $this->clock->microtime(); + $totalTimeMs = ($finishedAt - $this->runs[$key]['timeStarted']); + + $this->runs[$key]['ts_started'] = date('Y-m-d H:i:s', (int) $this->runs[$key]['timeStarted']); + $this->runs[$key]['totalTime'] = $totalTimeMs; + $this->runs[$key]['timeFinished'] = $finishedAt; + $this->runs[$key]['ts_finished'] = date('Y-m-d H:i:s', (int) $this->runs[$key]['timeFinished']); + + $exclusiveTimeMs = $this->calculateExclusiveTime($key); + $this->runs[$key]['exclusiveTime'] = $exclusiveTimeMs; + + /** @var Context $storedContext */ + $storedContext = $this->runs[$key]['context']; + $archiveName = Rules::getDoneStringFlagFor( + [$storedContext->idSite], + $storedContext->segment, + $storedContext->period->getLabel(), + $storedContext->plugin + ); + $this->writer->write($storedContext, [ + 'idarchive' => $idArchive, + 'archive_name' => $archiveName, + 'ts_started' => $this->runs[$key]['ts_started'], + 'ts_finished' => $this->runs[$key]['ts_finished'], + 'total_time' => (int) round($totalTimeMs * 1000), + 'total_time_exclusive' => (int) round($exclusiveTimeMs * 1000), + ]); + } + + private function isApplicableForTiming(Context $context): bool + { + if (false === $this->isArchivePhpTriggered) { + return false; + } + + $doneFlag = Rules::getDoneStringFlagFor( + [$context->idSite], + $context->segment, + $context->period->getLabel(), + $context->plugin + ); + if (strpos($doneFlag, '.') !== false) { + return false; + } + + return true; + } + + private function calculateExclusiveTime(string $currentKey): float + { + if (empty($this->runs[$currentKey])) { + return 0.0; + } + + $current = $this->runs[$currentKey]; + + $totalTimeMs = $current['totalTime']; + + // If the key is last in the array then it's probably not a nested archive so this calculation doesn't matter + if ($currentKey === array_key_last($this->runs)) { + return $totalTimeMs; + } + + $childTotalMs = 0.0; + + foreach ($this->runs as $otherKey => $run) { + if ($otherKey === $currentKey) { + continue; + } + if (empty($run['exclusiveTime'])) { + continue; + } + + $childFinished = $run['timeFinished'] ?? null; + $childStarted = $run['timeStarted'] ?? null; + if ($childFinished === null || $childStarted === null) { + continue; + } + + if ($childFinished <= $current['timeFinished'] && $childStarted >= $current['timeStarted']) { + $childTotalMs += $run['exclusiveTime']; + } + } + + $exclusive = $totalTimeMs - $childTotalMs; + if ($exclusive < 0) { + return $totalTimeMs; + } + + return $exclusive; + } +} diff --git a/plugins/ArchivingMetrics/Writer/DbWriter.php b/plugins/ArchivingMetrics/Writer/DbWriter.php new file mode 100644 index 00000000000..316f9e790ab --- /dev/null +++ b/plugins/ArchivingMetrics/Writer/DbWriter.php @@ -0,0 +1,39 @@ +idSite, + $timing['archive_name'], + $context->period->getDateTimeStart()->toString('Y-m-d'), + $context->period->getDateTimeEnd()->toString('Y-m-d'), + $context->period->getId(), + $timing['ts_started'], + $timing['ts_finished'], + $timing['total_time'], + $timing['total_time_exclusive'], + ] + ); + } +} diff --git a/plugins/ArchivingMetrics/Writer/WriterInterface.php b/plugins/ArchivingMetrics/Writer/WriterInterface.php new file mode 100644 index 00000000000..00ae4a74240 --- /dev/null +++ b/plugins/ArchivingMetrics/Writer/WriterInterface.php @@ -0,0 +1,19 @@ +ArchivingMetrics = ['retention_days' => 30]; + + $this->insertRow(Date::now()->subDay(31)->getDatetime()); + $this->insertRow(Date::now()->subDay(5)->getDatetime()); + + $task = new Tasks(); + $task->purgeOldMetrics(); + + $count = (int) Db::fetchOne('SELECT COUNT(*) FROM ' . Common::prefixTable('archiving_metrics')); + $this->assertSame(1, $count); + } + + public function testPurgeOldMetricsDisabledKeepsRows(): void + { + $config = Config::getInstance(); + $config->ArchivingMetrics = ['retention_days' => 0]; + + $this->insertRow(Date::now()->subDay(400)->getDatetime()); + + $task = new Tasks(); + $task->purgeOldMetrics(); + + $count = (int) Db::fetchOne('SELECT COUNT(*) FROM ' . Common::prefixTable('archiving_metrics')); + $this->assertSame(1, $count); + } + + public function testPurgeMetricsForDeletedSitesRemovesOrphanedRows(): void + { + $idSite = Fixture::createWebsite('2024-01-01 00:00:00'); + + $this->insertRow(Date::now()->subDay(5)->getDatetime(), $idSite); + $this->insertRow(Date::now()->subDay(5)->getDatetime(), 9999); + + $task = new Tasks(); + $task->purgeMetricsForDeletedSites(); + + $count = (int) Db::fetchOne('SELECT COUNT(*) FROM ' . Common::prefixTable('archiving_metrics')); + $this->assertSame(1, $count); + } + + private function insertRow(string $tsStarted, int $idSite = 1): void + { + $table = Common::prefixTable('archiving_metrics'); + Db::query( + "INSERT INTO {$table} (idarchive, idsite, archive_name, date1, date2, period, ts_started, ts_finished, total_time, total_time_exclusive) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [ + 1, + $idSite, + 'done', + '2025-01-01', + '2025-01-01', + 1, + $tsStarted, + $tsStarted, + 123, + 100, + ] + ); + } +} diff --git a/plugins/ArchivingMetrics/tests/Integration/TimerDbTest.php b/plugins/ArchivingMetrics/tests/Integration/TimerDbTest.php new file mode 100644 index 00000000000..c2b68ec6d83 --- /dev/null +++ b/plugins/ArchivingMetrics/tests/Integration/TimerDbTest.php @@ -0,0 +1,57 @@ +start($context); + $timer->complete($context, [999], false); + + $rows = Db::fetchAll('SELECT * FROM ' . Common::prefixTable('archiving_metrics')); + + self::assertCount(1, $rows, 'Expected archiving_metrics table to have exactly one record.'); + self::assertSame(999, (int) $rows[0]['idarchive']); + self::assertSame(1, (int) $rows[0]['idsite']); + self::assertNotEmpty($rows[0]['archive_name']); + self::assertSame('2025-11-01', $rows[0]['date1']); + self::assertSame('2025-11-01', $rows[0]['date2']); + self::assertIsNumeric($rows[0]['period']); + self::assertNotEmpty($rows[0]['ts_started']); + self::assertNotEmpty($rows[0]['ts_finished']); + self::assertIsNumeric($rows[0]['total_time']); + self::assertIsNumeric($rows[0]['total_time_exclusive']); + } +} diff --git a/plugins/ArchivingMetrics/tests/Unit/TimerTest.php b/plugins/ArchivingMetrics/tests/Unit/TimerTest.php new file mode 100644 index 00000000000..58a533eb253 --- /dev/null +++ b/plugins/ArchivingMetrics/tests/Unit/TimerTest.php @@ -0,0 +1,451 @@ +createMock(ClockInterface::class); + $clock->method('microtime')->willReturnOnConsecutiveCalls(...$microtimes); + $timer = new Timer(true, $clock, $writer); + + foreach ($events as $event) { + $context = $this->createContext($event['context']); + if ($event['action'] === 'start') { + $timer->start($context); + continue; + } + + $timer->complete( + $context, + $event['idArchives'], + $event['cached'] + ); + } + + $this->assertSame($expectedRecords, $writer->records); + } + + public function testItSkipsWhenArchivePhpNotTriggered(): void + { + $writer = new InMemoryWriter(); + $clock = $this->createMock(ClockInterface::class); + $clock->method('microtime')->willReturnOnConsecutiveCalls(0.0); + $timer = new Timer(false, $clock, $writer); + + $context = $this->createContext([ + 'idSite' => 1, + 'segment' => '', + 'plugin' => '', + 'date1' => '2024-01-01', + 'date2' => '2024-01-01', + 'period' => 'day', + ]); + + $timer->start($context); + $timer->complete($context, [123], false); + + $this->assertSame([], $writer->records); + } + + public function timerProvider(): array + { + // Blank segment ensures Rules::getDoneStringFlagFor returns "done" so the timer is active. + $base = [ + 'idSite' => 1, + 'segment' => '', + 'plugin' => '', + 'date1' => '2024-01-01', + 'date2' => '2024-01-01', + ]; + + return [ + 'single period' => [ + 'events' => [ + ['action' => 'start', 'context' => array_merge($base, ['period' => 'day'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'day']), 'idArchives' => [101], 'cached' => false], + ], + 'microtimes' => [ + strtotime('2024-01-01 00:00:00'), + strtotime('2024-01-01 00:00:00') + 1.2, + ], + 'expectedRecords' => [ + [ + 'idarchive' => 101, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-01-01', + 'date2' => '2024-01-01', + 'period' => 1, + 'ts_started' => '2024-01-01 00:00:00', + 'ts_finished' => '2024-01-01 00:00:01', + 'total_time' => 1200, + 'total_time_exclusive' => 1200, + ], + ], + ], + 'mix of events with no nesting' => [ + 'events' => [ + ['action' => 'start', 'context' => array_merge($base, ['period' => 'year', 'date1' => '2024-01-01', 'date2' => '2024-12-31'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'year', 'date1' => '2024-01-01', 'date2' => '2024-12-31']), 'idArchives' => [303], 'cached' => false], + ['action' => 'start', 'context' => array_merge($base, ['period' => 'day', 'date1' => '2024-02-01', 'date2' => '2024-02-01'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'day', 'date1' => '2024-02-01', 'date2' => '2024-02-01']), 'idArchives' => [204], 'cached' => false], + ['action' => 'start', 'context' => array_merge($base, ['period' => 'month', 'date1' => '2024-02-01', 'date2' => '2024-02-29'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'month', 'date1' => '2024-02-01', 'date2' => '2024-02-29']), 'idArchives' => [202], 'cached' => false], + ], + 'microtimes' => [ + strtotime('2024-01-01 00:00:00'), + strtotime('2024-01-01 00:00:00') + 6.3, + strtotime('2024-02-01 00:00:01'), + strtotime('2024-02-01 00:00:01') + 5.4, + strtotime('2024-02-01 00:00:01'), + strtotime('2024-02-01 00:00:01') + 12.3, + ], + 'expectedRecords' => [ + [ + 'idarchive' => 303, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-01-01', + 'date2' => '2024-12-31', + 'period' => 4, + 'ts_started' => '2024-01-01 00:00:00', + 'ts_finished' => '2024-01-01 00:00:06', + 'total_time' => 6300, + 'total_time_exclusive' => 6300, + ], + [ + 'idarchive' => 204, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-02-01', + 'date2' => '2024-02-01', + 'period' => 1, + 'ts_started' => '2024-02-01 00:00:01', + 'ts_finished' => '2024-02-01 00:00:06', + 'total_time' => 5400, + 'total_time_exclusive' => 5400, + ], + [ + 'idarchive' => 202, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-02-01', + 'date2' => '2024-02-29', + 'period' => 3, + 'ts_started' => '2024-02-01 00:00:01', + 'ts_finished' => '2024-02-01 00:00:13', + 'total_time' => 12300, + 'total_time_exclusive' => 12300, + ], + ], + ], + 'mix of events with no nesting and some fetched from cache' => [ + 'events' => [ + ['action' => 'start', 'context' => array_merge($base, ['period' => 'year', 'date1' => '2024-01-01', 'date2' => '2024-12-31'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'year', 'date1' => '2024-01-01', 'date2' => '2024-12-31']), 'idArchives' => [303], 'cached' => true], + ['action' => 'start', 'context' => array_merge($base, ['period' => 'day', 'date1' => '2024-02-01', 'date2' => '2024-02-01'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'day', 'date1' => '2024-02-01', 'date2' => '2024-02-01']), 'idArchives' => [204], 'cached' => false], + ['action' => 'start', 'context' => array_merge($base, ['period' => 'month', 'date1' => '2024-02-01', 'date2' => '2024-02-29'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'month', 'date1' => '2024-02-01', 'date2' => '2024-02-29']), 'idArchives' => [202], 'cached' => true], + ], + 'microtimes' => [ + strtotime('2024-01-01 00:00:00'), + strtotime('2024-02-01 00:00:00'), + strtotime('2024-02-01 00:00:00') + 3.0, + strtotime('2024-02-01 00:00:01'), + ], + 'expectedRecords' => [ + [ + 'idarchive' => 204, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-02-01', + 'date2' => '2024-02-01', + 'period' => 1, + 'ts_started' => '2024-02-01 00:00:00', + 'ts_finished' => '2024-02-01 00:00:03', + 'total_time' => 3000, + 'total_time_exclusive' => 3000, + ], + ], + ], + 'nested day inside week' => [ + 'events' => [ + ['action' => 'start', 'context' => array_merge($base, ['period' => 'week', 'date1' => '2024-01-08', 'date2' => '2024-01-14'])], + ['action' => 'start', 'context' => array_merge($base, ['period' => 'day', 'date1' => '2024-01-10', 'date2' => '2024-01-10'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'day', 'date1' => '2024-01-10', 'date2' => '2024-01-10']), 'idArchives' => [202], 'cached' => false], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'week', 'date1' => '2024-01-08', 'date2' => '2024-01-14']), 'idArchives' => [303], 'cached' => false], + ], + 'microtimes' => [ + strtotime('2024-01-01 00:00:00'), + strtotime('2024-01-01 00:00:00') + 0.5, + strtotime('2024-01-01 00:00:00') + 1.1, + strtotime('2024-01-01 00:00:00') + 2.5, + ], + 'expectedRecords' => [ + [ + 'idarchive' => 202, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-01-10', + 'date2' => '2024-01-10', + 'period' => 1, + 'ts_started' => '2024-01-01 00:00:00', + 'ts_finished' => '2024-01-01 00:00:01', + 'total_time' => 600, + 'total_time_exclusive' => 600, + ], + [ + 'idarchive' => 303, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-01-08', + 'date2' => '2024-01-14', + 'period' => 2, + 'ts_started' => '2024-01-01 00:00:00', + 'ts_finished' => '2024-01-01 00:00:02', + 'total_time' => 2500, + 'total_time_exclusive' => 1900, + ], + ], + ], + 'full cascade year <-> day' => [ + 'events' => [ + ['action' => 'start', 'context' => array_merge($base, ['period' => 'year', 'date1' => '2024-01-01', 'date2' => '2024-12-31'])], + ['action' => 'start', 'context' => array_merge($base, ['period' => 'month', 'date1' => '2024-01-01', 'date2' => '2024-01-31'])], + ['action' => 'start', 'context' => array_merge($base, ['period' => 'week', 'date1' => '2024-01-08', 'date2' => '2024-01-14'])], + ['action' => 'start', 'context' => array_merge($base, ['period' => 'day', 'date1' => '2024-01-10', 'date2' => '2024-01-10'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'day', 'date1' => '2024-01-10', 'date2' => '2024-01-10']), 'idArchives' => [202], 'cached' => false], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'week', 'date1' => '2024-01-08', 'date2' => '2024-01-14']), 'idArchives' => [303], 'cached' => false], + ['action' => 'start', 'context' => array_merge($base, ['period' => 'week', 'date1' => '2024-01-15', 'date2' => '2024-01-22'])], + ['action' => 'start', 'context' => array_merge($base, ['period' => 'day', 'date1' => '2024-01-20', 'date2' => '2024-01-20'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'day', 'date1' => '2024-01-20', 'date2' => '2024-01-20']), 'idArchives' => [404], 'cached' => false], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'week', 'date1' => '2024-01-15', 'date2' => '2024-01-21']), 'idArchives' => [505], 'cached' => false], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'month', 'date1' => '2024-01-01', 'date2' => '2024-01-31']), 'idArchives' => [606], 'cached' => false], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'year', 'date1' => '2024-01-01', 'date2' => '2024-12-31']), 'idArchives' => [707], 'cached' => false], + ], + 'microtimes' => [ + strtotime('2024-01-01 00:00:00'), + strtotime('2024-01-01 00:00:00') + 1, + strtotime('2024-01-01 00:00:00') + 2, + strtotime('2024-01-01 00:00:00') + 4, + strtotime('2024-01-01 00:00:00') + 8, + strtotime('2024-01-01 00:00:00') + 16, + strtotime('2024-01-01 00:00:00') + 32, + strtotime('2024-01-01 00:00:00') + 64, + strtotime('2024-01-01 00:00:00') + 128, + strtotime('2024-01-01 00:00:00') + 256, + strtotime('2024-01-01 00:00:00') + 512, + strtotime('2024-01-01 00:00:00') + 1024, + ], + 'expectedRecords' => [ + [ + 'idarchive' => 202, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-01-10', + 'date2' => '2024-01-10', + 'period' => 1, + 'ts_started' => '2024-01-01 00:00:04', + 'ts_finished' => '2024-01-01 00:00:08', + 'total_time' => 4000, + 'total_time_exclusive' => 4000, + ], + [ + 'idarchive' => 303, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-01-08', + 'date2' => '2024-01-14', + 'period' => 2, + 'ts_started' => '2024-01-01 00:00:02', + 'ts_finished' => '2024-01-01 00:00:16', + 'total_time' => 14000, + 'total_time_exclusive' => 10000, + ], + [ + 'idarchive' => 404, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-01-20', + 'date2' => '2024-01-20', + 'period' => 1, + 'ts_started' => '2024-01-01 00:01:04', + 'ts_finished' => '2024-01-01 00:02:08', + 'total_time' => 64000, + 'total_time_exclusive' => 64000, + ], + [ + 'idarchive' => 505, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-01-15', + 'date2' => '2024-01-21', + 'period' => 2, + 'ts_started' => '2024-01-01 00:00:32', + 'ts_finished' => '2024-01-01 00:04:16', + 'total_time' => 224000, + 'total_time_exclusive' => 160000, + ], + [ + 'idarchive' => 606, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-01-01', + 'date2' => '2024-01-31', + 'period' => 3, + 'ts_started' => '2024-01-01 00:00:01', + 'ts_finished' => '2024-01-01 00:08:32', + 'total_time' => 511000, + 'total_time_exclusive' => 273000, + ], + [ + 'idarchive' => 707, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-01-01', + 'date2' => '2024-12-31', + 'period' => 4, + 'ts_started' => '2024-01-01 00:00:00', + 'ts_finished' => '2024-01-01 00:17:04', + 'total_time' => 1024000, + 'total_time_exclusive' => 513000, + ], + ], + ], + 'segments have timings recorded' => [ + 'events' => [ + ['action' => 'start', 'context' => array_merge($base, ['period' => 'year', 'date1' => '2024-01-01', 'date2' => '2024-12-31', 'segment' => 'browserCode==FF'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'year', 'date1' => '2024-01-01', 'date2' => '2024-12-31', 'segment' => 'browserCode==FF']), 'idArchives' => [303], 'cached' => false], + ['action' => 'start', 'context' => array_merge($base, ['period' => 'day', 'date1' => '2024-02-01', 'date2' => '2024-02-01'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'day', 'date1' => '2024-02-01', 'date2' => '2024-02-01']), 'idArchives' => [204], 'cached' => false], + ['action' => 'start', 'context' => array_merge($base, ['period' => 'month', 'date1' => '2024-02-01', 'date2' => '2024-02-29'])], + ['action' => 'complete', 'context' => array_merge($base, ['period' => 'month', 'date1' => '2024-02-01', 'date2' => '2024-02-29']), 'idArchives' => [202], 'cached' => false], + ], + 'microtimes' => [ + strtotime('2024-01-01 00:00:00'), + strtotime('2024-01-01 00:00:00') + 6.3, + strtotime('2024-02-01 00:00:01'), + strtotime('2024-02-01 00:00:01') + 5.4, + strtotime('2024-02-01 00:00:01'), + strtotime('2024-02-01 00:00:01') + 12.3, + ], + 'expectedRecords' => [ + [ + 'idarchive' => 303, + 'idsite' => 1, + 'archive_name' => 'done' . md5('browserCode==FF'), + 'date1' => '2024-01-01', + 'date2' => '2024-12-31', + 'period' => 4, + 'ts_started' => '2024-01-01 00:00:00', + 'ts_finished' => '2024-01-01 00:00:06', + 'total_time' => 6300, + 'total_time_exclusive' => 6300, + ], + [ + 'idarchive' => 204, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-02-01', + 'date2' => '2024-02-01', + 'period' => 1, + 'ts_started' => '2024-02-01 00:00:01', + 'ts_finished' => '2024-02-01 00:00:06', + 'total_time' => 5400, + 'total_time_exclusive' => 5400, + ], + [ + 'idarchive' => 202, + 'idsite' => 1, + 'archive_name' => 'done', + 'date1' => '2024-02-01', + 'date2' => '2024-02-29', + 'period' => 3, + 'ts_started' => '2024-02-01 00:00:01', + 'ts_finished' => '2024-02-01 00:00:13', + 'total_time' => 12300, + 'total_time_exclusive' => 12300, + ], + ], + ], + ]; + } + + private function createContext(array $data): Context + { + $period = Factory::build($data['period'], $data['date1']); + + $segment = $this->createSegment($data['segment']); + + return new Context( + $data['idSite'], + $period, + $segment, + $data['plugin'], + $data['report'] ?? false + ); + } + private function createSegment(string $segmentString): Segment + { + $segment = $this->createMock(Segment::class); + $segment->method('getString')->willReturn($segmentString); + $segment->method('getHash')->willReturn($segmentString === '' ? '' : md5(urldecode($segmentString))); + $segment->method('isEmpty')->willReturn($segmentString === ''); + return $segment; + } +} + +class InMemoryWriter implements WriterInterface +{ + public $records = []; + + public function write(Context $context, array $timing): void + { + $this->records[] = array_merge( + [ + 'idarchive' => $timing['idarchive'], + 'idsite' => $context->idSite, + 'archive_name' => Rules::getDoneStringFlagFor( + [$context->idSite], + $context->segment, + $context->period->getLabel(), + $context->plugin + ), + 'date1' => $context->period->getDateTimeStart()->toString('Y-m-d'), + 'date2' => $context->period->getDateTimeEnd()->toString('Y-m-d'), + 'period' => $context->period->getId(), + ], + $timing + ); + } +} diff --git a/plugins/CoreAdminHome/API.php b/plugins/CoreAdminHome/API.php index ec377bb1fa6..9dbab08351a 100644 --- a/plugins/CoreAdminHome/API.php +++ b/plugins/CoreAdminHome/API.php @@ -299,20 +299,38 @@ public function archiveReports($idSite, $period, $date, $segment = false, $plugi $period = Factory::build($period, $date); $site = new Site($idSite); + $segmentObj = new Segment( + $segment, + [$idSite], + $period->getDateTimeStart()->setTimezone($site->getTimezone()), + $period->getDateTimeEnd()->setTimezone($site->getTimezone()) + ); $parameters = new ArchiveProcessor\Parameters( $site, $period, - new Segment( - $segment, - [$idSite], - $period->getDateTimeStart()->setTimezone($site->getTimezone()), - $period->getDateTimeEnd()->setTimezone($site->getTimezone()) - ) + $segmentObj ); if ($report) { $parameters->setArchiveOnlyReport($report); } + /** + * Triggered before a full archiveReports run starts. + * + * Usage example: + * Piwik::addAction('CoreAdminHome.archiveReports.start', function ($idSite, $period, $segment, $plugin, $report, $isArchivePhpTriggered) { ... }); + * + * @internal + */ + Piwik::postEvent('CoreAdminHome.archiveReports.start', [ + $idSite, + $period, + $segmentObj, + (string) $plugin, + $report, + $isArchivePhpTriggered, + ]); + // TODO: need to test case when there are multiple plugin archives w/ only some data each. does purging remove some that we need? $archiveLoader = new ArchiveProcessor\Loader($parameters, $invalidateBeforeArchiving); @@ -323,6 +341,29 @@ public function archiveReports($idSite, $period, $date, $segment = false, $plugi 'nb_visits' => $result[1], ]; } + + $idArchives = isset($result['idarchives']) ? (array) $result['idarchives'] : []; + $wasCached = $archiveLoader->didReuseArchive(); + + /** + * Triggered after a full archiveReports run completes. + * + * Usage example: + * Piwik::addAction('CoreAdminHome.archiveReports.complete', function ($idSite, $period, $segment, $plugin, $report, $isArchivePhpTriggered, $idArchives, $wasCached) { ... }); + * + * @internal + */ + Piwik::postEvent('CoreAdminHome.archiveReports.complete', [ + $idSite, + $period, + $segmentObj, + (string) $plugin, + $report, + $isArchivePhpTriggered, + $idArchives, + $wasCached, + ]); + return $result; } diff --git a/plugins/CoreAdminHome/tests/Integration/ArchiveReportsMetricsTimerTest.php b/plugins/CoreAdminHome/tests/Integration/ArchiveReportsMetricsTimerTest.php new file mode 100644 index 00000000000..b01ecb0d126 --- /dev/null +++ b/plugins/CoreAdminHome/tests/Integration/ArchiveReportsMetricsTimerTest.php @@ -0,0 +1,64 @@ +extraPluginsToLoad[] = 'ArchivingMetrics'; + + parent::setUpBeforeClass(); + } + + public function setUp(): void + { + parent::setUp(); + + Timer::resetInstanceForTests(); + } + + public function testArchiveReportsWritesMetricsOnceAndDoesNotWriteAgainWhenReusingDbArchive(): void + { + Fixture::createSuperUser(true); + $_GET['trigger'] = 'archivephp'; + + $idSite = Fixture::createWebsite('2024-01-01 00:00:00'); + + $t = Fixture::getTracker($idSite, '2024-01-01 12:00:00'); + $t->setUrl('http://example.com/'); + Fixture::checkResponse($t->doTrackPageView('test')); + + CoreAdminHomeAPI::getInstance()->archiveReports($idSite, 'day', '2024-01-01'); + $this->assertSame(1, $this->getMetricsCount(), 'Expected archiving_metrics to have 1 row after first archiveReports call.'); + + CoreAdminHomeAPI::getInstance()->archiveReports($idSite, 'day', '2024-01-01'); + $this->assertSame(1, $this->getMetricsCount(), 'Expected archiving_metrics to still have 1 row after reusing the same DB archive.'); + } + + private function getMetricsCount(): int + { + return (int) Db::fetchOne('SELECT COUNT(*) FROM ' . Common::prefixTable('archiving_metrics')); + } +} diff --git a/plugins/CorePluginsAdmin/tests/UI/expected-screenshots/PluginsAdmin_plugins.png b/plugins/CorePluginsAdmin/tests/UI/expected-screenshots/PluginsAdmin_plugins.png index d730be33f9f..c78c675be12 100644 --- a/plugins/CorePluginsAdmin/tests/UI/expected-screenshots/PluginsAdmin_plugins.png +++ b/plugins/CorePluginsAdmin/tests/UI/expected-screenshots/PluginsAdmin_plugins.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b114139cd6c972332bb187930604b219f34eb5dd3adb79669bbcf5bb2a0b733c -size 1119126 +oid sha256:fbed4f5508a0193eefd8dd2356911297884ed4f5c4c42b41ae0c7d8d091c6c36 +size 1127887 diff --git a/plugins/CorePluginsAdmin/tests/UI/expected-screenshots/PluginsAdmin_plugins_admin_disabled.png b/plugins/CorePluginsAdmin/tests/UI/expected-screenshots/PluginsAdmin_plugins_admin_disabled.png index 539b658b85f..e307cf1e3dd 100644 --- a/plugins/CorePluginsAdmin/tests/UI/expected-screenshots/PluginsAdmin_plugins_admin_disabled.png +++ b/plugins/CorePluginsAdmin/tests/UI/expected-screenshots/PluginsAdmin_plugins_admin_disabled.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9330b0d1231bae845a06cf448b426af987c3bc4c43a9b5f3504a0b743dd2e543 -size 988735 +oid sha256:6ec8dcf594877697e79e74b65d5712891e6e95e86e2f25e056317bb9c9e6d748 +size 996417 diff --git a/plugins/CorePluginsAdmin/tests/UI/expected-screenshots/PluginsAdmin_plugins_no_internet.png b/plugins/CorePluginsAdmin/tests/UI/expected-screenshots/PluginsAdmin_plugins_no_internet.png index bd6a592c31e..72bec94827c 100644 --- a/plugins/CorePluginsAdmin/tests/UI/expected-screenshots/PluginsAdmin_plugins_no_internet.png +++ b/plugins/CorePluginsAdmin/tests/UI/expected-screenshots/PluginsAdmin_plugins_no_internet.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f6b9a13cd88d39fc7ba6d83e6b3536152cfeca2c2b76b4bebf44e22d66a46c4b -size 1060902 +oid sha256:6847277a9c8ff94c18df746dc104fd93e0475ae58bc9232c90eb330fe82795b3 +size 1071092 diff --git a/plugins/Diagnostics/tests/UI/expected-screenshots/Diagnostics_page.png b/plugins/Diagnostics/tests/UI/expected-screenshots/Diagnostics_page.png index 8b28cc585d2..815c0440afa 100644 --- a/plugins/Diagnostics/tests/UI/expected-screenshots/Diagnostics_page.png +++ b/plugins/Diagnostics/tests/UI/expected-screenshots/Diagnostics_page.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:009eb3245feebfd59994c03f406ee6f7546f00d4d78f959d3577395332c5f7cf -size 413595 +oid sha256:ac6a33265dc49a6d9ee653fb4b374f239ade184c7b7dad058d202db5b793211f +size 413941 diff --git a/plugins/Installation/tests/UI/expected-screenshots/Installation_db_existing.png b/plugins/Installation/tests/UI/expected-screenshots/Installation_db_existing.png index 6ea0616aa01..4160a2127a3 100644 --- a/plugins/Installation/tests/UI/expected-screenshots/Installation_db_existing.png +++ b/plugins/Installation/tests/UI/expected-screenshots/Installation_db_existing.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5dbd987b1b5de1c42454f57fa601b1697dd924f286fef1263a39fd569614d3f9 -size 139355 +oid sha256:ca44b94072b7f6ec85a9e65717de89f57a0e4d5447aa143cf15dd11ac67aa5f1 +size 141592 diff --git a/plugins/Marketplace/tests/UI/expected-screenshots/Marketplace_paid_plugins_with_license_managePluginsUrl_superuser.png b/plugins/Marketplace/tests/UI/expected-screenshots/Marketplace_paid_plugins_with_license_managePluginsUrl_superuser.png index 5911aa0b3d1..418bd917504 100644 --- a/plugins/Marketplace/tests/UI/expected-screenshots/Marketplace_paid_plugins_with_license_managePluginsUrl_superuser.png +++ b/plugins/Marketplace/tests/UI/expected-screenshots/Marketplace_paid_plugins_with_license_managePluginsUrl_superuser.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:648c0b1e60f8563ff507a171c345b7b0b9617a70a8246bc01d3b41cecab040c1 -size 1094866 +oid sha256:3f97518db19c2e4e9f660275e1d1131d1f465e21cd9e621796c97df4fad917e3 +size 1104638 diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php index 6bf6a6597bb..787b986717a 100644 --- a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php +++ b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php @@ -48,6 +48,38 @@ protected static function beforeTableDataCached() Fixture::createWebsite('2012-02-03 00:00:00'); } + public function testDidReuseArchiveFlagIsOnlySetWhenReusingArchiveFromDb() + { + $_GET['trigger'] = 'archivephp'; + + $idSite = 1; + $dateTime = '2024-01-01 12:00:00'; + $date = '2024-01-01'; + + $t = Fixture::getTracker($idSite, $dateTime); + $t->setUrl('http://example.com/'); + Fixture::checkResponse($t->doTrackPageView('test')); + + $periodObj = Factory::build('day', $date); + + $params = new Parameters(new Site($idSite), $periodObj, new Segment('', [$idSite])); + $loader = new Loader($params); + $result = $loader->prepareArchive('VisitsSummary'); + + $this->assertFalse($loader->didReuseArchive(), 'Expected first archiving run to generate a new archive.'); + $this->assertNotEmpty($result); + $this->assertNotEmpty($result[0]); + + Cache::flushAll(); + + $params = new Parameters(new Site($idSite), $periodObj, new Segment('', [$idSite])); + $loader = new Loader($params); + $result = $loader->prepareArchive('VisitsSummary'); + + $this->assertTrue($loader->didReuseArchive(), 'Expected second archiving run to reuse the existing DB archive.'); + $this->assertSame(1, (int) $result[0][0], 'Expected second archiving run to return the same archive ids.'); + } + public function testPluginOnlyArchivingDoesNotRelaunchChildArchives() { $_GET['pluginOnly'] = 1; diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_admin_diagnostics_configfile.png b/tests/UI/expected-screenshots/UIIntegrationTest_admin_diagnostics_configfile.png index 25df15bb7f2..51a8420c080 100644 --- a/tests/UI/expected-screenshots/UIIntegrationTest_admin_diagnostics_configfile.png +++ b/tests/UI/expected-screenshots/UIIntegrationTest_admin_diagnostics_configfile.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6a5cf42836a6900398fcccf7e16398bc482bd7cf74c23f379b8b13c15e6c67d4 -size 5209642 +oid sha256:dd9f1c191d119dd0e6d9f5d08c03c81c5b93fb043b9f48dabbcb17759c09dc99 +size 5222833