From 9f2d16b7c406700b2b69f2852a20bf9887e5f306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20S=CC=8Ctekl?= Date: Wed, 26 Nov 2014 22:42:45 +0100 Subject: [PATCH] Improved performance of journal cleaning --- src/Kdyby/Redis/RedisJournal.php | 67 +++++++++++++++++++++-- src/Kdyby/Redis/scripts/common.lua | 48 ++++++++++++++++ src/Kdyby/Redis/scripts/journal.clean.lua | 7 +-- tests/KdybyTests/Redis/RedisJournal.phpt | 21 +------ 4 files changed, 113 insertions(+), 30 deletions(-) diff --git a/src/Kdyby/Redis/RedisJournal.php b/src/Kdyby/Redis/RedisJournal.php index 28f5210..060a919 100644 --- a/src/Kdyby/Redis/RedisJournal.php +++ b/src/Kdyby/Redis/RedisJournal.php @@ -31,6 +31,9 @@ class RedisJournal extends Nette\Object implements Nette\Caching\Storages\IJourn TAGS = 'tags', KEYS = 'keys'; + /** @internal batch delete size */ + const BATCH_SIZE = 8000; + /** * @var RedisClient */ @@ -108,34 +111,86 @@ private function cleanEntry($keys) * Cleans entries from journal. * * @param array $conds + * @param \Nette\Caching\IStorage $storage * * @return array of removed items or NULL when performing a full cleanup */ - public function clean(array $conds) + public function clean(array $conds, Nette\Caching\IStorage $storage = NULL) { if (!empty($conds[Cache::ALL])) { $all = $this->client->keys(self::NS_NETTE . ':*'); + if ($storage instanceof RedisStorage) { + $all = array_merge($all, $this->client->keys(RedisStorage::NS_NETTE . ':*')); + } - $this->client->multi(); call_user_func_array(array($this->client, 'del'), $all); - $this->client->exec(); return NULL; } $entries = array(); if (!empty($conds[Cache::TAGS])) { + $removingTagKeys = array(); foreach ((array)$conds[Cache::TAGS] as $tag) { - $this->cleanEntry($found = $this->tagEntries($tag)); + $found = $this->tagEntries($tag); + $removingTagKeys[] = $this->formatKey($tag, self::KEYS); $entries = array_merge($entries, $found); } + if ($removingTagKeys) { + call_user_func_array(array($this->client, 'del'), $removingTagKeys); + } } if (isset($conds[Cache::PRIORITY])) { - $this->cleanEntry($found = $this->priorityEntries($conds[Cache::PRIORITY])); + $found = $this->priorityEntries($conds[Cache::PRIORITY]); + call_user_func_array(array($this->client, 'zRemRangeByScore'), array($this->formatKey(self::PRIORITY), 0, (int)$conds[Cache::PRIORITY])); $entries = array_merge($entries, $found); } - return array_unique($entries); + $entries = array_unique($entries); + + $removingKeys = array(); + $removingKeyTags = array(); + $removingKeyPriorities = array(); + foreach ($entries as $key) { + if ($storage instanceof RedisStorage) { + $removingKeys[] = $key; + } + $removingKeyTags[] = $this->formatKey($key, self::TAGS); + $removingKeyPriorities[] = $this->formatKey($key, self::PRIORITY); + if (count($removingKeyTags) >= self::BATCH_SIZE) { + $this->cleanBatchData($removingKeys, $removingKeyPriorities, $removingKeyTags, $entries); + $removingKeys = array(); + $removingKeyTags = array(); + $removingKeyPriorities = array(); + } + } + + $this->cleanBatchData($removingKeys, $removingKeyPriorities, $removingKeyTags, $entries); + + return $storage instanceof RedisStorage ? array() : $entries; + } + + + + private function cleanBatchData(array $removingKeys, array $removingKeyPriorities, array $removingKeyTags, array $keys) + { + if ($removingKeyTags) { + if ($keys) { + $affectedTags = call_user_func_array(array($this->client, 'sunion'), array($removingKeyTags)); + foreach ($affectedTags as $tag) { + if ($tag) { + call_user_func_array(array($this->client, 'sRem'), array_merge(array($this->formatKey($tag, self::KEYS)), $keys)); + } + } + } + call_user_func_array(array($this->client, 'del'), $removingKeyTags); + } + if ($removingKeyPriorities) { + call_user_func_array(array($this->client, 'del'), $removingKeyPriorities); + } + if ($removingKeys) { + call_user_func_array(array($this->client, 'del'), $removingKeys); + } } diff --git a/src/Kdyby/Redis/scripts/common.lua b/src/Kdyby/Redis/scripts/common.lua index a1a3b22..8a852ab 100644 --- a/src/Kdyby/Redis/scripts/common.lua +++ b/src/Kdyby/Redis/scripts/common.lua @@ -29,6 +29,23 @@ local tagEntries = function (tag) return redis.call('sMembers', formatKey(tag, "keys")) end +local range = function (from, to, step) + step = step or 1 + local f = + step > 0 and + function(_, lastvalue) + local nextvalue = lastvalue + step + if nextvalue <= to then return nextvalue end + end or + step < 0 and + function(_, lastvalue) + local nextvalue = lastvalue + step + if nextvalue >= to then return nextvalue end + end or + function(_, lastvalue) return lastvalue end + return f, nil, from - step +end + local cleanEntry = function (keys) for i, key in pairs(keys) do local tags = entryTags(key) @@ -45,3 +62,34 @@ local cleanEntry = function (keys) -- redis.call('exec') end end + +local mergeTables = function (first, second) + for i, key in pairs(second) do + first[#first + 1] = key + end + return first +end + +local batch = function (keys, callback) + if #keys > 0 then + -- redis.call('multi') + -- the magic number 7998 becomes from Lua limitations, see http://stackoverflow.com/questions/19202367/how-to-avoid-redis-calls-in-lua-script-limitations + local tmp = {} + for i,key in pairs(keys) do + tmp[#tmp + 1] = key + if #tmp >= 7998 then + callback(tmp) + tmp = {} + end + end + callback(tmp) + -- redis.call('exec') + end +end + +local batchDelete = function(keys) + local delete = function (tmp) + redis.call('del', unpack(tmp)) + end + batch(keys, delete) +end diff --git a/src/Kdyby/Redis/scripts/journal.clean.lua b/src/Kdyby/Redis/scripts/journal.clean.lua index abe74c1..b525862 100644 --- a/src/Kdyby/Redis/scripts/journal.clean.lua +++ b/src/Kdyby/Redis/scripts/journal.clean.lua @@ -2,11 +2,10 @@ local conds = cjson.decode(ARGV[1]) if conds["all"] ~= nil then - -- redis.call('multi') - for i, value in pairs(redis.call('keys', "Nette.Journal:*")) do - redis.call('del', value) + batchDelete(redis.call('keys', "Nette.Journal:*")) + if conds["delete-entries"] ~= nil then + batchDelete(redis.call('keys', "Nette.Storage:*")) end - -- redis.call('exec') return redis.status_reply("Ok") end diff --git a/tests/KdybyTests/Redis/RedisJournal.phpt b/tests/KdybyTests/Redis/RedisJournal.phpt index 9b4372f..fec1d2c 100644 --- a/tests/KdybyTests/Redis/RedisJournal.phpt +++ b/tests/KdybyTests/Redis/RedisJournal.phpt @@ -284,7 +284,7 @@ class RedisJournalTest extends AbstractRedisTestCase Assert::null($result); $result2 = $this->journal->clean(array(Cache::TAGS => 'test:all')); - Assert::true(empty($result2)); + Assert::equal(array(), $result2); } @@ -362,26 +362,7 @@ LUA; private function cacheGeneratorScripts() { $script = file_get_contents(__DIR__ . '/../../../src/Kdyby/Redis/scripts/common.lua'); - $script .= << 0 and - function(_, lastvalue) - local nextvalue = lastvalue + step - if nextvalue <= to then return nextvalue end - end or - step < 0 and - function(_, lastvalue) - local nextvalue = lastvalue + step - if nextvalue >= to then return nextvalue end - end or - function(_, lastvalue) return lastvalue end - return f, nil, from - step -end - -LUA; return $script; }