Skip to content

Commit

Permalink
MINIFICPP-2168 Fix Github Actions cache issues
Browse files Browse the repository at this point in the history 
  - Environment variable for docker cache only used after defined
  - Fix mistake of comparing int with string representation of PR id

Closes #1613
Signed-off-by: Martin Zink <[email protected]>
  • Loading branch information
@lordgamez @martinzink
lordgamez authored and martinzink committed Jul 31, 2023
1 parent 9b6fb59 commit a266a30
Show file tree
Hide file tree
Showing 3 changed files with 103 additions and 14 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ on: [push, pull_request, workflow_dispatch]
env:
DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \
-DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \
-DENABLE_ELASTICSEARCH=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache
-DENABLE_ELASTICSEARCH=OFF -DDOCKER_BUILD_ONLY=ON
SCCACHE_GHA_ENABLE: true
jobs:
macos_xcode:
Expand Down Expand Up @@ -352,7 +352,7 @@ jobs:
if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
mkdir build
cd build
cmake ${DOCKER_CMAKE_FLAGS} ..
cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache ..
make docker
- name: cache save
uses: actions/cache/save@v3
Expand Down
2 changes: 1 addition & 1 deletion github_scripts/github_actions_cache_cleanup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def _get_cache_entries(self) -> List[CacheEntry]:

def _is_pr_already_closed(self, entry, open_tickets):
match = re.search(r'refs/pull/([\d]+)/merge', entry.key)
return match and match[1] not in open_tickets
return match and int(match[1]) not in open_tickets

def _remove_non_latest_branch_caches(self, entry: CacheEntry, latest_branch_cache_map: Dict[str, CacheEntry], removable_entries: List[str]):
cache_mapping_key = "-".join(entry.key.split("-")[0:-1])
Expand Down
111 changes: 100 additions & 11 deletions github_scripts/github_actions_cache_cleanup_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,51 +11,51 @@ def create_mock_github_request_sender(self):
mock.list_open_pull_requests = MagicMock()
open_pull_requests = [
{
"number": "227",
"number": 227,
"title": "MINIFICPP-13712 TEST1",
},
{
"number": "228",
"number": 228,
"title": "MINIFICPP-9999 TEST2",
},
{
"number": "229",
"number": 229,
"title": "MINIFICPP-123 TEST3",
}
]
mock.list_open_pull_requests.return_value = open_pull_requests
caches = {
"actions_caches": [
{
"id": "999",
"id": 999,
"key": "macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123",
},
{
"id": "11111",
"id": 11111,
"key": "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
},
{
"id": "11112",
"id": 11112,
"key": "ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487",
},
{
"id": "12345",
"id": 12345,
"key": "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536",
},
{
"id": "22221",
"id": 22221,
"key": "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456",
},
{
"id": "22222",
"id": 22222,
"key": "macos-xcode-ccache-refs/heads/MINIFICPP-9999-8f4d283f5bc894af8dfc295e5976a5f1b4664123",
},
{
"id": "44444",
"id": 44444,
"key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456",
},
{
"id": "55555",
"id": 55555,
"key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567",
}
]
Expand All @@ -65,6 +65,77 @@ def create_mock_github_request_sender(self):
mock.delete_cache = MagicMock()
return mock

def create_empty_open_pr_mock_github_request_sender(self):
mock = MagicMock()
mock.list_open_pull_requests = MagicMock()
mock.list_open_pull_requests.return_value = []
caches = {
"actions_caches": [
{
"id": 999,
"key": "macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123",
},
{
"id": 11111,
"key": "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
},
{
"id": 11112,
"key": "ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487",
},
{
"id": 12345,
"key": "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536",
},
{
"id": 22221,
"key": "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456",
},
{
"id": 22222,
"key": "macos-xcode-ccache-refs/heads/MINIFICPP-9999-8f4d283f5bc894af8dfc295e5976a5f1b4664123",
},
{
"id": 44444,
"key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456",
},
{
"id": 55555,
"key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567",
}
]
}
mock.list_caches = MagicMock()
mock.list_caches.return_value = caches
mock.delete_cache = MagicMock()
return mock

def create_empty_caches_mock_github_request_sender(self):
mock = MagicMock()
mock.list_open_pull_requests = MagicMock()
open_pull_requests = [
{
"number": 227,
"title": "MINIFICPP-13712 TEST1",
},
{
"number": 228,
"title": "MINIFICPP-9999 TEST2",
},
{
"number": 229,
"title": "MINIFICPP-123 TEST3",
}
]
mock.list_open_pull_requests.return_value = open_pull_requests
mock.list_caches = MagicMock()
caches = {
"actions_caches": []
}
mock.list_caches.return_value = caches
mock.delete_cache = MagicMock()
return mock

def test_cache_cleanup(self):
cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp")
cleaner.github_request_sender = self.create_mock_github_request_sender()
Expand All @@ -75,6 +146,24 @@ def test_cache_cleanup(self):
"ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
"ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"})

def test_cache_cleanup_with_zero_open_prs(self):
cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp")
cleaner.github_request_sender = self.create_empty_open_pr_mock_github_request_sender()
cleaner.remove_obsolete_cache_entries()
self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]),
{"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123",
"ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
"ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487",
"macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536",
"macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456",
"ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"})

def test_cache_cleanup_with_zero_action_caches(self):
cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp")
cleaner.github_request_sender = self.create_empty_caches_mock_github_request_sender()
cleaner.remove_obsolete_cache_entries()
self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), set())


if __name__ == '__main__':
unittest.main()

0 comments on commit a266a30

Please sign in to comment.