diff --git a/src/main/java/org/triplea/http/client/github/GithubClient.java b/src/main/java/org/triplea/http/client/github/GithubClient.java index 97378cf..eda5a40 100644 --- a/src/main/java/org/triplea/http/client/github/GithubClient.java +++ b/src/main/java/org/triplea/http/client/github/GithubClient.java @@ -4,9 +4,16 @@ import java.time.Instant; import java.util.Collection; import java.util.Optional; +import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; public interface GithubClient { + /// Max authenticated api requests per hour [2026-April] + static final int GITHUB_MAX_REQUESTS_PER_HOUR = 5000; + + /// Minimum ms between requests to stay within the hourly rate limit. + static final int GITHUB_MIN_REQUEST_INTERVAL_MS = + (int) (TimeUnit.HOURS.toMillis(1) / GITHUB_MAX_REQUESTS_PER_HOUR); static GithubClient build(String authToken, @Nonnull String org) { return new GithubApiClient(URI.create("https://api.github.com"), authToken, org); diff --git a/src/main/java/org/triplea/maps/indexing/MapIndexingTaskRunner.java b/src/main/java/org/triplea/maps/indexing/MapIndexingTaskRunner.java index a1922cd..4a08aa9 100644 --- a/src/main/java/org/triplea/maps/indexing/MapIndexingTaskRunner.java +++ b/src/main/java/org/triplea/maps/indexing/MapIndexingTaskRunner.java @@ -3,10 +3,8 @@ import com.google.common.annotations.VisibleForTesting; import java.net.URI; import java.time.Instant; -import java.util.ArrayDeque; import java.util.Collection; import java.util.Comparator; -import java.util.Deque; import java.util.List; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -61,16 +59,20 @@ public void run() { // create a stack of maps that we might need to index, pull one off at a time and do the // indexing. // Sleep between iterations to avoid rate limits. - final Deque reposToIndex = new ArrayDeque<>(mapUris); int mapsIndexed = 0; - while (!reposToIndex.isEmpty()) { - var listing = reposToIndex.pop(); - mapsIndexed++; - IndexingResult result = index(listing); + int errors = 0; + for (var listing : mapUris) { log.info("Indexing map: {}", listing.getUri()); - mapIndexDao.recordIndexingStatus(listing, result); try { - Thread.sleep(indexingTaskDelaySeconds * 1000L); + IndexingResult result = index(listing); + mapIndexDao.recordIndexingStatus(listing, result); + mapsIndexed++; + } catch (Exception e) { + log.error("Error indexing map: " + listing.getUri(), e); + errors++; + } + try { + Thread.sleep(GithubClient.GITHUB_MIN_REQUEST_INTERVAL_MS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("System aborted, process terminated while sleeping"); @@ -78,11 +80,16 @@ public void run() { } log.info( - "Map indexing finished in {} ms, repos found: {}, repos with map.yml: {}, maps deleted: {}", + "Map indexing finished in {} ms," + + " repos found: {}," + + " repos with map.yml: {}," + + " maps deleted: {}," + + " errors encountered: {}", (System.currentTimeMillis() - startTimeEpochMillis), totalNumberMaps, mapsIndexed, - mapsDeleted); + mapsDeleted, + errors); } // TODO: test me diff --git a/src/test/java/org/triplea/http/client/github/GithubClientTest.java b/src/test/java/org/triplea/http/client/github/GithubClientTest.java index cedcf14..0e2ec33 100644 --- a/src/test/java/org/triplea/http/client/github/GithubClientTest.java +++ b/src/test/java/org/triplea/http/client/github/GithubClientTest.java @@ -25,6 +25,13 @@ @ExtendWith({WiremockResolver.class, WiremockUriResolver.class}) class GithubClientTest { + @Test + @DisplayName("Request rate constant should throttle to stay within the GitHub rate limit") + void githubMaxRequestRate_isExpectedThrottleDelay() { + // 3_600_000 ms per hour / 5000 requests per hour = 720 ms per request + assertThat(GithubClient.GITHUB_MIN_REQUEST_INTERVAL, is(720)); + } + @Test void repoListing(@WiremockResolver.Wiremock final WireMockServer server) { stubRepoListingResponse(