Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,16 @@
import java.time.Instant;
import java.util.Collection;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;

public interface GithubClient {
/// Max authenticated api requests per hour [2026-April]
static final int GITHUB_MAX_REQUESTS_PER_HOUR = 5000;

/// Minimum ms between requests to stay within the hourly rate limit.
static final int GITHUB_MIN_REQUEST_INTERVAL_MS =
(int) (TimeUnit.HOURS.toMillis(1) / GITHUB_MAX_REQUESTS_PER_HOUR);

static GithubClient build(String authToken, @Nonnull String org) {
return new GithubApiClient(URI.create("https://api.github.com"), authToken, org);
Expand Down
29 changes: 18 additions & 11 deletions src/main/java/org/triplea/maps/indexing/MapIndexingTaskRunner.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,8 @@
import com.google.common.annotations.VisibleForTesting;
import java.net.URI;
import java.time.Instant;
import java.util.ArrayDeque;
import java.util.Collection;
import java.util.Comparator;
import java.util.Deque;
import java.util.List;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
Expand Down Expand Up @@ -61,28 +59,37 @@ public void run() {
// create a stack of maps that we might need to index, pull one off at a time and do the
// indexing.
// Sleep between iterations to avoid rate limits.
final Deque<MapRepoListing> reposToIndex = new ArrayDeque<>(mapUris);
int mapsIndexed = 0;
while (!reposToIndex.isEmpty()) {
var listing = reposToIndex.pop();
mapsIndexed++;
IndexingResult result = index(listing);
int errors = 0;
for (var listing : mapUris) {
log.info("Indexing map: {}", listing.getUri());
mapIndexDao.recordIndexingStatus(listing, result);
try {
Thread.sleep(indexingTaskDelaySeconds * 1000L);
IndexingResult result = index(listing);
mapIndexDao.recordIndexingStatus(listing, result);
mapsIndexed++;
} catch (Exception e) {
log.error("Error indexing map: " + listing.getUri(), e);
errors++;
}
try {
Thread.sleep(GithubClient.GITHUB_MIN_REQUEST_INTERVAL_MS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException("System aborted, process terminated while sleeping");
}
}

log.info(
"Map indexing finished in {} ms, repos found: {}, repos with map.yml: {}, maps deleted: {}",
"Map indexing finished in {} ms,"
+ " repos found: {},"
+ " repos with map.yml: {},"
+ " maps deleted: {},"
+ " errors encountered: {}",
(System.currentTimeMillis() - startTimeEpochMillis),
totalNumberMaps,
mapsIndexed,
mapsDeleted);
mapsDeleted,
errors);
}

// TODO: test me
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,13 @@
@ExtendWith({WiremockResolver.class, WiremockUriResolver.class})
class GithubClientTest {

@Test
@DisplayName("Request rate constant should throttle to stay within the GitHub rate limit")
void githubMaxRequestRate_isExpectedThrottleDelay() {
// 3_600_000 ms per hour / 5000 requests per hour = 720 ms per request
assertThat(GithubClient.GITHUB_MIN_REQUEST_INTERVAL, is(720));
}

@Test
void repoListing(@WiremockResolver.Wiremock final WireMockServer server) {
stubRepoListingResponse(
Expand Down
Loading