From e1da3fcbb9d195f2bde8d32612f5cf266e53a326 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Mon, 8 Dec 2025 17:58:38 -0700 Subject: [PATCH 01/25] traffic components --- .../traffic/OptOutTrafficCalculator.java | 643 +++++++ .../optout/traffic/OptOutTrafficFilter.java | 184 ++ .../traffic/OptOutTrafficCalculatorTest.java | 1577 +++++++++++++++++ .../traffic/OptOutTrafficFilterTest.java | 428 +++++ 4 files changed, 2832 insertions(+) create mode 100644 src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java create mode 100644 src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java create mode 100644 src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java create mode 100644 src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java new file mode 100644 index 0000000..ab6a2c0 --- /dev/null +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java @@ -0,0 +1,643 @@ +package com.uid2.optout.traffic; + +import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.optout.OptOutCollection; +import com.uid2.shared.optout.OptOutEntry; +import com.uid2.shared.optout.OptOutUtils; +import com.uid2.optout.Const; +import com.uid2.optout.sqs.SqsMessageOperations; + +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; + +import java.nio.charset.StandardCharsets; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.io.InputStream; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * Calculates opt-out traffic patterns to determine DEFAULT or DELAYED_PROCESSING status. + * + * Compares recent ~24h traffic (sumCurrent) against a configurable baseline (baselineTraffic) of expected traffic in 24 hours. + * The baseline is multiplied by (thresholdMultiplier) to determine the threshold. + * sumCurrent excludes records in allowlist ranges (surge windows determined by engineers). + * + * Returns DELAYED_PROCESSING if sumCurrent >= thresholdMultiplier * baselineTraffic, indicating abnormal traffic spike. + */ +public class OptOutTrafficCalculator { + private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficCalculator.class); + + private static final int HOURS_24 = 24 * 3600; // 24 hours in seconds + + private final Map deltaFileCache = new ConcurrentHashMap<>(); + private final ICloudStorage cloudStorage; + private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") + private final String trafficCalcConfigPath; + private int baselineTraffic; + private int thresholdMultiplier; + private int evaluationWindowSeconds; + private List> allowlistRanges; + + public enum TrafficStatus { + DELAYED_PROCESSING, + DEFAULT + } + + /** + * Cache entry for a delta file containing all record timestamps. + * + * Memory usage: ~8 bytes per timestamp (long) + * 1GB of memory can store ~130 million timestamps (1024^3)/8 + */ + private static class FileRecordCache { + final List timestamps; // All non-sentinel record timestamps + final long newestTimestamp; // evict delta from cache based on oldest record timestamp + + FileRecordCache(List timestamps) { + this.timestamps = timestamps; + this.newestTimestamp = timestamps.isEmpty() ? 0 : Collections.max(timestamps); + } + } + + /** + * Exception thrown by malformed traffic calculator config + */ + public static class MalformedTrafficCalcConfigException extends Exception { + public MalformedTrafficCalcConfigException(String message) { + super(message); + } + } + + /** + * Constructor for OptOutTrafficCalculator + * + * @param cloudStorage Cloud storage for reading delta files + * @param s3DeltaPrefix S3 prefix for delta files + * @param trafficCalcConfigS3Path S3 path for traffic calc config + */ + public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigPath) throws MalformedTrafficCalcConfigException { + this.cloudStorage = cloudStorage; + this.s3DeltaPrefix = s3DeltaPrefix; + this.trafficCalcConfigPath = trafficCalcConfigPath; + reloadTrafficCalcConfig(); // Load ConfigMap + + LOGGER.info("initialized: s3DeltaPrefix={}, threshold={}x", + s3DeltaPrefix, thresholdMultiplier); + } + + /** + * Reload traffic calc config from ConfigMap. + * Expected format: + * { + * "traffic_calc_evaluation_window_seconds": 86400, + * "traffic_calc_baseline_traffic": 100, + * "traffic_calc_threshold_multiplier": 5, + * "traffic_calc_allowlist_ranges": [ + * [startTimestamp1, endTimestamp1], + * [startTimestamp2, endTimestamp2] + * ], + * } + * + * Can be called periodically to pick up config changes without restarting. + */ + public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException { + LOGGER.info("loading traffic calc config from configmap"); + try (InputStream is = Files.newInputStream(Paths.get(trafficCalcConfigPath))) { + String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); + JsonObject trafficCalcConfig = new JsonObject(content); + + // Validate required fields exist + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_evaluation_window_seconds"); + } + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_baseline_traffic"); + } + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_threshold_multiplier"); + } + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_allowlist_ranges"); + } + + this.evaluationWindowSeconds = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp); + this.baselineTraffic = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcBaselineTrafficProp); + this.thresholdMultiplier = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcThresholdMultiplierProp); + + List> ranges = parseAllowlistRanges(trafficCalcConfig); + this.allowlistRanges = ranges; + + LOGGER.info("loaded traffic calc config: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", + this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size()); + + } catch (MalformedTrafficCalcConfigException e) { + LOGGER.error("circuit_breaker_config_error: config is malformed, configPath={}", trafficCalcConfigPath, e); + throw e; + } catch (Exception e) { + LOGGER.error("circuit_breaker_config_error: config is malformed or missing, configPath={}", trafficCalcConfigPath, e); + throw new MalformedTrafficCalcConfigException("failed to load traffic calc config: " + e.getMessage()); + } + } + + /** + * Parse allowlist ranges from JSON config + */ + List> parseAllowlistRanges(JsonObject config) throws MalformedTrafficCalcConfigException { + List> ranges = new ArrayList<>(); + + try { + var rangesArray = config.getJsonArray(Const.Config.OptOutTrafficCalcAllowlistRangesProp); + if (rangesArray != null) { + for (int i = 0; i < rangesArray.size(); i++) { + var rangeArray = rangesArray.getJsonArray(i); + if (rangeArray != null && rangeArray.size() >= 2) { + long start = rangeArray.getLong(0); + long end = rangeArray.getLong(1); + + if(start >= end) { + LOGGER.error("circuit_breaker_config_error: allowlist range start must be less than end, range=[{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": start must be less than end"); + } + + if (end - start > 86400) { + LOGGER.error("circuit_breaker_config_error: allowlist range must be less than 24 hours, range=[{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": range must be less than 24 hours"); + } + + List range = Arrays.asList(start, end); + ranges.add(range); + LOGGER.info("loaded allowlist range: [{}, {}]", start, end); + } + } + } + + ranges.sort(Comparator.comparing(range -> range.get(0))); + + // Validate no overlapping ranges + for (int i = 0; i < ranges.size() - 1; i++) { + long currentEnd = ranges.get(i).get(1); + long nextStart = ranges.get(i + 1).get(0); + if (currentEnd >= nextStart) { + LOGGER.error("circuit_breaker_config_error: overlapping allowlist ranges, range=[{}, {}] overlaps with range=[{}, {}]", + ranges.get(i).get(0), currentEnd, nextStart, ranges.get(i + 1).get(1)); + throw new MalformedTrafficCalcConfigException( + "overlapping allowlist ranges detected at indices " + i + " and " + (i + 1)); + } + } + + } catch (MalformedTrafficCalcConfigException e) { + throw e; + } catch (Exception e) { + LOGGER.error("circuit_breaker_config_error: failed to parse allowlist ranges", e); + throw new MalformedTrafficCalcConfigException("failed to parse allowlist ranges: " + e.getMessage()); + } + + return ranges; + } + + /** + * Calculate traffic status based on delta files and SQS queue messages. + * + * Uses the newest delta file timestamp to anchor the 24-hour delta traffic window, + * and the oldest queue timestamp to anchor the 5-minute queue window. + * + * Counts: + * - Delta file records (with allowlist filtering) + * - SQS messages passed in (with allowlist filtering) + * - Invisible messages from other consumers (from queue attributes, avoiding double count) + * + * @param sqsMessages List of SQS messages this consumer has read (non-denylisted) + * @param queueAttributes Queue attributes including invisible message count (can be null) + * @param denylistedCount Number of denylisted messages read by this consumer + * @param filteredAsTooRecentCount Number of messages filtered as "too recent" by window reader + * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) + */ + public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes, int denylistedCount, int filteredAsTooRecentCount) { + + try { + // Get list of delta files from S3 (sorted newest to oldest) + List deltaS3Paths = listDeltaFiles(); + + if (deltaS3Paths.isEmpty()) { + LOGGER.error("s3_error: no delta files found in s3 at prefix={}", s3DeltaPrefix); + throw new RuntimeException("no delta files found in s3 at prefix=" + s3DeltaPrefix); + } + + // Find newest delta file timestamp for delta traffic window + long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths); + LOGGER.info("traffic calculation: newestDeltaTs={}", newestDeltaTs); + + // Find oldest SQS queue message timestamp for queue window + long oldestQueueTs = findOldestQueueTimestamp(sqsMessages); + LOGGER.info("traffic calculation: oldestQueueTs={}", oldestQueueTs); + + // Define start time of the delta evaluation window + // We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend + // the window to account for any allowlist ranges in the extended portion + long deltaWindowStart = calculateWindowStartWithAllowlist(newestDeltaTs, this.evaluationWindowSeconds); + + // Evict old cache entries (older than delta window start) + evictOldCacheEntries(deltaWindowStart); + + // Process delta files and count records in [deltaWindowStart, newestDeltaTs] + // Files are sorted newest to oldest, records within files are sorted newest to oldest + // Stop when the newest record in a file is older than the window + int sum = 0; + int deltaRecordsCount = 0; + int deltaAllowlistedCount = 0; + int filesProcessed = 0; + int cacheHits = 0; + int cacheMisses = 0; + + for (String s3Path : deltaS3Paths) { + boolean wasCached = isCached(s3Path); + if (wasCached) { + cacheHits++; + } else { + cacheMisses++; + } + + List timestamps = getTimestampsFromFile(s3Path); + filesProcessed++; + + // Check newest record in file - if older than window, stop processing remaining files + long newestRecordTs = timestamps.get(0); + if (newestRecordTs < deltaWindowStart) { + break; + } + + for (long ts : timestamps) { + // Stop condition: record is older than our window + if (ts < deltaWindowStart) { + break; + } + + // skip records in allowlisted ranges + if (isInAllowlist(ts)) { + deltaAllowlistedCount++; + continue; + } + + // increment sum if record is in delta window + if (ts >= deltaWindowStart) { + deltaRecordsCount++; + sum++; + } + } + } + + LOGGER.info("delta files: processed={}, deltaRecords={}, allowlisted={}, cache hits={}, misses={}, cacheSize={}", + filesProcessed, deltaRecordsCount, deltaAllowlistedCount, cacheHits, cacheMisses, deltaFileCache.size()); + + // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] with allowlist filtering + int sqsCount = 0; + if (sqsMessages != null && !sqsMessages.isEmpty()) { + sqsCount = countSqsMessages(sqsMessages, oldestQueueTs); + sum += sqsCount; + } + + // Add invisible messages being processed by OTHER consumers + // (notVisible count includes our messages, so subtract what we've read to avoid double counting) + // ourMessages = delta messages + denylisted messages + filtered "too recent" messages + int otherConsumersMessages = 0; + if (queueAttributes != null) { + int totalInvisible = queueAttributes.getApproximateNumberOfMessagesNotVisible(); + int ourMessages = (sqsMessages != null ? sqsMessages.size() : 0) + denylistedCount + filteredAsTooRecentCount; + otherConsumersMessages = Math.max(0, totalInvisible - ourMessages); + sum += otherConsumersMessages; + LOGGER.info("traffic calculation: adding {} invisible messages from other consumers (totalInvisible={}, ourMessages={})", + otherConsumersMessages, totalInvisible, ourMessages); + } + + // Determine status + TrafficStatus status = determineStatus(sum, this.baselineTraffic); + + LOGGER.info("traffic calculation complete: sum={} (deltaRecords={} + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}", + sum, deltaRecordsCount, sqsCount, otherConsumersMessages, this.baselineTraffic, this.thresholdMultiplier, status); + + return status; + + } catch (Exception e) { + LOGGER.error("delta_job_failed: error calculating traffic status", e); + throw new RuntimeException("error calculating traffic status", e); + } + } + + /** + * Find the newest timestamp from delta files. + * Reads the newest delta file and returns its maximum timestamp. + */ + private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOException { + if (deltaS3Paths == null || deltaS3Paths.isEmpty()) { + return System.currentTimeMillis() / 1000; + } + + // Delta files are sorted (ISO 8601 format, lexicographically sortable) so first file is newest + String newestDeltaPath = deltaS3Paths.get(0); + List timestamps = getTimestampsFromFile(newestDeltaPath); + + if (timestamps.isEmpty()) { + LOGGER.error("s3_error: newest delta file has no timestamps, path={}", newestDeltaPath); + return System.currentTimeMillis() / 1000; + } + + long newestTs = Collections.max(timestamps); + LOGGER.info("found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); + return newestTs; + } + + /** + * List all delta files from S3, sorted newest to oldest + */ + private List listDeltaFiles() { + try { + // List all objects with the delta prefix + List allFiles = cloudStorage.list(s3DeltaPrefix); + + // Filter to only .dat delta files and sort newest to oldest + List deltaFiles = allFiles.stream() + .filter(OptOutUtils::isDeltaFile) + .sorted(OptOutUtils.DeltaFilenameComparatorDescending) + .collect(Collectors.toList()); + + LOGGER.info("listed {} delta files from s3 (prefix={})", deltaFiles.size(), s3DeltaPrefix); + return deltaFiles; + + } catch (Exception e) { + LOGGER.error("s3_error: failed to list delta files at prefix={}", s3DeltaPrefix, e); + return Collections.emptyList(); + } + } + + /** + * Check if a delta file is already cached + */ + private boolean isCached(String s3Path) { + String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); + return deltaFileCache.containsKey(filename); + } + + /** + * Get timestamps from a delta file (S3 path), using cache if available + */ + private List getTimestampsFromFile(String s3Path) throws IOException { + // Extract filename from S3 path for cache key + String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); + + // Check cache first + FileRecordCache cached = deltaFileCache.get(filename); + if (cached != null) { + return cached.timestamps; + } + + // Cache miss - download from S3 + List timestamps = readTimestampsFromS3(s3Path); + + // Store in cache + deltaFileCache.put(filename, new FileRecordCache(timestamps)); + + return timestamps; + } + + /** + * Read all non-sentinel record timestamps from a delta file in S3 + */ + private List readTimestampsFromS3(String s3Path) throws IOException { + try (InputStream is = cloudStorage.download(s3Path)) { + byte[] data = is.readAllBytes(); + OptOutCollection collection = new OptOutCollection(data); + + List timestamps = new ArrayList<>(); + for (int i = 0; i < collection.size(); i++) { + OptOutEntry entry = collection.get(i); + + // Skip sentinel entries + if (entry.isSpecialHash()) { + continue; + } + + timestamps.add(entry.timestamp); + } + + return timestamps; + } catch (Exception e) { + LOGGER.error("s3_error: failed to read delta file at path={}", s3Path, e); + throw new IOException("failed to read delta file from s3: " + s3Path, e); + } + } + + /** + * Calculate total duration of allowlist ranges that overlap with the given time window. + */ + long getAllowlistDuration(long t, long windowStart) { + long totalDuration = 0; + for (List range : this.allowlistRanges) { + long start = range.get(0); + long end = range.get(1); + + // Clip range to window boundaries + if (start < windowStart) { + start = windowStart; + } + if (end > t) { + end = t; + } + + // Only add duration if there's actual overlap (start < end) + if (start < end) { + totalDuration += end - start; + } + } + return totalDuration; + } + + /** + * Calculate the window start time that provides evaluationWindowSeconds of non-allowlisted time. + * Iteratively extends the window to account for allowlist ranges that may fall in extended portions. + */ + long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowSeconds) { + long allowlistDuration = getAllowlistDuration(newestDeltaTs, newestDeltaTs - evaluationWindowSeconds); + + // Each iteration discovers at least one new allowlist range, so max iterations = number of ranges + int maxIterations = this.allowlistRanges.size() + 1; + + for (int i = 0; i < maxIterations && allowlistDuration > 0; i++) { + long newWindowStart = newestDeltaTs - evaluationWindowSeconds - allowlistDuration; + long newAllowlistDuration = getAllowlistDuration(newestDeltaTs, newWindowStart); + + if (newAllowlistDuration == allowlistDuration) { + // No new allowlist time in extended portion, we've converged + break; + } + + allowlistDuration = newAllowlistDuration; + } + + return newestDeltaTs - evaluationWindowSeconds - allowlistDuration; + } + + /** + * Find the oldest SQS queue message timestamp + */ + private long findOldestQueueTimestamp(List sqsMessages) throws IOException { + long oldest = System.currentTimeMillis() / 1000; + + if (sqsMessages != null && !sqsMessages.isEmpty()) { + for (Message msg : sqsMessages) { + Long ts = extractTimestampFromMessage(msg); + if (ts != null && ts < oldest) { + oldest = ts; + } + } + } + + return oldest; + } + + /** + * Extract timestamp from SQS message (from SentTimestamp attribute) + */ + private Long extractTimestampFromMessage(Message msg) { + // Get SentTimestamp attribute (milliseconds) + String sentTimestamp = msg.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); + if (sentTimestamp != null) { + try { + return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds + } catch (NumberFormatException e) { + LOGGER.error("sqs_error: invalid sentTimestamp, messageId={}, sentTimestamp={}", msg.messageId(), sentTimestamp); + } + } + + // Fallback: use current time + return System.currentTimeMillis() / 1000; + } + + /** + * Count SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes + */ + private int countSqsMessages(List sqsMessages, long oldestQueueTs) { + + int count = 0; + int allowlistedCount = 0; + long windowEnd = oldestQueueTs + 5 * 60; + + for (Message msg : sqsMessages) { + Long ts = extractTimestampFromMessage(msg); + + if (ts < oldestQueueTs || ts > windowEnd) { + continue; + } + + if (isInAllowlist(ts)) { + allowlistedCount++; + continue; + } + count++; + + } + + LOGGER.info("sqs messages: {} in window, {} allowlisted [oldestQueueTs={}, oldestQueueTs+5m={}]", count, allowlistedCount, oldestQueueTs, windowEnd); + return count; + } + + /** + * Check if a timestamp falls within any allowlist range + */ + boolean isInAllowlist(long timestamp) { + if (allowlistRanges == null || allowlistRanges.isEmpty()) { + return false; + } + + for (List range : allowlistRanges) { + if (range.size() < 2) { + continue; + } + + long start = range.get(0); + long end = range.get(1); + + if (timestamp >= start && timestamp <= end) { + return true; + } + } + + return false; + } + + /** + * Evict cache entries with data older than the cutoff timestamp + */ + private void evictOldCacheEntries(long cutoffTimestamp) { + int beforeSize = deltaFileCache.size(); + + deltaFileCache.entrySet().removeIf(entry -> + entry.getValue().newestTimestamp < cutoffTimestamp + ); + + int afterSize = deltaFileCache.size(); + if (beforeSize != afterSize) { + LOGGER.info("evicted {} old cache entries (before={}, after={})", + beforeSize - afterSize, beforeSize, afterSize); + } + } + + /** + * Determine traffic status based on current vs baseline traffic. + * Logs warnings at 50%, 75%, and 90% of the circuit breaker threshold. + */ + TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { + if (baselineTraffic == 0 || thresholdMultiplier == 0) { + LOGGER.error("circuit_breaker_config_error: baselineTraffic is 0 or thresholdMultiplier is 0"); + throw new RuntimeException("invalid circuit breaker config: baselineTraffic=" + baselineTraffic + ", thresholdMultiplier=" + thresholdMultiplier); + } + + int threshold = thresholdMultiplier * baselineTraffic; + double thresholdPercent = (double) sumCurrent / threshold * 100; + + // Log warnings at increasing thresholds before circuit breaker triggers + if (thresholdPercent >= 90.0) { + LOGGER.warn("high_message_volume: 90% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + } else if (thresholdPercent >= 75.0) { + LOGGER.warn("high_message_volume: 75% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + } else if (thresholdPercent >= 50.0) { + LOGGER.warn("high_message_volume: 50% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + } + + if (sumCurrent >= threshold) { + LOGGER.error("circuit_breaker_triggered: traffic threshold breached, sumCurrent={}, threshold={} ({}x{})", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic); + return TrafficStatus.DELAYED_PROCESSING; + } + + LOGGER.info("traffic within normal range: sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + return TrafficStatus.DEFAULT; + } + + /** + * Get cache statistics for monitoring + */ + public Map getCacheStats() { + Map stats = new HashMap<>(); + stats.put("cached_files", deltaFileCache.size()); + + int totalTimestamps = deltaFileCache.values().stream() + .mapToInt(cache -> cache.timestamps.size()) + .sum(); + stats.put("total_cached_timestamps", totalTimestamps); + + return stats; + } + +} \ No newline at end of file diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java new file mode 100644 index 0000000..59a60a9 --- /dev/null +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java @@ -0,0 +1,184 @@ +package com.uid2.optout.traffic; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.uid2.optout.sqs.SqsParsedMessage; + +import java.util.ArrayList; +import java.util.List; +import java.util.Collections; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.charset.StandardCharsets; +import io.vertx.core.json.JsonObject; +import io.vertx.core.json.JsonArray; + +public class OptOutTrafficFilter { + private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficFilter.class); + + private final String trafficFilterConfigPath; + List filterRules; + + /** + * Traffic filter rule defining a time range and a list of IP addresses to exclude + */ + private static class TrafficFilterRule { + private final List range; + private final List ipAddresses; + + TrafficFilterRule(List range, List ipAddresses) { + this.range = range; + this.ipAddresses = ipAddresses; + } + + public long getRangeStart() { + return range.get(0); + } + public long getRangeEnd() { + return range.get(1); + } + public List getIpAddresses() { + return ipAddresses; + } + } + + public static class MalformedTrafficFilterConfigException extends Exception { + public MalformedTrafficFilterConfigException(String message) { + super(message); + } + } + + /** + * Constructor for OptOutTrafficFilter + * + * @param trafficFilterConfigPath S3 path for traffic filter config + * @throws MalformedTrafficFilterConfigException if the traffic filter config is invalid + */ + public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTrafficFilterConfigException { + this.trafficFilterConfigPath = trafficFilterConfigPath; + // Initial filter rules load + this.filterRules = Collections.emptyList(); // start empty + reloadTrafficFilterConfig(); // load ConfigMap + + LOGGER.info("initialized: filterRules={}", filterRules.size()); + } + + /** + * Reload traffic filter config from ConfigMap. + * Expected format: + * { + * "denylist_requests": [ + * {range: [startTimestamp, endTimestamp], IPs: ["ip1"]}, + * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip2"]}, + * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip3"]}, + * ] + * } + * + * Can be called periodically to pick up config changes without restarting. + */ + public void reloadTrafficFilterConfig() throws MalformedTrafficFilterConfigException { + LOGGER.info("loading traffic filter config"); + try (InputStream is = Files.newInputStream(Paths.get(trafficFilterConfigPath))) { + String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); + JsonObject filterConfigJson = new JsonObject(content); + + this.filterRules = parseFilterRules(filterConfigJson); + + LOGGER.info("loaded traffic filter config: filterRules={}", filterRules.size()); + + } catch (Exception e) { + LOGGER.error("circuit_breaker_config_error: no traffic filter config found at {}", trafficFilterConfigPath, e); + throw new MalformedTrafficFilterConfigException(e.getMessage()); + } + } + + /** + * Parse request filtering rules from JSON config + */ + List parseFilterRules(JsonObject config) throws MalformedTrafficFilterConfigException { + List rules = new ArrayList<>(); + try { + JsonArray denylistRequests = config.getJsonArray("denylist_requests"); + if (denylistRequests == null) { + LOGGER.error("circuit_breaker_config_error: denylist_requests is null"); + throw new MalformedTrafficFilterConfigException("invalid traffic filter config: denylist_requests is null"); + } + for (int i = 0; i < denylistRequests.size(); i++) { + JsonObject ruleJson = denylistRequests.getJsonObject(i); + + // parse range + var rangeJson = ruleJson.getJsonArray("range"); + List range = new ArrayList<>(); + if (rangeJson != null && rangeJson.size() == 2) { + long start = rangeJson.getLong(0); + long end = rangeJson.getLong(1); + + if (start >= end) { + LOGGER.error("circuit_breaker_config_error: rule range start must be less than end, rule={}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range start must be less than end"); + } + range.add(start); + range.add(end); + } + + // log error and throw exception if range is not 2 elements + if (range.size() != 2) { + LOGGER.error("circuit_breaker_config_error: rule range is not 2 elements, rule={}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range is not 2 elements"); + } + + // parse IPs + var ipAddressesJson = ruleJson.getJsonArray("IPs"); + List ipAddresses = new ArrayList<>(); + if (ipAddressesJson != null) { + for (int j = 0; j < ipAddressesJson.size(); j++) { + ipAddresses.add(ipAddressesJson.getString(j)); + } + } + + // log error and throw exception if IPs is empty + if (ipAddresses.size() == 0) { + LOGGER.error("circuit_breaker_config_error: rule IPs is empty, rule={}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: IPs is empty"); + } + + // log error and throw exception if rule is invalid + if (range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less + LOGGER.error("circuit_breaker_config_error: rule range must be 24 hours or less, rule={}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range must be 24 hours or less"); + } + + TrafficFilterRule rule = new TrafficFilterRule(range, ipAddresses); + + LOGGER.info("loaded traffic filter rule: range=[{}, {}], IPs={}", rule.getRangeStart(), rule.getRangeEnd(), rule.getIpAddresses()); + rules.add(rule); + } + return rules; + } catch (Exception e) { + LOGGER.error("circuit_breaker_config_error: failed to parse rules, config={}, error={}", config.encode(), e.getMessage()); + throw new MalformedTrafficFilterConfigException(e.getMessage()); + } + } + + public boolean isDenylisted(SqsParsedMessage message) { + long timestamp = message.getTimestamp(); + String clientIp = message.getClientIp(); + + if (clientIp == null || clientIp.isEmpty()) { + LOGGER.error("sqs_error: request does not contain client ip, messageId={}", message.getOriginalMessage().messageId()); + return false; + } + + for (TrafficFilterRule rule : filterRules) { + if(timestamp >= rule.getRangeStart() && timestamp <= rule.getRangeEnd()) { + if(rule.getIpAddresses().contains(clientIp)) { + return true; + } + }; + } + return false; + } + +} \ No newline at end of file diff --git a/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java new file mode 100644 index 0000000..fd435a9 --- /dev/null +++ b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java @@ -0,0 +1,1577 @@ +package com.uid2.optout.traffic; + +import com.uid2.shared.cloud.CloudStorageException; +import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.optout.OptOutCollection; +import com.uid2.shared.optout.OptOutEntry; +import com.uid2.optout.sqs.SqsMessageOperations; +import com.uid2.optout.Const; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; + +import com.uid2.optout.traffic.OptOutTrafficCalculator; +import com.uid2.optout.traffic.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; +import java.io.ByteArrayInputStream; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +public class OptOutTrafficCalculatorTest { + + @Mock + private ICloudStorage cloudStorage; + + private static final String S3_DELTA_PREFIX = "optout-v2/delta/"; + private static final String TRAFFIC_CONFIG_PATH = "./traffic-config.json"; + private static final int BASELINE_TRAFFIC = 100; + private static final int THRESHOLD_MULTIPLIER = 5; + private static final int EVALUATION_WINDOW_SECONDS = 24 * 3600; + + @BeforeEach + void setUp() { + // default config + JsonObject config = new JsonObject(); + config.put(Const.Config.OptOutTrafficCalcBaselineTrafficProp, BASELINE_TRAFFIC); + config.put(Const.Config.OptOutTrafficCalcThresholdMultiplierProp, THRESHOLD_MULTIPLIER); + config.put(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp, EVALUATION_WINDOW_SECONDS); + config.put(Const.Config.OptOutTrafficCalcAllowlistRangesProp, new JsonArray()); + try { + createTrafficConfigFile(config.toString()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @AfterEach + void tearDown() { + if (Files.exists(Path.of(TRAFFIC_CONFIG_PATH))) { + try { + Files.delete(Path.of(TRAFFIC_CONFIG_PATH)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + private void createTrafficConfigFile(String content) { + try { + Path configPath = Path.of(TRAFFIC_CONFIG_PATH); + Files.writeString(configPath, content); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Helper to create config by merging partial JSON with defaults + */ + private void createConfigFromPartialJson(String partialJson) { + JsonObject partial = new JsonObject(partialJson); + JsonObject config = new JsonObject(); + + // Set defaults + if (!partial.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { + config.put(Const.Config.OptOutTrafficCalcBaselineTrafficProp, BASELINE_TRAFFIC); + } + if (!partial.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { + config.put(Const.Config.OptOutTrafficCalcThresholdMultiplierProp, THRESHOLD_MULTIPLIER); + } + if (!partial.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { + config.put(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp, EVALUATION_WINDOW_SECONDS); + } + if (!partial.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { + config.put(Const.Config.OptOutTrafficCalcAllowlistRangesProp, new JsonArray()); + } + + // Merge in partial config (overrides defaults) + partial.forEach(entry -> config.put(entry.getKey(), entry.getValue())); + + createTrafficConfigFile(config.toString()); + } + + /** + * Helper to create config with custom threshold + */ + private void createConfigWithThreshold(int threshold) { + createConfigFromPartialJson("{\"" + Const.Config.OptOutTrafficCalcThresholdMultiplierProp + "\": " + threshold + "}"); + } + + // ============================================================================ + // SECTION 1: Constructor & Initialization Tests + // ============================================================================ + + @Test + void testConstructor_defaultThreshold() throws Exception { + // Setup - default threshold of 5 + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); // 10 < 5*3 + + status = calculator.determineStatus(15, 3); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 15 >= 5*3 + } + + @Test + void testConstructor_customThreshold() throws Exception { + // Setup - custom threshold of 10 + createConfigWithThreshold(10); + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(49, 5); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); // 49 < 10*5 + status = calculator.determineStatus(50, 5); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 50 >= 10*5 + } + + @Test + void testConstructor_trafficCalcConfigLoadFailure() throws Exception { + // Setup - traffic calc config load failure + createTrafficConfigFile("Invalid JSON"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + }); + + // Create valid config to test reload failure + createConfigFromPartialJson("{}"); + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + createTrafficConfigFile("Invalid JSON"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + } + + // ============================================================================ + // SECTION 2: parseTrafficCalcConfigRanges() + // ============================================================================ + + @Test + void testParseTrafficCalcConfigRanges_emptyConfig() throws Exception { + // Setup - no config + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + JsonObject emptyConfig = new JsonObject(); + + // Act + List> ranges = calculator.parseAllowlistRanges(emptyConfig); + + // Assert - empty ranges + assertTrue(ranges.isEmpty()); + } + + @Test + void testParseTrafficCalcConfigRanges_singleRange() throws Exception { + // Setup - single range + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)); + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - single range + assertEquals(1, result.size()); + assertEquals(1000L, result.get(0).get(0)); + assertEquals(2000L, result.get(0).get(1)); + } + + @Test + void testParseTrafficCalcConfigRanges_multipleRanges() throws Exception { + // Setup - multiple ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(3000L).add(4000L)) + .add(new JsonArray().add(5000L).add(6000L)); + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - multiple ranges + assertEquals(3, result.size()); + assertEquals(1000L, result.get(0).get(0)); + assertEquals(3000L, result.get(1).get(0)); + assertEquals(5000L, result.get(2).get(0)); + } + + @Test + void testParseTrafficCalcConfigRanges_misorderedRange() throws Exception { + // Setup - range with end < start is malformed + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(2000L).add(1000L)); // End before start + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.parseAllowlistRanges(configWithRanges); + }); + } + + @Test + void testParseTrafficCalcConfigRanges_rangeTooLong() throws Exception { + // Setup - range longer than 24 hours is malformed + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(2000L).add(200000L)); // Longer than 24 hours + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.parseAllowlistRanges(configWithRanges); + }); + } + + @Test + void testParseTrafficCalcConfigRanges_sortsByStartTime() throws Exception { + // Setup - ranges added out of order + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(5000L).add(6000L)) + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(3000L).add(4000L)); + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - should be sorted by start time + assertEquals(3, result.size()); + assertEquals(1000L, result.get(0).get(0)); + assertEquals(3000L, result.get(1).get(0)); + assertEquals(5000L, result.get(2).get(0)); + } + + @Test + void testParseTrafficCalcConfigRanges_invalidRangeTooFewElements() throws Exception { + // Setup - invalid range with only 1 element; + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L)) // Only 1 element + .add(new JsonArray().add(2000L).add(3000L)); // Valid + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - should skip invalid range + assertEquals(1, result.size()); + assertEquals(2000L, result.get(0).get(0)); + } + + @Test + void testParseTrafficCalcConfigRanges_nullArray() throws Exception { + // Setup - null array + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + configWithRanges.put("traffic_calc_allowlist_ranges", (JsonArray) null); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - empty ranges + assertTrue(result.isEmpty()); + } + + @Test + void testParseTrafficCalcConfigRanges_overlappingRanges() throws Exception { + // Setup - overlapping ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(1500L).add(2500L)); // Overlaps with first range + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act & Assert - should throw exception due to overlap + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.parseAllowlistRanges(configWithRanges); + }); + } + + @Test + void testParseTrafficCalcConfigRanges_adjacentRangesWithSameBoundary() throws Exception { + // Setup - ranges where end of first equals start of second (touching but not overlapping semantically, but we treat as overlap) + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(2000L).add(3000L)); // Starts exactly where first ends + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act & Assert - should throw exception because ranges touch at boundary + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.parseAllowlistRanges(configWithRanges); + }); + } + + @Test + void testParseTrafficCalcConfigRanges_nonOverlappingRanges() throws Exception { + // Setup - ranges that don't overlap (with gap between them) + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(2001L).add(3000L)); // Starts after first ends + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - should succeed with 2 ranges + assertEquals(2, result.size()); + } + + // ============================================================================ + // SECTION 3: isInTrafficCalcConfig() + // ============================================================================ + + @Test + void testIsInTrafficCalcConfig_withinSingleRange() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - true when within range + assertTrue(calculator.isInAllowlist(1500L)); + } + + @Test + void testIsInTrafficCalcConfig_exactlyAtStart() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - true when exactly at start of range + assertTrue(calculator.isInAllowlist(1000L)); + } + + @Test + void testIsInTrafficCalcConfig_exactlyAtEnd() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - true when exactly at end of range + assertTrue(calculator.isInAllowlist(2000L)); + } + + @Test + void testIsInTrafficCalcConfig_beforeRange() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - false when before range + assertFalse(calculator.isInAllowlist(999L)); + } + + @Test + void testIsInTrafficCalcConfig_afterRange() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - false when after range + assertFalse(calculator.isInAllowlist(2001L)); + } + + @Test + void testIsInTrafficCalcConfig_betweenRanges() throws Exception { + // Setup - load traffic calc config with two ranges [1000, 2000] and [3000, 4000] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000], + [3000, 4000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - false when between ranges + assertFalse(calculator.isInAllowlist(2500L)); + } + + @Test + void testIsInTrafficCalcConfig_emptyRanges() throws Exception { + // Setup uses default config from setUp() which has empty traffic calc config ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - false when empty ranges + assertFalse(calculator.isInAllowlist(1500L)); + } + + @Test + void testIsInTrafficCalcConfig_nullRanges() throws Exception { + // Setup - no traffic calc config ranges loaded (will fail and set empty) + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": null + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - false when null/empty ranges + assertFalse(calculator.isInAllowlist(1500L)); + } + + @Test + void testIsInTrafficCalcConfig_invalidRangeSize() throws Exception { + // Setup - load traffic calc config with invalid range (only 1 element) and valid range + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000], + [2000, 3000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert + assertFalse(calculator.isInAllowlist(1500L)); // Should not match invalid range + assertTrue(calculator.isInAllowlist(2500L)); // Should match valid range + } + + @Test + void testIsInTrafficCalcConfig_multipleRanges() throws Exception { + // Setup - load traffic calc config with multiple ranges + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000], + [3000, 4000], + [5000, 6000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert + assertTrue(calculator.isInAllowlist(1500L)); // In first range + assertTrue(calculator.isInAllowlist(3500L)); // In second range + assertTrue(calculator.isInAllowlist(5500L)); // In third range + assertFalse(calculator.isInAllowlist(2500L)); // Between first and second + } + + // ============================================================================ + // SECTION 4: getTrafficCalcConfigDuration() + // ============================================================================ + + @Test + void testGetTrafficCalcConfigDuration_noRanges() throws Exception { + // Setup - no ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert + assertEquals(0L, calculator.getAllowlistDuration(10000L, 5000L)); // 0 duration when no ranges + } + + @Test + void testGetTrafficCalcConfigDuration_rangeFullyWithinWindow() throws Exception { + // Setup - range fully within window + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [6000, 7000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window [5000, 10000], range [6000, 7000] + long duration = calculator.getAllowlistDuration(10000L, 5000L); + + // Assert - full range duration + assertEquals(1000L, duration); + } + + @Test + void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsStart() throws Exception { + // Setup - range partially overlaps start of window + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [3000, 7000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window [5000, 10000], range [3000, 7000] + long duration = calculator.getAllowlistDuration(10000L, 5000L); + + // Assert - should clip to [5000, 7000] = 2000 + assertEquals(2000L, duration); + } + + @Test + void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsEnd() throws Exception { + // Setup - range partially overlaps end of window + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [8000, 12000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window [5000, 10000], range [8000, 12000] + long duration = calculator.getAllowlistDuration(10000L, 5000L); + + // Assert - should clip to [8000, 10000] = 2000 + assertEquals(2000L, duration); + } + + @Test + void testGetTrafficCalcConfigDuration_rangeCompletelyOutsideWindow() throws Exception { + // Setup - range completely outside window + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window [5000, 10000], range [1000, 2000] + long duration = calculator.getAllowlistDuration(10000L, 5000L); + + // Assert - 0 duration when range completely outside window + assertEquals(0L, duration); + } + + @Test + void testGetTrafficCalcConfigDuration_multipleRanges() throws Exception { + // Setup - multiple ranges + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [6000, 7000], + [8000, 9000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window [5000, 10000], ranges [6000, 7000] and [8000, 9000] + long duration = calculator.getAllowlistDuration(10000L, 5000L); + + // Assert - 1000 + 1000 = 2000 + assertEquals(2000L, duration); + } + + @Test + void testGetTrafficCalcConfigDuration_rangeSpansEntireWindow() throws Exception { + // Setup - range spans entire window + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [3000, 12000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window [5000, 10000], range [3000, 12000] + long duration = calculator.getAllowlistDuration(10000L, 5000L); + + // Assert - entire window is in traffic calc config ranges = 5000 + assertEquals(5000L, duration); + } + + // ============================================================================ + // SECTION 4.5: calculateWindowStartWithAllowlist() + // ============================================================================ + + @Test + void testCalculateWindowStartWithAllowlist_noAllowlist() throws Exception { + // Setup - no allowlist ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window should be [3, 8] with no extension + long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); + + // Assert - no allowlist, so window start is simply newestDeltaTs - evaluationWindowSeconds + assertEquals(3L, windowStart); + } + + @Test + void testCalculateWindowStartWithAllowlist_allowlistInOriginalWindowOnly() throws Exception { + // Setup - allowlist range only in original window, not in extended portion + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [6, 7] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - newestDeltaTs=8, evaluationWindow=5 + // Original window [3, 8] has [6,7] allowlisted (1 hour) + // Extended portion [2, 3] has no allowlist + // So window start should be 8 - 5 - 1 = 2 + long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); + + assertEquals(2L, windowStart); + } + + @Test + void testCalculateWindowStartWithAllowlist_allowlistInExtendedPortion() throws Exception { + // Setup - allowlist ranges in both original window AND extended portion + // This is the user's example: evaluationWindow=5, newestDeltaTs=8, allowlist={[2,3], [6,7]} + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [2, 3], + [6, 7] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + // Original window [3, 8]: [6,7] allowlisted = 1 hour + // First extension to [2, 8]: [2,3] and [6,7] allowlisted = 2 hours total + // Second extension to [1, 8]: still [2,3] and [6,7] = 2 hours (no new allowlist) + // Final: windowStart = 8 - 5 - 2 = 1 + long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); + + assertEquals(1L, windowStart); + } + + @Test + void testCalculateWindowStartWithAllowlist_allowlistBeforeWindow() throws Exception { + // Setup - allowlist range entirely before the initial window + // This tests that we don't over-extend when allowlist is old + // evaluationWindow=5, newestDeltaTs=20, allowlist=[10,13] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [10, 13] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + // Initial window [15, 20]: no allowlist overlap, allowlistDuration = 0 + // No extension needed + // Final: windowStart = 20 - 5 - 0 = 15 + long windowStart = calculator.calculateWindowStartWithAllowlist(20L, 5); + + // Verify: window [15, 20] has 5 hours, 0 allowlisted = 5 non-allowlisted + assertEquals(15L, windowStart); + } + + // ============================================================================ + // SECTION 5: determineStatus() + // ============================================================================ + + @Test + void testDetermineStatus_belowThreshold() throws Exception { + // Setup - below threshold + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - 10 < 5 * 3 + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); + + // Assert - DEFAULT when below threshold + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testDetermineStatus_atThreshold() throws Exception { + // Setup - at threshold + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - 15 == 5 * 3 + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(15, 3); + + // Assert - DELAYED_PROCESSING when at threshold + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testDetermineStatus_aboveThreshold() throws Exception { + // Setup - above threshold + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - 20 > 5 * 3 + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(20, 3); + + // Assert - DELAYED_PROCESSING when above threshold + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testDetermineStatus_sumPastZero() throws Exception { + // Setup - sumPast is 0 + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert - should throw exception for invalid config + assertThrows(RuntimeException.class, () -> calculator.determineStatus(100, 0)); + } + + @Test + void testDetermineStatus_bothZero() throws Exception { + // Setup - both sumCurrent and sumPast are 0; + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert - should throw exception for invalid config + assertThrows(RuntimeException.class, () -> calculator.determineStatus(0, 0)); + } + + @Test + void testDetermineStatus_sumCurrentZero() throws Exception { + // Setup - sumCurrent is 0 + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - 0 < 5 * 10 + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 10); + + // Assert - DEFAULT when sumCurrent is 0 + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @ParameterizedTest + @CsvSource({ + "1, 1, 1, DELAYED_PROCESSING", // threshold=1: 1 >= 1*1 + "2, 4, 2, DELAYED_PROCESSING", // threshold=2: 4 >= 2*2 + "5, 10, 2, DELAYED_PROCESSING", // threshold=5: 10 >= 5*2 + "10, 100, 10, DELAYED_PROCESSING", // threshold=10: 100 >= 10*10 + "5, 24, 5, DEFAULT", // threshold=5: 24 < 5*5 + "100, 1000, 11, DEFAULT" // threshold=100: 1000 < 100*11 + }) + void testDetermineStatus_variousThresholds(int threshold, int sumCurrent, int sumPast, String expectedStatus) throws Exception { + // Setup - various thresholds + createConfigWithThreshold(threshold); + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(sumCurrent, sumPast); + + // Assert + assertEquals(OptOutTrafficCalculator.TrafficStatus.valueOf(expectedStatus), status); + } + + @Test + void testDetermineStatus_largeNumbers() throws Exception { + // Setup - test with large numbers + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(1_000_000, 200_000); + + // Assert - 1M >= 5 * 200K = 1M + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + // ============================================================================ + // SECTION 6: S3 Config Reload Tests + // ============================================================================ + + @Test + void testReloadTrafficCalcConfig_success() throws Exception { + // Setup - initial traffic calc config + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000], + [3000, 4000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Change the traffic calc config to a new range + String newTrafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [5000, 6000] + ] + } + """; + createConfigFromPartialJson(newTrafficCalcConfigJson); + + // Act - reload the traffic calc config + calculator.reloadTrafficCalcConfig(); + + // Assert - verify new traffic calc config is loaded + assertTrue(calculator.isInAllowlist(5500L)); + } + + @Test + void testReloadTrafficCalcConfig_failure() throws Exception { + // Setup - initial traffic calc config + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Now make it fail + createTrafficConfigFile("Invalid JSON"); + + // Act - should not throw exception + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + + } + + @Test + public void testReloadTrafficCalcConfig_failure_missingKeys() throws Exception { + // Setup + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert missing threshold multiplier + createTrafficConfigFile("{\"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + + // Act & Assert missing evaluation window seconds + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + + // Act & Assert missing baseline traffic + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + + // Act & Assert missing traffic calc config ranges + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100}"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + } + + @Test + public void testReloadTrafficCalcConfig_failure_misorderedRanges() throws Exception { + // Setup - misordered ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + createConfigFromPartialJson("{\"traffic_calc_allowlist_ranges\": [ [2000, 1000] ]}"); + + // Act & Assert + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + } + + @Test + public void testReloadTrafficCalcConfig_failure_rangeTooLong() throws Exception { + // Setup - range greater than 24 hours + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + createConfigFromPartialJson("{\"traffic_calc_allowlist_ranges\": [ [1000, 200000] ]}"); + + // Act & Assert + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + } + + // ============================================================================ + // SECTION 7: Cache Management Tests (also tested in section 9) + // ============================================================================ + + @Test + void testGetCacheStats_emptyCache() throws Exception { + // Setup + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + Map stats = calculator.getCacheStats(); + + // Assert - should return empty stats + assertEquals(0, stats.get("cached_files")); + assertEquals(0, stats.get("total_cached_timestamps")); + } + + // ============================================================================ + // SECTION 8: Helper Methods for Test Data Creation + // ============================================================================ + + /** + * Create a mock SQS message with specified timestamp + */ + private Message createSqsMessage(long timestampSeconds) { + Map attributes = new HashMap<>(); + attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampSeconds * 1000)); + + return Message.builder() + .messageId("test-msg-" + timestampSeconds) + .body("{\"test\": \"data\"}") + .attributes(attributes) + .build(); + } + + /** + * Create a mock SQS message without timestamp + */ + private Message createSqsMessageWithoutTimestamp() { + return Message.builder() + .messageId("test-msg-no-timestamp") + .body("{\"test\": \"data\"}") + .attributes(new HashMap<>()) + .build(); + } + + /** + * Create delta file bytes with specified timestamps + */ + private byte[] createDeltaFileBytes(List timestamps) throws Exception { + // Create OptOutEntry objects using newTestEntry + List entries = new ArrayList<>(); + + long idCounter = 1000; // Use incrementing IDs for test entries + for (long timestamp : timestamps) { + entries.add(OptOutEntry.newTestEntry(idCounter++, timestamp)); + } + + // Create OptOutCollection + OptOutCollection collection = new OptOutCollection(entries.toArray(new OptOutEntry[0])); + return collection.getStore(); + } + + + // ============================================================================ + // SECTION 9: Tests for calculateStatus() + // ============================================================================ + + @Test + void testCalculateStatus_noDeltaFiles() throws Exception { + // Setup - no delta files + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Collections.emptyList()); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert - should throw exception when no delta files + assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); + } + + @Test + void testCalculateStatus_normalTraffic() throws Exception { + // Setup - setup time: current time + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Create delta files with timestamps distributed over 48 hours + List timestamps = new ArrayList<>(); + + // add 499 entries in current window + for (int i = 0; i < 49; i++) { + timestamps.add(t - 23*3600 + i * 60); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - 100+1 < 5 * 50 = 250, so should be DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_delayedProcessing() throws Exception { + // Setup - create delta files with spike in current window + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Create delta files with spike in current window + List timestamps = new ArrayList<>(); + + // add 500 entries in current window + for (int i = 0; i < 500; i++) { + timestamps.add(t - 23*3600 + i * 60); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - 100+1 >= 5 * 10 = 50, DELAYED_PROCESSING + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testCalculateStatus_noSqsMessages() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600, t - 7200); // Some entries + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - null SQS messages + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null, null, 0, 0); + + // Assert - should still calculate based on delta files, DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_emptySqsMessages() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - empty SQS messages + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0); + + // Assert - should still calculate based on delta files, DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_multipleSqsMessages() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = new ArrayList<>(); + // add 470 entries in window + for (int i = 0; i < 470; i++) { + timestamps.add(t - 24*3600 + i * 60); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Add 30 SQS entries in [t, t+5min] + List sqsMessages = new ArrayList<>(); + for (int i = 0; i < 30; i++) { + sqsMessages.add(createSqsMessage(t - i * 10)); + } + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - DELAYED_PROCESSING + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testCalculateStatus_withTrafficCalcConfig() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Traffic calc config that covers part of window + String trafficCalcConfigJson = String.format(""" + { + "traffic_calc_allowlist_ranges": [ + [%d, %d] + ] + } + """, t - 12*3600, t - 6*3600); + + List timestamps = new ArrayList<>(); + + // window - 600 entries (300 in traffic calc config range, 300 outside) + for (int i = 0; i < 300; i++) { + timestamps.add(t - 12*3600 + i); + } + for (int i = 0; i < 300; i++) { + timestamps.add(t - 3600 + i); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + createConfigFromPartialJson(trafficCalcConfigJson); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta-001_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta-001_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - should filter out entries in traffic calc config ranges + // Only 300 from window count (not in traffic calc config ranges) + 1 SQS = 301 + // 301 < 5*100, so DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_cacheUtilization() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600, t - 7200); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - first call should populate cache + List sqsMessages = Arrays.asList(createSqsMessage(t)); + calculator.calculateStatus(sqsMessages, null, 0, 0); + + Map stats = calculator.getCacheStats(); + int cachedFiles = (Integer) stats.get("cached_files"); + + // Second call should use cache (no additional S3 download) + calculator.calculateStatus(sqsMessages, null, 0, 0); + + Map stats2 = calculator.getCacheStats(); + int cachedFiles2 = (Integer) stats2.get("cached_files"); + + // Assert - cache should be populated and remain consistent + assertEquals(1, cachedFiles); + assertEquals(cachedFiles, cachedFiles2); + + // Verify S3 download was called only once per file + verify(cloudStorage, times(1)).download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat"); + } + + @Test + void testCalculateStatus_s3Exception() throws Exception { + // Setup - S3 list error + when(cloudStorage.list(S3_DELTA_PREFIX)).thenThrow(new RuntimeException("S3 error")); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert - should throw exception on S3 error + assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); + } + + @Test + void testCalculateStatus_deltaFileReadException() throws Exception { + // Setup - S3 download error + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenThrow(new CloudStorageException("Failed to download")); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert - should throw exception on S3 download error + assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); + } + + @Test + void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - SQS message without timestamp (should use current time) + List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp()); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_multipleDeltaFiles() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // File 1 - recent entries + List timestamps1 = new ArrayList<>(); + for (int i = 0; i < 50; i++) { + timestamps1.add(t - 12*3600 + i * 1000); + } + byte[] deltaFileBytes1 = createDeltaFileBytes(timestamps1); + + // File 2 - older entries + List timestamps2 = new ArrayList<>(); + for (int i = 0; i < 30; i++) { + timestamps2.add(t - 36*3600 + i * 1000); + } + byte[] deltaFileBytes2 = createDeltaFileBytes(timestamps2); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList( + "optout-v2/delta/optout-delta--01_2025-11-13T02.00.00Z_bbbbbbbb.dat", + "optout-v2/delta/optout-delta--01_2025-11-13T01.00.00Z_aaaaaaaa.dat" + )); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T02.00.00Z_bbbbbbbb.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes1)); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T01.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes2)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + + // Verify cache has both files + Map stats = calculator.getCacheStats(); + assertEquals(2, stats.get("cached_files")); + } + + @Test + void testCalculateStatus_windowBoundaryTimestamp() throws Exception { + // Setup - create delta file with timestamps at window boundary + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + long currentWindowStart = t - 24*3600; + List timestamps = new ArrayList<>(); + for (int i = 0; i < 250; i++) { + timestamps.add(t); + } + for (int i = 0; i < 250; i++) { + timestamps.add(currentWindowStart); + } + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testCalculateStatus_timestampsCached() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600, t - 7200); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + + // Cache should contain the timestamps + Map stats = calculator.getCacheStats(); + assertEquals(2, stats.get("total_cached_timestamps")); + } + + // ============================================================================ + // SECTION 10: Tests for queue attributes (invisible messages from other consumers) + // ============================================================================ + + @Test + void testCalculateStatus_delayedProcessingFromQueueAttributesOnly() throws Exception { + // Setup - delta files with low traffic (10 records) + // Threshold = 100 * 5 = 500 + // Queue attributes will have 600 invisible messages (other consumers processing) + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + timestamps.add(t - 3600 + i); // 10 entries from 1 hour ago + } + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - 1 message read by us, but 600 invisible messages from other consumers + List sqsMessages = Arrays.asList(createSqsMessage(t)); + + // QueueAttributes: 0 visible, 600 invisible (other consumers), 0 delayed + // Since we read 1 message, otherConsumers = 600 - 1 = 599 + // Total = 10 (delta) + 1 (our message) + 599 (other consumers) = 610 >= 500 threshold + SqsMessageOperations.QueueAttributes queueAttributes = + new SqsMessageOperations.QueueAttributes(0, 600, 0); + + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0); + + // Assert - DELAYED_PROCESSING due to high invisible message count from other consumers + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testCalculateStatus_delayedProcessingFromBothQueueAndMessages() throws Exception { + // Setup - delta files with moderate traffic (100 records) + // Threshold = 100 * 5 = 500 + // We'll have 200 messages + 250 invisible from other consumers = 550 > 500 + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + timestamps.add(t - 3600 + i); // 100 entries from 1 hour ago + } + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - 200 messages read by us + 450 invisible (200 are ours + 250 from others) + // Messages must be within 5-minute window to be counted, so use 1-second spacing + List sqsMessages = new ArrayList<>(); + for (int i = 0; i < 200; i++) { + sqsMessages.add(createSqsMessage(t - i)); // 1 second apart, all within 5-minute window + } + + // QueueAttributes: 0 visible, 450 invisible (200 ours + 250 others), 0 delayed + // otherConsumers = 450 - 200 = 250 + // Total = 100 (delta) + 200 (our messages) + 250 (other consumers) = 550 >= 500 threshold + SqsMessageOperations.QueueAttributes queueAttributes = + new SqsMessageOperations.QueueAttributes(0, 450, 0); + + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0); + + // Assert - DELAYED_PROCESSING due to combined count exceeding threshold + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + +} \ No newline at end of file diff --git a/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java new file mode 100644 index 0000000..127cd04 --- /dev/null +++ b/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java @@ -0,0 +1,428 @@ +package com.uid2.optout.traffic; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.uid2.optout.sqs.SqsParsedMessage; +import com.uid2.optout.traffic.OptOutTrafficFilter; + +import software.amazon.awssdk.services.sqs.model.Message; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.Assert.*; + +public class OptOutTrafficFilterTest { + + private static final String TEST_CONFIG_PATH = "./traffic-config.json"; + + @Before + public void setUp() { + try { + Files.deleteIfExists(Path.of(TEST_CONFIG_PATH)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @After + public void tearDown() { + try { + Files.deleteIfExists(Path.of(TEST_CONFIG_PATH)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void testParseFilterRules_emptyRules() throws Exception { + // Setup - empty denylist + String config = """ + { + "denylist_requests": [] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - no rules + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(0, filter.filterRules.size()); + } + + @Test + public void testParseFilterRules_singleRule() throws Exception { + // Setup - config with one rule + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - one rule + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(1, filter.filterRules.size()); + } + + @Test + public void testParseFilterRules_multipleRules() throws Exception { + // Setup - config with multiple rules + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + }, + { + "range": [1700010000, 1700013600], + "IPs": ["10.0.0.1", "10.0.0.2"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - two rules + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(2, filter.filterRules.size()); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_missingDenylistRequests() throws Exception { + // Setup - config without denylist_requests field + String config = """ + { + "other_field": "value" + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_invalidRange_startAfterEnd() throws Exception { + // Setup - range where start > end + String config = """ + { + "denylist_requests": [ + { + "range": [1700003600, 1700000000], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_invalidRange_startEqualsEnd() throws Exception { + // Setup - range where start == end + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700000000], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_rangeExceeds24Hours() throws Exception { + // Setup - range longer than 24 hours (86400 seconds) + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700086401], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_emptyIPs() throws Exception { + // Setup - rule with empty IP list + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": [] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_missingIPs() throws Exception { + // Setup - rule without IPs field + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test + public void testIsDenylisted_matchingIPAndTimestamp() throws Exception { + // Setup - filter with denylist rule + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1", "10.0.0.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + SqsParsedMessage message = createTestMessage(1700001800, "192.168.1.1"); + + // Act & Assert - denylisted + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertTrue(filter.isDenylisted(message)); + } + + @Test + public void testIsDenylisted_matchingIPOutsideTimeRange() throws Exception { + // Setup - filter with denylist rule + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - message before range not denylisted + SqsParsedMessage messageBefore = createTestMessage(1699999999, "192.168.1.1"); + assertFalse(filter.isDenylisted(messageBefore)); + // Act & Assert - message after range not denylisted + SqsParsedMessage messageAfter = createTestMessage(1700003601, "192.168.1.1"); + assertFalse(filter.isDenylisted(messageAfter)); + } + + @Test + public void testIsDenylisted_nonMatchingIP() throws Exception { + // Setup - filter with denylist rule + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - non-matching IP not denylisted + SqsParsedMessage message = createTestMessage(1700001800, "10.0.0.1"); + assertFalse(filter.isDenylisted(message)); + } + + @Test + public void testIsDenylisted_atRangeBoundaries() throws Exception { + // Setup - filter with denylist rule + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - message at start boundary (inclusive) denylisted + SqsParsedMessage messageAtStart = createTestMessage(1700000000, "192.168.1.1"); + assertTrue(filter.isDenylisted(messageAtStart)); + + // Act & Assert - message at end boundary (inclusive) denylisted + SqsParsedMessage messageAtEnd = createTestMessage(1700003600, "192.168.1.1"); + assertTrue(filter.isDenylisted(messageAtEnd)); + } + + @Test + public void testIsDenylisted_multipleRules() throws Exception { + // Setup - multiple denylist rules + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + }, + { + "range": [1700010000, 1700013600], + "IPs": ["10.0.0.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - message matches first rule + SqsParsedMessage msg1 = createTestMessage(1700001800, "192.168.1.1"); + assertTrue(filter.isDenylisted(msg1)); + + // Act & Assert - message matches second rule + SqsParsedMessage msg2 = createTestMessage(1700011800, "10.0.0.1"); + assertTrue(filter.isDenylisted(msg2)); + + // Act & Assert - message matches neither rule + SqsParsedMessage msg3 = createTestMessage(1700005000, "172.16.0.1"); + assertFalse(filter.isDenylisted(msg3)); + } + + @Test + public void testIsDenylisted_nullClientIp() throws Exception { + // Setup - filter with denylist rule + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - message with null IP not denylisted + SqsParsedMessage message = createTestMessage(1700001800, null); + assertFalse(filter.isDenylisted(message)); + } + + @Test + public void testReloadTrafficFilterConfig_success() throws Exception { + // Setup - config with one rule + String initialConfig = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), initialConfig); + + // Act & Assert - one rule + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(1, filter.filterRules.size()); + + // Setup - update config + String updatedConfig = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + }, + { + "range": [1700010000, 1700013600], + "IPs": ["10.0.0.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), updatedConfig); + + // Act & Assert - two rules + filter.reloadTrafficFilterConfig(); + assertEquals(2, filter.filterRules.size()); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testReloadTrafficFilterConfig_fileNotFound() throws Exception { + // Setup, Act & Assert - try to create filter with non-existent config + new OptOutTrafficFilter("./non-existent-file.json"); + } + + @Test + public void testParseFilterRules_maxValidRange() throws Exception { + // Setup - range exactly 24 hours (86400 seconds) - should be valid + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700086400], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - one rule + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(1, filter.filterRules.size()); + } + + /** + * Helper method to create test SqsParsedMessage + */ + private SqsParsedMessage createTestMessage(long timestamp, String clientIp) { + Message mockMessage = Message.builder().build(); + byte[] hash = new byte[32]; + byte[] id = new byte[32]; + return new SqsParsedMessage(mockMessage, hash, id, timestamp, null, null, clientIp, null); + } +} From d6f6d16f44d330b07379ca2801fcb0917578c726 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 13:49:50 -0700 Subject: [PATCH 02/25] addressing PR comments --- .../OptOutTrafficCalculator.java | 623 ++++++++++++++++++ .../uid2/optout/sqs/SqsBatchProcessor.java | 4 +- .../uid2/optout/sqs/SqsMessageOperations.java | 72 +- .../com/uid2/optout/sqs/SqsMessageParser.java | 30 +- .../com/uid2/optout/sqs/SqsParsedMessage.java | 64 +- .../com/uid2/optout/sqs/SqsWindowReader.java | 8 +- .../optout/vertx/OptOutSqsLogProducer.java | 4 +- .../optout/vertx/OptOutTrafficFilter.java | 4 +- .../uid2/optout/vertx/SqsBatchProcessor.java | 6 +- .../uid2/optout/vertx/SqsMessageParser.java | 2 +- .../uid2/optout/vertx/SqsParsedMessage.java | 64 +- .../uid2/optout/vertx/SqsWindowReader.java | 2 +- .../optout/sqs/SqsBatchProcessorTest.java | 10 +- .../optout/sqs/SqsMessageOperationsTest.java | 316 +++++++++ .../uid2/optout/sqs/SqsMessageParserTest.java | 28 +- .../vertx/OptOutSqsLogProducerTest.java | 12 +- .../optout/vertx/SqsBatchProcessorTest.java | 10 +- .../optout/vertx/SqsMessageParserTest.java | 28 +- 18 files changed, 1086 insertions(+), 201 deletions(-) create mode 100644 src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java create mode 100644 src/test/java/com/uid2/optout/sqs/SqsMessageOperationsTest.java diff --git a/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java new file mode 100644 index 0000000..91da8f3 --- /dev/null +++ b/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java @@ -0,0 +1,623 @@ +package com.uid2.optout.circuitbreaker; + +import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.optout.OptOutCollection; +import com.uid2.shared.optout.OptOutEntry; +import com.uid2.shared.optout.OptOutUtils; +import com.uid2.optout.Const; +import com.uid2.optout.sqs.SqsMessageOperations; + +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; + +import java.nio.charset.StandardCharsets; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.io.InputStream; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * Calculates opt-out traffic patterns to determine DEFAULT or DELAYED_PROCESSING status. + * + * Compares recent ~24h traffic (sumCurrent) against a configurable baseline (baselineTraffic) of expected traffic in 24 hours. + * The baseline is multiplied by (thresholdMultiplier) to determine the threshold. + * sumCurrent excludes records in allowlist ranges (surge windows determined by engineers). + * + * Returns DELAYED_PROCESSING if sumCurrent >= thresholdMultiplier * baselineTraffic, indicating abnormal traffic spike. + */ +public class OptOutTrafficCalculator { + private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficCalculator.class); + + private static final int HOURS_24 = 24 * 3600; // 24 hours in seconds + + private final Map deltaFileCache = new ConcurrentHashMap<>(); + private final ICloudStorage cloudStorage; + private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") + private final String trafficCalcConfigPath; + private int baselineTraffic; + private int thresholdMultiplier; + private int evaluationWindowSeconds; + private List> allowlistRanges; + + public enum TrafficStatus { + DELAYED_PROCESSING, + DEFAULT + } + + /** + * Cache entry for a delta file containing all record timestamps. + * + * Memory usage: ~8 bytes per timestamp (long) + * 1GB of memory can store ~130 million timestamps (1024^3)/8 + */ + private static class FileRecordCache { + final List timestamps; // All non-sentinel record timestamps + final long newestTimestamp; // evict delta from cache based on oldest record timestamp + + FileRecordCache(List timestamps) { + this.timestamps = timestamps; + this.newestTimestamp = timestamps.isEmpty() ? 0 : Collections.max(timestamps); + } + } + + /** + * Exception thrown by malformed traffic calculator config + */ + public static class MalformedTrafficCalcConfigException extends Exception { + public MalformedTrafficCalcConfigException(String message) { + super(message); + } + } + + /** + * Constructor for OptOutTrafficCalculator + * + * @param cloudStorage Cloud storage for reading delta files + * @param s3DeltaPrefix S3 prefix for delta files + * @param trafficCalcConfigS3Path S3 path for traffic calc config + */ + public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigPath) throws MalformedTrafficCalcConfigException { + this.cloudStorage = cloudStorage; + this.s3DeltaPrefix = s3DeltaPrefix; + this.trafficCalcConfigPath = trafficCalcConfigPath; + reloadTrafficCalcConfig(); // Load ConfigMap + + LOGGER.info("initialized: s3DeltaPrefix={}, threshold={}x", + s3DeltaPrefix, thresholdMultiplier); + } + + /** + * Reload traffic calc config from ConfigMap. + * Expected format: + * { + * "traffic_calc_evaluation_window_seconds": 86400, + * "traffic_calc_baseline_traffic": 100, + * "traffic_calc_threshold_multiplier": 5, + * "traffic_calc_allowlist_ranges": [ + * [startTimestamp1, endTimestamp1], + * [startTimestamp2, endTimestamp2] + * ], + * } + * + * Can be called periodically to pick up config changes without restarting. + */ + public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException { + LOGGER.info("loading traffic calc config from configmap"); + try (InputStream is = Files.newInputStream(Paths.get(trafficCalcConfigPath))) { + String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); + JsonObject trafficCalcConfig = new JsonObject(content); + + // Validate required fields exist + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_evaluation_window_seconds"); + } + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_baseline_traffic"); + } + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_threshold_multiplier"); + } + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_allowlist_ranges"); + } + + this.evaluationWindowSeconds = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp); + this.baselineTraffic = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcBaselineTrafficProp); + this.thresholdMultiplier = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcThresholdMultiplierProp); + + List> ranges = parseAllowlistRanges(trafficCalcConfig); + this.allowlistRanges = ranges; + + LOGGER.info("loaded traffic calc config: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", + this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size()); + + } catch (MalformedTrafficCalcConfigException e) { + LOGGER.warn("failed to load traffic calc config, config is malformed: {}", trafficCalcConfigPath, e); + throw e; + } catch (Exception e) { + LOGGER.warn("failed to load traffic calc config, config is malformed or missing: {}", trafficCalcConfigPath, e); + throw new MalformedTrafficCalcConfigException("failed to load traffic calc config: " + e.getMessage()); + } + } + + /** + * Parse allowlist ranges from JSON config + */ + List> parseAllowlistRanges(JsonObject config) throws MalformedTrafficCalcConfigException { + List> ranges = new ArrayList<>(); + + try { + var rangesArray = config.getJsonArray(Const.Config.OptOutTrafficCalcAllowlistRangesProp); + if (rangesArray != null) { + for (int i = 0; i < rangesArray.size(); i++) { + var rangeArray = rangesArray.getJsonArray(i); + if (rangeArray != null && rangeArray.size() >= 2) { + long start = rangeArray.getLong(0); + long end = rangeArray.getLong(1); + + if(start >= end) { + LOGGER.error("invalid allowlist range: start must be less than end: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": start must be less than end"); + } + + if (end - start > 86400) { + LOGGER.error("invalid allowlist range: range must be less than 24 hours: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": range must be less than 24 hours"); + } + + List range = Arrays.asList(start, end); + ranges.add(range); + LOGGER.info("loaded allowlist range: [{}, {}]", start, end); + } + } + } + + ranges.sort(Comparator.comparing(range -> range.get(0))); + + // Validate no overlapping ranges + for (int i = 0; i < ranges.size() - 1; i++) { + long currentEnd = ranges.get(i).get(1); + long nextStart = ranges.get(i + 1).get(0); + if (currentEnd >= nextStart) { + LOGGER.error("overlapping allowlist ranges detected: [{}, {}] overlaps with [{}, {}]", + ranges.get(i).get(0), currentEnd, nextStart, ranges.get(i + 1).get(1)); + throw new MalformedTrafficCalcConfigException( + "overlapping allowlist ranges detected at indices " + i + " and " + (i + 1)); + } + } + + } catch (MalformedTrafficCalcConfigException e) { + throw e; + } catch (Exception e) { + LOGGER.error("failed to parse allowlist ranges", e); + throw new MalformedTrafficCalcConfigException("failed to parse allowlist ranges: " + e.getMessage()); + } + + return ranges; + } + + /** + * Calculate traffic status based on delta files and SQS queue messages. + * + * Uses the newest delta file timestamp to anchor the 24-hour delta traffic window, + * and the oldest queue timestamp to anchor the 5-minute queue window. + * + * Counts: + * - Delta file records (with allowlist filtering) + * - SQS messages passed in (with allowlist filtering) + * - Invisible messages from other consumers (from queue attributes, avoiding double count) + * + * @param sqsMessages List of SQS messages this consumer has read + * @param queueAttributes Queue attributes including invisible message count (can be null) + * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) + */ + public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes) { + + try { + // Get list of delta files from S3 (sorted newest to oldest) + List deltaS3Paths = listDeltaFiles(); + + if (deltaS3Paths.isEmpty()) { + LOGGER.warn("no delta files found in s3 with prefix: {}", s3DeltaPrefix); + return TrafficStatus.DEFAULT; + } + + // Find newest delta file timestamp for delta traffic window + long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths); + LOGGER.info("traffic calculation: newestDeltaTs={}", newestDeltaTs); + + // Find oldest SQS queue message timestamp for queue window + long oldestQueueTs = findOldestQueueTimestamp(sqsMessages); + LOGGER.info("traffic calculation: oldestQueueTs={}", oldestQueueTs); + + // Define start time of the delta evaluation window + // We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend + // the window to account for any allowlist ranges in the extended portion + long deltaWindowStart = calculateWindowStartWithAllowlist(newestDeltaTs, this.evaluationWindowSeconds); + + // Evict old cache entries (older than delta window start) + evictOldCacheEntries(deltaWindowStart); + + // Process delta files and count records in [deltaWindowStart, newestDeltaTs] + int sum = 0; + int deltaRecordsCount = 0; + int filesProcessed = 0; + int cacheHits = 0; + int cacheMisses = 0; + + for (String s3Path : deltaS3Paths) { + boolean wasCached = isCached(s3Path); + if (wasCached) { + cacheHits++; + } else { + cacheMisses++; + } + + List timestamps = getTimestampsFromFile(s3Path); + filesProcessed++; + + boolean shouldStop = false; + for (long ts : timestamps) { + // Stop condition: record is older than our window + if (ts < deltaWindowStart) { + LOGGER.info("stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); + break; + } + + // skip records in allowlisted ranges + if (isInAllowlist(ts)) { + continue; + } + + // increment sum if record is in delta window + if (ts >= deltaWindowStart) { + deltaRecordsCount++; + sum++; + } + + } + + if (shouldStop) { + break; + } + } + + LOGGER.info("delta files: processed={}, deltaRecords={}, cache hits={}, misses={}, cacheSize={}", + filesProcessed, deltaRecordsCount, cacheHits, cacheMisses, deltaFileCache.size()); + + // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] with allowlist filtering + int sqsCount = 0; + if (sqsMessages != null && !sqsMessages.isEmpty()) { + sqsCount = countSqsMessages(sqsMessages, oldestQueueTs); + sum += sqsCount; + } + + // Add invisible messages being processed by OTHER consumers + // (notVisible count includes our messages, so subtract what we've read to avoid double counting) + int otherConsumersMessages = 0; + if (queueAttributes != null) { + int totalInvisible = queueAttributes.getApproximateNumberOfMessagesNotVisible(); + int ourMessages = sqsMessages != null ? sqsMessages.size() : 0; + otherConsumersMessages = Math.max(0, totalInvisible - ourMessages); + sum += otherConsumersMessages; + LOGGER.info("traffic calculation: adding {} invisible messages from other consumers (totalInvisible={}, ourMessages={})", + otherConsumersMessages, totalInvisible, ourMessages); + } + + // Determine status + TrafficStatus status = determineStatus(sum, this.baselineTraffic); + + LOGGER.info("traffic calculation complete: sum={} (deltaRecords={} + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}", + sum, deltaRecordsCount, sqsCount, otherConsumersMessages, this.baselineTraffic, this.thresholdMultiplier, status); + + return status; + + } catch (Exception e) { + LOGGER.error("error calculating traffic status", e); + return TrafficStatus.DEFAULT; + } + } + + /** + * Find the newest timestamp from delta files. + * Reads the newest delta file and returns its maximum timestamp. + */ + private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOException { + if (deltaS3Paths == null || deltaS3Paths.isEmpty()) { + return System.currentTimeMillis() / 1000; + } + + // Delta files are sorted (ISO 8601 format, lexicographically sortable) so first file is newest + String newestDeltaPath = deltaS3Paths.get(0); + List timestamps = getTimestampsFromFile(newestDeltaPath); + + if (timestamps.isEmpty()) { + LOGGER.warn("newest delta file has no timestamps: {}", newestDeltaPath); + return System.currentTimeMillis() / 1000; + } + + long newestTs = Collections.max(timestamps); + LOGGER.info("found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); + return newestTs; + } + + /** + * List all delta files from S3, sorted newest to oldest + */ + private List listDeltaFiles() { + try { + // List all objects with the delta prefix + List allFiles = cloudStorage.list(s3DeltaPrefix); + + // Filter to only .dat delta files and sort newest to oldest + List deltaFiles = allFiles.stream() + .filter(OptOutUtils::isDeltaFile) + .sorted(OptOutUtils.DeltaFilenameComparatorDescending) + .collect(Collectors.toList()); + + LOGGER.info("listed {} delta files from s3 (prefix={})", deltaFiles.size(), s3DeltaPrefix); + return deltaFiles; + + } catch (Exception e) { + LOGGER.error("failed to list delta files from s3 with prefix: {}", s3DeltaPrefix, e); + return Collections.emptyList(); + } + } + + /** + * Check if a delta file is already cached + */ + private boolean isCached(String s3Path) { + String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); + return deltaFileCache.containsKey(filename); + } + + /** + * Get timestamps from a delta file (S3 path), using cache if available + */ + private List getTimestampsFromFile(String s3Path) throws IOException { + // Extract filename from S3 path for cache key + String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); + + // Check cache first + FileRecordCache cached = deltaFileCache.get(filename); + if (cached != null) { + LOGGER.info("using cached timestamps for file: {}", filename); + return cached.timestamps; + } + + // Cache miss - download from S3 + LOGGER.info("downloading and reading timestamps from s3: {}", s3Path); + List timestamps = readTimestampsFromS3(s3Path); + + // Store in cache + deltaFileCache.put(filename, new FileRecordCache(timestamps)); + LOGGER.info("cached delta file: {} ({} records)", filename, timestamps.size()); + + return timestamps; + } + + /** + * Read all non-sentinel record timestamps from a delta file in S3 + */ + private List readTimestampsFromS3(String s3Path) throws IOException { + try (InputStream is = cloudStorage.download(s3Path)) { + byte[] data = is.readAllBytes(); + OptOutCollection collection = new OptOutCollection(data); + + List timestamps = new ArrayList<>(); + for (int i = 0; i < collection.size(); i++) { + OptOutEntry entry = collection.get(i); + + // Skip sentinel entries + if (entry.isSpecialHash()) { + continue; + } + + timestamps.add(entry.timestamp); + } + + return timestamps; + } catch (Exception e) { + LOGGER.error("failed to read delta file from s3: {}", s3Path, e); + throw new IOException("failed to read delta file from s3: " + s3Path, e); + } + } + + /** + * Calculate total duration of allowlist ranges that overlap with the given time window. + */ + long getAllowlistDuration(long t, long windowStart) { + long totalDuration = 0; + for (List range : this.allowlistRanges) { + long start = range.get(0); + long end = range.get(1); + + // Clip range to window boundaries + if (start < windowStart) { + start = windowStart; + } + if (end > t) { + end = t; + } + + // Only add duration if there's actual overlap (start < end) + if (start < end) { + totalDuration += end - start; + } + } + return totalDuration; + } + + /** + * Calculate the window start time that provides evaluationWindowSeconds of non-allowlisted time. + * Iteratively extends the window to account for allowlist ranges that may fall in extended portions. + */ + long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowSeconds) { + long allowlistDuration = getAllowlistDuration(newestDeltaTs, newestDeltaTs - evaluationWindowSeconds); + + // Each iteration discovers at least one new allowlist range, so max iterations = number of ranges + int maxIterations = this.allowlistRanges.size() + 1; + + for (int i = 0; i < maxIterations && allowlistDuration > 0; i++) { + long newWindowStart = newestDeltaTs - evaluationWindowSeconds - allowlistDuration; + long newAllowlistDuration = getAllowlistDuration(newestDeltaTs, newWindowStart); + + if (newAllowlistDuration == allowlistDuration) { + // No new allowlist time in extended portion, we've converged + break; + } + + allowlistDuration = newAllowlistDuration; + } + + return newestDeltaTs - evaluationWindowSeconds - allowlistDuration; + } + + /** + * Find the oldest SQS queue message timestamp + */ + private long findOldestQueueTimestamp(List sqsMessages) throws IOException { + long oldest = System.currentTimeMillis() / 1000; + + if (sqsMessages != null && !sqsMessages.isEmpty()) { + for (Message msg : sqsMessages) { + Long ts = extractTimestampFromMessage(msg); + if (ts != null && ts < oldest) { + oldest = ts; + } + } + } + + return oldest; + } + + /** + * Extract timestamp from SQS message (from SentTimestamp attribute) + */ + private Long extractTimestampFromMessage(Message msg) { + // Get SentTimestamp attribute (milliseconds) + String sentTimestamp = msg.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); + if (sentTimestamp != null) { + try { + return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds + } catch (NumberFormatException e) { + LOGGER.warn("invalid sentTimestamp: {}", sentTimestamp); + } + } + + // Fallback: use current time + return System.currentTimeMillis() / 1000; + } + + /** + * Count SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes + */ + private int countSqsMessages(List sqsMessages, long oldestQueueTs) { + + int count = 0; + long windowEnd = oldestQueueTs + 5 * 60; + + for (Message msg : sqsMessages) { + Long ts = extractTimestampFromMessage(msg); + + if (ts < oldestQueueTs || ts > windowEnd) { + continue; + } + + if (isInAllowlist(ts)) { + continue; + } + count++; + + } + + LOGGER.info("sqs messages: {} in window [oldestQueueTs={}, oldestQueueTs+5m={}]", count, oldestQueueTs, windowEnd); + return count; + } + + /** + * Check if a timestamp falls within any allowlist range + */ + boolean isInAllowlist(long timestamp) { + if (allowlistRanges == null || allowlistRanges.isEmpty()) { + return false; + } + + for (List range : allowlistRanges) { + if (range.size() < 2) { + continue; + } + + long start = range.get(0); + long end = range.get(1); + + if (timestamp >= start && timestamp <= end) { + return true; + } + } + + return false; + } + + /** + * Evict cache entries with data older than the cutoff timestamp + */ + private void evictOldCacheEntries(long cutoffTimestamp) { + int beforeSize = deltaFileCache.size(); + + deltaFileCache.entrySet().removeIf(entry -> + entry.getValue().newestTimestamp < cutoffTimestamp + ); + + int afterSize = deltaFileCache.size(); + if (beforeSize != afterSize) { + LOGGER.info("evicted {} old cache entries (before={}, after={})", + beforeSize - afterSize, beforeSize, afterSize); + } + } + + /** + * Determine traffic status based on current vs past counts + */ + TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { + if (baselineTraffic == 0 || thresholdMultiplier == 0) { + // Avoid division by zero - if no baseline traffic, return DEFAULT status + LOGGER.warn("baselineTraffic is 0 or thresholdMultiplier is 0, returning default status"); + return TrafficStatus.DEFAULT; + } + + if (sumCurrent >= thresholdMultiplier * baselineTraffic) { + LOGGER.error("delayed_processing threshold breached: sumCurrent={} >= {}×baselineTraffic={}", + sumCurrent, thresholdMultiplier, baselineTraffic); + return TrafficStatus.DELAYED_PROCESSING; + } + + LOGGER.info("traffic within normal range: sumCurrent={} < {}×baselineTraffic={}", + sumCurrent, thresholdMultiplier, baselineTraffic); + return TrafficStatus.DEFAULT; + } + + /** + * Get cache statistics for monitoring + */ + public Map getCacheStats() { + Map stats = new HashMap<>(); + stats.put("cached_files", deltaFileCache.size()); + + int totalTimestamps = deltaFileCache.values().stream() + .mapToInt(cache -> cache.timestamps.size()) + .sum(); + stats.put("total_cached_timestamps", totalTimestamps); + + return stats; + } + +} \ No newline at end of file diff --git a/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java b/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java index d77f561..afe0861 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java +++ b/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java @@ -115,7 +115,7 @@ public BatchProcessingResult processBatch(List messageBatch, int batchN * @return true if the message is at least deltaWindowSeconds old */ private boolean isMessageEligible(SqsParsedMessage message, long currentTime) { - return currentTime - message.getTimestamp() >= this.deltaWindowSeconds; + return currentTime - message.timestamp() >= this.deltaWindowSeconds; } /** @@ -140,7 +140,7 @@ List filterEligibleMessages(List messages, l */ private List identifyInvalidMessages(List originalBatch, List parsedBatch) { Set validIds = parsedBatch.stream() - .map(p -> p.getOriginalMessage().messageId()) + .map(p -> p.originalMessage().messageId()) .collect(Collectors.toSet()); return originalBatch.stream() diff --git a/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java index a25806b..afd651c 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java +++ b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Utility class for SQS message operations. @@ -90,7 +91,7 @@ public static QueueAttributes getQueueAttributes(SqsClient sqsClient, String que return queueAttributes; } catch (Exception e) { - LOGGER.info("error getting queue attributes", e); + LOGGER.info("sqs_error: error getting queue attributes", e); return null; } } @@ -126,8 +127,8 @@ public static List receiveMessagesFromSqs( .queueUrl(queueUrl) .maxNumberOfMessages(maxMessages) .visibilityTimeout(visibilityTimeout) - .waitTimeSeconds(0) // Non-blocking poll - .messageSystemAttributeNames(MessageSystemAttributeName.SENT_TIMESTAMP) // Request SQS system timestamp + .waitTimeSeconds(0) // non-blocking poll + .messageSystemAttributeNames(MessageSystemAttributeName.SENT_TIMESTAMP) // request sqs system timestamp .build(); ReceiveMessageResponse response = sqsClient.receiveMessage(receiveRequest); @@ -143,6 +144,7 @@ public static List receiveMessagesFromSqs( /** * Deletes messages from SQS in batches (max 10 per batch). + * Retries failed deletes as long as progress is being made. * * @param sqsClient The SQS client * @param queueUrl The queue URL @@ -154,40 +156,58 @@ public static void deleteMessagesFromSqs(SqsClient sqsClient, String queueUrl, L } try { - List entries = new ArrayList<>(); - int batchId = 0; int totalDeleted = 0; + List batch = new ArrayList<>(); - for (Message msg : messages) { - entries.add(DeleteMessageBatchRequestEntry.builder() - .id(String.valueOf(batchId++)) - .receiptHandle(msg.receiptHandle()) + for (int i = 0; i < messages.size(); i++) { + batch.add(DeleteMessageBatchRequestEntry.builder() + .id(String.valueOf(i)) + .receiptHandle(messages.get(i).receiptHandle()) .build()); - // Send batch when we reach 10 messages or at the end - if (entries.size() == SQS_MAX_DELETE_BATCH_SIZE || batchId == messages.size()) { - DeleteMessageBatchRequest deleteRequest = DeleteMessageBatchRequest.builder() - .queueUrl(queueUrl) - .entries(entries) - .build(); + // send batch when we reach 10 messages or end of list + if (batch.size() == SQS_MAX_DELETE_BATCH_SIZE || i == messages.size() - 1) { + totalDeleted += deleteBatchWithRetry(sqsClient, queueUrl, batch); + batch.clear(); + } + } - DeleteMessageBatchResponse deleteResponse = sqsClient.deleteMessageBatch(deleteRequest); + LOGGER.info("deleted {} messages", totalDeleted); + } catch (Exception e) { + LOGGER.error("sqs_error: error deleting messages", e); + } + } - if (!deleteResponse.failed().isEmpty()) { - LOGGER.error("sqs_error: failed to delete {} messages", deleteResponse.failed().size()); - } else { - totalDeleted += entries.size(); - } + /** Deletes batch, retrying failed entries. Retries once unconditionally, then only while making progress. */ + private static int deleteBatchWithRetry(SqsClient sqsClient, String queueUrl, List entries) { + int deleted = 0; + List toDelete = entries; + boolean retriedOnce = false; - entries.clear(); - } + while (!toDelete.isEmpty()) { + DeleteMessageBatchResponse response = sqsClient.deleteMessageBatch( + DeleteMessageBatchRequest.builder().queueUrl(queueUrl).entries(toDelete).build()); + + int succeeded = response.successful().size(); + deleted += succeeded; + + if (response.failed().isEmpty()) { + break; // all done } - LOGGER.info("deleted {} messages", totalDeleted); + // retry once unconditionally, then only if making progress + if (retriedOnce && succeeded == 0) { + LOGGER.error("sqs_error: {} deletes failed with no progress", response.failed().size()); + break; + } + retriedOnce = true; - } catch (Exception e) { - LOGGER.error("sqs_error: exception during message deletion", e); + // retry deletion on the failed messages only + var failedIds = response.failed().stream().map(BatchResultErrorEntry::id).collect(Collectors.toSet()); + toDelete = toDelete.stream().filter(e -> failedIds.contains(e.id())).toList(); } + + return deleted; } } diff --git a/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java b/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java index 315ff61..ac948b9 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java +++ b/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java @@ -23,24 +23,28 @@ public class SqsMessageParser { * @return List of parsed messages sorted by timestamp (oldest first) */ public static List parseAndSortMessages(List messages) { - List parsedMessages = new ArrayList<>(); + List parsedMessages = new ArrayList<>(messages.size()); for (Message message : messages) { - try { - // Extract SQS system timestamp (in milliseconds), or use current time as fallback - long timestampSeconds = extractTimestamp(message); - // Parse message body + String traceId = null; + + try { + // parse message body JsonObject body = new JsonObject(message.body()); + traceId = body.getString("trace_id"); + String identityHash = body.getString("identity_hash"); String advertisingId = body.getString("advertising_id"); - String traceId = body.getString("trace_id"); String clientIp = body.getString("client_ip"); String email = body.getString("email"); String phone = body.getString("phone"); + // extract sqs system timestamp (in milliseconds), or use current time as fallback + long timestampSeconds = extractTimestamp(message, traceId); + if (identityHash == null || advertisingId == null) { - LOGGER.error("sqs_error: invalid message format: {}", message.body()); + LOGGER.error("sqs_error: invalid message format, messageId={}, traceId={}", message.messageId(), traceId); continue; } @@ -48,18 +52,18 @@ public static List parseAndSortMessages(List messages byte[] idBytes = OptOutUtils.base64StringTobyteArray(advertisingId); if (hashBytes == null || idBytes == null) { - LOGGER.error("sqs_error: invalid base64 encoding"); + LOGGER.error("sqs_error: invalid base64 encoding, messageId={}, traceId={}", message.messageId(), traceId); continue; } parsedMessages.add(new SqsParsedMessage(message, hashBytes, idBytes, timestampSeconds, email, phone, clientIp, traceId)); } catch (Exception e) { - LOGGER.error("sqs_error: error parsing message", e); + LOGGER.error("sqs_error: error parsing message, messageId={}, traceId={}", message.messageId(), traceId, e); } } - // Sort by timestamp - parsedMessages.sort((a, b) -> Long.compare(a.getTimestamp(), b.getTimestamp())); + // sort by timestamp + parsedMessages.sort((a, b) -> Long.compare(a.timestamp(), b.timestamp())); return parsedMessages; } @@ -70,10 +74,10 @@ public static List parseAndSortMessages(List messages * @param message The SQS message * @return Timestamp in seconds */ - private static long extractTimestamp(Message message) { + private static long extractTimestamp(Message message, String traceId) { String sentTimestampStr = message.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); if (sentTimestampStr == null) { - LOGGER.info("message missing SentTimestamp, using current time"); + LOGGER.info("message missing SentTimestamp, using current time instead, messageId={}, traceId={}", message.messageId(), traceId); return OptOutUtils.nowEpochSeconds(); } return Long.parseLong(sentTimestampStr) / 1000; // ms to seconds diff --git a/src/main/java/com/uid2/optout/sqs/SqsParsedMessage.java b/src/main/java/com/uid2/optout/sqs/SqsParsedMessage.java index 850cd7a..e061144 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsParsedMessage.java +++ b/src/main/java/com/uid2/optout/sqs/SqsParsedMessage.java @@ -5,57 +5,13 @@ /** * Represents a parsed SQS message containing opt-out data. */ -public class SqsParsedMessage { - private final Message originalMessage; - private final byte[] hashBytes; - private final byte[] idBytes; - private final long timestamp; - private final String email; - private final String phone; - private final String clientIp; - private final String traceId; - - public SqsParsedMessage(Message originalMessage, byte[] hashBytes, byte[] idBytes, long timestamp, String email, String phone, String clientIp, String traceId) { - this.originalMessage = originalMessage; - this.hashBytes = hashBytes; - this.idBytes = idBytes; - this.timestamp = timestamp; - this.email = email; - this.phone = phone; - this.clientIp = clientIp; - this.traceId = traceId; - } - - public Message getOriginalMessage() { - return originalMessage; - } - - public byte[] getHashBytes() { - return hashBytes; - } - - public byte[] getIdBytes() { - return idBytes; - } - - public long getTimestamp() { - return timestamp; - } - - public String getEmail() { - return email; - } - - public String getPhone() { - return phone; - } - - public String getClientIp() { - return clientIp; - } - - public String getTraceId() { - return traceId; - } -} - +public record SqsParsedMessage( + Message originalMessage, + byte[] hashBytes, + byte[] idBytes, + long timestamp, + String email, + String phone, + String clientIp, + String traceId +) {} diff --git a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java index f5f74d8..f28c93d 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java +++ b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java @@ -58,8 +58,8 @@ public static WindowReadResult withMessages(List messages, lon return new WindowReadResult(messages, windowStart, StopReason.NONE, rawMessagesRead); } - public static WindowReadResult queueEmpty(List messages, long windowStart, int rawMessagesRead) { - return new WindowReadResult(messages, windowStart, StopReason.QUEUE_EMPTY, rawMessagesRead); + public static WindowReadResult queueEmpty(List messages, long windowStart) { + return new WindowReadResult(messages, windowStart, StopReason.QUEUE_EMPTY, 0); } public static WindowReadResult messagesTooRecent(List messages, long windowStart, int rawMessagesRead) { @@ -105,7 +105,7 @@ public WindowReadResult readWindow() { this.sqsClient, this.queueUrl, this.maxMessagesPerPoll, this.visibilityTimeout); if (rawBatch.isEmpty()) { - return WindowReadResult.queueEmpty(windowMessages, currentWindowStart, rawMessagesRead); + return WindowReadResult.queueEmpty(windowMessages, currentWindowStart); } rawMessagesRead += rawBatch.size(); @@ -124,7 +124,7 @@ public WindowReadResult readWindow() { // Add eligible messages to current window boolean newWindow = false; for (SqsParsedMessage msg : batchResult.getMessages()) { - long msgWindowStart = msg.getTimestamp(); + long msgWindowStart = msg.timestamp(); // Discover start of window if (currentWindowStart == 0) { diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index 910a2c6..f84556b 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -409,8 +409,8 @@ private DeltaProductionResult produceBatchedDeltas() throws IOException { // Write all messages List sqsMessages = new ArrayList<>(); for (SqsParsedMessage msg : messages) { - writeOptOutEntry(deltaStream, msg.getHashBytes(), msg.getIdBytes(), msg.getTimestamp()); - sqsMessages.add(msg.getOriginalMessage()); + writeOptOutEntry(deltaStream, msg.hashBytes(), msg.idBytes(), msg.timestamp()); + sqsMessages.add(msg.originalMessage()); } // Upload and delete diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java index e8bd04b..866f5aa 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -151,8 +151,8 @@ List parseFilterRules(JsonObject config) throws MalformedTraf } public boolean isDenylisted(SqsParsedMessage message) { - long timestamp = message.getTimestamp(); - String clientIp = message.getClientIp(); + long timestamp = message.timestamp(); + String clientIp = message.clientIp(); if (clientIp == null || clientIp.isEmpty()) { LOGGER.error("Request does not contain client IP, timestamp={}", timestamp); diff --git a/src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java b/src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java index 0a656e9..4444faf 100644 --- a/src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java +++ b/src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java @@ -97,7 +97,7 @@ public BatchProcessingResult processBatch(List messageBatch, int batchN // Check if the oldest message in this batch is too recent long currentTime = OptOutUtils.nowEpochSeconds(); SqsParsedMessage oldestMessage = parsedBatch.get(0); - long messageAge = currentTime - oldestMessage.getTimestamp(); + long messageAge = currentTime - oldestMessage.timestamp(); if (messageAge < this.deltaWindowSeconds) { // Signal to stop processing - messages are too recent @@ -129,7 +129,7 @@ public List filterEligibleMessages( List eligibleMessages = new ArrayList<>(); for (SqsParsedMessage pm : messages) { - if (currentTime - pm.getTimestamp() >= this.deltaWindowSeconds) { + if (currentTime - pm.timestamp() >= this.deltaWindowSeconds) { eligibleMessages.add(pm); } } @@ -148,7 +148,7 @@ private List identifyInvalidMessages(List originalBatch, List< // Create a set of message IDs from successfully parsed messages Set validMessageIds = new HashSet<>(); for (SqsParsedMessage parsed : parsedBatch) { - validMessageIds.add(parsed.getOriginalMessage().messageId()); + validMessageIds.add(parsed.originalMessage().messageId()); } // Find messages that were not successfully parsed diff --git a/src/main/java/com/uid2/optout/vertx/SqsMessageParser.java b/src/main/java/com/uid2/optout/vertx/SqsMessageParser.java index 44a6c5e..279f6b2 100644 --- a/src/main/java/com/uid2/optout/vertx/SqsMessageParser.java +++ b/src/main/java/com/uid2/optout/vertx/SqsMessageParser.java @@ -59,7 +59,7 @@ public static List parseAndSortMessages(List messages } // Sort by timestamp - parsedMessages.sort((a, b) -> Long.compare(a.getTimestamp(), b.getTimestamp())); + parsedMessages.sort((a, b) -> Long.compare(a.timestamp(), b.timestamp())); return parsedMessages; } diff --git a/src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java b/src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java index 1ad8ba7..1714b56 100644 --- a/src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java +++ b/src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java @@ -5,57 +5,13 @@ /** * Represents a parsed SQS message containing opt-out data. */ -public class SqsParsedMessage { - private final Message originalMessage; - private final byte[] hashBytes; - private final byte[] idBytes; - private final long timestamp; - private final String email; - private final String phone; - private final String clientIp; - private final String traceId; - - public SqsParsedMessage(Message originalMessage, byte[] hashBytes, byte[] idBytes, long timestamp, String email, String phone, String clientIp, String traceId) { - this.originalMessage = originalMessage; - this.hashBytes = hashBytes; - this.idBytes = idBytes; - this.timestamp = timestamp; - this.email = email; - this.phone = phone; - this.clientIp = clientIp; - this.traceId = traceId; - } - - public Message getOriginalMessage() { - return originalMessage; - } - - public byte[] getHashBytes() { - return hashBytes; - } - - public byte[] getIdBytes() { - return idBytes; - } - - public long getTimestamp() { - return timestamp; - } - - public String getEmail() { - return email; - } - - public String getPhone() { - return phone; - } - - public String getClientIp() { - return clientIp; - } - - public String getTraceId() { - return traceId; - } -} - +public record SqsParsedMessage( + Message originalMessage, + byte[] hashBytes, + byte[] idBytes, + long timestamp, + String email, + String phone, + String clientIp, + String traceId +) {} diff --git a/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java b/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java index 75368c6..355669b 100644 --- a/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java +++ b/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java @@ -104,7 +104,7 @@ public WindowReadResult readWindow() { // Add eligible messages to current window boolean newWindow = false; for (SqsParsedMessage msg : batchResult.getEligibleMessages()) { - long msgWindowStart = (msg.getTimestamp() / this.deltaWindowSeconds) * this.deltaWindowSeconds; + long msgWindowStart = (msg.timestamp() / this.deltaWindowSeconds) * this.deltaWindowSeconds; // discover start of window if (currentWindowStart == 0) { diff --git a/src/test/java/com/uid2/optout/sqs/SqsBatchProcessorTest.java b/src/test/java/com/uid2/optout/sqs/SqsBatchProcessorTest.java index 778f858..312eda0 100644 --- a/src/test/java/com/uid2/optout/sqs/SqsBatchProcessorTest.java +++ b/src/test/java/com/uid2/optout/sqs/SqsBatchProcessorTest.java @@ -127,8 +127,8 @@ public void testFilterEligibleMessages_boundaryCases() { // Should only include the last two (>= threshold) assertEquals(2, result.size()); - assertEquals(currentTime - windowSeconds, result.get(0).getTimestamp()); - assertEquals(currentTime - windowSeconds - 1, result.get(1).getTimestamp()); + assertEquals(currentTime - windowSeconds, result.get(0).timestamp()); + assertEquals(currentTime - windowSeconds - 1, result.get(1).timestamp()); } @Test @@ -148,9 +148,9 @@ public void testFilterEligibleMessages_preservesOrder() { assertEquals(3, result.size()); // Verify order is preserved - assertEquals(100, result.get(0).getTimestamp()); - assertEquals(200, result.get(1).getTimestamp()); - assertEquals(300, result.get(2).getTimestamp()); + assertEquals(100, result.get(0).timestamp()); + assertEquals(200, result.get(1).timestamp()); + assertEquals(300, result.get(2).timestamp()); } @Test diff --git a/src/test/java/com/uid2/optout/sqs/SqsMessageOperationsTest.java b/src/test/java/com/uid2/optout/sqs/SqsMessageOperationsTest.java new file mode 100644 index 0000000..0988c2e --- /dev/null +++ b/src/test/java/com/uid2/optout/sqs/SqsMessageOperationsTest.java @@ -0,0 +1,316 @@ +package com.uid2.optout.sqs; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +public class SqsMessageOperationsTest { + + private SqsClient mockSqsClient; + private static final String TEST_QUEUE_URL = "https://sqs.us-east-1.amazonaws.com/123456789/test-queue"; + + @BeforeEach + void setUp() { + mockSqsClient = mock(SqsClient.class); + } + + // ==================== getQueueAttributes tests ==================== + + @Test + void testGetQueueAttributes_success() { + GetQueueAttributesResponse response = GetQueueAttributesResponse.builder() + .attributes(Map.of( + QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES, "100", + QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_NOT_VISIBLE, "50", + QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED, "25" + )) + .build(); + when(mockSqsClient.getQueueAttributes(any(GetQueueAttributesRequest.class))).thenReturn(response); + + SqsMessageOperations.QueueAttributes attrs = SqsMessageOperations.getQueueAttributes(mockSqsClient, TEST_QUEUE_URL); + + assertNotNull(attrs); + assertEquals(100, attrs.getApproximateNumberOfMessages()); + assertEquals(50, attrs.getApproximateNumberOfMessagesNotVisible()); + assertEquals(25, attrs.getApproximateNumberOfMessagesDelayed()); + assertEquals(175, attrs.getTotalMessages()); + } + + @Test + void testGetQueueAttributes_exception() { + when(mockSqsClient.getQueueAttributes(any(GetQueueAttributesRequest.class))) + .thenThrow(new RuntimeException("SQS error")); + + SqsMessageOperations.QueueAttributes attrs = SqsMessageOperations.getQueueAttributes(mockSqsClient, TEST_QUEUE_URL); + + assertNull(attrs); + } + + @Test + void testGetQueueAttributes_missingAttributes() { + GetQueueAttributesResponse response = GetQueueAttributesResponse.builder() + .attributes(Map.of()) // empty + .build(); + when(mockSqsClient.getQueueAttributes(any(GetQueueAttributesRequest.class))).thenReturn(response); + + SqsMessageOperations.QueueAttributes attrs = SqsMessageOperations.getQueueAttributes(mockSqsClient, TEST_QUEUE_URL); + + assertNotNull(attrs); + assertEquals(0, attrs.getApproximateNumberOfMessages()); + assertEquals(0, attrs.getTotalMessages()); + } + + // ==================== receiveMessagesFromSqs tests ==================== + + @Test + void testReceiveMessages_success() { + List messages = List.of( + Message.builder().messageId("1").receiptHandle("r1").build(), + Message.builder().messageId("2").receiptHandle("r2").build() + ); + ReceiveMessageResponse response = ReceiveMessageResponse.builder().messages(messages).build(); + when(mockSqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(response); + + List result = SqsMessageOperations.receiveMessagesFromSqs(mockSqsClient, TEST_QUEUE_URL, 10, 30); + + assertEquals(2, result.size()); + } + + @Test + void testReceiveMessages_exception() { + when(mockSqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenThrow(new RuntimeException("SQS error")); + + List result = SqsMessageOperations.receiveMessagesFromSqs(mockSqsClient, TEST_QUEUE_URL, 10, 30); + + assertTrue(result.isEmpty()); + } + + // ==================== deleteMessagesFromSqs tests ==================== + + @Test + void testDeleteMessages_emptyList() { + SqsMessageOperations.deleteMessagesFromSqs(mockSqsClient, TEST_QUEUE_URL, List.of()); + + verify(mockSqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); + } + + @Test + void testDeleteMessages_allSucceed() { + List messages = createMessages(3); + DeleteMessageBatchResponse response = DeleteMessageBatchResponse.builder() + .successful( + DeleteMessageBatchResultEntry.builder().id("0").build(), + DeleteMessageBatchResultEntry.builder().id("1").build(), + DeleteMessageBatchResultEntry.builder().id("2").build() + ) + .failed(List.of()) + .build(); + when(mockSqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))).thenReturn(response); + + SqsMessageOperations.deleteMessagesFromSqs(mockSqsClient, TEST_QUEUE_URL, messages); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); + verify(mockSqsClient, times(1)).deleteMessageBatch(captor.capture()); + assertEquals(3, captor.getValue().entries().size()); + } + + @Test + void testDeleteMessages_someFailThenSucceedOnRetry() { + List messages = createMessages(3); + + // First call: 2 succeed, 1 fails + DeleteMessageBatchResponse firstResponse = DeleteMessageBatchResponse.builder() + .successful( + DeleteMessageBatchResultEntry.builder().id("0").build(), + DeleteMessageBatchResultEntry.builder().id("1").build() + ) + .failed(BatchResultErrorEntry.builder().id("2").code("Error").build()) + .build(); + + // Second call (retry): the failed one succeeds + DeleteMessageBatchResponse secondResponse = DeleteMessageBatchResponse.builder() + .successful(DeleteMessageBatchResultEntry.builder().id("2").build()) + .failed(List.of()) + .build(); + + when(mockSqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(firstResponse) + .thenReturn(secondResponse); + + SqsMessageOperations.deleteMessagesFromSqs(mockSqsClient, TEST_QUEUE_URL, messages); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); + verify(mockSqsClient, times(2)).deleteMessageBatch(captor.capture()); + List requests = captor.getAllValues(); + assertEquals(3, requests.get(0).entries().size()); // first attempt: all 3 + assertEquals(1, requests.get(1).entries().size()); // retry: only the failed one + } + + @Test + void testDeleteMessages_allFailRetryOnceUnconditionally() { + List messages = createMessages(2); + + // All fail on first and second attempt + DeleteMessageBatchResponse failResponse = DeleteMessageBatchResponse.builder() + .successful(List.of()) + .failed( + BatchResultErrorEntry.builder().id("0").code("Error").build(), + BatchResultErrorEntry.builder().id("1").code("Error").build() + ) + .build(); + + when(mockSqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))).thenReturn(failResponse); + + SqsMessageOperations.deleteMessagesFromSqs(mockSqsClient, TEST_QUEUE_URL, messages); + + // Should retry once even with no progress, then stop + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); + verify(mockSqsClient, times(2)).deleteMessageBatch(captor.capture()); + List requests = captor.getAllValues(); + assertEquals(2, requests.get(0).entries().size()); // first attempt: all 2 + assertEquals(2, requests.get(1).entries().size()); // retry: still all 2 (none succeeded) + } + + @Test + void testDeleteMessages_retryWhileMakingProgress() { + List messages = createMessages(3); + + // First call: 1 succeeds, 2 fail + DeleteMessageBatchResponse first = DeleteMessageBatchResponse.builder() + .successful(DeleteMessageBatchResultEntry.builder().id("0").build()) + .failed( + BatchResultErrorEntry.builder().id("1").code("Error").build(), + BatchResultErrorEntry.builder().id("2").code("Error").build() + ) + .build(); + + // Second call: 1 succeeds, 1 fails (still making progress) + DeleteMessageBatchResponse second = DeleteMessageBatchResponse.builder() + .successful(DeleteMessageBatchResultEntry.builder().id("1").build()) + .failed(BatchResultErrorEntry.builder().id("2").code("Error").build()) + .build(); + + // Third call: last one succeeds + DeleteMessageBatchResponse third = DeleteMessageBatchResponse.builder() + .successful(DeleteMessageBatchResultEntry.builder().id("2").build()) + .failed(List.of()) + .build(); + + when(mockSqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(first) + .thenReturn(second) + .thenReturn(third); + + SqsMessageOperations.deleteMessagesFromSqs(mockSqsClient, TEST_QUEUE_URL, messages); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); + verify(mockSqsClient, times(3)).deleteMessageBatch(captor.capture()); + List requests = captor.getAllValues(); + assertEquals(3, requests.get(0).entries().size()); // first: all 3 + assertEquals(2, requests.get(1).entries().size()); // second: 2 failed + assertEquals(1, requests.get(2).entries().size()); // third: 1 failed + } + + @Test + void testDeleteMessages_stopWhenNoProgressAfterRetry() { + List messages = createMessages(2); + + // First call: 1 succeeds, 1 fails + DeleteMessageBatchResponse first = DeleteMessageBatchResponse.builder() + .successful(DeleteMessageBatchResultEntry.builder().id("0").build()) + .failed(BatchResultErrorEntry.builder().id("1").code("Error").build()) + .build(); + + // Second call (retry): still fails, no progress + DeleteMessageBatchResponse second = DeleteMessageBatchResponse.builder() + .successful(List.of()) + .failed(BatchResultErrorEntry.builder().id("1").code("Error").build()) + .build(); + + when(mockSqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(first) + .thenReturn(second); + + SqsMessageOperations.deleteMessagesFromSqs(mockSqsClient, TEST_QUEUE_URL, messages); + + // Should stop after second call since no progress was made + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); + verify(mockSqsClient, times(2)).deleteMessageBatch(captor.capture()); + List requests = captor.getAllValues(); + assertEquals(2, requests.get(0).entries().size()); // first: all 2 + assertEquals(1, requests.get(1).entries().size()); // retry: only the failed one + } + + @Test + void testDeleteMessages_batchesMoreThan10Messages() { + List messages = createMessages(15); + + DeleteMessageBatchResponse successResponse = DeleteMessageBatchResponse.builder() + .successful(List.of( + DeleteMessageBatchResultEntry.builder().id("0").build() + )) + .failed(List.of()) + .build(); + + when(mockSqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))).thenReturn(successResponse); + + SqsMessageOperations.deleteMessagesFromSqs(mockSqsClient, TEST_QUEUE_URL, messages); + + // Should be called twice: once for 10 messages, once for 5 + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); + verify(mockSqsClient, times(2)).deleteMessageBatch(captor.capture()); + + List requests = captor.getAllValues(); + assertEquals(10, requests.get(0).entries().size()); + assertEquals(5, requests.get(1).entries().size()); + } + + @Test + void testDeleteMessages_exception() { + List messages = createMessages(3); + when(mockSqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenThrow(new RuntimeException("SQS error")); + + // Should not throw, just log error + assertDoesNotThrow(() -> + SqsMessageOperations.deleteMessagesFromSqs(mockSqsClient, TEST_QUEUE_URL, messages)); + } + + // ==================== QueueAttributes tests ==================== + + @Test + void testQueueAttributes_toString() { + SqsMessageOperations.QueueAttributes attrs = new SqsMessageOperations.QueueAttributes(100, 50, 25); + + String str = attrs.toString(); + + assertTrue(str.contains("visible=100")); + assertTrue(str.contains("invisible=50")); + assertTrue(str.contains("delayed=25")); + assertTrue(str.contains("total=175")); + } + + // ==================== Helper methods ==================== + + private List createMessages(int count) { + List messages = new ArrayList<>(); + for (int i = 0; i < count; i++) { + messages.add(Message.builder() + .messageId("msg-" + i) + .receiptHandle("receipt-" + i) + .build()); + } + return messages; + } +} diff --git a/src/test/java/com/uid2/optout/sqs/SqsMessageParserTest.java b/src/test/java/com/uid2/optout/sqs/SqsMessageParserTest.java index 5d6810d..0f93c8b 100644 --- a/src/test/java/com/uid2/optout/sqs/SqsMessageParserTest.java +++ b/src/test/java/com/uid2/optout/sqs/SqsMessageParserTest.java @@ -58,9 +58,9 @@ public void testParseAndSortMessages_validMessages() { assertEquals(3, result.size()); // Verify sorting (oldest first) - assertEquals(TEST_TIMESTAMP_SEC, result.get(0).getTimestamp()); - assertEquals(TEST_TIMESTAMP_SEC + 1, result.get(1).getTimestamp()); - assertEquals(TEST_TIMESTAMP_SEC + 2, result.get(2).getTimestamp()); + assertEquals(TEST_TIMESTAMP_SEC, result.get(0).timestamp()); + assertEquals(TEST_TIMESTAMP_SEC + 1, result.get(1).timestamp()); + assertEquals(TEST_TIMESTAMP_SEC + 2, result.get(2).timestamp()); } @Test @@ -159,7 +159,7 @@ public void testParseAndSortMessages_missingTimestamp() { assertEquals(1, result.size()); // Timestamp should be close to current time (within 10 seconds) long currentTime = System.currentTimeMillis() / 1000; - assertTrue(Math.abs(result.get(0).getTimestamp() - currentTime) < 10); + assertTrue(Math.abs(result.get(0).timestamp() - currentTime) < 10); } @Test @@ -173,8 +173,8 @@ public void testParseAndSortMessages_mixValidAndInvalid() { List result = SqsMessageParser.parseAndSortMessages(messages); assertEquals(2, result.size()); // Only valid messages - assertEquals(TEST_TIMESTAMP_SEC, result.get(0).getTimestamp()); - assertEquals(TEST_TIMESTAMP_SEC + 1, result.get(1).getTimestamp()); + assertEquals(TEST_TIMESTAMP_SEC, result.get(0).timestamp()); + assertEquals(TEST_TIMESTAMP_SEC + 1, result.get(1).timestamp()); } @Test @@ -185,7 +185,7 @@ public void testParseAndSortMessages_timestampConversion() { List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(message)); assertEquals(1, result.size()); - assertEquals(1699308912L, result.get(0).getTimestamp()); // Should be in seconds + assertEquals(1699308912L, result.get(0).timestamp()); // Should be in seconds } @Test @@ -195,7 +195,7 @@ public void testParseAndSortMessages_preservesOriginalMessage() { List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(originalMessage)); assertEquals(1, result.size()); - assertSame(originalMessage, result.get(0).getOriginalMessage()); + assertSame(originalMessage, result.get(0).originalMessage()); } @Test @@ -214,7 +214,7 @@ public void testParseAndSortMessages_sortingOrder() { assertEquals(5, result.size()); // Verify ascending order for (int i = 1; i < result.size(); i++) { - assertTrue(result.get(i - 1).getTimestamp() <= result.get(i).getTimestamp(), + assertTrue(result.get(i - 1).timestamp() <= result.get(i).timestamp(), "Messages should be sorted in ascending order by timestamp"); } } @@ -226,10 +226,10 @@ public void testParseAndSortMessages_parsesHashAndIdBytes() { List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(message)); assertEquals(1, result.size()); - assertNotNull(result.get(0).getHashBytes()); - assertNotNull(result.get(0).getIdBytes()); - assertEquals(32, result.get(0).getHashBytes().length); - assertEquals(32, result.get(0).getIdBytes().length); + assertNotNull(result.get(0).hashBytes()); + assertNotNull(result.get(0).idBytes()); + assertEquals(32, result.get(0).hashBytes().length); + assertEquals(32, result.get(0).idBytes().length); } @Test @@ -265,7 +265,7 @@ public void testParseAndSortMessages_multipleValidMessages() { assertEquals(100, result.size()); // Verify all are sorted for (int i = 1; i < result.size(); i++) { - assertTrue(result.get(i - 1).getTimestamp() <= result.get(i).getTimestamp()); + assertTrue(result.get(i - 1).timestamp() <= result.get(i).timestamp()); } } diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index a46023f..13e9c9c 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -16,6 +16,7 @@ import java.io.InputStream; import java.util.*; +import java.util.concurrent.CountDownLatch; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -325,6 +326,9 @@ public void testDeltaProduceEndpoint_unauthorized(TestContext context) { public void testDeltaProduceEndpoint_concurrentJobPrevention(TestContext context) throws Exception { Async async = context.async(); + // Latch to keep job running until we verify the conflict response + CountDownLatch uploadLatch = new CountDownLatch(1); + // Create messages that will take some time to process long oldTime = System.currentTimeMillis() - 400_000; List messages = Arrays.asList( @@ -339,7 +343,11 @@ public void testDeltaProduceEndpoint_concurrentJobPrevention(TestContext context when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) .thenReturn(DeleteMessageBatchResponse.builder().build()); - doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString()); + // Block upload until latch is released + doAnswer(inv -> { + uploadLatch.await(); + return null; + }).when(cloudStorage).upload(any(InputStream.class), anyString()); int port = Const.Port.ServicePortForOptOut + 1; @@ -375,6 +383,8 @@ public void testDeltaProduceEndpoint_concurrentJobPrevention(TestContext context context.assertEquals("conflict", response.getString("status")); context.assertTrue(response.getString("message").contains("already running")); + // Release the latch to let the first job complete + uploadLatch.countDown(); async.complete(); })); } diff --git a/src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java b/src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java index 6fc1c86..3e371cd 100644 --- a/src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java +++ b/src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java @@ -126,8 +126,8 @@ public void testFilterEligibleMessages_boundaryCases() { // Should only include the last two (>= threshold) assertEquals(2, result.size()); - assertEquals(currentTime - windowSeconds, result.get(0).getTimestamp()); - assertEquals(currentTime - windowSeconds - 1, result.get(1).getTimestamp()); + assertEquals(currentTime - windowSeconds, result.get(0).timestamp()); + assertEquals(currentTime - windowSeconds - 1, result.get(1).timestamp()); } @Test @@ -147,9 +147,9 @@ public void testFilterEligibleMessages_preservesOrder() { assertEquals(3, result.size()); // Verify order is preserved - assertEquals(100, result.get(0).getTimestamp()); - assertEquals(200, result.get(1).getTimestamp()); - assertEquals(300, result.get(2).getTimestamp()); + assertEquals(100, result.get(0).timestamp()); + assertEquals(200, result.get(1).timestamp()); + assertEquals(300, result.get(2).timestamp()); } @Test diff --git a/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java b/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java index 810a7a4..bcc615f 100644 --- a/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java +++ b/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java @@ -57,9 +57,9 @@ public void testParseAndSortMessages_validMessages() { assertEquals(3, result.size()); // Verify sorting (oldest first) - assertEquals(TEST_TIMESTAMP_SEC, result.get(0).getTimestamp()); - assertEquals(TEST_TIMESTAMP_SEC + 1, result.get(1).getTimestamp()); - assertEquals(TEST_TIMESTAMP_SEC + 2, result.get(2).getTimestamp()); + assertEquals(TEST_TIMESTAMP_SEC, result.get(0).timestamp()); + assertEquals(TEST_TIMESTAMP_SEC + 1, result.get(1).timestamp()); + assertEquals(TEST_TIMESTAMP_SEC + 2, result.get(2).timestamp()); } @Test @@ -158,7 +158,7 @@ public void testParseAndSortMessages_missingTimestamp() { assertEquals(1, result.size()); // Timestamp should be close to current time (within 10 seconds) long currentTime = System.currentTimeMillis() / 1000; - assertTrue(Math.abs(result.get(0).getTimestamp() - currentTime) < 10); + assertTrue(Math.abs(result.get(0).timestamp() - currentTime) < 10); } @Test @@ -172,8 +172,8 @@ public void testParseAndSortMessages_mixValidAndInvalid() { List result = SqsMessageParser.parseAndSortMessages(messages); assertEquals(2, result.size()); // Only valid messages - assertEquals(TEST_TIMESTAMP_SEC, result.get(0).getTimestamp()); - assertEquals(TEST_TIMESTAMP_SEC + 1, result.get(1).getTimestamp()); + assertEquals(TEST_TIMESTAMP_SEC, result.get(0).timestamp()); + assertEquals(TEST_TIMESTAMP_SEC + 1, result.get(1).timestamp()); } @Test @@ -184,7 +184,7 @@ public void testParseAndSortMessages_timestampConversion() { List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(message)); assertEquals(1, result.size()); - assertEquals(1699308912L, result.get(0).getTimestamp()); // Should be in seconds + assertEquals(1699308912L, result.get(0).timestamp()); // Should be in seconds } @Test @@ -194,7 +194,7 @@ public void testParseAndSortMessages_preservesOriginalMessage() { List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(originalMessage)); assertEquals(1, result.size()); - assertSame(originalMessage, result.get(0).getOriginalMessage()); + assertSame(originalMessage, result.get(0).originalMessage()); } @Test @@ -213,7 +213,7 @@ public void testParseAndSortMessages_sortingOrder() { assertEquals(5, result.size()); // Verify ascending order for (int i = 1; i < result.size(); i++) { - assertTrue(result.get(i - 1).getTimestamp() <= result.get(i).getTimestamp(), + assertTrue(result.get(i - 1).timestamp() <= result.get(i).timestamp(), "Messages should be sorted in ascending order by timestamp"); } } @@ -225,10 +225,10 @@ public void testParseAndSortMessages_parsesHashAndIdBytes() { List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(message)); assertEquals(1, result.size()); - assertNotNull(result.get(0).getHashBytes()); - assertNotNull(result.get(0).getIdBytes()); - assertEquals(32, result.get(0).getHashBytes().length); - assertEquals(32, result.get(0).getIdBytes().length); + assertNotNull(result.get(0).hashBytes()); + assertNotNull(result.get(0).idBytes()); + assertEquals(32, result.get(0).hashBytes().length); + assertEquals(32, result.get(0).idBytes().length); } @Test @@ -264,7 +264,7 @@ public void testParseAndSortMessages_multipleValidMessages() { assertEquals(100, result.size()); // Verify all are sorted for (int i = 1; i < result.size(); i++) { - assertTrue(result.get(i - 1).getTimestamp() <= result.get(i).getTimestamp()); + assertTrue(result.get(i - 1).timestamp() <= result.get(i).timestamp()); } } From 193ab199fec8b43498a729c703445cd194e578bd Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 14:21:05 -0700 Subject: [PATCH 03/25] update tests --- .../com/uid2/optout/sqs/SqsWindowReader.java | 6 +- .../uid2/optout/sqs/SqsWindowReaderTest.java | 285 ++++++++++++++++++ .../vertx/OptOutSqsLogProducerTest.java | 2 +- 3 files changed, 289 insertions(+), 4 deletions(-) create mode 100644 src/test/java/com/uid2/optout/sqs/SqsWindowReaderTest.java diff --git a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java index f28c93d..3a9fe71 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java +++ b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java @@ -58,8 +58,8 @@ public static WindowReadResult withMessages(List messages, lon return new WindowReadResult(messages, windowStart, StopReason.NONE, rawMessagesRead); } - public static WindowReadResult queueEmpty(List messages, long windowStart) { - return new WindowReadResult(messages, windowStart, StopReason.QUEUE_EMPTY, 0); + public static WindowReadResult queueEmpty(List messages, long windowStart, int rawMessagesRead) { + return new WindowReadResult(messages, windowStart, StopReason.QUEUE_EMPTY, rawMessagesRead); } public static WindowReadResult messagesTooRecent(List messages, long windowStart, int rawMessagesRead) { @@ -105,7 +105,7 @@ public WindowReadResult readWindow() { this.sqsClient, this.queueUrl, this.maxMessagesPerPoll, this.visibilityTimeout); if (rawBatch.isEmpty()) { - return WindowReadResult.queueEmpty(windowMessages, currentWindowStart); + return WindowReadResult.queueEmpty(windowMessages, currentWindowStart, rawMessagesRead); } rawMessagesRead += rawBatch.size(); diff --git a/src/test/java/com/uid2/optout/sqs/SqsWindowReaderTest.java b/src/test/java/com/uid2/optout/sqs/SqsWindowReaderTest.java new file mode 100644 index 0000000..e1c2539 --- /dev/null +++ b/src/test/java/com/uid2/optout/sqs/SqsWindowReaderTest.java @@ -0,0 +1,285 @@ +package com.uid2.optout.sqs; + +import com.uid2.optout.delta.StopReason; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.*; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +public class SqsWindowReaderTest { + + private SqsClient mockSqsClient; + private SqsWindowReader windowReader; + + private static final String TEST_QUEUE_URL = "https://sqs.test.amazonaws.com/123456789/test"; + private static final String VALID_HASH_BASE64 = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; // 32 bytes + private static final String VALID_ID_BASE64 = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE="; // 32 bytes + private static final int MAX_MESSAGES_PER_POLL = 10; + private static final int VISIBILITY_TIMEOUT = 240; + private static final int DELTA_WINDOW_SECONDS = 300; // 5 minutes + private static final int MAX_MESSAGES_PER_WINDOW = 100; + + @BeforeEach + void setUp() { + mockSqsClient = mock(SqsClient.class); + windowReader = new SqsWindowReader( + mockSqsClient, TEST_QUEUE_URL, MAX_MESSAGES_PER_POLL, + VISIBILITY_TIMEOUT, DELTA_WINDOW_SECONDS, MAX_MESSAGES_PER_WINDOW + ); + } + + @Test + void testReadWindow_emptyQueue() { + when(mockSqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(List.of()).build()); + + SqsWindowReader.WindowReadResult result = windowReader.readWindow(); + + assertTrue(result.isEmpty()); + assertEquals(StopReason.QUEUE_EMPTY, result.getStopReason()); + assertEquals(0, result.getRawMessagesRead()); + } + + @Test + void testReadWindow_singleBatchSingleWindow() { + long windowStartSeconds = System.currentTimeMillis() / 1000 - 600; // 10 minutes ago + List messages = Arrays.asList( + createMessage(windowStartSeconds + 10), + createMessage(windowStartSeconds + 50), + createMessage(windowStartSeconds + 100) + ); + + when(mockSqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(List.of()).build()); + + SqsWindowReader.WindowReadResult result = windowReader.readWindow(); + + assertEquals(3, result.getMessages().size()); + assertEquals(StopReason.QUEUE_EMPTY, result.getStopReason()); + assertEquals(3, result.getRawMessagesRead()); + } + + @Test + void testReadWindow_multipleBatchesSameWindow() { + long windowStartSeconds = System.currentTimeMillis() / 1000 - 600; // 10 minutes ago + + List batch1 = Arrays.asList( + createMessage(windowStartSeconds + 10), + createMessage(windowStartSeconds + 20) + ); + List batch2 = Arrays.asList( + createMessage(windowStartSeconds + 100), + createMessage(windowStartSeconds + 150) + ); + + when(mockSqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(batch1).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(batch2).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(List.of()).build()); + + SqsWindowReader.WindowReadResult result = windowReader.readWindow(); + + assertEquals(4, result.getMessages().size()); + assertEquals(StopReason.QUEUE_EMPTY, result.getStopReason()); + assertEquals(4, result.getRawMessagesRead()); + } + + @Test + void testReadWindow_messagesTooRecent() { + long currentTimeMs = System.currentTimeMillis(); + List messages = Arrays.asList( + createMessageWithTimestampMs(currentTimeMs - 1000), // 1 second ago + createMessageWithTimestampMs(currentTimeMs - 2000) // 2 seconds ago + ); + + when(mockSqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()); + + SqsWindowReader.WindowReadResult result = windowReader.readWindow(); + + assertTrue(result.isEmpty()); + assertEquals(StopReason.MESSAGES_TOO_RECENT, result.getStopReason()); + assertEquals(2, result.getRawMessagesRead()); + } + + @Test + void testReadWindow_messageLimitExceeded() { + SqsWindowReader smallLimitReader = new SqsWindowReader( + mockSqsClient, TEST_QUEUE_URL, MAX_MESSAGES_PER_POLL, + VISIBILITY_TIMEOUT, DELTA_WINDOW_SECONDS, 5 // Only 5 messages max + ); + + long windowStartSeconds = System.currentTimeMillis() / 1000 - 600; + List batch = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + batch.add(createMessage(windowStartSeconds + i * 10)); + } + + when(mockSqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(batch).build()); + + SqsWindowReader.WindowReadResult result = smallLimitReader.readWindow(); + + assertEquals(StopReason.MESSAGE_LIMIT_EXCEEDED, result.getStopReason()); + assertTrue(result.getMessages().size() >= 5); + } + + @Test + void testReadWindow_discoversNewWindow() { + long window1StartSeconds = System.currentTimeMillis() / 1000 - 900; // 15 minutes ago + long window2StartSeconds = window1StartSeconds + DELTA_WINDOW_SECONDS + 100; // Next window + + List messages = Arrays.asList( + createMessage(window1StartSeconds + 10), + createMessage(window1StartSeconds + 50), + createMessage(window2StartSeconds + 10) // Next window + ); + + when(mockSqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()); + + SqsWindowReader.WindowReadResult result = windowReader.readWindow(); + + assertEquals(3, result.getMessages().size()); + assertEquals(StopReason.NONE, result.getStopReason()); + assertEquals(window1StartSeconds + 10, result.getWindowStart()); + } + + @Test + void testReadWindow_multipleWindowsMultipleBatchesPerWindow() { + // Window 1: 2 batches, then discovers window 2 + // Window 2: 2 batches (must be > 5 min old for eligibility) + long window1StartSeconds = System.currentTimeMillis() / 1000 - 1200; // 20 minutes ago + long window2StartSeconds = window1StartSeconds + DELTA_WINDOW_SECONDS + 100; // ~12 minutes ago + + List window1Batch1 = Arrays.asList( + createMessage(window1StartSeconds + 10), + createMessage(window1StartSeconds + 20), + createMessage(window1StartSeconds + 30) + ); + + List window1Batch2 = Arrays.asList( + createMessage(window1StartSeconds + 100), + createMessage(window1StartSeconds + 150), + createMessage(window1StartSeconds + 200) + ); + + // Mixed batch triggers new window detection + List mixedBatch = Arrays.asList( + createMessage(window1StartSeconds + 250), + createMessage(window2StartSeconds + 10), + createMessage(window2StartSeconds + 20) + ); + + List window2Batch1 = Arrays.asList( + createMessage(window2StartSeconds + 50), + createMessage(window2StartSeconds + 80) + ); + + List window2Batch2 = Arrays.asList( + createMessage(window2StartSeconds + 120), + createMessage(window2StartSeconds + 150) + ); + + when(mockSqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(window1Batch1).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(window1Batch2).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(mixedBatch).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(window2Batch1).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(window2Batch2).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(List.of()).build()); + + // First readWindow() returns window 1 + mixed batch (new window detected) + SqsWindowReader.WindowReadResult result1 = windowReader.readWindow(); + + assertEquals(9, result1.getMessages().size()); + assertEquals(StopReason.NONE, result1.getStopReason()); + assertEquals(window1StartSeconds + 10, result1.getWindowStart()); + assertEquals(9, result1.getRawMessagesRead()); + + // Second readWindow() processes window 2 + SqsWindowReader.WindowReadResult result2 = windowReader.readWindow(); + + assertEquals(4, result2.getMessages().size()); + assertEquals(StopReason.QUEUE_EMPTY, result2.getStopReason()); + assertEquals(window2StartSeconds + 50, result2.getWindowStart()); + assertEquals(4, result2.getRawMessagesRead()); + } + + @Test + void testReadWindow_corruptMessagesSkipped() { + long windowStartSeconds = System.currentTimeMillis() / 1000 - 600; + + // Corrupt message (missing required fields) + Message corruptMessage = Message.builder() + .body("{}") + .attributes(Map.of(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf((windowStartSeconds + 10) * 1000))) + .messageId("corrupt-1") + .receiptHandle("receipt-1") + .build(); + + List validBatch = Arrays.asList(createMessage(windowStartSeconds + 100)); + + when(mockSqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(List.of(corruptMessage)).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(validBatch).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(List.of()).build()); + + SqsWindowReader.WindowReadResult result = windowReader.readWindow(); + + assertEquals(1, result.getMessages().size()); + assertEquals(StopReason.QUEUE_EMPTY, result.getStopReason()); + } + + @Test + void testWindowReadResult_factoryMethods() { + List messages = List.of(); + long windowStart = 1700000000L; + + SqsWindowReader.WindowReadResult empty = SqsWindowReader.WindowReadResult.queueEmpty(messages, windowStart, 5); + assertEquals(StopReason.QUEUE_EMPTY, empty.getStopReason()); + assertEquals(5, empty.getRawMessagesRead()); + + SqsWindowReader.WindowReadResult tooRecent = SqsWindowReader.WindowReadResult.messagesTooRecent(messages, windowStart, 5); + assertEquals(StopReason.MESSAGES_TOO_RECENT, tooRecent.getStopReason()); + assertEquals(5, tooRecent.getRawMessagesRead()); + + SqsWindowReader.WindowReadResult limitExceeded = SqsWindowReader.WindowReadResult.messageLimitExceeded(messages, windowStart, 100); + assertEquals(StopReason.MESSAGE_LIMIT_EXCEEDED, limitExceeded.getStopReason()); + assertEquals(100, limitExceeded.getRawMessagesRead()); + + SqsWindowReader.WindowReadResult withMessages = SqsWindowReader.WindowReadResult.withMessages(messages, windowStart, 10); + assertEquals(StopReason.NONE, withMessages.getStopReason()); + assertEquals(10, withMessages.getRawMessagesRead()); + } + + // ==================== Helper methods ==================== + + private Message createMessage(long timestampSeconds) { + return createMessageWithTimestampMs(timestampSeconds * 1000); + } + + private Message createMessageWithTimestampMs(long timestampMs) { + JsonObject body = new JsonObject() + .put("identity_hash", VALID_HASH_BASE64) + .put("advertising_id", VALID_ID_BASE64); + + Map attributes = new HashMap<>(); + attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampMs)); + + return Message.builder() + .body(body.encode()) + .attributes(attributes) + .messageId("msg-" + UUID.randomUUID()) + .receiptHandle("receipt-" + UUID.randomUUID()) + .build(); + } +} diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index 13e9c9c..85ee532 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -78,7 +78,7 @@ public void tearDown(TestContext context) { vertx.close(context.asyncAssertSuccess()); } } - + private Message createMessage(String hash, String id, long timestampMs) { JsonObject body = new JsonObject() .put("identity_hash", hash) From 8acd1a81704c276e84b71a810b716db0407e1fe8 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 15:20:58 -0700 Subject: [PATCH 04/25] clean up files --- .../OptOutTrafficCalculator.java | 623 ------------------ .../optout/vertx/OptOutSqsLogProducer.java | 6 +- .../optout/vertx/OptOutTrafficFilter.java | 2 + .../uid2/optout/vertx/SqsBatchProcessor.java | 165 ----- .../optout/vertx/SqsMessageOperations.java | 140 ---- .../uid2/optout/vertx/SqsMessageParser.java | 82 --- .../uid2/optout/vertx/SqsParsedMessage.java | 17 - .../uid2/optout/vertx/SqsWindowReader.java | 129 ---- .../optout/vertx/OptOutTrafficFilterTest.java | 2 + .../optout/vertx/SqsBatchProcessorTest.java | 171 ----- .../optout/vertx/SqsMessageParserTest.java | 271 -------- 11 files changed, 9 insertions(+), 1599 deletions(-) delete mode 100644 src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java delete mode 100644 src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java delete mode 100644 src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java delete mode 100644 src/main/java/com/uid2/optout/vertx/SqsMessageParser.java delete mode 100644 src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java delete mode 100644 src/main/java/com/uid2/optout/vertx/SqsWindowReader.java delete mode 100644 src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java delete mode 100644 src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java diff --git a/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java deleted file mode 100644 index 91da8f3..0000000 --- a/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java +++ /dev/null @@ -1,623 +0,0 @@ -package com.uid2.optout.circuitbreaker; - -import com.uid2.shared.cloud.ICloudStorage; -import com.uid2.shared.optout.OptOutCollection; -import com.uid2.shared.optout.OptOutEntry; -import com.uid2.shared.optout.OptOutUtils; -import com.uid2.optout.Const; -import com.uid2.optout.sqs.SqsMessageOperations; - -import io.vertx.core.json.JsonObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; - -import java.nio.charset.StandardCharsets; - -import java.nio.file.Files; -import java.nio.file.Paths; -import java.io.InputStream; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -/** - * Calculates opt-out traffic patterns to determine DEFAULT or DELAYED_PROCESSING status. - * - * Compares recent ~24h traffic (sumCurrent) against a configurable baseline (baselineTraffic) of expected traffic in 24 hours. - * The baseline is multiplied by (thresholdMultiplier) to determine the threshold. - * sumCurrent excludes records in allowlist ranges (surge windows determined by engineers). - * - * Returns DELAYED_PROCESSING if sumCurrent >= thresholdMultiplier * baselineTraffic, indicating abnormal traffic spike. - */ -public class OptOutTrafficCalculator { - private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficCalculator.class); - - private static final int HOURS_24 = 24 * 3600; // 24 hours in seconds - - private final Map deltaFileCache = new ConcurrentHashMap<>(); - private final ICloudStorage cloudStorage; - private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") - private final String trafficCalcConfigPath; - private int baselineTraffic; - private int thresholdMultiplier; - private int evaluationWindowSeconds; - private List> allowlistRanges; - - public enum TrafficStatus { - DELAYED_PROCESSING, - DEFAULT - } - - /** - * Cache entry for a delta file containing all record timestamps. - * - * Memory usage: ~8 bytes per timestamp (long) - * 1GB of memory can store ~130 million timestamps (1024^3)/8 - */ - private static class FileRecordCache { - final List timestamps; // All non-sentinel record timestamps - final long newestTimestamp; // evict delta from cache based on oldest record timestamp - - FileRecordCache(List timestamps) { - this.timestamps = timestamps; - this.newestTimestamp = timestamps.isEmpty() ? 0 : Collections.max(timestamps); - } - } - - /** - * Exception thrown by malformed traffic calculator config - */ - public static class MalformedTrafficCalcConfigException extends Exception { - public MalformedTrafficCalcConfigException(String message) { - super(message); - } - } - - /** - * Constructor for OptOutTrafficCalculator - * - * @param cloudStorage Cloud storage for reading delta files - * @param s3DeltaPrefix S3 prefix for delta files - * @param trafficCalcConfigS3Path S3 path for traffic calc config - */ - public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigPath) throws MalformedTrafficCalcConfigException { - this.cloudStorage = cloudStorage; - this.s3DeltaPrefix = s3DeltaPrefix; - this.trafficCalcConfigPath = trafficCalcConfigPath; - reloadTrafficCalcConfig(); // Load ConfigMap - - LOGGER.info("initialized: s3DeltaPrefix={}, threshold={}x", - s3DeltaPrefix, thresholdMultiplier); - } - - /** - * Reload traffic calc config from ConfigMap. - * Expected format: - * { - * "traffic_calc_evaluation_window_seconds": 86400, - * "traffic_calc_baseline_traffic": 100, - * "traffic_calc_threshold_multiplier": 5, - * "traffic_calc_allowlist_ranges": [ - * [startTimestamp1, endTimestamp1], - * [startTimestamp2, endTimestamp2] - * ], - * } - * - * Can be called periodically to pick up config changes without restarting. - */ - public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException { - LOGGER.info("loading traffic calc config from configmap"); - try (InputStream is = Files.newInputStream(Paths.get(trafficCalcConfigPath))) { - String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); - JsonObject trafficCalcConfig = new JsonObject(content); - - // Validate required fields exist - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { - throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_evaluation_window_seconds"); - } - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { - throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_baseline_traffic"); - } - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { - throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_threshold_multiplier"); - } - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { - throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_allowlist_ranges"); - } - - this.evaluationWindowSeconds = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp); - this.baselineTraffic = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcBaselineTrafficProp); - this.thresholdMultiplier = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcThresholdMultiplierProp); - - List> ranges = parseAllowlistRanges(trafficCalcConfig); - this.allowlistRanges = ranges; - - LOGGER.info("loaded traffic calc config: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", - this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size()); - - } catch (MalformedTrafficCalcConfigException e) { - LOGGER.warn("failed to load traffic calc config, config is malformed: {}", trafficCalcConfigPath, e); - throw e; - } catch (Exception e) { - LOGGER.warn("failed to load traffic calc config, config is malformed or missing: {}", trafficCalcConfigPath, e); - throw new MalformedTrafficCalcConfigException("failed to load traffic calc config: " + e.getMessage()); - } - } - - /** - * Parse allowlist ranges from JSON config - */ - List> parseAllowlistRanges(JsonObject config) throws MalformedTrafficCalcConfigException { - List> ranges = new ArrayList<>(); - - try { - var rangesArray = config.getJsonArray(Const.Config.OptOutTrafficCalcAllowlistRangesProp); - if (rangesArray != null) { - for (int i = 0; i < rangesArray.size(); i++) { - var rangeArray = rangesArray.getJsonArray(i); - if (rangeArray != null && rangeArray.size() >= 2) { - long start = rangeArray.getLong(0); - long end = rangeArray.getLong(1); - - if(start >= end) { - LOGGER.error("invalid allowlist range: start must be less than end: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": start must be less than end"); - } - - if (end - start > 86400) { - LOGGER.error("invalid allowlist range: range must be less than 24 hours: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": range must be less than 24 hours"); - } - - List range = Arrays.asList(start, end); - ranges.add(range); - LOGGER.info("loaded allowlist range: [{}, {}]", start, end); - } - } - } - - ranges.sort(Comparator.comparing(range -> range.get(0))); - - // Validate no overlapping ranges - for (int i = 0; i < ranges.size() - 1; i++) { - long currentEnd = ranges.get(i).get(1); - long nextStart = ranges.get(i + 1).get(0); - if (currentEnd >= nextStart) { - LOGGER.error("overlapping allowlist ranges detected: [{}, {}] overlaps with [{}, {}]", - ranges.get(i).get(0), currentEnd, nextStart, ranges.get(i + 1).get(1)); - throw new MalformedTrafficCalcConfigException( - "overlapping allowlist ranges detected at indices " + i + " and " + (i + 1)); - } - } - - } catch (MalformedTrafficCalcConfigException e) { - throw e; - } catch (Exception e) { - LOGGER.error("failed to parse allowlist ranges", e); - throw new MalformedTrafficCalcConfigException("failed to parse allowlist ranges: " + e.getMessage()); - } - - return ranges; - } - - /** - * Calculate traffic status based on delta files and SQS queue messages. - * - * Uses the newest delta file timestamp to anchor the 24-hour delta traffic window, - * and the oldest queue timestamp to anchor the 5-minute queue window. - * - * Counts: - * - Delta file records (with allowlist filtering) - * - SQS messages passed in (with allowlist filtering) - * - Invisible messages from other consumers (from queue attributes, avoiding double count) - * - * @param sqsMessages List of SQS messages this consumer has read - * @param queueAttributes Queue attributes including invisible message count (can be null) - * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) - */ - public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes) { - - try { - // Get list of delta files from S3 (sorted newest to oldest) - List deltaS3Paths = listDeltaFiles(); - - if (deltaS3Paths.isEmpty()) { - LOGGER.warn("no delta files found in s3 with prefix: {}", s3DeltaPrefix); - return TrafficStatus.DEFAULT; - } - - // Find newest delta file timestamp for delta traffic window - long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths); - LOGGER.info("traffic calculation: newestDeltaTs={}", newestDeltaTs); - - // Find oldest SQS queue message timestamp for queue window - long oldestQueueTs = findOldestQueueTimestamp(sqsMessages); - LOGGER.info("traffic calculation: oldestQueueTs={}", oldestQueueTs); - - // Define start time of the delta evaluation window - // We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend - // the window to account for any allowlist ranges in the extended portion - long deltaWindowStart = calculateWindowStartWithAllowlist(newestDeltaTs, this.evaluationWindowSeconds); - - // Evict old cache entries (older than delta window start) - evictOldCacheEntries(deltaWindowStart); - - // Process delta files and count records in [deltaWindowStart, newestDeltaTs] - int sum = 0; - int deltaRecordsCount = 0; - int filesProcessed = 0; - int cacheHits = 0; - int cacheMisses = 0; - - for (String s3Path : deltaS3Paths) { - boolean wasCached = isCached(s3Path); - if (wasCached) { - cacheHits++; - } else { - cacheMisses++; - } - - List timestamps = getTimestampsFromFile(s3Path); - filesProcessed++; - - boolean shouldStop = false; - for (long ts : timestamps) { - // Stop condition: record is older than our window - if (ts < deltaWindowStart) { - LOGGER.info("stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); - break; - } - - // skip records in allowlisted ranges - if (isInAllowlist(ts)) { - continue; - } - - // increment sum if record is in delta window - if (ts >= deltaWindowStart) { - deltaRecordsCount++; - sum++; - } - - } - - if (shouldStop) { - break; - } - } - - LOGGER.info("delta files: processed={}, deltaRecords={}, cache hits={}, misses={}, cacheSize={}", - filesProcessed, deltaRecordsCount, cacheHits, cacheMisses, deltaFileCache.size()); - - // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] with allowlist filtering - int sqsCount = 0; - if (sqsMessages != null && !sqsMessages.isEmpty()) { - sqsCount = countSqsMessages(sqsMessages, oldestQueueTs); - sum += sqsCount; - } - - // Add invisible messages being processed by OTHER consumers - // (notVisible count includes our messages, so subtract what we've read to avoid double counting) - int otherConsumersMessages = 0; - if (queueAttributes != null) { - int totalInvisible = queueAttributes.getApproximateNumberOfMessagesNotVisible(); - int ourMessages = sqsMessages != null ? sqsMessages.size() : 0; - otherConsumersMessages = Math.max(0, totalInvisible - ourMessages); - sum += otherConsumersMessages; - LOGGER.info("traffic calculation: adding {} invisible messages from other consumers (totalInvisible={}, ourMessages={})", - otherConsumersMessages, totalInvisible, ourMessages); - } - - // Determine status - TrafficStatus status = determineStatus(sum, this.baselineTraffic); - - LOGGER.info("traffic calculation complete: sum={} (deltaRecords={} + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}", - sum, deltaRecordsCount, sqsCount, otherConsumersMessages, this.baselineTraffic, this.thresholdMultiplier, status); - - return status; - - } catch (Exception e) { - LOGGER.error("error calculating traffic status", e); - return TrafficStatus.DEFAULT; - } - } - - /** - * Find the newest timestamp from delta files. - * Reads the newest delta file and returns its maximum timestamp. - */ - private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOException { - if (deltaS3Paths == null || deltaS3Paths.isEmpty()) { - return System.currentTimeMillis() / 1000; - } - - // Delta files are sorted (ISO 8601 format, lexicographically sortable) so first file is newest - String newestDeltaPath = deltaS3Paths.get(0); - List timestamps = getTimestampsFromFile(newestDeltaPath); - - if (timestamps.isEmpty()) { - LOGGER.warn("newest delta file has no timestamps: {}", newestDeltaPath); - return System.currentTimeMillis() / 1000; - } - - long newestTs = Collections.max(timestamps); - LOGGER.info("found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); - return newestTs; - } - - /** - * List all delta files from S3, sorted newest to oldest - */ - private List listDeltaFiles() { - try { - // List all objects with the delta prefix - List allFiles = cloudStorage.list(s3DeltaPrefix); - - // Filter to only .dat delta files and sort newest to oldest - List deltaFiles = allFiles.stream() - .filter(OptOutUtils::isDeltaFile) - .sorted(OptOutUtils.DeltaFilenameComparatorDescending) - .collect(Collectors.toList()); - - LOGGER.info("listed {} delta files from s3 (prefix={})", deltaFiles.size(), s3DeltaPrefix); - return deltaFiles; - - } catch (Exception e) { - LOGGER.error("failed to list delta files from s3 with prefix: {}", s3DeltaPrefix, e); - return Collections.emptyList(); - } - } - - /** - * Check if a delta file is already cached - */ - private boolean isCached(String s3Path) { - String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); - return deltaFileCache.containsKey(filename); - } - - /** - * Get timestamps from a delta file (S3 path), using cache if available - */ - private List getTimestampsFromFile(String s3Path) throws IOException { - // Extract filename from S3 path for cache key - String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); - - // Check cache first - FileRecordCache cached = deltaFileCache.get(filename); - if (cached != null) { - LOGGER.info("using cached timestamps for file: {}", filename); - return cached.timestamps; - } - - // Cache miss - download from S3 - LOGGER.info("downloading and reading timestamps from s3: {}", s3Path); - List timestamps = readTimestampsFromS3(s3Path); - - // Store in cache - deltaFileCache.put(filename, new FileRecordCache(timestamps)); - LOGGER.info("cached delta file: {} ({} records)", filename, timestamps.size()); - - return timestamps; - } - - /** - * Read all non-sentinel record timestamps from a delta file in S3 - */ - private List readTimestampsFromS3(String s3Path) throws IOException { - try (InputStream is = cloudStorage.download(s3Path)) { - byte[] data = is.readAllBytes(); - OptOutCollection collection = new OptOutCollection(data); - - List timestamps = new ArrayList<>(); - for (int i = 0; i < collection.size(); i++) { - OptOutEntry entry = collection.get(i); - - // Skip sentinel entries - if (entry.isSpecialHash()) { - continue; - } - - timestamps.add(entry.timestamp); - } - - return timestamps; - } catch (Exception e) { - LOGGER.error("failed to read delta file from s3: {}", s3Path, e); - throw new IOException("failed to read delta file from s3: " + s3Path, e); - } - } - - /** - * Calculate total duration of allowlist ranges that overlap with the given time window. - */ - long getAllowlistDuration(long t, long windowStart) { - long totalDuration = 0; - for (List range : this.allowlistRanges) { - long start = range.get(0); - long end = range.get(1); - - // Clip range to window boundaries - if (start < windowStart) { - start = windowStart; - } - if (end > t) { - end = t; - } - - // Only add duration if there's actual overlap (start < end) - if (start < end) { - totalDuration += end - start; - } - } - return totalDuration; - } - - /** - * Calculate the window start time that provides evaluationWindowSeconds of non-allowlisted time. - * Iteratively extends the window to account for allowlist ranges that may fall in extended portions. - */ - long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowSeconds) { - long allowlistDuration = getAllowlistDuration(newestDeltaTs, newestDeltaTs - evaluationWindowSeconds); - - // Each iteration discovers at least one new allowlist range, so max iterations = number of ranges - int maxIterations = this.allowlistRanges.size() + 1; - - for (int i = 0; i < maxIterations && allowlistDuration > 0; i++) { - long newWindowStart = newestDeltaTs - evaluationWindowSeconds - allowlistDuration; - long newAllowlistDuration = getAllowlistDuration(newestDeltaTs, newWindowStart); - - if (newAllowlistDuration == allowlistDuration) { - // No new allowlist time in extended portion, we've converged - break; - } - - allowlistDuration = newAllowlistDuration; - } - - return newestDeltaTs - evaluationWindowSeconds - allowlistDuration; - } - - /** - * Find the oldest SQS queue message timestamp - */ - private long findOldestQueueTimestamp(List sqsMessages) throws IOException { - long oldest = System.currentTimeMillis() / 1000; - - if (sqsMessages != null && !sqsMessages.isEmpty()) { - for (Message msg : sqsMessages) { - Long ts = extractTimestampFromMessage(msg); - if (ts != null && ts < oldest) { - oldest = ts; - } - } - } - - return oldest; - } - - /** - * Extract timestamp from SQS message (from SentTimestamp attribute) - */ - private Long extractTimestampFromMessage(Message msg) { - // Get SentTimestamp attribute (milliseconds) - String sentTimestamp = msg.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); - if (sentTimestamp != null) { - try { - return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds - } catch (NumberFormatException e) { - LOGGER.warn("invalid sentTimestamp: {}", sentTimestamp); - } - } - - // Fallback: use current time - return System.currentTimeMillis() / 1000; - } - - /** - * Count SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes - */ - private int countSqsMessages(List sqsMessages, long oldestQueueTs) { - - int count = 0; - long windowEnd = oldestQueueTs + 5 * 60; - - for (Message msg : sqsMessages) { - Long ts = extractTimestampFromMessage(msg); - - if (ts < oldestQueueTs || ts > windowEnd) { - continue; - } - - if (isInAllowlist(ts)) { - continue; - } - count++; - - } - - LOGGER.info("sqs messages: {} in window [oldestQueueTs={}, oldestQueueTs+5m={}]", count, oldestQueueTs, windowEnd); - return count; - } - - /** - * Check if a timestamp falls within any allowlist range - */ - boolean isInAllowlist(long timestamp) { - if (allowlistRanges == null || allowlistRanges.isEmpty()) { - return false; - } - - for (List range : allowlistRanges) { - if (range.size() < 2) { - continue; - } - - long start = range.get(0); - long end = range.get(1); - - if (timestamp >= start && timestamp <= end) { - return true; - } - } - - return false; - } - - /** - * Evict cache entries with data older than the cutoff timestamp - */ - private void evictOldCacheEntries(long cutoffTimestamp) { - int beforeSize = deltaFileCache.size(); - - deltaFileCache.entrySet().removeIf(entry -> - entry.getValue().newestTimestamp < cutoffTimestamp - ); - - int afterSize = deltaFileCache.size(); - if (beforeSize != afterSize) { - LOGGER.info("evicted {} old cache entries (before={}, after={})", - beforeSize - afterSize, beforeSize, afterSize); - } - } - - /** - * Determine traffic status based on current vs past counts - */ - TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { - if (baselineTraffic == 0 || thresholdMultiplier == 0) { - // Avoid division by zero - if no baseline traffic, return DEFAULT status - LOGGER.warn("baselineTraffic is 0 or thresholdMultiplier is 0, returning default status"); - return TrafficStatus.DEFAULT; - } - - if (sumCurrent >= thresholdMultiplier * baselineTraffic) { - LOGGER.error("delayed_processing threshold breached: sumCurrent={} >= {}×baselineTraffic={}", - sumCurrent, thresholdMultiplier, baselineTraffic); - return TrafficStatus.DELAYED_PROCESSING; - } - - LOGGER.info("traffic within normal range: sumCurrent={} < {}×baselineTraffic={}", - sumCurrent, thresholdMultiplier, baselineTraffic); - return TrafficStatus.DEFAULT; - } - - /** - * Get cache statistics for monitoring - */ - public Map getCacheStats() { - Map stats = new HashMap<>(); - stats.put("cached_files", deltaFileCache.size()); - - int totalTimestamps = deltaFileCache.values().stream() - .mapToInt(cache -> cache.timestamps.size()) - .sum(); - stats.put("total_cached_timestamps", totalTimestamps); - - return stats; - } - -} \ No newline at end of file diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index f84556b..f098690 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -2,6 +2,10 @@ import com.uid2.optout.Const; import com.uid2.optout.auth.InternalAuthMiddleware; +import com.uid2.optout.sqs.SqsWindowReader; +import com.uid2.optout.sqs.SqsParsedMessage; +import com.uid2.optout.sqs.SqsMessageOperations; +import com.uid2.optout.delta.StopReason; import com.uid2.shared.Utils; import com.uid2.shared.cloud.ICloudStorage; import com.uid2.shared.health.HealthComponent; @@ -392,7 +396,7 @@ private DeltaProductionResult produceBatchedDeltas() throws IOException { // If no messages, we're done (queue empty or messages too recent) if (windowResult.isEmpty()) { - stoppedDueToMessagesTooRecent = windowResult.stoppedDueToMessagesTooRecent(); + stoppedDueToMessagesTooRecent = windowResult.getStopReason() == StopReason.MESSAGES_TOO_RECENT; LOGGER.info("Delta production complete - no more eligible messages"); break; } diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java index 866f5aa..d0d5cd6 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -1,5 +1,7 @@ package com.uid2.optout.vertx; +import com.uid2.optout.sqs.SqsParsedMessage; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java b/src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java deleted file mode 100644 index 4444faf..0000000 --- a/src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java +++ /dev/null @@ -1,165 +0,0 @@ -package com.uid2.optout.vertx; - -import com.uid2.shared.optout.OptOutUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.sqs.SqsClient; -import software.amazon.awssdk.services.sqs.model.Message; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * Applies parsing, validation, filtering, and deletion of corrupted SQS messages. - * Used by SqsWindowReader - */ -public class SqsBatchProcessor { - private static final Logger LOGGER = LoggerFactory.getLogger(SqsBatchProcessor.class); - - private final SqsClient sqsClient; - private final String queueUrl; - private final int deltaWindowSeconds; - - public SqsBatchProcessor(SqsClient sqsClient, String queueUrl, int deltaWindowSeconds) { - this.sqsClient = sqsClient; - this.queueUrl = queueUrl; - this.deltaWindowSeconds = deltaWindowSeconds; - } - - /** - * Result of processing a batch of messages from SQS. - * Encapsulates eligible messages and metadata about the processing. - */ - public static class BatchProcessingResult { - private final List eligibleMessages; - private final boolean shouldStopProcessing; - - private BatchProcessingResult(List eligibleMessages, boolean shouldStopProcessing) { - this.eligibleMessages = eligibleMessages; - this.shouldStopProcessing = shouldStopProcessing; - } - - public static BatchProcessingResult withEligibleMessages(List messages) { - return new BatchProcessingResult(messages, false); - } - - public static BatchProcessingResult stopProcessing() { - return new BatchProcessingResult(new ArrayList<>(), true); - } - - public static BatchProcessingResult empty() { - return new BatchProcessingResult(new ArrayList<>(), false); - } - - public boolean isEmpty() { - return eligibleMessages.isEmpty(); - } - - public boolean shouldStopProcessing() { - return shouldStopProcessing; - } - - public List getEligibleMessages() { - return eligibleMessages; - } - } - - /** - * Processes a batch of messages: parses, validates, cleans up invalid messages, - * and filters for eligible messages based on age threshold (message is less than 5 minutes old) - * - * @param messageBatch Raw messages from SQS - * @param batchNumber The batch number (for logging) - * @return BatchProcessingResult containing eligible messages and processing metadata - */ - public BatchProcessingResult processBatch(List messageBatch, int batchNumber) { - // Parse and sort messages by timestamp - List parsedBatch = SqsMessageParser.parseAndSortMessages(messageBatch); - - // Identify and delete corrupt messages - if (parsedBatch.size() < messageBatch.size()) { - List invalidMessages = identifyInvalidMessages(messageBatch, parsedBatch); - if (!invalidMessages.isEmpty()) { - LOGGER.error("Found {} invalid messages in batch {} (failed parsing). Deleting from queue.", - invalidMessages.size(), batchNumber); - SqsMessageOperations.deleteMessagesFromSqs(this.sqsClient, this.queueUrl, invalidMessages); - } - } - - // If no valid messages, return empty result - if (parsedBatch.isEmpty()) { - LOGGER.warn("No valid messages in batch {} (all failed parsing)", batchNumber); - return BatchProcessingResult.empty(); - } - - // Check if the oldest message in this batch is too recent - long currentTime = OptOutUtils.nowEpochSeconds(); - SqsParsedMessage oldestMessage = parsedBatch.get(0); - long messageAge = currentTime - oldestMessage.timestamp(); - - if (messageAge < this.deltaWindowSeconds) { - // Signal to stop processing - messages are too recent - return BatchProcessingResult.stopProcessing(); - } - - // Filter for eligible messages (>= 5 minutes old) - List eligibleMessages = filterEligibleMessages(parsedBatch, currentTime); - - if (eligibleMessages.isEmpty()) { - LOGGER.debug("No eligible messages in batch {} (all too recent)", batchNumber); - return BatchProcessingResult.empty(); - } - - return BatchProcessingResult.withEligibleMessages(eligibleMessages); - } - - /** - * Filters messages to only include those where sufficient time has elapsed. -. * - * @param messages List of parsed messages - * @param currentTime Current time in seconds - * @return List of messages that meet the time threshold - */ - public List filterEligibleMessages( - List messages, - long currentTime) { - - List eligibleMessages = new ArrayList<>(); - - for (SqsParsedMessage pm : messages) { - if (currentTime - pm.timestamp() >= this.deltaWindowSeconds) { - eligibleMessages.add(pm); - } - } - - return eligibleMessages; - } - - /** - * Identifies messages that failed to parse by comparing the original batch with parsed results. - * - * @param originalBatch The original list of messages from SQS - * @param parsedBatch The list of successfully parsed messages - * @return List of messages that failed to parse - */ - private List identifyInvalidMessages(List originalBatch, List parsedBatch) { - // Create a set of message IDs from successfully parsed messages - Set validMessageIds = new HashSet<>(); - for (SqsParsedMessage parsed : parsedBatch) { - validMessageIds.add(parsed.originalMessage().messageId()); - } - - // Find messages that were not successfully parsed - List invalidMessages = new ArrayList<>(); - for (Message msg : originalBatch) { - if (!validMessageIds.contains(msg.messageId())) { - invalidMessages.add(msg); - } - } - - return invalidMessages; - } -} - diff --git a/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java b/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java deleted file mode 100644 index 6c2715b..0000000 --- a/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java +++ /dev/null @@ -1,140 +0,0 @@ -package com.uid2.optout.vertx; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.sqs.SqsClient; -import software.amazon.awssdk.services.sqs.model.*; - -import java.util.ArrayList; -import java.util.List; - -/** - * Utility class for SQS message operations. - */ -public class SqsMessageOperations { - private static final Logger LOGGER = LoggerFactory.getLogger(SqsMessageOperations.class); - private static final int SQS_MAX_DELETE_BATCH_SIZE = 10; - - /** - * Receives all available messages from an SQS queue up to a maximum number of batches. - * - * @param sqsClient The SQS client - * @param queueUrl The queue URL - * @param maxMessagesPerPoll Maximum messages to receive per poll (max 10) - * @param visibilityTimeout Visibility timeout in seconds - * @param maxBatches Maximum number of receive batches - * @return List of all received messages - */ - public static List receiveAllAvailableMessages( - SqsClient sqsClient, - String queueUrl, - int maxMessagesPerPoll, - int visibilityTimeout, - int maxBatches) { - - List allMessages = new ArrayList<>(); - int batchCount = 0; - - // Keep receiving messages until we get an empty batch or hit the limit - while (batchCount < maxBatches) { - List batch = receiveMessagesFromSqs(sqsClient, queueUrl, maxMessagesPerPoll, visibilityTimeout); - if (batch.isEmpty()) { - break; - } - allMessages.addAll(batch); - batchCount++; - - // If we got fewer messages than the max (of 10), the queue is likely empty - if (batch.size() < maxMessagesPerPoll) { - break; - } - } - - return allMessages; - } - - /** - * Receives a batch of messages from SQS. - * - * @param sqsClient The SQS client - * @param queueUrl The queue URL - * @param maxMessages Maximum number of messages to receive (max 10) - * @param visibilityTimeout Visibility timeout in seconds - * @return List of received messages - */ - public static List receiveMessagesFromSqs( - SqsClient sqsClient, - String queueUrl, - int maxMessages, - int visibilityTimeout) { - - try { - ReceiveMessageRequest receiveRequest = ReceiveMessageRequest.builder() - .queueUrl(queueUrl) - .maxNumberOfMessages(maxMessages) - .visibilityTimeout(visibilityTimeout) - .waitTimeSeconds(0) // Non-blocking poll - .messageSystemAttributeNames(MessageSystemAttributeName.SENT_TIMESTAMP) // Request SQS system timestamp - .build(); - - ReceiveMessageResponse response = sqsClient.receiveMessage(receiveRequest); - - LOGGER.debug("Received {} messages from SQS", response.messages().size()); - return response.messages(); - - } catch (Exception e) { - LOGGER.error("Error receiving messages from SQS", e); - return new ArrayList<>(); - } - } - - /** - * Deletes messages from SQS in batches (max 10 per batch). - * - * @param sqsClient The SQS client - * @param queueUrl The queue URL - * @param messages Messages to delete - */ - public static void deleteMessagesFromSqs(SqsClient sqsClient, String queueUrl, List messages) { - if (messages.isEmpty()) { - return; - } - - try { - List entries = new ArrayList<>(); - int batchId = 0; - int totalDeleted = 0; - - for (Message msg : messages) { - entries.add(DeleteMessageBatchRequestEntry.builder() - .id(String.valueOf(batchId++)) - .receiptHandle(msg.receiptHandle()) - .build()); - - // Send batch when we reach 10 messages or at the end - if (entries.size() == SQS_MAX_DELETE_BATCH_SIZE || batchId == messages.size()) { - DeleteMessageBatchRequest deleteRequest = DeleteMessageBatchRequest.builder() - .queueUrl(queueUrl) - .entries(entries) - .build(); - - DeleteMessageBatchResponse deleteResponse = sqsClient.deleteMessageBatch(deleteRequest); - - if (!deleteResponse.failed().isEmpty()) { - LOGGER.error("Failed to delete {} messages from SQS", deleteResponse.failed().size()); - } else { - totalDeleted += entries.size(); - } - - entries.clear(); - } - } - - LOGGER.info("Deleted {} messages from SQS", totalDeleted); - - } catch (Exception e) { - LOGGER.error("Error deleting messages from SQS", e); - } - } -} - diff --git a/src/main/java/com/uid2/optout/vertx/SqsMessageParser.java b/src/main/java/com/uid2/optout/vertx/SqsMessageParser.java deleted file mode 100644 index 279f6b2..0000000 --- a/src/main/java/com/uid2/optout/vertx/SqsMessageParser.java +++ /dev/null @@ -1,82 +0,0 @@ -package com.uid2.optout.vertx; - -import com.uid2.shared.optout.OptOutUtils; -import io.vertx.core.json.JsonObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; - -import java.util.ArrayList; -import java.util.List; - -/** - * Utility class for parsing SQS messages containing opt-out data. - */ -public class SqsMessageParser { - private static final Logger LOGGER = LoggerFactory.getLogger(SqsMessageParser.class); - - /** - * Parses and sorts a list of SQS messages by timestamp. - * - * @param messages List of raw SQS messages - * @return List of parsed messages sorted by timestamp (oldest first) - */ - public static List parseAndSortMessages(List messages) { - List parsedMessages = new ArrayList<>(); - - for (Message message : messages) { - try { - // Extract SQS system timestamp (in milliseconds), or use current time as fallback - long timestampSeconds = extractTimestamp(message); - - // Parse message body - JsonObject body = new JsonObject(message.body()); - String identityHash = body.getString("identity_hash"); - String advertisingId = body.getString("advertising_id"); - String traceId = body.getString("trace_id"); - String clientIp = body.getString("client_ip"); - String email = body.getString("email"); - String phone = body.getString("phone"); - - if (identityHash == null || advertisingId == null) { - LOGGER.error("Invalid message format, skipping: {}", message.body()); - continue; - } - - byte[] hashBytes = OptOutUtils.base64StringTobyteArray(identityHash); - byte[] idBytes = OptOutUtils.base64StringTobyteArray(advertisingId); - - if (hashBytes == null || idBytes == null) { - LOGGER.error("Invalid base64 encoding, skipping message"); - continue; - } - - parsedMessages.add(new SqsParsedMessage(message, hashBytes, idBytes, timestampSeconds, email, phone, clientIp, traceId)); - } catch (Exception e) { - LOGGER.error("Error parsing SQS message", e); - } - } - - // Sort by timestamp - parsedMessages.sort((a, b) -> Long.compare(a.timestamp(), b.timestamp())); - - return parsedMessages; - } - - /** - * Extracts timestamp from SQS message attributes, falling back to current time if unavailable. - * - * @param message The SQS message - * @return Timestamp in seconds - */ - private static long extractTimestamp(Message message) { - String sentTimestampStr = message.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); - if (sentTimestampStr == null) { - LOGGER.warn("Message missing SentTimestamp attribute, using current time"); - return OptOutUtils.nowEpochSeconds(); - } - return Long.parseLong(sentTimestampStr) / 1000; // ms to seconds - } -} - diff --git a/src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java b/src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java deleted file mode 100644 index 1714b56..0000000 --- a/src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.uid2.optout.vertx; - -import software.amazon.awssdk.services.sqs.model.Message; - -/** - * Represents a parsed SQS message containing opt-out data. - */ -public record SqsParsedMessage( - Message originalMessage, - byte[] hashBytes, - byte[] idBytes, - long timestamp, - String email, - String phone, - String clientIp, - String traceId -) {} diff --git a/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java b/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java deleted file mode 100644 index 355669b..0000000 --- a/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java +++ /dev/null @@ -1,129 +0,0 @@ -package com.uid2.optout.vertx; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.sqs.SqsClient; -import software.amazon.awssdk.services.sqs.model.Message; - -import java.util.ArrayList; -import java.util.List; - -/** - * Reads messages from SQS for complete 5-minute time windows. - * Handles accumulation of all messages for a window before returning. - * Limits messages per window to prevent memory issues. - */ -public class SqsWindowReader { - private static final Logger LOGGER = LoggerFactory.getLogger(SqsWindowReader.class); - - private final SqsClient sqsClient; - private final String queueUrl; - private final int maxMessagesPerPoll; - private final int visibilityTimeout; - private final int deltaWindowSeconds; - private final int maxMessagesPerFile; - private final SqsBatchProcessor batchProcessor; - - public SqsWindowReader(SqsClient sqsClient, String queueUrl, int maxMessagesPerPoll, - int visibilityTimeout, int deltaWindowSeconds, int maxMessagesPerFile) { - this.sqsClient = sqsClient; - this.queueUrl = queueUrl; - this.maxMessagesPerPoll = maxMessagesPerPoll; - this.visibilityTimeout = visibilityTimeout; - this.deltaWindowSeconds = deltaWindowSeconds; - this.maxMessagesPerFile = maxMessagesPerFile; - this.batchProcessor = new SqsBatchProcessor(sqsClient, queueUrl, deltaWindowSeconds); - LOGGER.info("SqsWindowReader initialized with: maxMessagesPerFile: {}, maxMessagesPerPoll: {}, visibilityTimeout: {}, deltaWindowSeconds: {}", - maxMessagesPerFile, maxMessagesPerPoll, visibilityTimeout, deltaWindowSeconds); - } - - /** - * Result of reading messages for a 5-minute window. - */ - public static class WindowReadResult { - private final List messages; - private final long windowStart; - private final boolean stoppedDueToMessagesTooRecent; - - public WindowReadResult(List messages, long windowStart, - boolean stoppedDueToMessagesTooRecent) { - this.messages = messages; - this.windowStart = windowStart; - this.stoppedDueToMessagesTooRecent = stoppedDueToMessagesTooRecent; - } - - public List getMessages() { return messages; } - public long getWindowStart() { return windowStart; } - public boolean isEmpty() { return messages.isEmpty(); } - public boolean stoppedDueToMessagesTooRecent() { return stoppedDueToMessagesTooRecent; } - } - - /** - * Reads messages from SQS for one complete 5-minute window. - * Keeps reading batches and accumulating messages until: - * - We discover the next window - * - Queue is empty (no more messages) - * - Messages are too recent (all messages younger than 5 minutes) - * - Message limit is reached (memory protection) - * - * @return WindowReadResult with messages for the window, or empty if done - */ - public WindowReadResult readWindow() { - List windowMessages = new ArrayList<>(); - long currentWindowStart = 0; - - while (true) { - // Check if we've hit the message limit - if (windowMessages.size() >= this.maxMessagesPerFile) { - LOGGER.warn("Window message limit reached ({} messages). Truncating window starting at {} for memory protection.", - this.maxMessagesPerFile, currentWindowStart); - return new WindowReadResult(windowMessages, currentWindowStart, false); - } - - // Read one batch from SQS (up to 10 messages) - List rawBatch = SqsMessageOperations.receiveMessagesFromSqs( - this.sqsClient, this.queueUrl, this.maxMessagesPerPoll, this.visibilityTimeout); - - if (rawBatch.isEmpty()) { - // Queue empty - return what we have - return new WindowReadResult(windowMessages, currentWindowStart, false); - } - - // Process batch: parse, validate, filter - SqsBatchProcessor.BatchProcessingResult batchResult = batchProcessor.processBatch(rawBatch, 0); - - if (batchResult.isEmpty()) { - if (batchResult.shouldStopProcessing()) { - // Messages too recent - return what we have - return new WindowReadResult(windowMessages, currentWindowStart, true); - } - // corrupt messages deleted, read next messages - continue; - } - - // Add eligible messages to current window - boolean newWindow = false; - for (SqsParsedMessage msg : batchResult.getEligibleMessages()) { - long msgWindowStart = (msg.timestamp() / this.deltaWindowSeconds) * this.deltaWindowSeconds; - - // discover start of window - if (currentWindowStart == 0) { - currentWindowStart = msgWindowStart; - } - - // discover new window - if (msgWindowStart > currentWindowStart + this.deltaWindowSeconds) { - newWindow = true; - } - - windowMessages.add(msg); - } - - if (newWindow) { - // close current window and return - return new WindowReadResult(windowMessages, currentWindowStart, false); - } - } - } -} - diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java index 63f6807..88881c5 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java @@ -1,5 +1,7 @@ package com.uid2.optout.vertx; +import com.uid2.optout.sqs.SqsParsedMessage; + import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java b/src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java deleted file mode 100644 index 3e371cd..0000000 --- a/src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java +++ /dev/null @@ -1,171 +0,0 @@ -package com.uid2.optout.vertx; - -import io.vertx.core.json.JsonObject; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; - -import java.util.*; - -import static org.junit.jupiter.api.Assertions.*; - -public class SqsBatchProcessorTest { - - private static final String VALID_HASH_BASE64 = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; // 32 bytes - private static final String VALID_ID_BASE64 = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE="; // 32 bytes - private static final long TEST_TIMESTAMP_MS = 1699308900000L; // Nov 7, 2023 in ms - - private static final int DEFAULT_WINDOW_SECONDS = 300; // 5 minutes - - private SqsBatchProcessor batchProcessor; - - @BeforeEach - public void setUp() { - // Pass null for SqsClient - filterEligibleMessages doesn't use it - batchProcessor = new SqsBatchProcessor(null, "test-queue-url", DEFAULT_WINDOW_SECONDS); - } - - private Message createValidMessage(String identityHash, String advertisingId, long timestampMs) { - JsonObject body = new JsonObject() - .put("identity_hash", identityHash) - .put("advertising_id", advertisingId); - - Map attributes = new HashMap<>(); - attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampMs)); - - return Message.builder() - .body(body.encode()) - .attributes(attributes) - .messageId("test-message-id-" + timestampMs) - .receiptHandle("test-receipt-handle") - .build(); - } - - @Test - public void testFilterEligibleMessages_allEligible() { - List messages = new ArrayList<>(); - Message mockMsg = createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS); - - // Create messages from 10 minutes ago (600 seconds) - long currentTime = 1000L; - long oldTimestamp = currentTime - 600; - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], oldTimestamp, null, null, null, null)); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], oldTimestamp - 100, null, null, null, null)); - - List result = batchProcessor.filterEligibleMessages(messages, currentTime); - - assertEquals(2, result.size()); // All should be eligible (> 5 minutes old) - } - - @Test - public void testFilterEligibleMessages_noneEligible() { - List messages = new ArrayList<>(); - Message mockMsg = createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS); - - // Create messages from 1 minute ago (too recent) - long currentTime = 1000L; - long recentTimestamp = currentTime - 60; - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], recentTimestamp, null, null, null, null)); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], recentTimestamp + 10, null, null, null, null)); - - List result = batchProcessor.filterEligibleMessages(messages, currentTime); - - assertEquals(0, result.size()); // None should be eligible (< 5 minutes old) - } - - @Test - public void testFilterEligibleMessages_mixedEligibility() { - List messages = new ArrayList<>(); - Message mockMsg = createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS); - - long currentTime = 1000L; - - // Old enough (600 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 600, null, null, null, null)); - - // Exactly at threshold (300 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 300, null, null, null, null)); - - // Too recent (100 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 100, null, null, null, null)); - - List result = batchProcessor.filterEligibleMessages(messages, currentTime); - - assertEquals(2, result.size()); // First two are eligible (>= 300 seconds old) - } - - @Test - public void testFilterEligibleMessages_emptyList() { - List messages = new ArrayList<>(); - long currentTime = 1000L; - - List result = batchProcessor.filterEligibleMessages(messages, currentTime); - - assertEquals(0, result.size()); - } - - @Test - public void testFilterEligibleMessages_boundaryCases() { - List messages = new ArrayList<>(); - Message mockMsg = createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS); - - long currentTime = 1000L; - int windowSeconds = DEFAULT_WINDOW_SECONDS; // 300 - - // One second too new (299 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds + 1, null, null, null, null)); - - // Exactly at threshold (300 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds, null, null, null, null)); - - // One second past threshold (301 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds - 1, null, null, null, null)); - - List result = batchProcessor.filterEligibleMessages(messages, currentTime); - - // Should only include the last two (>= threshold) - assertEquals(2, result.size()); - assertEquals(currentTime - windowSeconds, result.get(0).timestamp()); - assertEquals(currentTime - windowSeconds - 1, result.get(1).timestamp()); - } - - @Test - public void testFilterEligibleMessages_preservesOrder() { - List messages = new ArrayList<>(); - Message mockMsg = createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS); - - long currentTime = 1000L; - - // Add eligible messages in specific order (all older than 300 seconds) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 100, null, null, null, null)); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 200, null, null, null, null)); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 300, null, null, null, null)); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 900, null, null, null, null)); // Too recent (100 seconds ago) - - List result = batchProcessor.filterEligibleMessages(messages, currentTime); - - assertEquals(3, result.size()); - // Verify order is preserved - assertEquals(100, result.get(0).timestamp()); - assertEquals(200, result.get(1).timestamp()); - assertEquals(300, result.get(2).timestamp()); - } - - @Test - public void testFilterEligibleMessages_zeroWindowSeconds() { - // Create processor with 0 window seconds - SqsBatchProcessor zeroWindowProcessor = new SqsBatchProcessor(null, "test-queue-url", 0); - - List messages = new ArrayList<>(); - Message mockMsg = createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS); - - long currentTime = 1000L; - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime, null, null, null, null)); - - List result = zeroWindowProcessor.filterEligibleMessages(messages, currentTime); - - assertEquals(1, result.size()); // With 0 window, current time messages should be eligible - } -} - diff --git a/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java b/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java deleted file mode 100644 index bcc615f..0000000 --- a/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java +++ /dev/null @@ -1,271 +0,0 @@ -package com.uid2.optout.vertx; - -import io.vertx.core.json.JsonObject; -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; -import java.util.*; - -import static org.junit.jupiter.api.Assertions.*; - -public class SqsMessageParserTest { - - private static final String VALID_HASH_BASE64 = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; // 32 bytes - private static final String VALID_ID_BASE64 = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE="; // 32 bytes - private static final long TEST_TIMESTAMP_MS = 1699308900000L; // Nov 7, 2023 in ms - private static final long TEST_TIMESTAMP_SEC = 1699308900L; // Nov 7, 2023 in seconds - - private Message createValidMessage(String identityHash, String advertisingId, long timestampMs) { - JsonObject body = new JsonObject() - .put("identity_hash", identityHash) - .put("advertising_id", advertisingId); - - Map attributes = new HashMap<>(); - attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampMs)); - - return Message.builder() - .body(body.encode()) - .attributes(attributes) - .messageId("test-message-id") - .receiptHandle("test-receipt-handle") - .build(); - } - - private Message createMessageWithoutTimestamp(String identityHash, String advertisingId) { - JsonObject body = new JsonObject() - .put("identity_hash", identityHash) - .put("advertising_id", advertisingId); - - return Message.builder() - .body(body.encode()) - .attributes(new HashMap<>()) - .messageId("test-message-id") - .receiptHandle("test-receipt-handle") - .build(); - } - - @Test - public void testParseAndSortMessages_validMessages() { - // Create messages with different timestamps - List messages = Arrays.asList( - createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS + 2000), - createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS), - createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS + 1000) - ); - - List result = SqsMessageParser.parseAndSortMessages(messages); - - assertEquals(3, result.size()); - // Verify sorting (oldest first) - assertEquals(TEST_TIMESTAMP_SEC, result.get(0).timestamp()); - assertEquals(TEST_TIMESTAMP_SEC + 1, result.get(1).timestamp()); - assertEquals(TEST_TIMESTAMP_SEC + 2, result.get(2).timestamp()); - } - - @Test - public void testParseAndSortMessages_emptyList() { - List messages = new ArrayList<>(); - List result = SqsMessageParser.parseAndSortMessages(messages); - - assertEquals(0, result.size()); - } - - @Test - public void testParseAndSortMessages_missingIdentityHash() { - JsonObject body = new JsonObject() - .put("advertising_id", VALID_ID_BASE64); - - Map attributes = new HashMap<>(); - attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(TEST_TIMESTAMP_MS)); - - Message message = Message.builder() - .body(body.encode()) - .attributes(attributes) - .messageId("test") - .receiptHandle("test") - .build(); - - List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(message)); - - assertEquals(0, result.size()); // Should skip invalid message - } - - @Test - public void testParseAndSortMessages_missingAdvertisingId() { - JsonObject body = new JsonObject() - .put("identity_hash", VALID_HASH_BASE64); - - Map attributes = new HashMap<>(); - attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(TEST_TIMESTAMP_MS)); - - Message message = Message.builder() - .body(body.encode()) - .attributes(attributes) - .messageId("test") - .receiptHandle("test") - .build(); - - List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(message)); - - assertEquals(0, result.size()); // Should skip invalid message - } - - @Test - public void testParseAndSortMessages_invalidBase64() { - JsonObject body = new JsonObject() - .put("identity_hash", "not-valid-base64!!!") - .put("advertising_id", VALID_ID_BASE64); - - Map attributes = new HashMap<>(); - attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(TEST_TIMESTAMP_MS)); - - Message message = Message.builder() - .body(body.encode()) - .attributes(attributes) - .messageId("test") - .receiptHandle("test") - .build(); - - List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(message)); - - assertEquals(0, result.size()); // Should skip message with invalid base64 - } - - @Test - public void testParseAndSortMessages_invalidJson() { - Map attributes = new HashMap<>(); - attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(TEST_TIMESTAMP_MS)); - - Message message = Message.builder() - .body("not valid json") - .attributes(attributes) - .messageId("test") - .receiptHandle("test") - .build(); - - List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(message)); - - assertEquals(0, result.size()); // Should skip message with invalid JSON - } - - @Test - public void testParseAndSortMessages_missingTimestamp() { - // Message without SentTimestamp should use current time as fallback - Message message = createMessageWithoutTimestamp(VALID_HASH_BASE64, VALID_ID_BASE64); - - List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(message)); - - assertEquals(1, result.size()); - // Timestamp should be close to current time (within 10 seconds) - long currentTime = System.currentTimeMillis() / 1000; - assertTrue(Math.abs(result.get(0).timestamp() - currentTime) < 10); - } - - @Test - public void testParseAndSortMessages_mixValidAndInvalid() { - List messages = Arrays.asList( - createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS), - createMessageWithoutTimestamp("invalid-base64", VALID_ID_BASE64), // Invalid - createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS + 1000) - ); - - List result = SqsMessageParser.parseAndSortMessages(messages); - - assertEquals(2, result.size()); // Only valid messages - assertEquals(TEST_TIMESTAMP_SEC, result.get(0).timestamp()); - assertEquals(TEST_TIMESTAMP_SEC + 1, result.get(1).timestamp()); - } - - @Test - public void testParseAndSortMessages_timestampConversion() { - // Verify milliseconds to seconds conversion - Message message = createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, 1699308912345L); // ms - - List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(message)); - - assertEquals(1, result.size()); - assertEquals(1699308912L, result.get(0).timestamp()); // Should be in seconds - } - - @Test - public void testParseAndSortMessages_preservesOriginalMessage() { - Message originalMessage = createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS); - - List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(originalMessage)); - - assertEquals(1, result.size()); - assertSame(originalMessage, result.get(0).originalMessage()); - } - - @Test - public void testParseAndSortMessages_sortingOrder() { - // Create messages in random order - List messages = Arrays.asList( - createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, 1699308900000L), // timestamp 0 - createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, 1699308950000L), // timestamp 50 - createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, 1699308920000L), // timestamp 20 - createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, 1699308910000L), // timestamp 10 - createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, 1699308940000L) // timestamp 40 - ); - - List result = SqsMessageParser.parseAndSortMessages(messages); - - assertEquals(5, result.size()); - // Verify ascending order - for (int i = 1; i < result.size(); i++) { - assertTrue(result.get(i - 1).timestamp() <= result.get(i).timestamp(), - "Messages should be sorted in ascending order by timestamp"); - } - } - - @Test - public void testParseAndSortMessages_parsesHashAndIdBytes() { - Message message = createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS); - - List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(message)); - - assertEquals(1, result.size()); - assertNotNull(result.get(0).hashBytes()); - assertNotNull(result.get(0).idBytes()); - assertEquals(32, result.get(0).hashBytes().length); - assertEquals(32, result.get(0).idBytes().length); - } - - @Test - public void testParseAndSortMessages_nullBodyFields() { - JsonObject body = new JsonObject() - .put("identity_hash", (String) null) - .put("advertising_id", VALID_ID_BASE64); - - Map attributes = new HashMap<>(); - attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(TEST_TIMESTAMP_MS)); - - Message message = Message.builder() - .body(body.encode()) - .attributes(attributes) - .messageId("test") - .receiptHandle("test") - .build(); - - List result = SqsMessageParser.parseAndSortMessages(Arrays.asList(message)); - - assertEquals(0, result.size()); // Should skip message with null fields - } - - @Test - public void testParseAndSortMessages_multipleValidMessages() { - List messages = new ArrayList<>(); - for (int i = 0; i < 100; i++) { - messages.add(createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS + i * 1000)); - } - - List result = SqsMessageParser.parseAndSortMessages(messages); - - assertEquals(100, result.size()); - // Verify all are sorted - for (int i = 1; i < result.size(); i++) { - assertTrue(result.get(i - 1).timestamp() <= result.get(i).timestamp()); - } - } - -} From d9e264147674a3e5bd205b79f88c70164fda865e Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 15:52:22 -0700 Subject: [PATCH 05/25] test fix --- .../java/com/uid2/optout/sqs/SqsWindowReaderTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/uid2/optout/sqs/SqsWindowReaderTest.java b/src/test/java/com/uid2/optout/sqs/SqsWindowReaderTest.java index e1c2539..044be5a 100644 --- a/src/test/java/com/uid2/optout/sqs/SqsWindowReaderTest.java +++ b/src/test/java/com/uid2/optout/sqs/SqsWindowReaderTest.java @@ -122,14 +122,19 @@ void testReadWindow_messageLimitExceeded() { for (int i = 0; i < 10; i++) { batch.add(createMessage(windowStartSeconds + i * 10)); } + List batch2 = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + batch2.add(createMessage(windowStartSeconds + i * 10)); + } when(mockSqsClient.receiveMessage(any(ReceiveMessageRequest.class))) - .thenReturn(ReceiveMessageResponse.builder().messages(batch).build()); + .thenReturn(ReceiveMessageResponse.builder().messages(batch).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(batch2).build()); SqsWindowReader.WindowReadResult result = smallLimitReader.readWindow(); assertEquals(StopReason.MESSAGE_LIMIT_EXCEEDED, result.getStopReason()); - assertTrue(result.getMessages().size() >= 5); + assertEquals(10, result.getMessages().size()); } @Test From 609c3471aa7efc491172433dba67cd96eb68a966 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 16:02:31 -0700 Subject: [PATCH 06/25] clean up files --- .../optout/traffic/OptOutTrafficFilter.java | 6 +- .../optout/vertx/OptOutTrafficCalculator.java | 574 ------- .../optout/vertx/OptOutTrafficFilter.java | 174 -- .../vertx/OptOutTrafficCalculatorTest.java | 1510 ----------------- .../optout/vertx/OptOutTrafficFilterTest.java | 426 ----- 5 files changed, 3 insertions(+), 2687 deletions(-) delete mode 100644 src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java delete mode 100644 src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java delete mode 100644 src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java delete mode 100644 src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java index 59a60a9..445f032 100644 --- a/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java @@ -163,11 +163,11 @@ List parseFilterRules(JsonObject config) throws MalformedTraf } public boolean isDenylisted(SqsParsedMessage message) { - long timestamp = message.getTimestamp(); - String clientIp = message.getClientIp(); + long timestamp = message.timestamp(); + String clientIp = message.clientIp(); if (clientIp == null || clientIp.isEmpty()) { - LOGGER.error("sqs_error: request does not contain client ip, messageId={}", message.getOriginalMessage().messageId()); + LOGGER.error("sqs_error: request does not contain client ip, messageId={}", message.originalMessage().messageId()); return false; } diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java deleted file mode 100644 index f0c7a7c..0000000 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ /dev/null @@ -1,574 +0,0 @@ -package com.uid2.optout.vertx; - -import com.uid2.shared.cloud.ICloudStorage; -import com.uid2.shared.optout.OptOutCollection; -import com.uid2.shared.optout.OptOutEntry; -import com.uid2.shared.optout.OptOutUtils; -import com.uid2.optout.Const; -import io.vertx.core.json.JsonObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; - -import java.nio.charset.StandardCharsets; - -import java.nio.file.Files; -import java.nio.file.Paths; -import java.io.InputStream; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -/** - * Calculates opt-out traffic patterns to determine DEFAULT or DELAYED_PROCESSING status. - * - * Compares recent ~24h traffic (sumCurrent) against a configurable baseline (baselineTraffic) of expected traffic in 24 hours. - * The baseline is multiplied by (thresholdMultiplier) to determine the threshold. - * sumCurrent excludes records in allowlist ranges (surge windows determined by engineers). - * - * Returns DELAYED_PROCESSING if sumCurrent >= thresholdMultiplier * baselineTraffic, indicating abnormal traffic spike. - */ -public class OptOutTrafficCalculator { - private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficCalculator.class); - - private static final int HOURS_24 = 24 * 3600; // 24 hours in seconds - - private final Map deltaFileCache = new ConcurrentHashMap<>(); - private final ICloudStorage cloudStorage; - private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") - private final String trafficCalcConfigPath; - private int baselineTraffic; - private int thresholdMultiplier; - private int evaluationWindowSeconds; - private List> allowlistRanges; - - public enum TrafficStatus { - DELAYED_PROCESSING, - DEFAULT - } - - /** - * Cache entry for a delta file containing all record timestamps. - * - * Memory usage: ~8 bytes per timestamp (long) - * 1GB of memory can store ~130 million timestamps (1024^3)/8 - */ - private static class FileRecordCache { - final List timestamps; // All non-sentinel record timestamps - final long newestTimestamp; // evict delta from cache based on oldest record timestamp - - FileRecordCache(List timestamps) { - this.timestamps = timestamps; - this.newestTimestamp = timestamps.isEmpty() ? 0 : Collections.max(timestamps); - } - } - - /** - * Exception thrown by malformed traffic calculator config - */ - public static class MalformedTrafficCalcConfigException extends Exception { - public MalformedTrafficCalcConfigException(String message) { - super(message); - } - } - - /** - * Constructor for OptOutTrafficCalculator - * - * @param cloudStorage Cloud storage for reading delta files - * @param s3DeltaPrefix S3 prefix for delta files - * @param trafficCalcConfigS3Path S3 path for traffic calc config - */ - public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigPath) throws MalformedTrafficCalcConfigException { - this.cloudStorage = cloudStorage; - this.s3DeltaPrefix = s3DeltaPrefix; - this.trafficCalcConfigPath = trafficCalcConfigPath; - reloadTrafficCalcConfig(); // Load ConfigMap - - LOGGER.info("OptOutTrafficCalculator initialized: s3DeltaPrefix={}, threshold={}x", - s3DeltaPrefix, thresholdMultiplier); - } - - /** - * Reload traffic calc config from ConfigMap. - * Expected format: - * { - * "traffic_calc_evaluation_window_seconds": 86400, - * "traffic_calc_baseline_traffic": 100, - * "traffic_calc_threshold_multiplier": 5, - * "traffic_calc_allowlist_ranges": [ - * [startTimestamp1, endTimestamp1], - * [startTimestamp2, endTimestamp2] - * ], - * } - * - * Can be called periodically to pick up config changes without restarting. - */ - public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException { - LOGGER.info("Loading traffic calc config from ConfigMap"); - try (InputStream is = Files.newInputStream(Paths.get(trafficCalcConfigPath))) { - String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); - JsonObject trafficCalcConfig = new JsonObject(content); - - // Validate required fields exist - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_evaluation_window_seconds"); - } - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_baseline_traffic"); - } - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_threshold_multiplier"); - } - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_allowlist_ranges"); - } - - this.evaluationWindowSeconds = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp); - this.baselineTraffic = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcBaselineTrafficProp); - this.thresholdMultiplier = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcThresholdMultiplierProp); - - List> ranges = parseAllowlistRanges(trafficCalcConfig); - this.allowlistRanges = ranges; - - LOGGER.info("Successfully loaded traffic calc config from ConfigMap: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", - this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size()); - - } catch (MalformedTrafficCalcConfigException e) { - LOGGER.warn("Failed to load traffic calc config. Config is malformed: {}", trafficCalcConfigPath, e); - throw e; - } catch (Exception e) { - LOGGER.warn("Failed to load traffic calc config. Config is malformed or missing: {}", trafficCalcConfigPath, e); - throw new MalformedTrafficCalcConfigException("Failed to load traffic calc config: " + e.getMessage()); - } - } - - /** - * Parse allowlist ranges from JSON config - */ - List> parseAllowlistRanges(JsonObject config) throws MalformedTrafficCalcConfigException { - List> ranges = new ArrayList<>(); - - try { - var rangesArray = config.getJsonArray(Const.Config.OptOutTrafficCalcAllowlistRangesProp); - if (rangesArray != null) { - for (int i = 0; i < rangesArray.size(); i++) { - var rangeArray = rangesArray.getJsonArray(i); - if (rangeArray != null && rangeArray.size() >= 2) { - long start = rangeArray.getLong(0); - long end = rangeArray.getLong(1); - - if(start >= end) { - LOGGER.error("Invalid allowlist range: start must be less than end: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("Invalid allowlist range at index " + i + ": start must be less than end"); - } - - if (end - start > 86400) { - LOGGER.error("Invalid allowlist range: range must be less than 24 hours: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("Invalid allowlist range at index " + i + ": range must be less than 24 hours"); - } - - List range = Arrays.asList(start, end); - ranges.add(range); - LOGGER.info("Loaded allowlist range: [{}, {}]", start, end); - } - } - } - - ranges.sort(Comparator.comparing(range -> range.get(0))); - - // Validate no overlapping ranges - for (int i = 0; i < ranges.size() - 1; i++) { - long currentEnd = ranges.get(i).get(1); - long nextStart = ranges.get(i + 1).get(0); - if (currentEnd >= nextStart) { - LOGGER.error("Overlapping allowlist ranges detected: [{}, {}] overlaps with [{}, {}]", - ranges.get(i).get(0), currentEnd, nextStart, ranges.get(i + 1).get(1)); - throw new MalformedTrafficCalcConfigException( - "Overlapping allowlist ranges detected at indices " + i + " and " + (i + 1)); - } - } - - } catch (MalformedTrafficCalcConfigException e) { - throw e; - } catch (Exception e) { - LOGGER.error("Failed to parse allowlist ranges", e); - throw new MalformedTrafficCalcConfigException("Failed to parse allowlist ranges: " + e.getMessage()); - } - - return ranges; - } - - /** - * Calculate traffic status based on delta files and SQS queue messages. - * - * Uses the newest delta file timestamp to anchor the 24-hour delta traffic window, - * and the oldest queue timestamp to anchor the 5-minute queue window. - * - * @param sqsMessages List of SQS messages - * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) - */ - public TrafficStatus calculateStatus(List sqsMessages) { - - try { - // Get list of delta files from S3 (sorted newest to oldest) - List deltaS3Paths = listDeltaFiles(); - - if (deltaS3Paths.isEmpty()) { - LOGGER.warn("No delta files found in S3 with prefix: {}", s3DeltaPrefix); - return TrafficStatus.DEFAULT; - } - - // Find newest delta file timestamp for delta traffic window - long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths); - LOGGER.info("Traffic calculation: newestDeltaTs={}", newestDeltaTs); - - // Find oldest SQS queue message timestamp for queue window - long oldestQueueTs = findOldestQueueTimestamp(sqsMessages); - LOGGER.info("Traffic calculation: oldestQueueTs={}", oldestQueueTs); - - // Define start time of the delta evaluation window - // We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend - // the window to account for any allowlist ranges in the extended portion - long deltaWindowStart = calculateWindowStartWithAllowlist(newestDeltaTs, this.evaluationWindowSeconds); - - // Evict old cache entries (older than delta window start) - evictOldCacheEntries(deltaWindowStart); - - // Process delta files and count records in [deltaWindowStart, newestDeltaTs] - int sum = 0; - - for (String s3Path : deltaS3Paths) { - List timestamps = getTimestampsFromFile(s3Path); - - boolean shouldStop = false; - for (long ts : timestamps) { - // Stop condition: record is older than our window - if (ts < deltaWindowStart) { - LOGGER.debug("Stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); - break; - } - - // skip records in allowlisted ranges - if (isInAllowlist(ts)) { - continue; - } - - // increment sum if record is in delta window - if (ts >= deltaWindowStart) { - sum++; - } - - } - - if (shouldStop) { - break; - } - } - - // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] - if (sqsMessages != null && !sqsMessages.isEmpty()) { - int sqsCount = countSqsMessages(sqsMessages, oldestQueueTs); - sum += sqsCount; - } - - // Determine status - TrafficStatus status = determineStatus(sum, this.baselineTraffic); - - LOGGER.info("Traffic calculation complete: sum={}, baselineTraffic={}, thresholdMultiplier={}, status={}", - sum, this.baselineTraffic, this.thresholdMultiplier, status); - - return status; - - } catch (Exception e) { - LOGGER.error("Error calculating traffic status", e); - return TrafficStatus.DEFAULT; - } - } - - /** - * Find the newest timestamp from delta files. - * Reads the newest delta file and returns its maximum timestamp. - */ - private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOException { - if (deltaS3Paths == null || deltaS3Paths.isEmpty()) { - return System.currentTimeMillis() / 1000; - } - - // Delta files are sorted (ISO 8601 format, lexicographically sortable) so first file is newest - String newestDeltaPath = deltaS3Paths.get(0); - List timestamps = getTimestampsFromFile(newestDeltaPath); - - if (timestamps.isEmpty()) { - LOGGER.warn("Newest delta file has no timestamps: {}", newestDeltaPath); - return System.currentTimeMillis() / 1000; - } - - long newestTs = Collections.max(timestamps); - LOGGER.debug("Found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); - return newestTs; - } - - /** - * List all delta files from S3, sorted newest to oldest - */ - private List listDeltaFiles() { - try { - // List all objects with the delta prefix - List allFiles = cloudStorage.list(s3DeltaPrefix); - - // Filter to only .dat delta files and sort newest to oldest - return allFiles.stream() - .filter(OptOutUtils::isDeltaFile) - .sorted(OptOutUtils.DeltaFilenameComparatorDescending) - .collect(Collectors.toList()); - - } catch (Exception e) { - LOGGER.error("Failed to list delta files from S3 with prefix: {}", s3DeltaPrefix, e); - return Collections.emptyList(); - } - } - - /** - * Get timestamps from a delta file (S3 path), using cache if available - */ - private List getTimestampsFromFile(String s3Path) throws IOException { - // Extract filename from S3 path for cache key - String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); - - // Check cache first - FileRecordCache cached = deltaFileCache.get(filename); - if (cached != null) { - LOGGER.debug("Using cached timestamps for file: {}", filename); - return cached.timestamps; - } - - // Cache miss - download from S3 - LOGGER.debug("Downloading and reading timestamps from S3: {}", s3Path); - List timestamps = readTimestampsFromS3(s3Path); - - // Store in cache - deltaFileCache.put(filename, new FileRecordCache(timestamps)); - - return timestamps; - } - - /** - * Read all non-sentinel record timestamps from a delta file in S3 - */ - private List readTimestampsFromS3(String s3Path) throws IOException { - try (InputStream is = cloudStorage.download(s3Path)) { - byte[] data = is.readAllBytes(); - OptOutCollection collection = new OptOutCollection(data); - - List timestamps = new ArrayList<>(); - for (int i = 0; i < collection.size(); i++) { - OptOutEntry entry = collection.get(i); - - // Skip sentinel entries - if (entry.isSpecialHash()) { - continue; - } - - timestamps.add(entry.timestamp); - } - - return timestamps; - } catch (Exception e) { - LOGGER.error("Failed to read delta file from S3: {}", s3Path, e); - throw new IOException("Failed to read delta file from S3: " + s3Path, e); - } - } - - /** - * Calculate total duration of allowlist ranges that overlap with the given time window. - */ - long getAllowlistDuration(long t, long windowStart) { - long totalDuration = 0; - for (List range : this.allowlistRanges) { - long start = range.get(0); - long end = range.get(1); - - // Clip range to window boundaries - if (start < windowStart) { - start = windowStart; - } - if (end > t) { - end = t; - } - - // Only add duration if there's actual overlap (start < end) - if (start < end) { - totalDuration += end - start; - } - } - return totalDuration; - } - - /** - * Calculate the window start time that provides evaluationWindowSeconds of non-allowlisted time. - * Iteratively extends the window to account for allowlist ranges that may fall in extended portions. - */ - long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowSeconds) { - long allowlistDuration = getAllowlistDuration(newestDeltaTs, newestDeltaTs - evaluationWindowSeconds); - - // Each iteration discovers at least one new allowlist range, so max iterations = number of ranges - int maxIterations = this.allowlistRanges.size() + 1; - - for (int i = 0; i < maxIterations && allowlistDuration > 0; i++) { - long newWindowStart = newestDeltaTs - evaluationWindowSeconds - allowlistDuration; - long newAllowlistDuration = getAllowlistDuration(newestDeltaTs, newWindowStart); - - if (newAllowlistDuration == allowlistDuration) { - // No new allowlist time in extended portion, we've converged - break; - } - - allowlistDuration = newAllowlistDuration; - } - - return newestDeltaTs - evaluationWindowSeconds - allowlistDuration; - } - - /** - * Find the oldest SQS queue message timestamp - */ - private long findOldestQueueTimestamp(List sqsMessages) throws IOException { - long oldest = System.currentTimeMillis() / 1000; - - if (sqsMessages != null && !sqsMessages.isEmpty()) { - for (Message msg : sqsMessages) { - Long ts = extractTimestampFromMessage(msg); - if (ts != null && ts < oldest) { - oldest = ts; - } - } - } - - return oldest; - } - - /** - * Extract timestamp from SQS message (from SentTimestamp attribute) - */ - private Long extractTimestampFromMessage(Message msg) { - // Get SentTimestamp attribute (milliseconds) - String sentTimestamp = msg.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); - if (sentTimestamp != null) { - try { - return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds - } catch (NumberFormatException e) { - LOGGER.debug("Invalid SentTimestamp: {}", sentTimestamp); - } - } - - // Fallback: use current time - return System.currentTimeMillis() / 1000; - } - - /** - * Count SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes - */ - private int countSqsMessages(List sqsMessages, long oldestQueueTs) { - - int count = 0; - long windowEnd = oldestQueueTs + 5 * 60; - - for (Message msg : sqsMessages) { - Long ts = extractTimestampFromMessage(msg); - - if (ts < oldestQueueTs || ts > windowEnd) { - continue; - } - - if (isInAllowlist(ts)) { - continue; - } - count++; - - } - - LOGGER.info("SQS messages: {} in window [oldestQueueTs={}, oldestQueueTs+5m={}]", count, oldestQueueTs, windowEnd); - return count; - } - - /** - * Check if a timestamp falls within any allowlist range - */ - boolean isInAllowlist(long timestamp) { - if (allowlistRanges == null || allowlistRanges.isEmpty()) { - return false; - } - - for (List range : allowlistRanges) { - if (range.size() < 2) { - continue; - } - - long start = range.get(0); - long end = range.get(1); - - if (timestamp >= start && timestamp <= end) { - return true; - } - } - - return false; - } - - /** - * Evict cache entries with data older than the cutoff timestamp - */ - private void evictOldCacheEntries(long cutoffTimestamp) { - int beforeSize = deltaFileCache.size(); - - deltaFileCache.entrySet().removeIf(entry -> - entry.getValue().newestTimestamp < cutoffTimestamp - ); - - int afterSize = deltaFileCache.size(); - if (beforeSize != afterSize) { - LOGGER.info("Evicted {} old cache entries (before={}, after={})", - beforeSize - afterSize, beforeSize, afterSize); - } - } - - /** - * Determine traffic status based on current vs past counts - */ - TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { - if (baselineTraffic == 0 || thresholdMultiplier == 0) { - // Avoid division by zero - if no baseline traffic, return DEFAULT status - LOGGER.warn("baselineTraffic is 0 or thresholdMultiplier is 0 returning DEFAULT status."); - return TrafficStatus.DEFAULT; - } - - if (sumCurrent >= thresholdMultiplier * baselineTraffic) { - LOGGER.warn("DELAYED_PROCESSING threshold breached: sumCurrent={} >= {}×baselineTraffic={}", - sumCurrent, thresholdMultiplier, baselineTraffic); - return TrafficStatus.DELAYED_PROCESSING; - } - - LOGGER.info("Traffic within normal range: sumCurrent={} < {}×baselineTraffic={}", - sumCurrent, thresholdMultiplier, baselineTraffic); - return TrafficStatus.DEFAULT; - } - - /** - * Get cache statistics for monitoring - */ - public Map getCacheStats() { - Map stats = new HashMap<>(); - stats.put("cached_files", deltaFileCache.size()); - - int totalTimestamps = deltaFileCache.values().stream() - .mapToInt(cache -> cache.timestamps.size()) - .sum(); - stats.put("total_cached_timestamps", totalTimestamps); - - return stats; - } - -} diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java deleted file mode 100644 index d0d5cd6..0000000 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java +++ /dev/null @@ -1,174 +0,0 @@ -package com.uid2.optout.vertx; - -import com.uid2.optout.sqs.SqsParsedMessage; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Collections; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.charset.StandardCharsets; -import io.vertx.core.json.JsonObject; -import io.vertx.core.json.JsonArray; - -public class OptOutTrafficFilter { - private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficFilter.class); - - private final String trafficFilterConfigPath; - List filterRules; - - /** - * Traffic filter rule defining a time range and a list of IP addresses to exclude - */ - private static class TrafficFilterRule { - private final List range; - private final List ipAddresses; - - TrafficFilterRule(List range, List ipAddresses) { - this.range = range; - this.ipAddresses = ipAddresses; - } - - public long getRangeStart() { - return range.get(0); - } - public long getRangeEnd() { - return range.get(1); - } - public List getIpAddresses() { - return ipAddresses; - } - } - - public static class MalformedTrafficFilterConfigException extends Exception { - public MalformedTrafficFilterConfigException(String message) { - super(message); - } - } - - /** - * Constructor for OptOutTrafficFilter - * - * @param trafficFilterConfigPath S3 path for traffic filter config - * @throws MalformedTrafficFilterConfigException if the traffic filter config is invalid - */ - public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTrafficFilterConfigException { - this.trafficFilterConfigPath = trafficFilterConfigPath; - // Initial filter rules load - this.filterRules = Collections.emptyList(); // start empty - reloadTrafficFilterConfig(); // load ConfigMap - - LOGGER.info("OptOutTrafficFilter initialized: filterRules={}", - filterRules.size()); - } - - /** - * Reload traffic filter config from ConfigMap. - * Expected format: - * { - * "denylist_requests": [ - * {range: [startTimestamp, endTimestamp], IPs: ["ip1"]}, - * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip2"]}, - * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip3"]}, - * ] - * } - * - * Can be called periodically to pick up config changes without restarting. - */ - public void reloadTrafficFilterConfig() throws MalformedTrafficFilterConfigException { - LOGGER.info("Loading traffic filter config from ConfigMap"); - try (InputStream is = Files.newInputStream(Paths.get(trafficFilterConfigPath))) { - String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); - JsonObject filterConfigJson = new JsonObject(content); - - this.filterRules = parseFilterRules(filterConfigJson); - - LOGGER.info("Successfully loaded traffic filter config from ConfigMap: filterRules={}", - filterRules.size()); - - } catch (Exception e) { - LOGGER.warn("No traffic filter config found at: {}", trafficFilterConfigPath, e); - throw new MalformedTrafficFilterConfigException(e.getMessage()); - } - } - - /** - * Parse request filtering rules from JSON config - */ - List parseFilterRules(JsonObject config) throws MalformedTrafficFilterConfigException { - List rules = new ArrayList<>(); - try { - JsonArray denylistRequests = config.getJsonArray("denylist_requests"); - if (denylistRequests == null) { - LOGGER.error("Invalid traffic filter config: denylist_requests is null"); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter config: denylist_requests is null"); - } - for (int i = 0; i < denylistRequests.size(); i++) { - JsonObject ruleJson = denylistRequests.getJsonObject(i); - - // parse range - var rangeJson = ruleJson.getJsonArray("range"); - List range = new ArrayList<>(); - if (rangeJson != null && rangeJson.size() == 2) { - long start = rangeJson.getLong(0); - long end = rangeJson.getLong(1); - - if (start >= end) { - LOGGER.error("Invalid traffic filter rule: range start must be less than end: {}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule: range start must be less than end"); - } - range.add(start); - range.add(end); - } - - // parse IPs - var ipAddressesJson = ruleJson.getJsonArray("IPs"); - List ipAddresses = new ArrayList<>(); - if (ipAddressesJson != null) { - for (int j = 0; j < ipAddressesJson.size(); j++) { - ipAddresses.add(ipAddressesJson.getString(j)); - } - } - - // log error and throw exception if rule is invalid - if (range.size() != 2 || ipAddresses.size() == 0 || range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less - LOGGER.error("Invalid traffic filter rule, range must be 24 hours or less: {}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule, range must be 24 hours or less"); - } - - TrafficFilterRule rule = new TrafficFilterRule(range, ipAddresses); - - LOGGER.info("Loaded traffic filter rule: range=[{}, {}], IPs={}", rule.getRangeStart(), rule.getRangeEnd(), rule.getIpAddresses()); - rules.add(rule); - } - return rules; - } catch (Exception e) { - LOGGER.error("Failed to parse traffic filter rules: config={}, error={}", config.encode(), e.getMessage()); - throw new MalformedTrafficFilterConfigException(e.getMessage()); - } - } - - public boolean isDenylisted(SqsParsedMessage message) { - long timestamp = message.timestamp(); - String clientIp = message.clientIp(); - - if (clientIp == null || clientIp.isEmpty()) { - LOGGER.error("Request does not contain client IP, timestamp={}", timestamp); - return false; - } - - for (TrafficFilterRule rule : filterRules) { - if(timestamp >= rule.getRangeStart() && timestamp <= rule.getRangeEnd()) { - if(rule.getIpAddresses().contains(clientIp)) { - return true; - } - }; - } - return false; - } - -} \ No newline at end of file diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java deleted file mode 100644 index f977233..0000000 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ /dev/null @@ -1,1510 +0,0 @@ -package com.uid2.optout.vertx; - -import com.uid2.shared.cloud.CloudStorageException; -import com.uid2.shared.cloud.ICloudStorage; -import com.uid2.shared.optout.OptOutCollection; -import com.uid2.shared.optout.OptOutEntry; -import com.uid2.optout.Const; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import java.nio.file.Files; -import java.nio.file.Path; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; -import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; - -import com.uid2.optout.vertx.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; -import java.io.ByteArrayInputStream; -import java.util.*; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; - - -@ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) -public class OptOutTrafficCalculatorTest { - - @Mock - private ICloudStorage cloudStorage; - - private static final String S3_DELTA_PREFIX = "optout-v2/delta/"; - private static final String TRAFFIC_CONFIG_PATH = "./traffic-config.json"; - private static final int BASELINE_TRAFFIC = 100; - private static final int THRESHOLD_MULTIPLIER = 5; - private static final int EVALUATION_WINDOW_SECONDS = 24 * 3600; - - @BeforeEach - void setUp() { - // default config - JsonObject config = new JsonObject(); - config.put(Const.Config.OptOutTrafficCalcBaselineTrafficProp, BASELINE_TRAFFIC); - config.put(Const.Config.OptOutTrafficCalcThresholdMultiplierProp, THRESHOLD_MULTIPLIER); - config.put(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp, EVALUATION_WINDOW_SECONDS); - config.put(Const.Config.OptOutTrafficCalcAllowlistRangesProp, new JsonArray()); - try { - createTrafficConfigFile(config.toString()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @AfterEach - void tearDown() { - if (Files.exists(Path.of(TRAFFIC_CONFIG_PATH))) { - try { - Files.delete(Path.of(TRAFFIC_CONFIG_PATH)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - - private void createTrafficConfigFile(String content) { - try { - Path configPath = Path.of(TRAFFIC_CONFIG_PATH); - Files.writeString(configPath, content); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * Helper to create config by merging partial JSON with defaults - */ - private void createConfigFromPartialJson(String partialJson) { - JsonObject partial = new JsonObject(partialJson); - JsonObject config = new JsonObject(); - - // Set defaults - if (!partial.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { - config.put(Const.Config.OptOutTrafficCalcBaselineTrafficProp, BASELINE_TRAFFIC); - } - if (!partial.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { - config.put(Const.Config.OptOutTrafficCalcThresholdMultiplierProp, THRESHOLD_MULTIPLIER); - } - if (!partial.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { - config.put(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp, EVALUATION_WINDOW_SECONDS); - } - if (!partial.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { - config.put(Const.Config.OptOutTrafficCalcAllowlistRangesProp, new JsonArray()); - } - - // Merge in partial config (overrides defaults) - partial.forEach(entry -> config.put(entry.getKey(), entry.getValue())); - - createTrafficConfigFile(config.toString()); - } - - /** - * Helper to create config with custom threshold - */ - private void createConfigWithThreshold(int threshold) { - createConfigFromPartialJson("{\"" + Const.Config.OptOutTrafficCalcThresholdMultiplierProp + "\": " + threshold + "}"); - } - - // ============================================================================ - // SECTION 1: Constructor & Initialization Tests - // ============================================================================ - - @Test - void testConstructor_defaultThreshold() throws Exception { - // Setup - default threshold of 5 - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); // 10 < 5*3 - - status = calculator.determineStatus(15, 3); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 15 >= 5*3 - } - - @Test - void testConstructor_customThreshold() throws Exception { - // Setup - custom threshold of 10 - createConfigWithThreshold(10); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(49, 5); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); // 49 < 10*5 - status = calculator.determineStatus(50, 5); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 50 >= 10*5 - } - - @Test - void testConstructor_trafficCalcConfigLoadFailure() throws Exception { - // Setup - traffic calc config load failure - createTrafficConfigFile("Invalid JSON"); - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - }); - - // Create valid config to test reload failure - createConfigFromPartialJson("{}"); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - createTrafficConfigFile("Invalid JSON"); - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - } - - // ============================================================================ - // SECTION 2: parseTrafficCalcConfigRanges() - // ============================================================================ - - @Test - void testParseTrafficCalcConfigRanges_emptyConfig() throws Exception { - // Setup - no config - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - JsonObject emptyConfig = new JsonObject(); - - // Act - List> ranges = calculator.parseAllowlistRanges(emptyConfig); - - // Assert - empty ranges - assertTrue(ranges.isEmpty()); - } - - @Test - void testParseTrafficCalcConfigRanges_singleRange() throws Exception { - // Setup - single range - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(1000L).add(2000L)); - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - List> result = calculator.parseAllowlistRanges(configWithRanges); - - // Assert - single range - assertEquals(1, result.size()); - assertEquals(1000L, result.get(0).get(0)); - assertEquals(2000L, result.get(0).get(1)); - } - - @Test - void testParseTrafficCalcConfigRanges_multipleRanges() throws Exception { - // Setup - multiple ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(1000L).add(2000L)) - .add(new JsonArray().add(3000L).add(4000L)) - .add(new JsonArray().add(5000L).add(6000L)); - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - List> result = calculator.parseAllowlistRanges(configWithRanges); - - // Assert - multiple ranges - assertEquals(3, result.size()); - assertEquals(1000L, result.get(0).get(0)); - assertEquals(3000L, result.get(1).get(0)); - assertEquals(5000L, result.get(2).get(0)); - } - - @Test - void testParseTrafficCalcConfigRanges_misorderedRange() throws Exception { - // Setup - range with end < start is malformed - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(2000L).add(1000L)); // End before start - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.parseAllowlistRanges(configWithRanges); - }); - } - - @Test - void testParseTrafficCalcConfigRanges_rangeTooLong() throws Exception { - // Setup - range longer than 24 hours is malformed - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(2000L).add(200000L)); // Longer than 24 hours - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.parseAllowlistRanges(configWithRanges); - }); - } - - @Test - void testParseTrafficCalcConfigRanges_sortsByStartTime() throws Exception { - // Setup - ranges added out of order - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(5000L).add(6000L)) - .add(new JsonArray().add(1000L).add(2000L)) - .add(new JsonArray().add(3000L).add(4000L)); - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - List> result = calculator.parseAllowlistRanges(configWithRanges); - - // Assert - should be sorted by start time - assertEquals(3, result.size()); - assertEquals(1000L, result.get(0).get(0)); - assertEquals(3000L, result.get(1).get(0)); - assertEquals(5000L, result.get(2).get(0)); - } - - @Test - void testParseTrafficCalcConfigRanges_invalidRangeTooFewElements() throws Exception { - // Setup - invalid range with only 1 element; - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(1000L)) // Only 1 element - .add(new JsonArray().add(2000L).add(3000L)); // Valid - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - List> result = calculator.parseAllowlistRanges(configWithRanges); - - // Assert - should skip invalid range - assertEquals(1, result.size()); - assertEquals(2000L, result.get(0).get(0)); - } - - @Test - void testParseTrafficCalcConfigRanges_nullArray() throws Exception { - // Setup - null array - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - configWithRanges.put("traffic_calc_allowlist_ranges", (JsonArray) null); - - // Act - List> result = calculator.parseAllowlistRanges(configWithRanges); - - // Assert - empty ranges - assertTrue(result.isEmpty()); - } - - @Test - void testParseTrafficCalcConfigRanges_overlappingRanges() throws Exception { - // Setup - overlapping ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(1000L).add(2000L)) - .add(new JsonArray().add(1500L).add(2500L)); // Overlaps with first range - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act & Assert - should throw exception due to overlap - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.parseAllowlistRanges(configWithRanges); - }); - } - - @Test - void testParseTrafficCalcConfigRanges_adjacentRangesWithSameBoundary() throws Exception { - // Setup - ranges where end of first equals start of second (touching but not overlapping semantically, but we treat as overlap) - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(1000L).add(2000L)) - .add(new JsonArray().add(2000L).add(3000L)); // Starts exactly where first ends - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act & Assert - should throw exception because ranges touch at boundary - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.parseAllowlistRanges(configWithRanges); - }); - } - - @Test - void testParseTrafficCalcConfigRanges_nonOverlappingRanges() throws Exception { - // Setup - ranges that don't overlap (with gap between them) - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(1000L).add(2000L)) - .add(new JsonArray().add(2001L).add(3000L)); // Starts after first ends - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - List> result = calculator.parseAllowlistRanges(configWithRanges); - - // Assert - should succeed with 2 ranges - assertEquals(2, result.size()); - } - - // ============================================================================ - // SECTION 3: isInTrafficCalcConfig() - // ============================================================================ - - @Test - void testIsInTrafficCalcConfig_withinSingleRange() throws Exception { - // Setup - load traffic calc config with single range [1000, 2000] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - true when within range - assertTrue(calculator.isInAllowlist(1500L)); - } - - @Test - void testIsInTrafficCalcConfig_exactlyAtStart() throws Exception { - // Setup - load traffic calc config with single range [1000, 2000] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - true when exactly at start of range - assertTrue(calculator.isInAllowlist(1000L)); - } - - @Test - void testIsInTrafficCalcConfig_exactlyAtEnd() throws Exception { - // Setup - load traffic calc config with single range [1000, 2000] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - true when exactly at end of range - assertTrue(calculator.isInAllowlist(2000L)); - } - - @Test - void testIsInTrafficCalcConfig_beforeRange() throws Exception { - // Setup - load traffic calc config with single range [1000, 2000] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - false when before range - assertFalse(calculator.isInAllowlist(999L)); - } - - @Test - void testIsInTrafficCalcConfig_afterRange() throws Exception { - // Setup - load traffic calc config with single range [1000, 2000] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - false when after range - assertFalse(calculator.isInAllowlist(2001L)); - } - - @Test - void testIsInTrafficCalcConfig_betweenRanges() throws Exception { - // Setup - load traffic calc config with two ranges [1000, 2000] and [3000, 4000] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000], - [3000, 4000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - false when between ranges - assertFalse(calculator.isInAllowlist(2500L)); - } - - @Test - void testIsInTrafficCalcConfig_emptyRanges() throws Exception { - // Setup uses default config from setUp() which has empty traffic calc config ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - false when empty ranges - assertFalse(calculator.isInAllowlist(1500L)); - } - - @Test - void testIsInTrafficCalcConfig_nullRanges() throws Exception { - // Setup - no traffic calc config ranges loaded (will fail and set empty) - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": null - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - false when null/empty ranges - assertFalse(calculator.isInAllowlist(1500L)); - } - - @Test - void testIsInTrafficCalcConfig_invalidRangeSize() throws Exception { - // Setup - load traffic calc config with invalid range (only 1 element) and valid range - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000], - [2000, 3000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - assertFalse(calculator.isInAllowlist(1500L)); // Should not match invalid range - assertTrue(calculator.isInAllowlist(2500L)); // Should match valid range - } - - @Test - void testIsInTrafficCalcConfig_multipleRanges() throws Exception { - // Setup - load traffic calc config with multiple ranges - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000], - [3000, 4000], - [5000, 6000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - assertTrue(calculator.isInAllowlist(1500L)); // In first range - assertTrue(calculator.isInAllowlist(3500L)); // In second range - assertTrue(calculator.isInAllowlist(5500L)); // In third range - assertFalse(calculator.isInAllowlist(2500L)); // Between first and second - } - - // ============================================================================ - // SECTION 4: getTrafficCalcConfigDuration() - // ============================================================================ - - @Test - void testGetTrafficCalcConfigDuration_noRanges() throws Exception { - // Setup - no ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - assertEquals(0L, calculator.getAllowlistDuration(10000L, 5000L)); // 0 duration when no ranges - } - - @Test - void testGetTrafficCalcConfigDuration_rangeFullyWithinWindow() throws Exception { - // Setup - range fully within window - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [6000, 7000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window [5000, 10000], range [6000, 7000] - long duration = calculator.getAllowlistDuration(10000L, 5000L); - - // Assert - full range duration - assertEquals(1000L, duration); - } - - @Test - void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsStart() throws Exception { - // Setup - range partially overlaps start of window - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [3000, 7000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window [5000, 10000], range [3000, 7000] - long duration = calculator.getAllowlistDuration(10000L, 5000L); - - // Assert - should clip to [5000, 7000] = 2000 - assertEquals(2000L, duration); - } - - @Test - void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsEnd() throws Exception { - // Setup - range partially overlaps end of window - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [8000, 12000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window [5000, 10000], range [8000, 12000] - long duration = calculator.getAllowlistDuration(10000L, 5000L); - - // Assert - should clip to [8000, 10000] = 2000 - assertEquals(2000L, duration); - } - - @Test - void testGetTrafficCalcConfigDuration_rangeCompletelyOutsideWindow() throws Exception { - // Setup - range completely outside window - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window [5000, 10000], range [1000, 2000] - long duration = calculator.getAllowlistDuration(10000L, 5000L); - - // Assert - 0 duration when range completely outside window - assertEquals(0L, duration); - } - - @Test - void testGetTrafficCalcConfigDuration_multipleRanges() throws Exception { - // Setup - multiple ranges - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [6000, 7000], - [8000, 9000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window [5000, 10000], ranges [6000, 7000] and [8000, 9000] - long duration = calculator.getAllowlistDuration(10000L, 5000L); - - // Assert - 1000 + 1000 = 2000 - assertEquals(2000L, duration); - } - - @Test - void testGetTrafficCalcConfigDuration_rangeSpansEntireWindow() throws Exception { - // Setup - range spans entire window - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [3000, 12000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window [5000, 10000], range [3000, 12000] - long duration = calculator.getAllowlistDuration(10000L, 5000L); - - // Assert - entire window is in traffic calc config ranges = 5000 - assertEquals(5000L, duration); - } - - // ============================================================================ - // SECTION 4.5: calculateWindowStartWithAllowlist() - // ============================================================================ - - @Test - void testCalculateWindowStartWithAllowlist_noAllowlist() throws Exception { - // Setup - no allowlist ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window should be [3, 8] with no extension - long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); - - // Assert - no allowlist, so window start is simply newestDeltaTs - evaluationWindowSeconds - assertEquals(3L, windowStart); - } - - @Test - void testCalculateWindowStartWithAllowlist_allowlistInOriginalWindowOnly() throws Exception { - // Setup - allowlist range only in original window, not in extended portion - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [6, 7] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - newestDeltaTs=8, evaluationWindow=5 - // Original window [3, 8] has [6,7] allowlisted (1 hour) - // Extended portion [2, 3] has no allowlist - // So window start should be 8 - 5 - 1 = 2 - long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); - - assertEquals(2L, windowStart); - } - - @Test - void testCalculateWindowStartWithAllowlist_allowlistInExtendedPortion() throws Exception { - // Setup - allowlist ranges in both original window AND extended portion - // This is the user's example: evaluationWindow=5, newestDeltaTs=8, allowlist={[2,3], [6,7]} - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [2, 3], - [6, 7] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - // Original window [3, 8]: [6,7] allowlisted = 1 hour - // First extension to [2, 8]: [2,3] and [6,7] allowlisted = 2 hours total - // Second extension to [1, 8]: still [2,3] and [6,7] = 2 hours (no new allowlist) - // Final: windowStart = 8 - 5 - 2 = 1 - long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); - - assertEquals(1L, windowStart); - } - - @Test - void testCalculateWindowStartWithAllowlist_allowlistBeforeWindow() throws Exception { - // Setup - allowlist range entirely before the initial window - // This tests that we don't over-extend when allowlist is old - // evaluationWindow=5, newestDeltaTs=20, allowlist=[10,13] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [10, 13] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - // Initial window [15, 20]: no allowlist overlap, allowlistDuration = 0 - // No extension needed - // Final: windowStart = 20 - 5 - 0 = 15 - long windowStart = calculator.calculateWindowStartWithAllowlist(20L, 5); - - // Verify: window [15, 20] has 5 hours, 0 allowlisted = 5 non-allowlisted - assertEquals(15L, windowStart); - } - - // ============================================================================ - // SECTION 5: determineStatus() - // ============================================================================ - - @Test - void testDetermineStatus_belowThreshold() throws Exception { - // Setup - below threshold - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - 10 < 5 * 3 - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); - - // Assert - DEFAULT when below threshold - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testDetermineStatus_atThreshold() throws Exception { - // Setup - at threshold - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - 15 == 5 * 3 - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(15, 3); - - // Assert - DELAYED_PROCESSING when at threshold - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - @Test - void testDetermineStatus_aboveThreshold() throws Exception { - // Setup - above threshold - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - 20 > 5 * 3 - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(20, 3); - - // Assert - DELAYED_PROCESSING when above threshold - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - @Test - void testDetermineStatus_sumPastZero() throws Exception { - // Setup - sumPast is 0 - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - should return DEFAULT to avoid crash - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(100, 0); - - // Assert - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testDetermineStatus_bothZero() throws Exception { - // Setup - both sumCurrent and sumPast are 0; - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - should return DEFAULT to avoid crash - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 0); - - // Assert - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testDetermineStatus_sumCurrentZero() throws Exception { - // Setup - sumCurrent is 0 - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - 0 < 5 * 10 - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 10); - - // Assert - DEFAULT when sumCurrent is 0 - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @ParameterizedTest - @CsvSource({ - "1, 1, 1, DELAYED_PROCESSING", // threshold=1: 1 >= 1*1 - "2, 4, 2, DELAYED_PROCESSING", // threshold=2: 4 >= 2*2 - "5, 10, 2, DELAYED_PROCESSING", // threshold=5: 10 >= 5*2 - "10, 100, 10, DELAYED_PROCESSING", // threshold=10: 100 >= 10*10 - "5, 24, 5, DEFAULT", // threshold=5: 24 < 5*5 - "100, 1000, 11, DEFAULT" // threshold=100: 1000 < 100*11 - }) - void testDetermineStatus_variousThresholds(int threshold, int sumCurrent, int sumPast, String expectedStatus) throws Exception { - // Setup - various thresholds - createConfigWithThreshold(threshold); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(sumCurrent, sumPast); - - // Assert - assertEquals(OptOutTrafficCalculator.TrafficStatus.valueOf(expectedStatus), status); - } - - @Test - void testDetermineStatus_largeNumbers() throws Exception { - // Setup - test with large numbers - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(1_000_000, 200_000); - - // Assert - 1M >= 5 * 200K = 1M - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - // ============================================================================ - // SECTION 6: S3 Config Reload Tests - // ============================================================================ - - @Test - void testReloadTrafficCalcConfig_success() throws Exception { - // Setup - initial traffic calc config - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000], - [3000, 4000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Change the traffic calc config to a new range - String newTrafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [5000, 6000] - ] - } - """; - createConfigFromPartialJson(newTrafficCalcConfigJson); - - // Act - reload the traffic calc config - calculator.reloadTrafficCalcConfig(); - - // Assert - verify new traffic calc config is loaded - assertTrue(calculator.isInAllowlist(5500L)); - } - - @Test - void testReloadTrafficCalcConfig_failure() throws Exception { - // Setup - initial traffic calc config - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Now make it fail - createTrafficConfigFile("Invalid JSON"); - - // Act - should not throw exception - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - - } - - @Test - public void testReloadTrafficCalcConfig_failure_missingKeys() throws Exception { - // Setup - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act & Assert missing threshold multiplier - createTrafficConfigFile("{\"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - - // Act & Assert missing evaluation window seconds - createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - - // Act & Assert missing baseline traffic - createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - - // Act & Assert missing traffic calc config ranges - createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100}"); - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - } - - @Test - public void testReloadTrafficCalcConfig_failure_misorderedRanges() throws Exception { - // Setup - misordered ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - createConfigFromPartialJson("{\"traffic_calc_allowlist_ranges\": [ [2000, 1000] ]}"); - - // Act & Assert - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - } - - @Test - public void testReloadTrafficCalcConfig_failure_rangeTooLong() throws Exception { - // Setup - range greater than 24 hours - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - createConfigFromPartialJson("{\"traffic_calc_allowlist_ranges\": [ [1000, 200000] ]}"); - - // Act & Assert - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - } - - // ============================================================================ - // SECTION 7: Cache Management Tests (also tested in section 9) - // ============================================================================ - - @Test - void testGetCacheStats_emptyCache() throws Exception { - // Setup - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - Map stats = calculator.getCacheStats(); - - // Assert - should return empty stats - assertEquals(0, stats.get("cached_files")); - assertEquals(0, stats.get("total_cached_timestamps")); - } - - // ============================================================================ - // SECTION 8: Helper Methods for Test Data Creation - // ============================================================================ - - /** - * Create a mock SQS message with specified timestamp - */ - private Message createSqsMessage(long timestampSeconds) { - Map attributes = new HashMap<>(); - attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampSeconds * 1000)); - - return Message.builder() - .messageId("test-msg-" + timestampSeconds) - .body("{\"test\": \"data\"}") - .attributes(attributes) - .build(); - } - - /** - * Create a mock SQS message without timestamp - */ - private Message createSqsMessageWithoutTimestamp() { - return Message.builder() - .messageId("test-msg-no-timestamp") - .body("{\"test\": \"data\"}") - .attributes(new HashMap<>()) - .build(); - } - - /** - * Create delta file bytes with specified timestamps - */ - private byte[] createDeltaFileBytes(List timestamps) throws Exception { - // Create OptOutEntry objects using newTestEntry - List entries = new ArrayList<>(); - - long idCounter = 1000; // Use incrementing IDs for test entries - for (long timestamp : timestamps) { - entries.add(OptOutEntry.newTestEntry(idCounter++, timestamp)); - } - - // Create OptOutCollection - OptOutCollection collection = new OptOutCollection(entries.toArray(new OptOutEntry[0])); - return collection.getStore(); - } - - - // ============================================================================ - // SECTION 9: Tests for calculateStatus() - // ============================================================================ - - @Test - void testCalculateStatus_noDeltaFiles() throws Exception { - // Setup - no delta files - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Collections.emptyList()); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); - - // Assert - should return DEFAULT when no delta files - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_normalTraffic() throws Exception { - // Setup - setup time: current time - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - // Create delta files with timestamps distributed over 48 hours - List timestamps = new ArrayList<>(); - - // add 499 entries in current window - for (int i = 0; i < 49; i++) { - timestamps.add(t - 23*3600 + i * 60); - } - - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); - - // Assert - 100+1 < 5 * 50 = 250, so should be DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_delayedProcessing() throws Exception { - // Setup - create delta files with spike in current window - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - // Create delta files with spike in current window - List timestamps = new ArrayList<>(); - - // add 500 entries in current window - for (int i = 0; i < 500; i++) { - timestamps.add(t - 23*3600 + i * 60); - } - - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); - - // Assert - 100+1 >= 5 * 10 = 50, DELAYED_PROCESSING - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - @Test - void testCalculateStatus_noSqsMessages() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600, t - 7200); // Some entries - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - null SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null); - - // Assert - should still calculate based on delta files, DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_emptySqsMessages() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600); - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - empty SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); - - // Assert - should still calculate based on delta files, DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_multipleSqsMessages() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = new ArrayList<>(); - // add 470 entries in window - for (int i = 0; i < 470; i++) { - timestamps.add(t - 24*3600 + i * 60); - } - - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Add 30 SQS entries in [t, t+5min] - List sqsMessages = new ArrayList<>(); - for (int i = 0; i < 30; i++) { - sqsMessages.add(createSqsMessage(t - i * 10)); - } - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); - - // Assert - DELAYED_PROCESSING - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - @Test - void testCalculateStatus_withTrafficCalcConfig() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - // Traffic calc config that covers part of window - String trafficCalcConfigJson = String.format(""" - { - "traffic_calc_allowlist_ranges": [ - [%d, %d] - ] - } - """, t - 12*3600, t - 6*3600); - - List timestamps = new ArrayList<>(); - - // window - 600 entries (300 in traffic calc config range, 300 outside) - for (int i = 0; i < 300; i++) { - timestamps.add(t - 12*3600 + i); - } - for (int i = 0; i < 300; i++) { - timestamps.add(t - 3600 + i); - } - - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - createConfigFromPartialJson(trafficCalcConfigJson); - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/delta-001.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); - - // Assert - should filter out entries in traffic calc config ranges - // Only 300 from window count (not in traffic calc config ranges) + 1 SQS = 301 - // 301 < 5*100, so DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_cacheUtilization() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600, t - 7200); - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - first call should populate cache - List sqsMessages = Arrays.asList(createSqsMessage(t)); - calculator.calculateStatus(sqsMessages); - - Map stats = calculator.getCacheStats(); - int cachedFiles = (Integer) stats.get("cached_files"); - - // Second call should use cache (no additional S3 download) - calculator.calculateStatus(sqsMessages); - - Map stats2 = calculator.getCacheStats(); - int cachedFiles2 = (Integer) stats2.get("cached_files"); - - // Assert - cache should be populated and remain consistent - assertEquals(1, cachedFiles); - assertEquals(cachedFiles, cachedFiles2); - - // Verify S3 download was called only once per file - verify(cloudStorage, times(1)).download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat"); - } - - @Test - void testCalculateStatus_s3Exception() throws Exception { - // Setup - S3 list error - when(cloudStorage.list(S3_DELTA_PREFIX)).thenThrow(new RuntimeException("S3 error")); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - should not throw exception - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); - - // Assert - DEFAULT on error - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_deltaFileReadException() throws Exception { - // Setup - S3 download error - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenThrow(new CloudStorageException("Failed to download")); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - empty SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); - - // Assert - DEFAULT on error - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600); - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - SQS message without timestamp (should use current time) - List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp()); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); - - // Assert - DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_multipleDeltaFiles() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - // File 1 - recent entries - List timestamps1 = new ArrayList<>(); - for (int i = 0; i < 50; i++) { - timestamps1.add(t - 12*3600 + i * 1000); - } - byte[] deltaFileBytes1 = createDeltaFileBytes(timestamps1); - - // File 2 - older entries - List timestamps2 = new ArrayList<>(); - for (int i = 0; i < 30; i++) { - timestamps2.add(t - 36*3600 + i * 1000); - } - byte[] deltaFileBytes2 = createDeltaFileBytes(timestamps2); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList( - "optout-v2/delta/optout-delta--01_2025-11-13T02.00.00Z_bbbbbbbb.dat", - "optout-v2/delta/optout-delta--01_2025-11-13T01.00.00Z_aaaaaaaa.dat" - )); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T02.00.00Z_bbbbbbbb.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes1)); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T01.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes2)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); - - // Assert - DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - - // Verify cache has both files - Map stats = calculator.getCacheStats(); - assertEquals(2, stats.get("cached_files")); - } - - @Test - void testCalculateStatus_windowBoundaryTimestamp() throws Exception { - // Setup - create delta file with timestamps at window boundary - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - long currentWindowStart = t - 24*3600; - List timestamps = new ArrayList<>(); - for (int i = 0; i < 250; i++) { - timestamps.add(t); - } - for (int i = 0; i < 250; i++) { - timestamps.add(currentWindowStart); - } - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); - - // Assert - DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - @Test - void testCalculateStatus_timestampsCached() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600, t - 7200); - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); - - // Assert - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - - // Cache should contain the timestamps - Map stats = calculator.getCacheStats(); - assertEquals(2, stats.get("total_cached_timestamps")); - } - -} diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java deleted file mode 100644 index 88881c5..0000000 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java +++ /dev/null @@ -1,426 +0,0 @@ -package com.uid2.optout.vertx; - -import com.uid2.optout.sqs.SqsParsedMessage; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import software.amazon.awssdk.services.sqs.model.Message; - -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.junit.Assert.*; - -public class OptOutTrafficFilterTest { - - private static final String TEST_CONFIG_PATH = "./traffic-config.json"; - - @Before - public void setUp() { - try { - Files.deleteIfExists(Path.of(TEST_CONFIG_PATH)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @After - public void tearDown() { - try { - Files.deleteIfExists(Path.of(TEST_CONFIG_PATH)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Test - public void testParseFilterRules_emptyRules() throws Exception { - // Setup - empty denylist - String config = """ - { - "denylist_requests": [] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - no rules - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertEquals(0, filter.filterRules.size()); - } - - @Test - public void testParseFilterRules_singleRule() throws Exception { - // Setup - config with one rule - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - one rule - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertEquals(1, filter.filterRules.size()); - } - - @Test - public void testParseFilterRules_multipleRules() throws Exception { - // Setup - config with multiple rules - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - }, - { - "range": [1700010000, 1700013600], - "IPs": ["10.0.0.1", "10.0.0.2"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - two rules - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertEquals(2, filter.filterRules.size()); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_missingDenylistRequests() throws Exception { - // Setup - config without denylist_requests field - String config = """ - { - "other_field": "value" - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_invalidRange_startAfterEnd() throws Exception { - // Setup - range where start > end - String config = """ - { - "denylist_requests": [ - { - "range": [1700003600, 1700000000], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_invalidRange_startEqualsEnd() throws Exception { - // Setup - range where start == end - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700000000], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_rangeExceeds24Hours() throws Exception { - // Setup - range longer than 24 hours (86400 seconds) - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700086401], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_emptyIPs() throws Exception { - // Setup - rule with empty IP list - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": [] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_missingIPs() throws Exception { - // Setup - rule without IPs field - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); - } - - @Test - public void testIsDenylisted_matchingIPAndTimestamp() throws Exception { - // Setup - filter with denylist rule - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1", "10.0.0.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - SqsParsedMessage message = createTestMessage(1700001800, "192.168.1.1"); - - // Act & Assert - denylisted - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertTrue(filter.isDenylisted(message)); - } - - @Test - public void testIsDenylisted_matchingIPOutsideTimeRange() throws Exception { - // Setup - filter with denylist rule - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - - // Act & Assert - message before range not denylisted - SqsParsedMessage messageBefore = createTestMessage(1699999999, "192.168.1.1"); - assertFalse(filter.isDenylisted(messageBefore)); - // Act & Assert - message after range not denylisted - SqsParsedMessage messageAfter = createTestMessage(1700003601, "192.168.1.1"); - assertFalse(filter.isDenylisted(messageAfter)); - } - - @Test - public void testIsDenylisted_nonMatchingIP() throws Exception { - // Setup - filter with denylist rule - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - - // Act & Assert - non-matching IP not denylisted - SqsParsedMessage message = createTestMessage(1700001800, "10.0.0.1"); - assertFalse(filter.isDenylisted(message)); - } - - @Test - public void testIsDenylisted_atRangeBoundaries() throws Exception { - // Setup - filter with denylist rule - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - - // Act & Assert - message at start boundary (inclusive) denylisted - SqsParsedMessage messageAtStart = createTestMessage(1700000000, "192.168.1.1"); - assertTrue(filter.isDenylisted(messageAtStart)); - - // Act & Assert - message at end boundary (inclusive) denylisted - SqsParsedMessage messageAtEnd = createTestMessage(1700003600, "192.168.1.1"); - assertTrue(filter.isDenylisted(messageAtEnd)); - } - - @Test - public void testIsDenylisted_multipleRules() throws Exception { - // Setup - multiple denylist rules - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - }, - { - "range": [1700010000, 1700013600], - "IPs": ["10.0.0.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - - // Act & Assert - message matches first rule - SqsParsedMessage msg1 = createTestMessage(1700001800, "192.168.1.1"); - assertTrue(filter.isDenylisted(msg1)); - - // Act & Assert - message matches second rule - SqsParsedMessage msg2 = createTestMessage(1700011800, "10.0.0.1"); - assertTrue(filter.isDenylisted(msg2)); - - // Act & Assert - message matches neither rule - SqsParsedMessage msg3 = createTestMessage(1700005000, "172.16.0.1"); - assertFalse(filter.isDenylisted(msg3)); - } - - @Test - public void testIsDenylisted_nullClientIp() throws Exception { - // Setup - filter with denylist rule - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - - // Act & Assert - message with null IP not denylisted - SqsParsedMessage message = createTestMessage(1700001800, null); - assertFalse(filter.isDenylisted(message)); - } - - @Test - public void testReloadTrafficFilterConfig_success() throws Exception { - // Setup - config with one rule - String initialConfig = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), initialConfig); - - // Act & Assert - one rule - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertEquals(1, filter.filterRules.size()); - - // Setup - update config - String updatedConfig = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - }, - { - "range": [1700010000, 1700013600], - "IPs": ["10.0.0.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), updatedConfig); - - // Act & Assert - two rules - filter.reloadTrafficFilterConfig(); - assertEquals(2, filter.filterRules.size()); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testReloadTrafficFilterConfig_fileNotFound() throws Exception { - // Setup, Act & Assert - try to create filter with non-existent config - new OptOutTrafficFilter("./non-existent-file.json"); - } - - @Test - public void testParseFilterRules_maxValidRange() throws Exception { - // Setup - range exactly 24 hours (86400 seconds) - should be valid - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700086400], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - one rule - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertEquals(1, filter.filterRules.size()); - } - - /** - * Helper method to create test SqsParsedMessage - */ - private SqsParsedMessage createTestMessage(long timestamp, String clientIp) { - Message mockMessage = Message.builder().build(); - byte[] hash = new byte[32]; - byte[] id = new byte[32]; - return new SqsParsedMessage(mockMessage, hash, id, timestamp, null, null, clientIp, null); - } -} From 950af13f4692308326ae06b0442b4ea95479cf9a Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 16:35:48 -0700 Subject: [PATCH 07/25] try make git render file recreated --- .../traffic/OptOutTrafficCalculator.java | 643 ------- .../optout/traffic/OptOutTrafficFilter.java | 184 -- .../traffic/OptOutTrafficCalculatorTest.java | 1577 ----------------- .../traffic/OptOutTrafficFilterTest.java | 428 ----- 4 files changed, 2832 deletions(-) delete mode 100644 src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java delete mode 100644 src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java delete mode 100644 src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java delete mode 100644 src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java deleted file mode 100644 index ab6a2c0..0000000 --- a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java +++ /dev/null @@ -1,643 +0,0 @@ -package com.uid2.optout.traffic; - -import com.uid2.shared.cloud.ICloudStorage; -import com.uid2.shared.optout.OptOutCollection; -import com.uid2.shared.optout.OptOutEntry; -import com.uid2.shared.optout.OptOutUtils; -import com.uid2.optout.Const; -import com.uid2.optout.sqs.SqsMessageOperations; - -import io.vertx.core.json.JsonObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; - -import java.nio.charset.StandardCharsets; - -import java.nio.file.Files; -import java.nio.file.Paths; -import java.io.InputStream; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -/** - * Calculates opt-out traffic patterns to determine DEFAULT or DELAYED_PROCESSING status. - * - * Compares recent ~24h traffic (sumCurrent) against a configurable baseline (baselineTraffic) of expected traffic in 24 hours. - * The baseline is multiplied by (thresholdMultiplier) to determine the threshold. - * sumCurrent excludes records in allowlist ranges (surge windows determined by engineers). - * - * Returns DELAYED_PROCESSING if sumCurrent >= thresholdMultiplier * baselineTraffic, indicating abnormal traffic spike. - */ -public class OptOutTrafficCalculator { - private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficCalculator.class); - - private static final int HOURS_24 = 24 * 3600; // 24 hours in seconds - - private final Map deltaFileCache = new ConcurrentHashMap<>(); - private final ICloudStorage cloudStorage; - private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") - private final String trafficCalcConfigPath; - private int baselineTraffic; - private int thresholdMultiplier; - private int evaluationWindowSeconds; - private List> allowlistRanges; - - public enum TrafficStatus { - DELAYED_PROCESSING, - DEFAULT - } - - /** - * Cache entry for a delta file containing all record timestamps. - * - * Memory usage: ~8 bytes per timestamp (long) - * 1GB of memory can store ~130 million timestamps (1024^3)/8 - */ - private static class FileRecordCache { - final List timestamps; // All non-sentinel record timestamps - final long newestTimestamp; // evict delta from cache based on oldest record timestamp - - FileRecordCache(List timestamps) { - this.timestamps = timestamps; - this.newestTimestamp = timestamps.isEmpty() ? 0 : Collections.max(timestamps); - } - } - - /** - * Exception thrown by malformed traffic calculator config - */ - public static class MalformedTrafficCalcConfigException extends Exception { - public MalformedTrafficCalcConfigException(String message) { - super(message); - } - } - - /** - * Constructor for OptOutTrafficCalculator - * - * @param cloudStorage Cloud storage for reading delta files - * @param s3DeltaPrefix S3 prefix for delta files - * @param trafficCalcConfigS3Path S3 path for traffic calc config - */ - public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigPath) throws MalformedTrafficCalcConfigException { - this.cloudStorage = cloudStorage; - this.s3DeltaPrefix = s3DeltaPrefix; - this.trafficCalcConfigPath = trafficCalcConfigPath; - reloadTrafficCalcConfig(); // Load ConfigMap - - LOGGER.info("initialized: s3DeltaPrefix={}, threshold={}x", - s3DeltaPrefix, thresholdMultiplier); - } - - /** - * Reload traffic calc config from ConfigMap. - * Expected format: - * { - * "traffic_calc_evaluation_window_seconds": 86400, - * "traffic_calc_baseline_traffic": 100, - * "traffic_calc_threshold_multiplier": 5, - * "traffic_calc_allowlist_ranges": [ - * [startTimestamp1, endTimestamp1], - * [startTimestamp2, endTimestamp2] - * ], - * } - * - * Can be called periodically to pick up config changes without restarting. - */ - public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException { - LOGGER.info("loading traffic calc config from configmap"); - try (InputStream is = Files.newInputStream(Paths.get(trafficCalcConfigPath))) { - String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); - JsonObject trafficCalcConfig = new JsonObject(content); - - // Validate required fields exist - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { - throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_evaluation_window_seconds"); - } - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { - throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_baseline_traffic"); - } - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { - throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_threshold_multiplier"); - } - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { - throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_allowlist_ranges"); - } - - this.evaluationWindowSeconds = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp); - this.baselineTraffic = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcBaselineTrafficProp); - this.thresholdMultiplier = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcThresholdMultiplierProp); - - List> ranges = parseAllowlistRanges(trafficCalcConfig); - this.allowlistRanges = ranges; - - LOGGER.info("loaded traffic calc config: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", - this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size()); - - } catch (MalformedTrafficCalcConfigException e) { - LOGGER.error("circuit_breaker_config_error: config is malformed, configPath={}", trafficCalcConfigPath, e); - throw e; - } catch (Exception e) { - LOGGER.error("circuit_breaker_config_error: config is malformed or missing, configPath={}", trafficCalcConfigPath, e); - throw new MalformedTrafficCalcConfigException("failed to load traffic calc config: " + e.getMessage()); - } - } - - /** - * Parse allowlist ranges from JSON config - */ - List> parseAllowlistRanges(JsonObject config) throws MalformedTrafficCalcConfigException { - List> ranges = new ArrayList<>(); - - try { - var rangesArray = config.getJsonArray(Const.Config.OptOutTrafficCalcAllowlistRangesProp); - if (rangesArray != null) { - for (int i = 0; i < rangesArray.size(); i++) { - var rangeArray = rangesArray.getJsonArray(i); - if (rangeArray != null && rangeArray.size() >= 2) { - long start = rangeArray.getLong(0); - long end = rangeArray.getLong(1); - - if(start >= end) { - LOGGER.error("circuit_breaker_config_error: allowlist range start must be less than end, range=[{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": start must be less than end"); - } - - if (end - start > 86400) { - LOGGER.error("circuit_breaker_config_error: allowlist range must be less than 24 hours, range=[{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": range must be less than 24 hours"); - } - - List range = Arrays.asList(start, end); - ranges.add(range); - LOGGER.info("loaded allowlist range: [{}, {}]", start, end); - } - } - } - - ranges.sort(Comparator.comparing(range -> range.get(0))); - - // Validate no overlapping ranges - for (int i = 0; i < ranges.size() - 1; i++) { - long currentEnd = ranges.get(i).get(1); - long nextStart = ranges.get(i + 1).get(0); - if (currentEnd >= nextStart) { - LOGGER.error("circuit_breaker_config_error: overlapping allowlist ranges, range=[{}, {}] overlaps with range=[{}, {}]", - ranges.get(i).get(0), currentEnd, nextStart, ranges.get(i + 1).get(1)); - throw new MalformedTrafficCalcConfigException( - "overlapping allowlist ranges detected at indices " + i + " and " + (i + 1)); - } - } - - } catch (MalformedTrafficCalcConfigException e) { - throw e; - } catch (Exception e) { - LOGGER.error("circuit_breaker_config_error: failed to parse allowlist ranges", e); - throw new MalformedTrafficCalcConfigException("failed to parse allowlist ranges: " + e.getMessage()); - } - - return ranges; - } - - /** - * Calculate traffic status based on delta files and SQS queue messages. - * - * Uses the newest delta file timestamp to anchor the 24-hour delta traffic window, - * and the oldest queue timestamp to anchor the 5-minute queue window. - * - * Counts: - * - Delta file records (with allowlist filtering) - * - SQS messages passed in (with allowlist filtering) - * - Invisible messages from other consumers (from queue attributes, avoiding double count) - * - * @param sqsMessages List of SQS messages this consumer has read (non-denylisted) - * @param queueAttributes Queue attributes including invisible message count (can be null) - * @param denylistedCount Number of denylisted messages read by this consumer - * @param filteredAsTooRecentCount Number of messages filtered as "too recent" by window reader - * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) - */ - public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes, int denylistedCount, int filteredAsTooRecentCount) { - - try { - // Get list of delta files from S3 (sorted newest to oldest) - List deltaS3Paths = listDeltaFiles(); - - if (deltaS3Paths.isEmpty()) { - LOGGER.error("s3_error: no delta files found in s3 at prefix={}", s3DeltaPrefix); - throw new RuntimeException("no delta files found in s3 at prefix=" + s3DeltaPrefix); - } - - // Find newest delta file timestamp for delta traffic window - long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths); - LOGGER.info("traffic calculation: newestDeltaTs={}", newestDeltaTs); - - // Find oldest SQS queue message timestamp for queue window - long oldestQueueTs = findOldestQueueTimestamp(sqsMessages); - LOGGER.info("traffic calculation: oldestQueueTs={}", oldestQueueTs); - - // Define start time of the delta evaluation window - // We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend - // the window to account for any allowlist ranges in the extended portion - long deltaWindowStart = calculateWindowStartWithAllowlist(newestDeltaTs, this.evaluationWindowSeconds); - - // Evict old cache entries (older than delta window start) - evictOldCacheEntries(deltaWindowStart); - - // Process delta files and count records in [deltaWindowStart, newestDeltaTs] - // Files are sorted newest to oldest, records within files are sorted newest to oldest - // Stop when the newest record in a file is older than the window - int sum = 0; - int deltaRecordsCount = 0; - int deltaAllowlistedCount = 0; - int filesProcessed = 0; - int cacheHits = 0; - int cacheMisses = 0; - - for (String s3Path : deltaS3Paths) { - boolean wasCached = isCached(s3Path); - if (wasCached) { - cacheHits++; - } else { - cacheMisses++; - } - - List timestamps = getTimestampsFromFile(s3Path); - filesProcessed++; - - // Check newest record in file - if older than window, stop processing remaining files - long newestRecordTs = timestamps.get(0); - if (newestRecordTs < deltaWindowStart) { - break; - } - - for (long ts : timestamps) { - // Stop condition: record is older than our window - if (ts < deltaWindowStart) { - break; - } - - // skip records in allowlisted ranges - if (isInAllowlist(ts)) { - deltaAllowlistedCount++; - continue; - } - - // increment sum if record is in delta window - if (ts >= deltaWindowStart) { - deltaRecordsCount++; - sum++; - } - } - } - - LOGGER.info("delta files: processed={}, deltaRecords={}, allowlisted={}, cache hits={}, misses={}, cacheSize={}", - filesProcessed, deltaRecordsCount, deltaAllowlistedCount, cacheHits, cacheMisses, deltaFileCache.size()); - - // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] with allowlist filtering - int sqsCount = 0; - if (sqsMessages != null && !sqsMessages.isEmpty()) { - sqsCount = countSqsMessages(sqsMessages, oldestQueueTs); - sum += sqsCount; - } - - // Add invisible messages being processed by OTHER consumers - // (notVisible count includes our messages, so subtract what we've read to avoid double counting) - // ourMessages = delta messages + denylisted messages + filtered "too recent" messages - int otherConsumersMessages = 0; - if (queueAttributes != null) { - int totalInvisible = queueAttributes.getApproximateNumberOfMessagesNotVisible(); - int ourMessages = (sqsMessages != null ? sqsMessages.size() : 0) + denylistedCount + filteredAsTooRecentCount; - otherConsumersMessages = Math.max(0, totalInvisible - ourMessages); - sum += otherConsumersMessages; - LOGGER.info("traffic calculation: adding {} invisible messages from other consumers (totalInvisible={}, ourMessages={})", - otherConsumersMessages, totalInvisible, ourMessages); - } - - // Determine status - TrafficStatus status = determineStatus(sum, this.baselineTraffic); - - LOGGER.info("traffic calculation complete: sum={} (deltaRecords={} + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}", - sum, deltaRecordsCount, sqsCount, otherConsumersMessages, this.baselineTraffic, this.thresholdMultiplier, status); - - return status; - - } catch (Exception e) { - LOGGER.error("delta_job_failed: error calculating traffic status", e); - throw new RuntimeException("error calculating traffic status", e); - } - } - - /** - * Find the newest timestamp from delta files. - * Reads the newest delta file and returns its maximum timestamp. - */ - private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOException { - if (deltaS3Paths == null || deltaS3Paths.isEmpty()) { - return System.currentTimeMillis() / 1000; - } - - // Delta files are sorted (ISO 8601 format, lexicographically sortable) so first file is newest - String newestDeltaPath = deltaS3Paths.get(0); - List timestamps = getTimestampsFromFile(newestDeltaPath); - - if (timestamps.isEmpty()) { - LOGGER.error("s3_error: newest delta file has no timestamps, path={}", newestDeltaPath); - return System.currentTimeMillis() / 1000; - } - - long newestTs = Collections.max(timestamps); - LOGGER.info("found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); - return newestTs; - } - - /** - * List all delta files from S3, sorted newest to oldest - */ - private List listDeltaFiles() { - try { - // List all objects with the delta prefix - List allFiles = cloudStorage.list(s3DeltaPrefix); - - // Filter to only .dat delta files and sort newest to oldest - List deltaFiles = allFiles.stream() - .filter(OptOutUtils::isDeltaFile) - .sorted(OptOutUtils.DeltaFilenameComparatorDescending) - .collect(Collectors.toList()); - - LOGGER.info("listed {} delta files from s3 (prefix={})", deltaFiles.size(), s3DeltaPrefix); - return deltaFiles; - - } catch (Exception e) { - LOGGER.error("s3_error: failed to list delta files at prefix={}", s3DeltaPrefix, e); - return Collections.emptyList(); - } - } - - /** - * Check if a delta file is already cached - */ - private boolean isCached(String s3Path) { - String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); - return deltaFileCache.containsKey(filename); - } - - /** - * Get timestamps from a delta file (S3 path), using cache if available - */ - private List getTimestampsFromFile(String s3Path) throws IOException { - // Extract filename from S3 path for cache key - String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); - - // Check cache first - FileRecordCache cached = deltaFileCache.get(filename); - if (cached != null) { - return cached.timestamps; - } - - // Cache miss - download from S3 - List timestamps = readTimestampsFromS3(s3Path); - - // Store in cache - deltaFileCache.put(filename, new FileRecordCache(timestamps)); - - return timestamps; - } - - /** - * Read all non-sentinel record timestamps from a delta file in S3 - */ - private List readTimestampsFromS3(String s3Path) throws IOException { - try (InputStream is = cloudStorage.download(s3Path)) { - byte[] data = is.readAllBytes(); - OptOutCollection collection = new OptOutCollection(data); - - List timestamps = new ArrayList<>(); - for (int i = 0; i < collection.size(); i++) { - OptOutEntry entry = collection.get(i); - - // Skip sentinel entries - if (entry.isSpecialHash()) { - continue; - } - - timestamps.add(entry.timestamp); - } - - return timestamps; - } catch (Exception e) { - LOGGER.error("s3_error: failed to read delta file at path={}", s3Path, e); - throw new IOException("failed to read delta file from s3: " + s3Path, e); - } - } - - /** - * Calculate total duration of allowlist ranges that overlap with the given time window. - */ - long getAllowlistDuration(long t, long windowStart) { - long totalDuration = 0; - for (List range : this.allowlistRanges) { - long start = range.get(0); - long end = range.get(1); - - // Clip range to window boundaries - if (start < windowStart) { - start = windowStart; - } - if (end > t) { - end = t; - } - - // Only add duration if there's actual overlap (start < end) - if (start < end) { - totalDuration += end - start; - } - } - return totalDuration; - } - - /** - * Calculate the window start time that provides evaluationWindowSeconds of non-allowlisted time. - * Iteratively extends the window to account for allowlist ranges that may fall in extended portions. - */ - long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowSeconds) { - long allowlistDuration = getAllowlistDuration(newestDeltaTs, newestDeltaTs - evaluationWindowSeconds); - - // Each iteration discovers at least one new allowlist range, so max iterations = number of ranges - int maxIterations = this.allowlistRanges.size() + 1; - - for (int i = 0; i < maxIterations && allowlistDuration > 0; i++) { - long newWindowStart = newestDeltaTs - evaluationWindowSeconds - allowlistDuration; - long newAllowlistDuration = getAllowlistDuration(newestDeltaTs, newWindowStart); - - if (newAllowlistDuration == allowlistDuration) { - // No new allowlist time in extended portion, we've converged - break; - } - - allowlistDuration = newAllowlistDuration; - } - - return newestDeltaTs - evaluationWindowSeconds - allowlistDuration; - } - - /** - * Find the oldest SQS queue message timestamp - */ - private long findOldestQueueTimestamp(List sqsMessages) throws IOException { - long oldest = System.currentTimeMillis() / 1000; - - if (sqsMessages != null && !sqsMessages.isEmpty()) { - for (Message msg : sqsMessages) { - Long ts = extractTimestampFromMessage(msg); - if (ts != null && ts < oldest) { - oldest = ts; - } - } - } - - return oldest; - } - - /** - * Extract timestamp from SQS message (from SentTimestamp attribute) - */ - private Long extractTimestampFromMessage(Message msg) { - // Get SentTimestamp attribute (milliseconds) - String sentTimestamp = msg.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); - if (sentTimestamp != null) { - try { - return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds - } catch (NumberFormatException e) { - LOGGER.error("sqs_error: invalid sentTimestamp, messageId={}, sentTimestamp={}", msg.messageId(), sentTimestamp); - } - } - - // Fallback: use current time - return System.currentTimeMillis() / 1000; - } - - /** - * Count SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes - */ - private int countSqsMessages(List sqsMessages, long oldestQueueTs) { - - int count = 0; - int allowlistedCount = 0; - long windowEnd = oldestQueueTs + 5 * 60; - - for (Message msg : sqsMessages) { - Long ts = extractTimestampFromMessage(msg); - - if (ts < oldestQueueTs || ts > windowEnd) { - continue; - } - - if (isInAllowlist(ts)) { - allowlistedCount++; - continue; - } - count++; - - } - - LOGGER.info("sqs messages: {} in window, {} allowlisted [oldestQueueTs={}, oldestQueueTs+5m={}]", count, allowlistedCount, oldestQueueTs, windowEnd); - return count; - } - - /** - * Check if a timestamp falls within any allowlist range - */ - boolean isInAllowlist(long timestamp) { - if (allowlistRanges == null || allowlistRanges.isEmpty()) { - return false; - } - - for (List range : allowlistRanges) { - if (range.size() < 2) { - continue; - } - - long start = range.get(0); - long end = range.get(1); - - if (timestamp >= start && timestamp <= end) { - return true; - } - } - - return false; - } - - /** - * Evict cache entries with data older than the cutoff timestamp - */ - private void evictOldCacheEntries(long cutoffTimestamp) { - int beforeSize = deltaFileCache.size(); - - deltaFileCache.entrySet().removeIf(entry -> - entry.getValue().newestTimestamp < cutoffTimestamp - ); - - int afterSize = deltaFileCache.size(); - if (beforeSize != afterSize) { - LOGGER.info("evicted {} old cache entries (before={}, after={})", - beforeSize - afterSize, beforeSize, afterSize); - } - } - - /** - * Determine traffic status based on current vs baseline traffic. - * Logs warnings at 50%, 75%, and 90% of the circuit breaker threshold. - */ - TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { - if (baselineTraffic == 0 || thresholdMultiplier == 0) { - LOGGER.error("circuit_breaker_config_error: baselineTraffic is 0 or thresholdMultiplier is 0"); - throw new RuntimeException("invalid circuit breaker config: baselineTraffic=" + baselineTraffic + ", thresholdMultiplier=" + thresholdMultiplier); - } - - int threshold = thresholdMultiplier * baselineTraffic; - double thresholdPercent = (double) sumCurrent / threshold * 100; - - // Log warnings at increasing thresholds before circuit breaker triggers - if (thresholdPercent >= 90.0) { - LOGGER.warn("high_message_volume: 90% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", - sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); - } else if (thresholdPercent >= 75.0) { - LOGGER.warn("high_message_volume: 75% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", - sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); - } else if (thresholdPercent >= 50.0) { - LOGGER.warn("high_message_volume: 50% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", - sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); - } - - if (sumCurrent >= threshold) { - LOGGER.error("circuit_breaker_triggered: traffic threshold breached, sumCurrent={}, threshold={} ({}x{})", - sumCurrent, threshold, thresholdMultiplier, baselineTraffic); - return TrafficStatus.DELAYED_PROCESSING; - } - - LOGGER.info("traffic within normal range: sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", - sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); - return TrafficStatus.DEFAULT; - } - - /** - * Get cache statistics for monitoring - */ - public Map getCacheStats() { - Map stats = new HashMap<>(); - stats.put("cached_files", deltaFileCache.size()); - - int totalTimestamps = deltaFileCache.values().stream() - .mapToInt(cache -> cache.timestamps.size()) - .sum(); - stats.put("total_cached_timestamps", totalTimestamps); - - return stats; - } - -} \ No newline at end of file diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java deleted file mode 100644 index 445f032..0000000 --- a/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java +++ /dev/null @@ -1,184 +0,0 @@ -package com.uid2.optout.traffic; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.uid2.optout.sqs.SqsParsedMessage; - -import java.util.ArrayList; -import java.util.List; -import java.util.Collections; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.charset.StandardCharsets; -import io.vertx.core.json.JsonObject; -import io.vertx.core.json.JsonArray; - -public class OptOutTrafficFilter { - private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficFilter.class); - - private final String trafficFilterConfigPath; - List filterRules; - - /** - * Traffic filter rule defining a time range and a list of IP addresses to exclude - */ - private static class TrafficFilterRule { - private final List range; - private final List ipAddresses; - - TrafficFilterRule(List range, List ipAddresses) { - this.range = range; - this.ipAddresses = ipAddresses; - } - - public long getRangeStart() { - return range.get(0); - } - public long getRangeEnd() { - return range.get(1); - } - public List getIpAddresses() { - return ipAddresses; - } - } - - public static class MalformedTrafficFilterConfigException extends Exception { - public MalformedTrafficFilterConfigException(String message) { - super(message); - } - } - - /** - * Constructor for OptOutTrafficFilter - * - * @param trafficFilterConfigPath S3 path for traffic filter config - * @throws MalformedTrafficFilterConfigException if the traffic filter config is invalid - */ - public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTrafficFilterConfigException { - this.trafficFilterConfigPath = trafficFilterConfigPath; - // Initial filter rules load - this.filterRules = Collections.emptyList(); // start empty - reloadTrafficFilterConfig(); // load ConfigMap - - LOGGER.info("initialized: filterRules={}", filterRules.size()); - } - - /** - * Reload traffic filter config from ConfigMap. - * Expected format: - * { - * "denylist_requests": [ - * {range: [startTimestamp, endTimestamp], IPs: ["ip1"]}, - * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip2"]}, - * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip3"]}, - * ] - * } - * - * Can be called periodically to pick up config changes without restarting. - */ - public void reloadTrafficFilterConfig() throws MalformedTrafficFilterConfigException { - LOGGER.info("loading traffic filter config"); - try (InputStream is = Files.newInputStream(Paths.get(trafficFilterConfigPath))) { - String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); - JsonObject filterConfigJson = new JsonObject(content); - - this.filterRules = parseFilterRules(filterConfigJson); - - LOGGER.info("loaded traffic filter config: filterRules={}", filterRules.size()); - - } catch (Exception e) { - LOGGER.error("circuit_breaker_config_error: no traffic filter config found at {}", trafficFilterConfigPath, e); - throw new MalformedTrafficFilterConfigException(e.getMessage()); - } - } - - /** - * Parse request filtering rules from JSON config - */ - List parseFilterRules(JsonObject config) throws MalformedTrafficFilterConfigException { - List rules = new ArrayList<>(); - try { - JsonArray denylistRequests = config.getJsonArray("denylist_requests"); - if (denylistRequests == null) { - LOGGER.error("circuit_breaker_config_error: denylist_requests is null"); - throw new MalformedTrafficFilterConfigException("invalid traffic filter config: denylist_requests is null"); - } - for (int i = 0; i < denylistRequests.size(); i++) { - JsonObject ruleJson = denylistRequests.getJsonObject(i); - - // parse range - var rangeJson = ruleJson.getJsonArray("range"); - List range = new ArrayList<>(); - if (rangeJson != null && rangeJson.size() == 2) { - long start = rangeJson.getLong(0); - long end = rangeJson.getLong(1); - - if (start >= end) { - LOGGER.error("circuit_breaker_config_error: rule range start must be less than end, rule={}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range start must be less than end"); - } - range.add(start); - range.add(end); - } - - // log error and throw exception if range is not 2 elements - if (range.size() != 2) { - LOGGER.error("circuit_breaker_config_error: rule range is not 2 elements, rule={}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range is not 2 elements"); - } - - // parse IPs - var ipAddressesJson = ruleJson.getJsonArray("IPs"); - List ipAddresses = new ArrayList<>(); - if (ipAddressesJson != null) { - for (int j = 0; j < ipAddressesJson.size(); j++) { - ipAddresses.add(ipAddressesJson.getString(j)); - } - } - - // log error and throw exception if IPs is empty - if (ipAddresses.size() == 0) { - LOGGER.error("circuit_breaker_config_error: rule IPs is empty, rule={}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: IPs is empty"); - } - - // log error and throw exception if rule is invalid - if (range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less - LOGGER.error("circuit_breaker_config_error: rule range must be 24 hours or less, rule={}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range must be 24 hours or less"); - } - - TrafficFilterRule rule = new TrafficFilterRule(range, ipAddresses); - - LOGGER.info("loaded traffic filter rule: range=[{}, {}], IPs={}", rule.getRangeStart(), rule.getRangeEnd(), rule.getIpAddresses()); - rules.add(rule); - } - return rules; - } catch (Exception e) { - LOGGER.error("circuit_breaker_config_error: failed to parse rules, config={}, error={}", config.encode(), e.getMessage()); - throw new MalformedTrafficFilterConfigException(e.getMessage()); - } - } - - public boolean isDenylisted(SqsParsedMessage message) { - long timestamp = message.timestamp(); - String clientIp = message.clientIp(); - - if (clientIp == null || clientIp.isEmpty()) { - LOGGER.error("sqs_error: request does not contain client ip, messageId={}", message.originalMessage().messageId()); - return false; - } - - for (TrafficFilterRule rule : filterRules) { - if(timestamp >= rule.getRangeStart() && timestamp <= rule.getRangeEnd()) { - if(rule.getIpAddresses().contains(clientIp)) { - return true; - } - }; - } - return false; - } - -} \ No newline at end of file diff --git a/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java deleted file mode 100644 index fd435a9..0000000 --- a/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java +++ /dev/null @@ -1,1577 +0,0 @@ -package com.uid2.optout.traffic; - -import com.uid2.shared.cloud.CloudStorageException; -import com.uid2.shared.cloud.ICloudStorage; -import com.uid2.shared.optout.OptOutCollection; -import com.uid2.shared.optout.OptOutEntry; -import com.uid2.optout.sqs.SqsMessageOperations; -import com.uid2.optout.Const; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import java.nio.file.Files; -import java.nio.file.Path; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; -import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; - -import com.uid2.optout.traffic.OptOutTrafficCalculator; -import com.uid2.optout.traffic.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; -import java.io.ByteArrayInputStream; -import java.util.*; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; - - -@ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) -public class OptOutTrafficCalculatorTest { - - @Mock - private ICloudStorage cloudStorage; - - private static final String S3_DELTA_PREFIX = "optout-v2/delta/"; - private static final String TRAFFIC_CONFIG_PATH = "./traffic-config.json"; - private static final int BASELINE_TRAFFIC = 100; - private static final int THRESHOLD_MULTIPLIER = 5; - private static final int EVALUATION_WINDOW_SECONDS = 24 * 3600; - - @BeforeEach - void setUp() { - // default config - JsonObject config = new JsonObject(); - config.put(Const.Config.OptOutTrafficCalcBaselineTrafficProp, BASELINE_TRAFFIC); - config.put(Const.Config.OptOutTrafficCalcThresholdMultiplierProp, THRESHOLD_MULTIPLIER); - config.put(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp, EVALUATION_WINDOW_SECONDS); - config.put(Const.Config.OptOutTrafficCalcAllowlistRangesProp, new JsonArray()); - try { - createTrafficConfigFile(config.toString()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @AfterEach - void tearDown() { - if (Files.exists(Path.of(TRAFFIC_CONFIG_PATH))) { - try { - Files.delete(Path.of(TRAFFIC_CONFIG_PATH)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - - private void createTrafficConfigFile(String content) { - try { - Path configPath = Path.of(TRAFFIC_CONFIG_PATH); - Files.writeString(configPath, content); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * Helper to create config by merging partial JSON with defaults - */ - private void createConfigFromPartialJson(String partialJson) { - JsonObject partial = new JsonObject(partialJson); - JsonObject config = new JsonObject(); - - // Set defaults - if (!partial.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { - config.put(Const.Config.OptOutTrafficCalcBaselineTrafficProp, BASELINE_TRAFFIC); - } - if (!partial.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { - config.put(Const.Config.OptOutTrafficCalcThresholdMultiplierProp, THRESHOLD_MULTIPLIER); - } - if (!partial.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { - config.put(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp, EVALUATION_WINDOW_SECONDS); - } - if (!partial.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { - config.put(Const.Config.OptOutTrafficCalcAllowlistRangesProp, new JsonArray()); - } - - // Merge in partial config (overrides defaults) - partial.forEach(entry -> config.put(entry.getKey(), entry.getValue())); - - createTrafficConfigFile(config.toString()); - } - - /** - * Helper to create config with custom threshold - */ - private void createConfigWithThreshold(int threshold) { - createConfigFromPartialJson("{\"" + Const.Config.OptOutTrafficCalcThresholdMultiplierProp + "\": " + threshold + "}"); - } - - // ============================================================================ - // SECTION 1: Constructor & Initialization Tests - // ============================================================================ - - @Test - void testConstructor_defaultThreshold() throws Exception { - // Setup - default threshold of 5 - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); // 10 < 5*3 - - status = calculator.determineStatus(15, 3); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 15 >= 5*3 - } - - @Test - void testConstructor_customThreshold() throws Exception { - // Setup - custom threshold of 10 - createConfigWithThreshold(10); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(49, 5); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); // 49 < 10*5 - status = calculator.determineStatus(50, 5); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 50 >= 10*5 - } - - @Test - void testConstructor_trafficCalcConfigLoadFailure() throws Exception { - // Setup - traffic calc config load failure - createTrafficConfigFile("Invalid JSON"); - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - }); - - // Create valid config to test reload failure - createConfigFromPartialJson("{}"); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - createTrafficConfigFile("Invalid JSON"); - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - } - - // ============================================================================ - // SECTION 2: parseTrafficCalcConfigRanges() - // ============================================================================ - - @Test - void testParseTrafficCalcConfigRanges_emptyConfig() throws Exception { - // Setup - no config - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - JsonObject emptyConfig = new JsonObject(); - - // Act - List> ranges = calculator.parseAllowlistRanges(emptyConfig); - - // Assert - empty ranges - assertTrue(ranges.isEmpty()); - } - - @Test - void testParseTrafficCalcConfigRanges_singleRange() throws Exception { - // Setup - single range - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(1000L).add(2000L)); - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - List> result = calculator.parseAllowlistRanges(configWithRanges); - - // Assert - single range - assertEquals(1, result.size()); - assertEquals(1000L, result.get(0).get(0)); - assertEquals(2000L, result.get(0).get(1)); - } - - @Test - void testParseTrafficCalcConfigRanges_multipleRanges() throws Exception { - // Setup - multiple ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(1000L).add(2000L)) - .add(new JsonArray().add(3000L).add(4000L)) - .add(new JsonArray().add(5000L).add(6000L)); - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - List> result = calculator.parseAllowlistRanges(configWithRanges); - - // Assert - multiple ranges - assertEquals(3, result.size()); - assertEquals(1000L, result.get(0).get(0)); - assertEquals(3000L, result.get(1).get(0)); - assertEquals(5000L, result.get(2).get(0)); - } - - @Test - void testParseTrafficCalcConfigRanges_misorderedRange() throws Exception { - // Setup - range with end < start is malformed - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(2000L).add(1000L)); // End before start - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.parseAllowlistRanges(configWithRanges); - }); - } - - @Test - void testParseTrafficCalcConfigRanges_rangeTooLong() throws Exception { - // Setup - range longer than 24 hours is malformed - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(2000L).add(200000L)); // Longer than 24 hours - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.parseAllowlistRanges(configWithRanges); - }); - } - - @Test - void testParseTrafficCalcConfigRanges_sortsByStartTime() throws Exception { - // Setup - ranges added out of order - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(5000L).add(6000L)) - .add(new JsonArray().add(1000L).add(2000L)) - .add(new JsonArray().add(3000L).add(4000L)); - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - List> result = calculator.parseAllowlistRanges(configWithRanges); - - // Assert - should be sorted by start time - assertEquals(3, result.size()); - assertEquals(1000L, result.get(0).get(0)); - assertEquals(3000L, result.get(1).get(0)); - assertEquals(5000L, result.get(2).get(0)); - } - - @Test - void testParseTrafficCalcConfigRanges_invalidRangeTooFewElements() throws Exception { - // Setup - invalid range with only 1 element; - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(1000L)) // Only 1 element - .add(new JsonArray().add(2000L).add(3000L)); // Valid - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - List> result = calculator.parseAllowlistRanges(configWithRanges); - - // Assert - should skip invalid range - assertEquals(1, result.size()); - assertEquals(2000L, result.get(0).get(0)); - } - - @Test - void testParseTrafficCalcConfigRanges_nullArray() throws Exception { - // Setup - null array - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - configWithRanges.put("traffic_calc_allowlist_ranges", (JsonArray) null); - - // Act - List> result = calculator.parseAllowlistRanges(configWithRanges); - - // Assert - empty ranges - assertTrue(result.isEmpty()); - } - - @Test - void testParseTrafficCalcConfigRanges_overlappingRanges() throws Exception { - // Setup - overlapping ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(1000L).add(2000L)) - .add(new JsonArray().add(1500L).add(2500L)); // Overlaps with first range - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act & Assert - should throw exception due to overlap - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.parseAllowlistRanges(configWithRanges); - }); - } - - @Test - void testParseTrafficCalcConfigRanges_adjacentRangesWithSameBoundary() throws Exception { - // Setup - ranges where end of first equals start of second (touching but not overlapping semantically, but we treat as overlap) - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(1000L).add(2000L)) - .add(new JsonArray().add(2000L).add(3000L)); // Starts exactly where first ends - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act & Assert - should throw exception because ranges touch at boundary - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.parseAllowlistRanges(configWithRanges); - }); - } - - @Test - void testParseTrafficCalcConfigRanges_nonOverlappingRanges() throws Exception { - // Setup - ranges that don't overlap (with gap between them) - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - JsonObject configWithRanges = new JsonObject(); - JsonArray ranges = new JsonArray() - .add(new JsonArray().add(1000L).add(2000L)) - .add(new JsonArray().add(2001L).add(3000L)); // Starts after first ends - configWithRanges.put("traffic_calc_allowlist_ranges", ranges); - - // Act - List> result = calculator.parseAllowlistRanges(configWithRanges); - - // Assert - should succeed with 2 ranges - assertEquals(2, result.size()); - } - - // ============================================================================ - // SECTION 3: isInTrafficCalcConfig() - // ============================================================================ - - @Test - void testIsInTrafficCalcConfig_withinSingleRange() throws Exception { - // Setup - load traffic calc config with single range [1000, 2000] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - true when within range - assertTrue(calculator.isInAllowlist(1500L)); - } - - @Test - void testIsInTrafficCalcConfig_exactlyAtStart() throws Exception { - // Setup - load traffic calc config with single range [1000, 2000] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - true when exactly at start of range - assertTrue(calculator.isInAllowlist(1000L)); - } - - @Test - void testIsInTrafficCalcConfig_exactlyAtEnd() throws Exception { - // Setup - load traffic calc config with single range [1000, 2000] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - true when exactly at end of range - assertTrue(calculator.isInAllowlist(2000L)); - } - - @Test - void testIsInTrafficCalcConfig_beforeRange() throws Exception { - // Setup - load traffic calc config with single range [1000, 2000] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - false when before range - assertFalse(calculator.isInAllowlist(999L)); - } - - @Test - void testIsInTrafficCalcConfig_afterRange() throws Exception { - // Setup - load traffic calc config with single range [1000, 2000] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - false when after range - assertFalse(calculator.isInAllowlist(2001L)); - } - - @Test - void testIsInTrafficCalcConfig_betweenRanges() throws Exception { - // Setup - load traffic calc config with two ranges [1000, 2000] and [3000, 4000] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000], - [3000, 4000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - false when between ranges - assertFalse(calculator.isInAllowlist(2500L)); - } - - @Test - void testIsInTrafficCalcConfig_emptyRanges() throws Exception { - // Setup uses default config from setUp() which has empty traffic calc config ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - false when empty ranges - assertFalse(calculator.isInAllowlist(1500L)); - } - - @Test - void testIsInTrafficCalcConfig_nullRanges() throws Exception { - // Setup - no traffic calc config ranges loaded (will fail and set empty) - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": null - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - false when null/empty ranges - assertFalse(calculator.isInAllowlist(1500L)); - } - - @Test - void testIsInTrafficCalcConfig_invalidRangeSize() throws Exception { - // Setup - load traffic calc config with invalid range (only 1 element) and valid range - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000], - [2000, 3000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - assertFalse(calculator.isInAllowlist(1500L)); // Should not match invalid range - assertTrue(calculator.isInAllowlist(2500L)); // Should match valid range - } - - @Test - void testIsInTrafficCalcConfig_multipleRanges() throws Exception { - // Setup - load traffic calc config with multiple ranges - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000], - [3000, 4000], - [5000, 6000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - assertTrue(calculator.isInAllowlist(1500L)); // In first range - assertTrue(calculator.isInAllowlist(3500L)); // In second range - assertTrue(calculator.isInAllowlist(5500L)); // In third range - assertFalse(calculator.isInAllowlist(2500L)); // Between first and second - } - - // ============================================================================ - // SECTION 4: getTrafficCalcConfigDuration() - // ============================================================================ - - @Test - void testGetTrafficCalcConfigDuration_noRanges() throws Exception { - // Setup - no ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Assert - assertEquals(0L, calculator.getAllowlistDuration(10000L, 5000L)); // 0 duration when no ranges - } - - @Test - void testGetTrafficCalcConfigDuration_rangeFullyWithinWindow() throws Exception { - // Setup - range fully within window - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [6000, 7000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window [5000, 10000], range [6000, 7000] - long duration = calculator.getAllowlistDuration(10000L, 5000L); - - // Assert - full range duration - assertEquals(1000L, duration); - } - - @Test - void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsStart() throws Exception { - // Setup - range partially overlaps start of window - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [3000, 7000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window [5000, 10000], range [3000, 7000] - long duration = calculator.getAllowlistDuration(10000L, 5000L); - - // Assert - should clip to [5000, 7000] = 2000 - assertEquals(2000L, duration); - } - - @Test - void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsEnd() throws Exception { - // Setup - range partially overlaps end of window - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [8000, 12000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window [5000, 10000], range [8000, 12000] - long duration = calculator.getAllowlistDuration(10000L, 5000L); - - // Assert - should clip to [8000, 10000] = 2000 - assertEquals(2000L, duration); - } - - @Test - void testGetTrafficCalcConfigDuration_rangeCompletelyOutsideWindow() throws Exception { - // Setup - range completely outside window - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window [5000, 10000], range [1000, 2000] - long duration = calculator.getAllowlistDuration(10000L, 5000L); - - // Assert - 0 duration when range completely outside window - assertEquals(0L, duration); - } - - @Test - void testGetTrafficCalcConfigDuration_multipleRanges() throws Exception { - // Setup - multiple ranges - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [6000, 7000], - [8000, 9000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window [5000, 10000], ranges [6000, 7000] and [8000, 9000] - long duration = calculator.getAllowlistDuration(10000L, 5000L); - - // Assert - 1000 + 1000 = 2000 - assertEquals(2000L, duration); - } - - @Test - void testGetTrafficCalcConfigDuration_rangeSpansEntireWindow() throws Exception { - // Setup - range spans entire window - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [3000, 12000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window [5000, 10000], range [3000, 12000] - long duration = calculator.getAllowlistDuration(10000L, 5000L); - - // Assert - entire window is in traffic calc config ranges = 5000 - assertEquals(5000L, duration); - } - - // ============================================================================ - // SECTION 4.5: calculateWindowStartWithAllowlist() - // ============================================================================ - - @Test - void testCalculateWindowStartWithAllowlist_noAllowlist() throws Exception { - // Setup - no allowlist ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - window should be [3, 8] with no extension - long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); - - // Assert - no allowlist, so window start is simply newestDeltaTs - evaluationWindowSeconds - assertEquals(3L, windowStart); - } - - @Test - void testCalculateWindowStartWithAllowlist_allowlistInOriginalWindowOnly() throws Exception { - // Setup - allowlist range only in original window, not in extended portion - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [6, 7] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - newestDeltaTs=8, evaluationWindow=5 - // Original window [3, 8] has [6,7] allowlisted (1 hour) - // Extended portion [2, 3] has no allowlist - // So window start should be 8 - 5 - 1 = 2 - long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); - - assertEquals(2L, windowStart); - } - - @Test - void testCalculateWindowStartWithAllowlist_allowlistInExtendedPortion() throws Exception { - // Setup - allowlist ranges in both original window AND extended portion - // This is the user's example: evaluationWindow=5, newestDeltaTs=8, allowlist={[2,3], [6,7]} - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [2, 3], - [6, 7] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - // Original window [3, 8]: [6,7] allowlisted = 1 hour - // First extension to [2, 8]: [2,3] and [6,7] allowlisted = 2 hours total - // Second extension to [1, 8]: still [2,3] and [6,7] = 2 hours (no new allowlist) - // Final: windowStart = 8 - 5 - 2 = 1 - long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); - - assertEquals(1L, windowStart); - } - - @Test - void testCalculateWindowStartWithAllowlist_allowlistBeforeWindow() throws Exception { - // Setup - allowlist range entirely before the initial window - // This tests that we don't over-extend when allowlist is old - // evaluationWindow=5, newestDeltaTs=20, allowlist=[10,13] - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [10, 13] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - // Initial window [15, 20]: no allowlist overlap, allowlistDuration = 0 - // No extension needed - // Final: windowStart = 20 - 5 - 0 = 15 - long windowStart = calculator.calculateWindowStartWithAllowlist(20L, 5); - - // Verify: window [15, 20] has 5 hours, 0 allowlisted = 5 non-allowlisted - assertEquals(15L, windowStart); - } - - // ============================================================================ - // SECTION 5: determineStatus() - // ============================================================================ - - @Test - void testDetermineStatus_belowThreshold() throws Exception { - // Setup - below threshold - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - 10 < 5 * 3 - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); - - // Assert - DEFAULT when below threshold - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testDetermineStatus_atThreshold() throws Exception { - // Setup - at threshold - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - 15 == 5 * 3 - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(15, 3); - - // Assert - DELAYED_PROCESSING when at threshold - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - @Test - void testDetermineStatus_aboveThreshold() throws Exception { - // Setup - above threshold - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - 20 > 5 * 3 - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(20, 3); - - // Assert - DELAYED_PROCESSING when above threshold - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - @Test - void testDetermineStatus_sumPastZero() throws Exception { - // Setup - sumPast is 0 - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act & Assert - should throw exception for invalid config - assertThrows(RuntimeException.class, () -> calculator.determineStatus(100, 0)); - } - - @Test - void testDetermineStatus_bothZero() throws Exception { - // Setup - both sumCurrent and sumPast are 0; - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act & Assert - should throw exception for invalid config - assertThrows(RuntimeException.class, () -> calculator.determineStatus(0, 0)); - } - - @Test - void testDetermineStatus_sumCurrentZero() throws Exception { - // Setup - sumCurrent is 0 - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - 0 < 5 * 10 - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 10); - - // Assert - DEFAULT when sumCurrent is 0 - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @ParameterizedTest - @CsvSource({ - "1, 1, 1, DELAYED_PROCESSING", // threshold=1: 1 >= 1*1 - "2, 4, 2, DELAYED_PROCESSING", // threshold=2: 4 >= 2*2 - "5, 10, 2, DELAYED_PROCESSING", // threshold=5: 10 >= 5*2 - "10, 100, 10, DELAYED_PROCESSING", // threshold=10: 100 >= 10*10 - "5, 24, 5, DEFAULT", // threshold=5: 24 < 5*5 - "100, 1000, 11, DEFAULT" // threshold=100: 1000 < 100*11 - }) - void testDetermineStatus_variousThresholds(int threshold, int sumCurrent, int sumPast, String expectedStatus) throws Exception { - // Setup - various thresholds - createConfigWithThreshold(threshold); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(sumCurrent, sumPast); - - // Assert - assertEquals(OptOutTrafficCalculator.TrafficStatus.valueOf(expectedStatus), status); - } - - @Test - void testDetermineStatus_largeNumbers() throws Exception { - // Setup - test with large numbers - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(1_000_000, 200_000); - - // Assert - 1M >= 5 * 200K = 1M - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - // ============================================================================ - // SECTION 6: S3 Config Reload Tests - // ============================================================================ - - @Test - void testReloadTrafficCalcConfig_success() throws Exception { - // Setup - initial traffic calc config - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000], - [3000, 4000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Change the traffic calc config to a new range - String newTrafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [5000, 6000] - ] - } - """; - createConfigFromPartialJson(newTrafficCalcConfigJson); - - // Act - reload the traffic calc config - calculator.reloadTrafficCalcConfig(); - - // Assert - verify new traffic calc config is loaded - assertTrue(calculator.isInAllowlist(5500L)); - } - - @Test - void testReloadTrafficCalcConfig_failure() throws Exception { - // Setup - initial traffic calc config - String trafficCalcConfigJson = """ - { - "traffic_calc_allowlist_ranges": [ - [1000, 2000] - ] - } - """; - createConfigFromPartialJson(trafficCalcConfigJson); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Now make it fail - createTrafficConfigFile("Invalid JSON"); - - // Act - should not throw exception - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - - } - - @Test - public void testReloadTrafficCalcConfig_failure_missingKeys() throws Exception { - // Setup - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act & Assert missing threshold multiplier - createTrafficConfigFile("{\"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - - // Act & Assert missing evaluation window seconds - createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - - // Act & Assert missing baseline traffic - createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - - // Act & Assert missing traffic calc config ranges - createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100}"); - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - } - - @Test - public void testReloadTrafficCalcConfig_failure_misorderedRanges() throws Exception { - // Setup - misordered ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - createConfigFromPartialJson("{\"traffic_calc_allowlist_ranges\": [ [2000, 1000] ]}"); - - // Act & Assert - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - } - - @Test - public void testReloadTrafficCalcConfig_failure_rangeTooLong() throws Exception { - // Setup - range greater than 24 hours - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - createConfigFromPartialJson("{\"traffic_calc_allowlist_ranges\": [ [1000, 200000] ]}"); - - // Act & Assert - assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.reloadTrafficCalcConfig(); - }); - } - - // ============================================================================ - // SECTION 7: Cache Management Tests (also tested in section 9) - // ============================================================================ - - @Test - void testGetCacheStats_emptyCache() throws Exception { - // Setup - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - Map stats = calculator.getCacheStats(); - - // Assert - should return empty stats - assertEquals(0, stats.get("cached_files")); - assertEquals(0, stats.get("total_cached_timestamps")); - } - - // ============================================================================ - // SECTION 8: Helper Methods for Test Data Creation - // ============================================================================ - - /** - * Create a mock SQS message with specified timestamp - */ - private Message createSqsMessage(long timestampSeconds) { - Map attributes = new HashMap<>(); - attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampSeconds * 1000)); - - return Message.builder() - .messageId("test-msg-" + timestampSeconds) - .body("{\"test\": \"data\"}") - .attributes(attributes) - .build(); - } - - /** - * Create a mock SQS message without timestamp - */ - private Message createSqsMessageWithoutTimestamp() { - return Message.builder() - .messageId("test-msg-no-timestamp") - .body("{\"test\": \"data\"}") - .attributes(new HashMap<>()) - .build(); - } - - /** - * Create delta file bytes with specified timestamps - */ - private byte[] createDeltaFileBytes(List timestamps) throws Exception { - // Create OptOutEntry objects using newTestEntry - List entries = new ArrayList<>(); - - long idCounter = 1000; // Use incrementing IDs for test entries - for (long timestamp : timestamps) { - entries.add(OptOutEntry.newTestEntry(idCounter++, timestamp)); - } - - // Create OptOutCollection - OptOutCollection collection = new OptOutCollection(entries.toArray(new OptOutEntry[0])); - return collection.getStore(); - } - - - // ============================================================================ - // SECTION 9: Tests for calculateStatus() - // ============================================================================ - - @Test - void testCalculateStatus_noDeltaFiles() throws Exception { - // Setup - no delta files - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Collections.emptyList()); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act & Assert - should throw exception when no delta files - assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); - } - - @Test - void testCalculateStatus_normalTraffic() throws Exception { - // Setup - setup time: current time - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - // Create delta files with timestamps distributed over 48 hours - List timestamps = new ArrayList<>(); - - // add 499 entries in current window - for (int i = 0; i < 49; i++) { - timestamps.add(t - 23*3600 + i * 60); - } - - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); - - // Assert - 100+1 < 5 * 50 = 250, so should be DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_delayedProcessing() throws Exception { - // Setup - create delta files with spike in current window - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - // Create delta files with spike in current window - List timestamps = new ArrayList<>(); - - // add 500 entries in current window - for (int i = 0; i < 500; i++) { - timestamps.add(t - 23*3600 + i * 60); - } - - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); - - // Assert - 100+1 >= 5 * 10 = 50, DELAYED_PROCESSING - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - @Test - void testCalculateStatus_noSqsMessages() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600, t - 7200); // Some entries - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - null SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null, null, 0, 0); - - // Assert - should still calculate based on delta files, DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_emptySqsMessages() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600); - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - empty SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0); - - // Assert - should still calculate based on delta files, DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_multipleSqsMessages() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = new ArrayList<>(); - // add 470 entries in window - for (int i = 0; i < 470; i++) { - timestamps.add(t - 24*3600 + i * 60); - } - - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Add 30 SQS entries in [t, t+5min] - List sqsMessages = new ArrayList<>(); - for (int i = 0; i < 30; i++) { - sqsMessages.add(createSqsMessage(t - i * 10)); - } - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); - - // Assert - DELAYED_PROCESSING - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - @Test - void testCalculateStatus_withTrafficCalcConfig() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - // Traffic calc config that covers part of window - String trafficCalcConfigJson = String.format(""" - { - "traffic_calc_allowlist_ranges": [ - [%d, %d] - ] - } - """, t - 12*3600, t - 6*3600); - - List timestamps = new ArrayList<>(); - - // window - 600 entries (300 in traffic calc config range, 300 outside) - for (int i = 0; i < 300; i++) { - timestamps.add(t - 12*3600 + i); - } - for (int i = 0; i < 300; i++) { - timestamps.add(t - 3600 + i); - } - - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - createConfigFromPartialJson(trafficCalcConfigJson); - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta-001_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta-001_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); - - // Assert - should filter out entries in traffic calc config ranges - // Only 300 from window count (not in traffic calc config ranges) + 1 SQS = 301 - // 301 < 5*100, so DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_cacheUtilization() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600, t - 7200); - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - first call should populate cache - List sqsMessages = Arrays.asList(createSqsMessage(t)); - calculator.calculateStatus(sqsMessages, null, 0, 0); - - Map stats = calculator.getCacheStats(); - int cachedFiles = (Integer) stats.get("cached_files"); - - // Second call should use cache (no additional S3 download) - calculator.calculateStatus(sqsMessages, null, 0, 0); - - Map stats2 = calculator.getCacheStats(); - int cachedFiles2 = (Integer) stats2.get("cached_files"); - - // Assert - cache should be populated and remain consistent - assertEquals(1, cachedFiles); - assertEquals(cachedFiles, cachedFiles2); - - // Verify S3 download was called only once per file - verify(cloudStorage, times(1)).download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat"); - } - - @Test - void testCalculateStatus_s3Exception() throws Exception { - // Setup - S3 list error - when(cloudStorage.list(S3_DELTA_PREFIX)).thenThrow(new RuntimeException("S3 error")); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act & Assert - should throw exception on S3 error - assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); - } - - @Test - void testCalculateStatus_deltaFileReadException() throws Exception { - // Setup - S3 download error - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenThrow(new CloudStorageException("Failed to download")); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act & Assert - should throw exception on S3 download error - assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); - } - - @Test - void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600); - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - SQS message without timestamp (should use current time) - List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp()); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); - - // Assert - DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_multipleDeltaFiles() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - // File 1 - recent entries - List timestamps1 = new ArrayList<>(); - for (int i = 0; i < 50; i++) { - timestamps1.add(t - 12*3600 + i * 1000); - } - byte[] deltaFileBytes1 = createDeltaFileBytes(timestamps1); - - // File 2 - older entries - List timestamps2 = new ArrayList<>(); - for (int i = 0; i < 30; i++) { - timestamps2.add(t - 36*3600 + i * 1000); - } - byte[] deltaFileBytes2 = createDeltaFileBytes(timestamps2); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList( - "optout-v2/delta/optout-delta--01_2025-11-13T02.00.00Z_bbbbbbbb.dat", - "optout-v2/delta/optout-delta--01_2025-11-13T01.00.00Z_aaaaaaaa.dat" - )); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T02.00.00Z_bbbbbbbb.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes1)); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T01.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes2)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); - - // Assert - DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - - // Verify cache has both files - Map stats = calculator.getCacheStats(); - assertEquals(2, stats.get("cached_files")); - } - - @Test - void testCalculateStatus_windowBoundaryTimestamp() throws Exception { - // Setup - create delta file with timestamps at window boundary - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - long currentWindowStart = t - 24*3600; - List timestamps = new ArrayList<>(); - for (int i = 0; i < 250; i++) { - timestamps.add(t); - } - for (int i = 0; i < 250; i++) { - timestamps.add(currentWindowStart); - } - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); - - // Assert - DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - @Test - void testCalculateStatus_timestampsCached() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600, t - 7200); - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); - - // Assert - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - - // Cache should contain the timestamps - Map stats = calculator.getCacheStats(); - assertEquals(2, stats.get("total_cached_timestamps")); - } - - // ============================================================================ - // SECTION 10: Tests for queue attributes (invisible messages from other consumers) - // ============================================================================ - - @Test - void testCalculateStatus_delayedProcessingFromQueueAttributesOnly() throws Exception { - // Setup - delta files with low traffic (10 records) - // Threshold = 100 * 5 = 500 - // Queue attributes will have 600 invisible messages (other consumers processing) - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - timestamps.add(t - 3600 + i); // 10 entries from 1 hour ago - } - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - 1 message read by us, but 600 invisible messages from other consumers - List sqsMessages = Arrays.asList(createSqsMessage(t)); - - // QueueAttributes: 0 visible, 600 invisible (other consumers), 0 delayed - // Since we read 1 message, otherConsumers = 600 - 1 = 599 - // Total = 10 (delta) + 1 (our message) + 599 (other consumers) = 610 >= 500 threshold - SqsMessageOperations.QueueAttributes queueAttributes = - new SqsMessageOperations.QueueAttributes(0, 600, 0); - - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0); - - // Assert - DELAYED_PROCESSING due to high invisible message count from other consumers - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - @Test - void testCalculateStatus_delayedProcessingFromBothQueueAndMessages() throws Exception { - // Setup - delta files with moderate traffic (100 records) - // Threshold = 100 * 5 = 500 - // We'll have 200 messages + 250 invisible from other consumers = 550 > 500 - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = new ArrayList<>(); - for (int i = 0; i < 100; i++) { - timestamps.add(t - 3600 + i); // 100 entries from 1 hour ago - } - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - 200 messages read by us + 450 invisible (200 are ours + 250 from others) - // Messages must be within 5-minute window to be counted, so use 1-second spacing - List sqsMessages = new ArrayList<>(); - for (int i = 0; i < 200; i++) { - sqsMessages.add(createSqsMessage(t - i)); // 1 second apart, all within 5-minute window - } - - // QueueAttributes: 0 visible, 450 invisible (200 ours + 250 others), 0 delayed - // otherConsumers = 450 - 200 = 250 - // Total = 100 (delta) + 200 (our messages) + 250 (other consumers) = 550 >= 500 threshold - SqsMessageOperations.QueueAttributes queueAttributes = - new SqsMessageOperations.QueueAttributes(0, 450, 0); - - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0); - - // Assert - DELAYED_PROCESSING due to combined count exceeding threshold - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - -} \ No newline at end of file diff --git a/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java deleted file mode 100644 index 127cd04..0000000 --- a/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java +++ /dev/null @@ -1,428 +0,0 @@ -package com.uid2.optout.traffic; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.uid2.optout.sqs.SqsParsedMessage; -import com.uid2.optout.traffic.OptOutTrafficFilter; - -import software.amazon.awssdk.services.sqs.model.Message; - -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.junit.Assert.*; - -public class OptOutTrafficFilterTest { - - private static final String TEST_CONFIG_PATH = "./traffic-config.json"; - - @Before - public void setUp() { - try { - Files.deleteIfExists(Path.of(TEST_CONFIG_PATH)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @After - public void tearDown() { - try { - Files.deleteIfExists(Path.of(TEST_CONFIG_PATH)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Test - public void testParseFilterRules_emptyRules() throws Exception { - // Setup - empty denylist - String config = """ - { - "denylist_requests": [] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - no rules - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertEquals(0, filter.filterRules.size()); - } - - @Test - public void testParseFilterRules_singleRule() throws Exception { - // Setup - config with one rule - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - one rule - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertEquals(1, filter.filterRules.size()); - } - - @Test - public void testParseFilterRules_multipleRules() throws Exception { - // Setup - config with multiple rules - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - }, - { - "range": [1700010000, 1700013600], - "IPs": ["10.0.0.1", "10.0.0.2"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - two rules - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertEquals(2, filter.filterRules.size()); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_missingDenylistRequests() throws Exception { - // Setup - config without denylist_requests field - String config = """ - { - "other_field": "value" - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_invalidRange_startAfterEnd() throws Exception { - // Setup - range where start > end - String config = """ - { - "denylist_requests": [ - { - "range": [1700003600, 1700000000], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_invalidRange_startEqualsEnd() throws Exception { - // Setup - range where start == end - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700000000], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_rangeExceeds24Hours() throws Exception { - // Setup - range longer than 24 hours (86400 seconds) - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700086401], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_emptyIPs() throws Exception { - // Setup - rule with empty IP list - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": [] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_missingIPs() throws Exception { - // Setup - rule without IPs field - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); - } - - @Test - public void testIsDenylisted_matchingIPAndTimestamp() throws Exception { - // Setup - filter with denylist rule - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1", "10.0.0.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - SqsParsedMessage message = createTestMessage(1700001800, "192.168.1.1"); - - // Act & Assert - denylisted - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertTrue(filter.isDenylisted(message)); - } - - @Test - public void testIsDenylisted_matchingIPOutsideTimeRange() throws Exception { - // Setup - filter with denylist rule - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - - // Act & Assert - message before range not denylisted - SqsParsedMessage messageBefore = createTestMessage(1699999999, "192.168.1.1"); - assertFalse(filter.isDenylisted(messageBefore)); - // Act & Assert - message after range not denylisted - SqsParsedMessage messageAfter = createTestMessage(1700003601, "192.168.1.1"); - assertFalse(filter.isDenylisted(messageAfter)); - } - - @Test - public void testIsDenylisted_nonMatchingIP() throws Exception { - // Setup - filter with denylist rule - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - - // Act & Assert - non-matching IP not denylisted - SqsParsedMessage message = createTestMessage(1700001800, "10.0.0.1"); - assertFalse(filter.isDenylisted(message)); - } - - @Test - public void testIsDenylisted_atRangeBoundaries() throws Exception { - // Setup - filter with denylist rule - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - - // Act & Assert - message at start boundary (inclusive) denylisted - SqsParsedMessage messageAtStart = createTestMessage(1700000000, "192.168.1.1"); - assertTrue(filter.isDenylisted(messageAtStart)); - - // Act & Assert - message at end boundary (inclusive) denylisted - SqsParsedMessage messageAtEnd = createTestMessage(1700003600, "192.168.1.1"); - assertTrue(filter.isDenylisted(messageAtEnd)); - } - - @Test - public void testIsDenylisted_multipleRules() throws Exception { - // Setup - multiple denylist rules - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - }, - { - "range": [1700010000, 1700013600], - "IPs": ["10.0.0.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - - // Act & Assert - message matches first rule - SqsParsedMessage msg1 = createTestMessage(1700001800, "192.168.1.1"); - assertTrue(filter.isDenylisted(msg1)); - - // Act & Assert - message matches second rule - SqsParsedMessage msg2 = createTestMessage(1700011800, "10.0.0.1"); - assertTrue(filter.isDenylisted(msg2)); - - // Act & Assert - message matches neither rule - SqsParsedMessage msg3 = createTestMessage(1700005000, "172.16.0.1"); - assertFalse(filter.isDenylisted(msg3)); - } - - @Test - public void testIsDenylisted_nullClientIp() throws Exception { - // Setup - filter with denylist rule - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - - // Act & Assert - message with null IP not denylisted - SqsParsedMessage message = createTestMessage(1700001800, null); - assertFalse(filter.isDenylisted(message)); - } - - @Test - public void testReloadTrafficFilterConfig_success() throws Exception { - // Setup - config with one rule - String initialConfig = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), initialConfig); - - // Act & Assert - one rule - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertEquals(1, filter.filterRules.size()); - - // Setup - update config - String updatedConfig = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700003600], - "IPs": ["192.168.1.1"] - }, - { - "range": [1700010000, 1700013600], - "IPs": ["10.0.0.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), updatedConfig); - - // Act & Assert - two rules - filter.reloadTrafficFilterConfig(); - assertEquals(2, filter.filterRules.size()); - } - - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testReloadTrafficFilterConfig_fileNotFound() throws Exception { - // Setup, Act & Assert - try to create filter with non-existent config - new OptOutTrafficFilter("./non-existent-file.json"); - } - - @Test - public void testParseFilterRules_maxValidRange() throws Exception { - // Setup - range exactly 24 hours (86400 seconds) - should be valid - String config = """ - { - "denylist_requests": [ - { - "range": [1700000000, 1700086400], - "IPs": ["192.168.1.1"] - } - ] - } - """; - Files.writeString(Path.of(TEST_CONFIG_PATH), config); - - // Act & Assert - one rule - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertEquals(1, filter.filterRules.size()); - } - - /** - * Helper method to create test SqsParsedMessage - */ - private SqsParsedMessage createTestMessage(long timestamp, String clientIp) { - Message mockMessage = Message.builder().build(); - byte[] hash = new byte[32]; - byte[] id = new byte[32]; - return new SqsParsedMessage(mockMessage, hash, id, timestamp, null, null, clientIp, null); - } -} From 7c49222b315929dcec649b9af191c2f8e77047b6 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 16:36:12 -0700 Subject: [PATCH 08/25] try make git render file recreated --- .../traffic/OptOutTrafficCalculator.java | 643 +++++++ .../optout/traffic/OptOutTrafficFilter.java | 184 ++ .../traffic/OptOutTrafficCalculatorTest.java | 1577 +++++++++++++++++ .../traffic/OptOutTrafficFilterTest.java | 428 +++++ 4 files changed, 2832 insertions(+) create mode 100644 src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java create mode 100644 src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java create mode 100644 src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java create mode 100644 src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java new file mode 100644 index 0000000..ab6a2c0 --- /dev/null +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java @@ -0,0 +1,643 @@ +package com.uid2.optout.traffic; + +import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.optout.OptOutCollection; +import com.uid2.shared.optout.OptOutEntry; +import com.uid2.shared.optout.OptOutUtils; +import com.uid2.optout.Const; +import com.uid2.optout.sqs.SqsMessageOperations; + +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; + +import java.nio.charset.StandardCharsets; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.io.InputStream; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * Calculates opt-out traffic patterns to determine DEFAULT or DELAYED_PROCESSING status. + * + * Compares recent ~24h traffic (sumCurrent) against a configurable baseline (baselineTraffic) of expected traffic in 24 hours. + * The baseline is multiplied by (thresholdMultiplier) to determine the threshold. + * sumCurrent excludes records in allowlist ranges (surge windows determined by engineers). + * + * Returns DELAYED_PROCESSING if sumCurrent >= thresholdMultiplier * baselineTraffic, indicating abnormal traffic spike. + */ +public class OptOutTrafficCalculator { + private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficCalculator.class); + + private static final int HOURS_24 = 24 * 3600; // 24 hours in seconds + + private final Map deltaFileCache = new ConcurrentHashMap<>(); + private final ICloudStorage cloudStorage; + private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") + private final String trafficCalcConfigPath; + private int baselineTraffic; + private int thresholdMultiplier; + private int evaluationWindowSeconds; + private List> allowlistRanges; + + public enum TrafficStatus { + DELAYED_PROCESSING, + DEFAULT + } + + /** + * Cache entry for a delta file containing all record timestamps. + * + * Memory usage: ~8 bytes per timestamp (long) + * 1GB of memory can store ~130 million timestamps (1024^3)/8 + */ + private static class FileRecordCache { + final List timestamps; // All non-sentinel record timestamps + final long newestTimestamp; // evict delta from cache based on oldest record timestamp + + FileRecordCache(List timestamps) { + this.timestamps = timestamps; + this.newestTimestamp = timestamps.isEmpty() ? 0 : Collections.max(timestamps); + } + } + + /** + * Exception thrown by malformed traffic calculator config + */ + public static class MalformedTrafficCalcConfigException extends Exception { + public MalformedTrafficCalcConfigException(String message) { + super(message); + } + } + + /** + * Constructor for OptOutTrafficCalculator + * + * @param cloudStorage Cloud storage for reading delta files + * @param s3DeltaPrefix S3 prefix for delta files + * @param trafficCalcConfigS3Path S3 path for traffic calc config + */ + public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigPath) throws MalformedTrafficCalcConfigException { + this.cloudStorage = cloudStorage; + this.s3DeltaPrefix = s3DeltaPrefix; + this.trafficCalcConfigPath = trafficCalcConfigPath; + reloadTrafficCalcConfig(); // Load ConfigMap + + LOGGER.info("initialized: s3DeltaPrefix={}, threshold={}x", + s3DeltaPrefix, thresholdMultiplier); + } + + /** + * Reload traffic calc config from ConfigMap. + * Expected format: + * { + * "traffic_calc_evaluation_window_seconds": 86400, + * "traffic_calc_baseline_traffic": 100, + * "traffic_calc_threshold_multiplier": 5, + * "traffic_calc_allowlist_ranges": [ + * [startTimestamp1, endTimestamp1], + * [startTimestamp2, endTimestamp2] + * ], + * } + * + * Can be called periodically to pick up config changes without restarting. + */ + public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException { + LOGGER.info("loading traffic calc config from configmap"); + try (InputStream is = Files.newInputStream(Paths.get(trafficCalcConfigPath))) { + String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); + JsonObject trafficCalcConfig = new JsonObject(content); + + // Validate required fields exist + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_evaluation_window_seconds"); + } + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_baseline_traffic"); + } + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_threshold_multiplier"); + } + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_allowlist_ranges"); + } + + this.evaluationWindowSeconds = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp); + this.baselineTraffic = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcBaselineTrafficProp); + this.thresholdMultiplier = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcThresholdMultiplierProp); + + List> ranges = parseAllowlistRanges(trafficCalcConfig); + this.allowlistRanges = ranges; + + LOGGER.info("loaded traffic calc config: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", + this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size()); + + } catch (MalformedTrafficCalcConfigException e) { + LOGGER.error("circuit_breaker_config_error: config is malformed, configPath={}", trafficCalcConfigPath, e); + throw e; + } catch (Exception e) { + LOGGER.error("circuit_breaker_config_error: config is malformed or missing, configPath={}", trafficCalcConfigPath, e); + throw new MalformedTrafficCalcConfigException("failed to load traffic calc config: " + e.getMessage()); + } + } + + /** + * Parse allowlist ranges from JSON config + */ + List> parseAllowlistRanges(JsonObject config) throws MalformedTrafficCalcConfigException { + List> ranges = new ArrayList<>(); + + try { + var rangesArray = config.getJsonArray(Const.Config.OptOutTrafficCalcAllowlistRangesProp); + if (rangesArray != null) { + for (int i = 0; i < rangesArray.size(); i++) { + var rangeArray = rangesArray.getJsonArray(i); + if (rangeArray != null && rangeArray.size() >= 2) { + long start = rangeArray.getLong(0); + long end = rangeArray.getLong(1); + + if(start >= end) { + LOGGER.error("circuit_breaker_config_error: allowlist range start must be less than end, range=[{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": start must be less than end"); + } + + if (end - start > 86400) { + LOGGER.error("circuit_breaker_config_error: allowlist range must be less than 24 hours, range=[{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": range must be less than 24 hours"); + } + + List range = Arrays.asList(start, end); + ranges.add(range); + LOGGER.info("loaded allowlist range: [{}, {}]", start, end); + } + } + } + + ranges.sort(Comparator.comparing(range -> range.get(0))); + + // Validate no overlapping ranges + for (int i = 0; i < ranges.size() - 1; i++) { + long currentEnd = ranges.get(i).get(1); + long nextStart = ranges.get(i + 1).get(0); + if (currentEnd >= nextStart) { + LOGGER.error("circuit_breaker_config_error: overlapping allowlist ranges, range=[{}, {}] overlaps with range=[{}, {}]", + ranges.get(i).get(0), currentEnd, nextStart, ranges.get(i + 1).get(1)); + throw new MalformedTrafficCalcConfigException( + "overlapping allowlist ranges detected at indices " + i + " and " + (i + 1)); + } + } + + } catch (MalformedTrafficCalcConfigException e) { + throw e; + } catch (Exception e) { + LOGGER.error("circuit_breaker_config_error: failed to parse allowlist ranges", e); + throw new MalformedTrafficCalcConfigException("failed to parse allowlist ranges: " + e.getMessage()); + } + + return ranges; + } + + /** + * Calculate traffic status based on delta files and SQS queue messages. + * + * Uses the newest delta file timestamp to anchor the 24-hour delta traffic window, + * and the oldest queue timestamp to anchor the 5-minute queue window. + * + * Counts: + * - Delta file records (with allowlist filtering) + * - SQS messages passed in (with allowlist filtering) + * - Invisible messages from other consumers (from queue attributes, avoiding double count) + * + * @param sqsMessages List of SQS messages this consumer has read (non-denylisted) + * @param queueAttributes Queue attributes including invisible message count (can be null) + * @param denylistedCount Number of denylisted messages read by this consumer + * @param filteredAsTooRecentCount Number of messages filtered as "too recent" by window reader + * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) + */ + public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes, int denylistedCount, int filteredAsTooRecentCount) { + + try { + // Get list of delta files from S3 (sorted newest to oldest) + List deltaS3Paths = listDeltaFiles(); + + if (deltaS3Paths.isEmpty()) { + LOGGER.error("s3_error: no delta files found in s3 at prefix={}", s3DeltaPrefix); + throw new RuntimeException("no delta files found in s3 at prefix=" + s3DeltaPrefix); + } + + // Find newest delta file timestamp for delta traffic window + long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths); + LOGGER.info("traffic calculation: newestDeltaTs={}", newestDeltaTs); + + // Find oldest SQS queue message timestamp for queue window + long oldestQueueTs = findOldestQueueTimestamp(sqsMessages); + LOGGER.info("traffic calculation: oldestQueueTs={}", oldestQueueTs); + + // Define start time of the delta evaluation window + // We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend + // the window to account for any allowlist ranges in the extended portion + long deltaWindowStart = calculateWindowStartWithAllowlist(newestDeltaTs, this.evaluationWindowSeconds); + + // Evict old cache entries (older than delta window start) + evictOldCacheEntries(deltaWindowStart); + + // Process delta files and count records in [deltaWindowStart, newestDeltaTs] + // Files are sorted newest to oldest, records within files are sorted newest to oldest + // Stop when the newest record in a file is older than the window + int sum = 0; + int deltaRecordsCount = 0; + int deltaAllowlistedCount = 0; + int filesProcessed = 0; + int cacheHits = 0; + int cacheMisses = 0; + + for (String s3Path : deltaS3Paths) { + boolean wasCached = isCached(s3Path); + if (wasCached) { + cacheHits++; + } else { + cacheMisses++; + } + + List timestamps = getTimestampsFromFile(s3Path); + filesProcessed++; + + // Check newest record in file - if older than window, stop processing remaining files + long newestRecordTs = timestamps.get(0); + if (newestRecordTs < deltaWindowStart) { + break; + } + + for (long ts : timestamps) { + // Stop condition: record is older than our window + if (ts < deltaWindowStart) { + break; + } + + // skip records in allowlisted ranges + if (isInAllowlist(ts)) { + deltaAllowlistedCount++; + continue; + } + + // increment sum if record is in delta window + if (ts >= deltaWindowStart) { + deltaRecordsCount++; + sum++; + } + } + } + + LOGGER.info("delta files: processed={}, deltaRecords={}, allowlisted={}, cache hits={}, misses={}, cacheSize={}", + filesProcessed, deltaRecordsCount, deltaAllowlistedCount, cacheHits, cacheMisses, deltaFileCache.size()); + + // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] with allowlist filtering + int sqsCount = 0; + if (sqsMessages != null && !sqsMessages.isEmpty()) { + sqsCount = countSqsMessages(sqsMessages, oldestQueueTs); + sum += sqsCount; + } + + // Add invisible messages being processed by OTHER consumers + // (notVisible count includes our messages, so subtract what we've read to avoid double counting) + // ourMessages = delta messages + denylisted messages + filtered "too recent" messages + int otherConsumersMessages = 0; + if (queueAttributes != null) { + int totalInvisible = queueAttributes.getApproximateNumberOfMessagesNotVisible(); + int ourMessages = (sqsMessages != null ? sqsMessages.size() : 0) + denylistedCount + filteredAsTooRecentCount; + otherConsumersMessages = Math.max(0, totalInvisible - ourMessages); + sum += otherConsumersMessages; + LOGGER.info("traffic calculation: adding {} invisible messages from other consumers (totalInvisible={}, ourMessages={})", + otherConsumersMessages, totalInvisible, ourMessages); + } + + // Determine status + TrafficStatus status = determineStatus(sum, this.baselineTraffic); + + LOGGER.info("traffic calculation complete: sum={} (deltaRecords={} + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}", + sum, deltaRecordsCount, sqsCount, otherConsumersMessages, this.baselineTraffic, this.thresholdMultiplier, status); + + return status; + + } catch (Exception e) { + LOGGER.error("delta_job_failed: error calculating traffic status", e); + throw new RuntimeException("error calculating traffic status", e); + } + } + + /** + * Find the newest timestamp from delta files. + * Reads the newest delta file and returns its maximum timestamp. + */ + private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOException { + if (deltaS3Paths == null || deltaS3Paths.isEmpty()) { + return System.currentTimeMillis() / 1000; + } + + // Delta files are sorted (ISO 8601 format, lexicographically sortable) so first file is newest + String newestDeltaPath = deltaS3Paths.get(0); + List timestamps = getTimestampsFromFile(newestDeltaPath); + + if (timestamps.isEmpty()) { + LOGGER.error("s3_error: newest delta file has no timestamps, path={}", newestDeltaPath); + return System.currentTimeMillis() / 1000; + } + + long newestTs = Collections.max(timestamps); + LOGGER.info("found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); + return newestTs; + } + + /** + * List all delta files from S3, sorted newest to oldest + */ + private List listDeltaFiles() { + try { + // List all objects with the delta prefix + List allFiles = cloudStorage.list(s3DeltaPrefix); + + // Filter to only .dat delta files and sort newest to oldest + List deltaFiles = allFiles.stream() + .filter(OptOutUtils::isDeltaFile) + .sorted(OptOutUtils.DeltaFilenameComparatorDescending) + .collect(Collectors.toList()); + + LOGGER.info("listed {} delta files from s3 (prefix={})", deltaFiles.size(), s3DeltaPrefix); + return deltaFiles; + + } catch (Exception e) { + LOGGER.error("s3_error: failed to list delta files at prefix={}", s3DeltaPrefix, e); + return Collections.emptyList(); + } + } + + /** + * Check if a delta file is already cached + */ + private boolean isCached(String s3Path) { + String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); + return deltaFileCache.containsKey(filename); + } + + /** + * Get timestamps from a delta file (S3 path), using cache if available + */ + private List getTimestampsFromFile(String s3Path) throws IOException { + // Extract filename from S3 path for cache key + String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); + + // Check cache first + FileRecordCache cached = deltaFileCache.get(filename); + if (cached != null) { + return cached.timestamps; + } + + // Cache miss - download from S3 + List timestamps = readTimestampsFromS3(s3Path); + + // Store in cache + deltaFileCache.put(filename, new FileRecordCache(timestamps)); + + return timestamps; + } + + /** + * Read all non-sentinel record timestamps from a delta file in S3 + */ + private List readTimestampsFromS3(String s3Path) throws IOException { + try (InputStream is = cloudStorage.download(s3Path)) { + byte[] data = is.readAllBytes(); + OptOutCollection collection = new OptOutCollection(data); + + List timestamps = new ArrayList<>(); + for (int i = 0; i < collection.size(); i++) { + OptOutEntry entry = collection.get(i); + + // Skip sentinel entries + if (entry.isSpecialHash()) { + continue; + } + + timestamps.add(entry.timestamp); + } + + return timestamps; + } catch (Exception e) { + LOGGER.error("s3_error: failed to read delta file at path={}", s3Path, e); + throw new IOException("failed to read delta file from s3: " + s3Path, e); + } + } + + /** + * Calculate total duration of allowlist ranges that overlap with the given time window. + */ + long getAllowlistDuration(long t, long windowStart) { + long totalDuration = 0; + for (List range : this.allowlistRanges) { + long start = range.get(0); + long end = range.get(1); + + // Clip range to window boundaries + if (start < windowStart) { + start = windowStart; + } + if (end > t) { + end = t; + } + + // Only add duration if there's actual overlap (start < end) + if (start < end) { + totalDuration += end - start; + } + } + return totalDuration; + } + + /** + * Calculate the window start time that provides evaluationWindowSeconds of non-allowlisted time. + * Iteratively extends the window to account for allowlist ranges that may fall in extended portions. + */ + long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowSeconds) { + long allowlistDuration = getAllowlistDuration(newestDeltaTs, newestDeltaTs - evaluationWindowSeconds); + + // Each iteration discovers at least one new allowlist range, so max iterations = number of ranges + int maxIterations = this.allowlistRanges.size() + 1; + + for (int i = 0; i < maxIterations && allowlistDuration > 0; i++) { + long newWindowStart = newestDeltaTs - evaluationWindowSeconds - allowlistDuration; + long newAllowlistDuration = getAllowlistDuration(newestDeltaTs, newWindowStart); + + if (newAllowlistDuration == allowlistDuration) { + // No new allowlist time in extended portion, we've converged + break; + } + + allowlistDuration = newAllowlistDuration; + } + + return newestDeltaTs - evaluationWindowSeconds - allowlistDuration; + } + + /** + * Find the oldest SQS queue message timestamp + */ + private long findOldestQueueTimestamp(List sqsMessages) throws IOException { + long oldest = System.currentTimeMillis() / 1000; + + if (sqsMessages != null && !sqsMessages.isEmpty()) { + for (Message msg : sqsMessages) { + Long ts = extractTimestampFromMessage(msg); + if (ts != null && ts < oldest) { + oldest = ts; + } + } + } + + return oldest; + } + + /** + * Extract timestamp from SQS message (from SentTimestamp attribute) + */ + private Long extractTimestampFromMessage(Message msg) { + // Get SentTimestamp attribute (milliseconds) + String sentTimestamp = msg.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); + if (sentTimestamp != null) { + try { + return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds + } catch (NumberFormatException e) { + LOGGER.error("sqs_error: invalid sentTimestamp, messageId={}, sentTimestamp={}", msg.messageId(), sentTimestamp); + } + } + + // Fallback: use current time + return System.currentTimeMillis() / 1000; + } + + /** + * Count SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes + */ + private int countSqsMessages(List sqsMessages, long oldestQueueTs) { + + int count = 0; + int allowlistedCount = 0; + long windowEnd = oldestQueueTs + 5 * 60; + + for (Message msg : sqsMessages) { + Long ts = extractTimestampFromMessage(msg); + + if (ts < oldestQueueTs || ts > windowEnd) { + continue; + } + + if (isInAllowlist(ts)) { + allowlistedCount++; + continue; + } + count++; + + } + + LOGGER.info("sqs messages: {} in window, {} allowlisted [oldestQueueTs={}, oldestQueueTs+5m={}]", count, allowlistedCount, oldestQueueTs, windowEnd); + return count; + } + + /** + * Check if a timestamp falls within any allowlist range + */ + boolean isInAllowlist(long timestamp) { + if (allowlistRanges == null || allowlistRanges.isEmpty()) { + return false; + } + + for (List range : allowlistRanges) { + if (range.size() < 2) { + continue; + } + + long start = range.get(0); + long end = range.get(1); + + if (timestamp >= start && timestamp <= end) { + return true; + } + } + + return false; + } + + /** + * Evict cache entries with data older than the cutoff timestamp + */ + private void evictOldCacheEntries(long cutoffTimestamp) { + int beforeSize = deltaFileCache.size(); + + deltaFileCache.entrySet().removeIf(entry -> + entry.getValue().newestTimestamp < cutoffTimestamp + ); + + int afterSize = deltaFileCache.size(); + if (beforeSize != afterSize) { + LOGGER.info("evicted {} old cache entries (before={}, after={})", + beforeSize - afterSize, beforeSize, afterSize); + } + } + + /** + * Determine traffic status based on current vs baseline traffic. + * Logs warnings at 50%, 75%, and 90% of the circuit breaker threshold. + */ + TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { + if (baselineTraffic == 0 || thresholdMultiplier == 0) { + LOGGER.error("circuit_breaker_config_error: baselineTraffic is 0 or thresholdMultiplier is 0"); + throw new RuntimeException("invalid circuit breaker config: baselineTraffic=" + baselineTraffic + ", thresholdMultiplier=" + thresholdMultiplier); + } + + int threshold = thresholdMultiplier * baselineTraffic; + double thresholdPercent = (double) sumCurrent / threshold * 100; + + // Log warnings at increasing thresholds before circuit breaker triggers + if (thresholdPercent >= 90.0) { + LOGGER.warn("high_message_volume: 90% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + } else if (thresholdPercent >= 75.0) { + LOGGER.warn("high_message_volume: 75% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + } else if (thresholdPercent >= 50.0) { + LOGGER.warn("high_message_volume: 50% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + } + + if (sumCurrent >= threshold) { + LOGGER.error("circuit_breaker_triggered: traffic threshold breached, sumCurrent={}, threshold={} ({}x{})", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic); + return TrafficStatus.DELAYED_PROCESSING; + } + + LOGGER.info("traffic within normal range: sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + return TrafficStatus.DEFAULT; + } + + /** + * Get cache statistics for monitoring + */ + public Map getCacheStats() { + Map stats = new HashMap<>(); + stats.put("cached_files", deltaFileCache.size()); + + int totalTimestamps = deltaFileCache.values().stream() + .mapToInt(cache -> cache.timestamps.size()) + .sum(); + stats.put("total_cached_timestamps", totalTimestamps); + + return stats; + } + +} \ No newline at end of file diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java new file mode 100644 index 0000000..445f032 --- /dev/null +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java @@ -0,0 +1,184 @@ +package com.uid2.optout.traffic; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.uid2.optout.sqs.SqsParsedMessage; + +import java.util.ArrayList; +import java.util.List; +import java.util.Collections; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.charset.StandardCharsets; +import io.vertx.core.json.JsonObject; +import io.vertx.core.json.JsonArray; + +public class OptOutTrafficFilter { + private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficFilter.class); + + private final String trafficFilterConfigPath; + List filterRules; + + /** + * Traffic filter rule defining a time range and a list of IP addresses to exclude + */ + private static class TrafficFilterRule { + private final List range; + private final List ipAddresses; + + TrafficFilterRule(List range, List ipAddresses) { + this.range = range; + this.ipAddresses = ipAddresses; + } + + public long getRangeStart() { + return range.get(0); + } + public long getRangeEnd() { + return range.get(1); + } + public List getIpAddresses() { + return ipAddresses; + } + } + + public static class MalformedTrafficFilterConfigException extends Exception { + public MalformedTrafficFilterConfigException(String message) { + super(message); + } + } + + /** + * Constructor for OptOutTrafficFilter + * + * @param trafficFilterConfigPath S3 path for traffic filter config + * @throws MalformedTrafficFilterConfigException if the traffic filter config is invalid + */ + public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTrafficFilterConfigException { + this.trafficFilterConfigPath = trafficFilterConfigPath; + // Initial filter rules load + this.filterRules = Collections.emptyList(); // start empty + reloadTrafficFilterConfig(); // load ConfigMap + + LOGGER.info("initialized: filterRules={}", filterRules.size()); + } + + /** + * Reload traffic filter config from ConfigMap. + * Expected format: + * { + * "denylist_requests": [ + * {range: [startTimestamp, endTimestamp], IPs: ["ip1"]}, + * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip2"]}, + * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip3"]}, + * ] + * } + * + * Can be called periodically to pick up config changes without restarting. + */ + public void reloadTrafficFilterConfig() throws MalformedTrafficFilterConfigException { + LOGGER.info("loading traffic filter config"); + try (InputStream is = Files.newInputStream(Paths.get(trafficFilterConfigPath))) { + String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); + JsonObject filterConfigJson = new JsonObject(content); + + this.filterRules = parseFilterRules(filterConfigJson); + + LOGGER.info("loaded traffic filter config: filterRules={}", filterRules.size()); + + } catch (Exception e) { + LOGGER.error("circuit_breaker_config_error: no traffic filter config found at {}", trafficFilterConfigPath, e); + throw new MalformedTrafficFilterConfigException(e.getMessage()); + } + } + + /** + * Parse request filtering rules from JSON config + */ + List parseFilterRules(JsonObject config) throws MalformedTrafficFilterConfigException { + List rules = new ArrayList<>(); + try { + JsonArray denylistRequests = config.getJsonArray("denylist_requests"); + if (denylistRequests == null) { + LOGGER.error("circuit_breaker_config_error: denylist_requests is null"); + throw new MalformedTrafficFilterConfigException("invalid traffic filter config: denylist_requests is null"); + } + for (int i = 0; i < denylistRequests.size(); i++) { + JsonObject ruleJson = denylistRequests.getJsonObject(i); + + // parse range + var rangeJson = ruleJson.getJsonArray("range"); + List range = new ArrayList<>(); + if (rangeJson != null && rangeJson.size() == 2) { + long start = rangeJson.getLong(0); + long end = rangeJson.getLong(1); + + if (start >= end) { + LOGGER.error("circuit_breaker_config_error: rule range start must be less than end, rule={}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range start must be less than end"); + } + range.add(start); + range.add(end); + } + + // log error and throw exception if range is not 2 elements + if (range.size() != 2) { + LOGGER.error("circuit_breaker_config_error: rule range is not 2 elements, rule={}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range is not 2 elements"); + } + + // parse IPs + var ipAddressesJson = ruleJson.getJsonArray("IPs"); + List ipAddresses = new ArrayList<>(); + if (ipAddressesJson != null) { + for (int j = 0; j < ipAddressesJson.size(); j++) { + ipAddresses.add(ipAddressesJson.getString(j)); + } + } + + // log error and throw exception if IPs is empty + if (ipAddresses.size() == 0) { + LOGGER.error("circuit_breaker_config_error: rule IPs is empty, rule={}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: IPs is empty"); + } + + // log error and throw exception if rule is invalid + if (range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less + LOGGER.error("circuit_breaker_config_error: rule range must be 24 hours or less, rule={}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range must be 24 hours or less"); + } + + TrafficFilterRule rule = new TrafficFilterRule(range, ipAddresses); + + LOGGER.info("loaded traffic filter rule: range=[{}, {}], IPs={}", rule.getRangeStart(), rule.getRangeEnd(), rule.getIpAddresses()); + rules.add(rule); + } + return rules; + } catch (Exception e) { + LOGGER.error("circuit_breaker_config_error: failed to parse rules, config={}, error={}", config.encode(), e.getMessage()); + throw new MalformedTrafficFilterConfigException(e.getMessage()); + } + } + + public boolean isDenylisted(SqsParsedMessage message) { + long timestamp = message.timestamp(); + String clientIp = message.clientIp(); + + if (clientIp == null || clientIp.isEmpty()) { + LOGGER.error("sqs_error: request does not contain client ip, messageId={}", message.originalMessage().messageId()); + return false; + } + + for (TrafficFilterRule rule : filterRules) { + if(timestamp >= rule.getRangeStart() && timestamp <= rule.getRangeEnd()) { + if(rule.getIpAddresses().contains(clientIp)) { + return true; + } + }; + } + return false; + } + +} \ No newline at end of file diff --git a/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java new file mode 100644 index 0000000..fd435a9 --- /dev/null +++ b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java @@ -0,0 +1,1577 @@ +package com.uid2.optout.traffic; + +import com.uid2.shared.cloud.CloudStorageException; +import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.optout.OptOutCollection; +import com.uid2.shared.optout.OptOutEntry; +import com.uid2.optout.sqs.SqsMessageOperations; +import com.uid2.optout.Const; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; + +import com.uid2.optout.traffic.OptOutTrafficCalculator; +import com.uid2.optout.traffic.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; +import java.io.ByteArrayInputStream; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +public class OptOutTrafficCalculatorTest { + + @Mock + private ICloudStorage cloudStorage; + + private static final String S3_DELTA_PREFIX = "optout-v2/delta/"; + private static final String TRAFFIC_CONFIG_PATH = "./traffic-config.json"; + private static final int BASELINE_TRAFFIC = 100; + private static final int THRESHOLD_MULTIPLIER = 5; + private static final int EVALUATION_WINDOW_SECONDS = 24 * 3600; + + @BeforeEach + void setUp() { + // default config + JsonObject config = new JsonObject(); + config.put(Const.Config.OptOutTrafficCalcBaselineTrafficProp, BASELINE_TRAFFIC); + config.put(Const.Config.OptOutTrafficCalcThresholdMultiplierProp, THRESHOLD_MULTIPLIER); + config.put(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp, EVALUATION_WINDOW_SECONDS); + config.put(Const.Config.OptOutTrafficCalcAllowlistRangesProp, new JsonArray()); + try { + createTrafficConfigFile(config.toString()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @AfterEach + void tearDown() { + if (Files.exists(Path.of(TRAFFIC_CONFIG_PATH))) { + try { + Files.delete(Path.of(TRAFFIC_CONFIG_PATH)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + private void createTrafficConfigFile(String content) { + try { + Path configPath = Path.of(TRAFFIC_CONFIG_PATH); + Files.writeString(configPath, content); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Helper to create config by merging partial JSON with defaults + */ + private void createConfigFromPartialJson(String partialJson) { + JsonObject partial = new JsonObject(partialJson); + JsonObject config = new JsonObject(); + + // Set defaults + if (!partial.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { + config.put(Const.Config.OptOutTrafficCalcBaselineTrafficProp, BASELINE_TRAFFIC); + } + if (!partial.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { + config.put(Const.Config.OptOutTrafficCalcThresholdMultiplierProp, THRESHOLD_MULTIPLIER); + } + if (!partial.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { + config.put(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp, EVALUATION_WINDOW_SECONDS); + } + if (!partial.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { + config.put(Const.Config.OptOutTrafficCalcAllowlistRangesProp, new JsonArray()); + } + + // Merge in partial config (overrides defaults) + partial.forEach(entry -> config.put(entry.getKey(), entry.getValue())); + + createTrafficConfigFile(config.toString()); + } + + /** + * Helper to create config with custom threshold + */ + private void createConfigWithThreshold(int threshold) { + createConfigFromPartialJson("{\"" + Const.Config.OptOutTrafficCalcThresholdMultiplierProp + "\": " + threshold + "}"); + } + + // ============================================================================ + // SECTION 1: Constructor & Initialization Tests + // ============================================================================ + + @Test + void testConstructor_defaultThreshold() throws Exception { + // Setup - default threshold of 5 + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); // 10 < 5*3 + + status = calculator.determineStatus(15, 3); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 15 >= 5*3 + } + + @Test + void testConstructor_customThreshold() throws Exception { + // Setup - custom threshold of 10 + createConfigWithThreshold(10); + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(49, 5); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); // 49 < 10*5 + status = calculator.determineStatus(50, 5); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 50 >= 10*5 + } + + @Test + void testConstructor_trafficCalcConfigLoadFailure() throws Exception { + // Setup - traffic calc config load failure + createTrafficConfigFile("Invalid JSON"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + }); + + // Create valid config to test reload failure + createConfigFromPartialJson("{}"); + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + createTrafficConfigFile("Invalid JSON"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + } + + // ============================================================================ + // SECTION 2: parseTrafficCalcConfigRanges() + // ============================================================================ + + @Test + void testParseTrafficCalcConfigRanges_emptyConfig() throws Exception { + // Setup - no config + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + JsonObject emptyConfig = new JsonObject(); + + // Act + List> ranges = calculator.parseAllowlistRanges(emptyConfig); + + // Assert - empty ranges + assertTrue(ranges.isEmpty()); + } + + @Test + void testParseTrafficCalcConfigRanges_singleRange() throws Exception { + // Setup - single range + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)); + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - single range + assertEquals(1, result.size()); + assertEquals(1000L, result.get(0).get(0)); + assertEquals(2000L, result.get(0).get(1)); + } + + @Test + void testParseTrafficCalcConfigRanges_multipleRanges() throws Exception { + // Setup - multiple ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(3000L).add(4000L)) + .add(new JsonArray().add(5000L).add(6000L)); + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - multiple ranges + assertEquals(3, result.size()); + assertEquals(1000L, result.get(0).get(0)); + assertEquals(3000L, result.get(1).get(0)); + assertEquals(5000L, result.get(2).get(0)); + } + + @Test + void testParseTrafficCalcConfigRanges_misorderedRange() throws Exception { + // Setup - range with end < start is malformed + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(2000L).add(1000L)); // End before start + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.parseAllowlistRanges(configWithRanges); + }); + } + + @Test + void testParseTrafficCalcConfigRanges_rangeTooLong() throws Exception { + // Setup - range longer than 24 hours is malformed + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(2000L).add(200000L)); // Longer than 24 hours + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.parseAllowlistRanges(configWithRanges); + }); + } + + @Test + void testParseTrafficCalcConfigRanges_sortsByStartTime() throws Exception { + // Setup - ranges added out of order + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(5000L).add(6000L)) + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(3000L).add(4000L)); + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - should be sorted by start time + assertEquals(3, result.size()); + assertEquals(1000L, result.get(0).get(0)); + assertEquals(3000L, result.get(1).get(0)); + assertEquals(5000L, result.get(2).get(0)); + } + + @Test + void testParseTrafficCalcConfigRanges_invalidRangeTooFewElements() throws Exception { + // Setup - invalid range with only 1 element; + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L)) // Only 1 element + .add(new JsonArray().add(2000L).add(3000L)); // Valid + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - should skip invalid range + assertEquals(1, result.size()); + assertEquals(2000L, result.get(0).get(0)); + } + + @Test + void testParseTrafficCalcConfigRanges_nullArray() throws Exception { + // Setup - null array + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + configWithRanges.put("traffic_calc_allowlist_ranges", (JsonArray) null); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - empty ranges + assertTrue(result.isEmpty()); + } + + @Test + void testParseTrafficCalcConfigRanges_overlappingRanges() throws Exception { + // Setup - overlapping ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(1500L).add(2500L)); // Overlaps with first range + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act & Assert - should throw exception due to overlap + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.parseAllowlistRanges(configWithRanges); + }); + } + + @Test + void testParseTrafficCalcConfigRanges_adjacentRangesWithSameBoundary() throws Exception { + // Setup - ranges where end of first equals start of second (touching but not overlapping semantically, but we treat as overlap) + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(2000L).add(3000L)); // Starts exactly where first ends + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act & Assert - should throw exception because ranges touch at boundary + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.parseAllowlistRanges(configWithRanges); + }); + } + + @Test + void testParseTrafficCalcConfigRanges_nonOverlappingRanges() throws Exception { + // Setup - ranges that don't overlap (with gap between them) + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(2001L).add(3000L)); // Starts after first ends + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - should succeed with 2 ranges + assertEquals(2, result.size()); + } + + // ============================================================================ + // SECTION 3: isInTrafficCalcConfig() + // ============================================================================ + + @Test + void testIsInTrafficCalcConfig_withinSingleRange() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - true when within range + assertTrue(calculator.isInAllowlist(1500L)); + } + + @Test + void testIsInTrafficCalcConfig_exactlyAtStart() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - true when exactly at start of range + assertTrue(calculator.isInAllowlist(1000L)); + } + + @Test + void testIsInTrafficCalcConfig_exactlyAtEnd() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - true when exactly at end of range + assertTrue(calculator.isInAllowlist(2000L)); + } + + @Test + void testIsInTrafficCalcConfig_beforeRange() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - false when before range + assertFalse(calculator.isInAllowlist(999L)); + } + + @Test + void testIsInTrafficCalcConfig_afterRange() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - false when after range + assertFalse(calculator.isInAllowlist(2001L)); + } + + @Test + void testIsInTrafficCalcConfig_betweenRanges() throws Exception { + // Setup - load traffic calc config with two ranges [1000, 2000] and [3000, 4000] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000], + [3000, 4000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - false when between ranges + assertFalse(calculator.isInAllowlist(2500L)); + } + + @Test + void testIsInTrafficCalcConfig_emptyRanges() throws Exception { + // Setup uses default config from setUp() which has empty traffic calc config ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - false when empty ranges + assertFalse(calculator.isInAllowlist(1500L)); + } + + @Test + void testIsInTrafficCalcConfig_nullRanges() throws Exception { + // Setup - no traffic calc config ranges loaded (will fail and set empty) + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": null + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert - false when null/empty ranges + assertFalse(calculator.isInAllowlist(1500L)); + } + + @Test + void testIsInTrafficCalcConfig_invalidRangeSize() throws Exception { + // Setup - load traffic calc config with invalid range (only 1 element) and valid range + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000], + [2000, 3000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert + assertFalse(calculator.isInAllowlist(1500L)); // Should not match invalid range + assertTrue(calculator.isInAllowlist(2500L)); // Should match valid range + } + + @Test + void testIsInTrafficCalcConfig_multipleRanges() throws Exception { + // Setup - load traffic calc config with multiple ranges + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000], + [3000, 4000], + [5000, 6000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert + assertTrue(calculator.isInAllowlist(1500L)); // In first range + assertTrue(calculator.isInAllowlist(3500L)); // In second range + assertTrue(calculator.isInAllowlist(5500L)); // In third range + assertFalse(calculator.isInAllowlist(2500L)); // Between first and second + } + + // ============================================================================ + // SECTION 4: getTrafficCalcConfigDuration() + // ============================================================================ + + @Test + void testGetTrafficCalcConfigDuration_noRanges() throws Exception { + // Setup - no ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Assert + assertEquals(0L, calculator.getAllowlistDuration(10000L, 5000L)); // 0 duration when no ranges + } + + @Test + void testGetTrafficCalcConfigDuration_rangeFullyWithinWindow() throws Exception { + // Setup - range fully within window + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [6000, 7000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window [5000, 10000], range [6000, 7000] + long duration = calculator.getAllowlistDuration(10000L, 5000L); + + // Assert - full range duration + assertEquals(1000L, duration); + } + + @Test + void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsStart() throws Exception { + // Setup - range partially overlaps start of window + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [3000, 7000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window [5000, 10000], range [3000, 7000] + long duration = calculator.getAllowlistDuration(10000L, 5000L); + + // Assert - should clip to [5000, 7000] = 2000 + assertEquals(2000L, duration); + } + + @Test + void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsEnd() throws Exception { + // Setup - range partially overlaps end of window + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [8000, 12000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window [5000, 10000], range [8000, 12000] + long duration = calculator.getAllowlistDuration(10000L, 5000L); + + // Assert - should clip to [8000, 10000] = 2000 + assertEquals(2000L, duration); + } + + @Test + void testGetTrafficCalcConfigDuration_rangeCompletelyOutsideWindow() throws Exception { + // Setup - range completely outside window + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window [5000, 10000], range [1000, 2000] + long duration = calculator.getAllowlistDuration(10000L, 5000L); + + // Assert - 0 duration when range completely outside window + assertEquals(0L, duration); + } + + @Test + void testGetTrafficCalcConfigDuration_multipleRanges() throws Exception { + // Setup - multiple ranges + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [6000, 7000], + [8000, 9000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window [5000, 10000], ranges [6000, 7000] and [8000, 9000] + long duration = calculator.getAllowlistDuration(10000L, 5000L); + + // Assert - 1000 + 1000 = 2000 + assertEquals(2000L, duration); + } + + @Test + void testGetTrafficCalcConfigDuration_rangeSpansEntireWindow() throws Exception { + // Setup - range spans entire window + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [3000, 12000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window [5000, 10000], range [3000, 12000] + long duration = calculator.getAllowlistDuration(10000L, 5000L); + + // Assert - entire window is in traffic calc config ranges = 5000 + assertEquals(5000L, duration); + } + + // ============================================================================ + // SECTION 4.5: calculateWindowStartWithAllowlist() + // ============================================================================ + + @Test + void testCalculateWindowStartWithAllowlist_noAllowlist() throws Exception { + // Setup - no allowlist ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window should be [3, 8] with no extension + long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); + + // Assert - no allowlist, so window start is simply newestDeltaTs - evaluationWindowSeconds + assertEquals(3L, windowStart); + } + + @Test + void testCalculateWindowStartWithAllowlist_allowlistInOriginalWindowOnly() throws Exception { + // Setup - allowlist range only in original window, not in extended portion + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [6, 7] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - newestDeltaTs=8, evaluationWindow=5 + // Original window [3, 8] has [6,7] allowlisted (1 hour) + // Extended portion [2, 3] has no allowlist + // So window start should be 8 - 5 - 1 = 2 + long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); + + assertEquals(2L, windowStart); + } + + @Test + void testCalculateWindowStartWithAllowlist_allowlistInExtendedPortion() throws Exception { + // Setup - allowlist ranges in both original window AND extended portion + // This is the user's example: evaluationWindow=5, newestDeltaTs=8, allowlist={[2,3], [6,7]} + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [2, 3], + [6, 7] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + // Original window [3, 8]: [6,7] allowlisted = 1 hour + // First extension to [2, 8]: [2,3] and [6,7] allowlisted = 2 hours total + // Second extension to [1, 8]: still [2,3] and [6,7] = 2 hours (no new allowlist) + // Final: windowStart = 8 - 5 - 2 = 1 + long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); + + assertEquals(1L, windowStart); + } + + @Test + void testCalculateWindowStartWithAllowlist_allowlistBeforeWindow() throws Exception { + // Setup - allowlist range entirely before the initial window + // This tests that we don't over-extend when allowlist is old + // evaluationWindow=5, newestDeltaTs=20, allowlist=[10,13] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [10, 13] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + // Initial window [15, 20]: no allowlist overlap, allowlistDuration = 0 + // No extension needed + // Final: windowStart = 20 - 5 - 0 = 15 + long windowStart = calculator.calculateWindowStartWithAllowlist(20L, 5); + + // Verify: window [15, 20] has 5 hours, 0 allowlisted = 5 non-allowlisted + assertEquals(15L, windowStart); + } + + // ============================================================================ + // SECTION 5: determineStatus() + // ============================================================================ + + @Test + void testDetermineStatus_belowThreshold() throws Exception { + // Setup - below threshold + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - 10 < 5 * 3 + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); + + // Assert - DEFAULT when below threshold + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testDetermineStatus_atThreshold() throws Exception { + // Setup - at threshold + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - 15 == 5 * 3 + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(15, 3); + + // Assert - DELAYED_PROCESSING when at threshold + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testDetermineStatus_aboveThreshold() throws Exception { + // Setup - above threshold + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - 20 > 5 * 3 + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(20, 3); + + // Assert - DELAYED_PROCESSING when above threshold + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testDetermineStatus_sumPastZero() throws Exception { + // Setup - sumPast is 0 + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert - should throw exception for invalid config + assertThrows(RuntimeException.class, () -> calculator.determineStatus(100, 0)); + } + + @Test + void testDetermineStatus_bothZero() throws Exception { + // Setup - both sumCurrent and sumPast are 0; + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert - should throw exception for invalid config + assertThrows(RuntimeException.class, () -> calculator.determineStatus(0, 0)); + } + + @Test + void testDetermineStatus_sumCurrentZero() throws Exception { + // Setup - sumCurrent is 0 + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - 0 < 5 * 10 + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 10); + + // Assert - DEFAULT when sumCurrent is 0 + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @ParameterizedTest + @CsvSource({ + "1, 1, 1, DELAYED_PROCESSING", // threshold=1: 1 >= 1*1 + "2, 4, 2, DELAYED_PROCESSING", // threshold=2: 4 >= 2*2 + "5, 10, 2, DELAYED_PROCESSING", // threshold=5: 10 >= 5*2 + "10, 100, 10, DELAYED_PROCESSING", // threshold=10: 100 >= 10*10 + "5, 24, 5, DEFAULT", // threshold=5: 24 < 5*5 + "100, 1000, 11, DEFAULT" // threshold=100: 1000 < 100*11 + }) + void testDetermineStatus_variousThresholds(int threshold, int sumCurrent, int sumPast, String expectedStatus) throws Exception { + // Setup - various thresholds + createConfigWithThreshold(threshold); + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(sumCurrent, sumPast); + + // Assert + assertEquals(OptOutTrafficCalculator.TrafficStatus.valueOf(expectedStatus), status); + } + + @Test + void testDetermineStatus_largeNumbers() throws Exception { + // Setup - test with large numbers + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(1_000_000, 200_000); + + // Assert - 1M >= 5 * 200K = 1M + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + // ============================================================================ + // SECTION 6: S3 Config Reload Tests + // ============================================================================ + + @Test + void testReloadTrafficCalcConfig_success() throws Exception { + // Setup - initial traffic calc config + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000], + [3000, 4000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Change the traffic calc config to a new range + String newTrafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [5000, 6000] + ] + } + """; + createConfigFromPartialJson(newTrafficCalcConfigJson); + + // Act - reload the traffic calc config + calculator.reloadTrafficCalcConfig(); + + // Assert - verify new traffic calc config is loaded + assertTrue(calculator.isInAllowlist(5500L)); + } + + @Test + void testReloadTrafficCalcConfig_failure() throws Exception { + // Setup - initial traffic calc config + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [1000, 2000] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Now make it fail + createTrafficConfigFile("Invalid JSON"); + + // Act - should not throw exception + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + + } + + @Test + public void testReloadTrafficCalcConfig_failure_missingKeys() throws Exception { + // Setup + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert missing threshold multiplier + createTrafficConfigFile("{\"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + + // Act & Assert missing evaluation window seconds + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + + // Act & Assert missing baseline traffic + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + + // Act & Assert missing traffic calc config ranges + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100}"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + } + + @Test + public void testReloadTrafficCalcConfig_failure_misorderedRanges() throws Exception { + // Setup - misordered ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + createConfigFromPartialJson("{\"traffic_calc_allowlist_ranges\": [ [2000, 1000] ]}"); + + // Act & Assert + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + } + + @Test + public void testReloadTrafficCalcConfig_failure_rangeTooLong() throws Exception { + // Setup - range greater than 24 hours + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + createConfigFromPartialJson("{\"traffic_calc_allowlist_ranges\": [ [1000, 200000] ]}"); + + // Act & Assert + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + } + + // ============================================================================ + // SECTION 7: Cache Management Tests (also tested in section 9) + // ============================================================================ + + @Test + void testGetCacheStats_emptyCache() throws Exception { + // Setup + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + Map stats = calculator.getCacheStats(); + + // Assert - should return empty stats + assertEquals(0, stats.get("cached_files")); + assertEquals(0, stats.get("total_cached_timestamps")); + } + + // ============================================================================ + // SECTION 8: Helper Methods for Test Data Creation + // ============================================================================ + + /** + * Create a mock SQS message with specified timestamp + */ + private Message createSqsMessage(long timestampSeconds) { + Map attributes = new HashMap<>(); + attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampSeconds * 1000)); + + return Message.builder() + .messageId("test-msg-" + timestampSeconds) + .body("{\"test\": \"data\"}") + .attributes(attributes) + .build(); + } + + /** + * Create a mock SQS message without timestamp + */ + private Message createSqsMessageWithoutTimestamp() { + return Message.builder() + .messageId("test-msg-no-timestamp") + .body("{\"test\": \"data\"}") + .attributes(new HashMap<>()) + .build(); + } + + /** + * Create delta file bytes with specified timestamps + */ + private byte[] createDeltaFileBytes(List timestamps) throws Exception { + // Create OptOutEntry objects using newTestEntry + List entries = new ArrayList<>(); + + long idCounter = 1000; // Use incrementing IDs for test entries + for (long timestamp : timestamps) { + entries.add(OptOutEntry.newTestEntry(idCounter++, timestamp)); + } + + // Create OptOutCollection + OptOutCollection collection = new OptOutCollection(entries.toArray(new OptOutEntry[0])); + return collection.getStore(); + } + + + // ============================================================================ + // SECTION 9: Tests for calculateStatus() + // ============================================================================ + + @Test + void testCalculateStatus_noDeltaFiles() throws Exception { + // Setup - no delta files + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Collections.emptyList()); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert - should throw exception when no delta files + assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); + } + + @Test + void testCalculateStatus_normalTraffic() throws Exception { + // Setup - setup time: current time + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Create delta files with timestamps distributed over 48 hours + List timestamps = new ArrayList<>(); + + // add 499 entries in current window + for (int i = 0; i < 49; i++) { + timestamps.add(t - 23*3600 + i * 60); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - 100+1 < 5 * 50 = 250, so should be DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_delayedProcessing() throws Exception { + // Setup - create delta files with spike in current window + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Create delta files with spike in current window + List timestamps = new ArrayList<>(); + + // add 500 entries in current window + for (int i = 0; i < 500; i++) { + timestamps.add(t - 23*3600 + i * 60); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - 100+1 >= 5 * 10 = 50, DELAYED_PROCESSING + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testCalculateStatus_noSqsMessages() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600, t - 7200); // Some entries + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - null SQS messages + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null, null, 0, 0); + + // Assert - should still calculate based on delta files, DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_emptySqsMessages() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - empty SQS messages + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0); + + // Assert - should still calculate based on delta files, DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_multipleSqsMessages() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = new ArrayList<>(); + // add 470 entries in window + for (int i = 0; i < 470; i++) { + timestamps.add(t - 24*3600 + i * 60); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Add 30 SQS entries in [t, t+5min] + List sqsMessages = new ArrayList<>(); + for (int i = 0; i < 30; i++) { + sqsMessages.add(createSqsMessage(t - i * 10)); + } + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - DELAYED_PROCESSING + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testCalculateStatus_withTrafficCalcConfig() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Traffic calc config that covers part of window + String trafficCalcConfigJson = String.format(""" + { + "traffic_calc_allowlist_ranges": [ + [%d, %d] + ] + } + """, t - 12*3600, t - 6*3600); + + List timestamps = new ArrayList<>(); + + // window - 600 entries (300 in traffic calc config range, 300 outside) + for (int i = 0; i < 300; i++) { + timestamps.add(t - 12*3600 + i); + } + for (int i = 0; i < 300; i++) { + timestamps.add(t - 3600 + i); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + createConfigFromPartialJson(trafficCalcConfigJson); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta-001_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta-001_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - should filter out entries in traffic calc config ranges + // Only 300 from window count (not in traffic calc config ranges) + 1 SQS = 301 + // 301 < 5*100, so DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_cacheUtilization() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600, t - 7200); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - first call should populate cache + List sqsMessages = Arrays.asList(createSqsMessage(t)); + calculator.calculateStatus(sqsMessages, null, 0, 0); + + Map stats = calculator.getCacheStats(); + int cachedFiles = (Integer) stats.get("cached_files"); + + // Second call should use cache (no additional S3 download) + calculator.calculateStatus(sqsMessages, null, 0, 0); + + Map stats2 = calculator.getCacheStats(); + int cachedFiles2 = (Integer) stats2.get("cached_files"); + + // Assert - cache should be populated and remain consistent + assertEquals(1, cachedFiles); + assertEquals(cachedFiles, cachedFiles2); + + // Verify S3 download was called only once per file + verify(cloudStorage, times(1)).download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat"); + } + + @Test + void testCalculateStatus_s3Exception() throws Exception { + // Setup - S3 list error + when(cloudStorage.list(S3_DELTA_PREFIX)).thenThrow(new RuntimeException("S3 error")); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert - should throw exception on S3 error + assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); + } + + @Test + void testCalculateStatus_deltaFileReadException() throws Exception { + // Setup - S3 download error + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenThrow(new CloudStorageException("Failed to download")); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert - should throw exception on S3 download error + assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); + } + + @Test + void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - SQS message without timestamp (should use current time) + List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp()); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_multipleDeltaFiles() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // File 1 - recent entries + List timestamps1 = new ArrayList<>(); + for (int i = 0; i < 50; i++) { + timestamps1.add(t - 12*3600 + i * 1000); + } + byte[] deltaFileBytes1 = createDeltaFileBytes(timestamps1); + + // File 2 - older entries + List timestamps2 = new ArrayList<>(); + for (int i = 0; i < 30; i++) { + timestamps2.add(t - 36*3600 + i * 1000); + } + byte[] deltaFileBytes2 = createDeltaFileBytes(timestamps2); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList( + "optout-v2/delta/optout-delta--01_2025-11-13T02.00.00Z_bbbbbbbb.dat", + "optout-v2/delta/optout-delta--01_2025-11-13T01.00.00Z_aaaaaaaa.dat" + )); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T02.00.00Z_bbbbbbbb.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes1)); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T01.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes2)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + + // Verify cache has both files + Map stats = calculator.getCacheStats(); + assertEquals(2, stats.get("cached_files")); + } + + @Test + void testCalculateStatus_windowBoundaryTimestamp() throws Exception { + // Setup - create delta file with timestamps at window boundary + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + long currentWindowStart = t - 24*3600; + List timestamps = new ArrayList<>(); + for (int i = 0; i < 250; i++) { + timestamps.add(t); + } + for (int i = 0; i < 250; i++) { + timestamps.add(currentWindowStart); + } + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert - DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testCalculateStatus_timestampsCached() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600, t - 7200); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + + // Assert + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + + // Cache should contain the timestamps + Map stats = calculator.getCacheStats(); + assertEquals(2, stats.get("total_cached_timestamps")); + } + + // ============================================================================ + // SECTION 10: Tests for queue attributes (invisible messages from other consumers) + // ============================================================================ + + @Test + void testCalculateStatus_delayedProcessingFromQueueAttributesOnly() throws Exception { + // Setup - delta files with low traffic (10 records) + // Threshold = 100 * 5 = 500 + // Queue attributes will have 600 invisible messages (other consumers processing) + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + timestamps.add(t - 3600 + i); // 10 entries from 1 hour ago + } + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - 1 message read by us, but 600 invisible messages from other consumers + List sqsMessages = Arrays.asList(createSqsMessage(t)); + + // QueueAttributes: 0 visible, 600 invisible (other consumers), 0 delayed + // Since we read 1 message, otherConsumers = 600 - 1 = 599 + // Total = 10 (delta) + 1 (our message) + 599 (other consumers) = 610 >= 500 threshold + SqsMessageOperations.QueueAttributes queueAttributes = + new SqsMessageOperations.QueueAttributes(0, 600, 0); + + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0); + + // Assert - DELAYED_PROCESSING due to high invisible message count from other consumers + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testCalculateStatus_delayedProcessingFromBothQueueAndMessages() throws Exception { + // Setup - delta files with moderate traffic (100 records) + // Threshold = 100 * 5 = 500 + // We'll have 200 messages + 250 invisible from other consumers = 550 > 500 + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + timestamps.add(t - 3600 + i); // 100 entries from 1 hour ago + } + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - 200 messages read by us + 450 invisible (200 are ours + 250 from others) + // Messages must be within 5-minute window to be counted, so use 1-second spacing + List sqsMessages = new ArrayList<>(); + for (int i = 0; i < 200; i++) { + sqsMessages.add(createSqsMessage(t - i)); // 1 second apart, all within 5-minute window + } + + // QueueAttributes: 0 visible, 450 invisible (200 ours + 250 others), 0 delayed + // otherConsumers = 450 - 200 = 250 + // Total = 100 (delta) + 200 (our messages) + 250 (other consumers) = 550 >= 500 threshold + SqsMessageOperations.QueueAttributes queueAttributes = + new SqsMessageOperations.QueueAttributes(0, 450, 0); + + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0); + + // Assert - DELAYED_PROCESSING due to combined count exceeding threshold + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + +} \ No newline at end of file diff --git a/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java new file mode 100644 index 0000000..127cd04 --- /dev/null +++ b/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java @@ -0,0 +1,428 @@ +package com.uid2.optout.traffic; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.uid2.optout.sqs.SqsParsedMessage; +import com.uid2.optout.traffic.OptOutTrafficFilter; + +import software.amazon.awssdk.services.sqs.model.Message; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.Assert.*; + +public class OptOutTrafficFilterTest { + + private static final String TEST_CONFIG_PATH = "./traffic-config.json"; + + @Before + public void setUp() { + try { + Files.deleteIfExists(Path.of(TEST_CONFIG_PATH)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @After + public void tearDown() { + try { + Files.deleteIfExists(Path.of(TEST_CONFIG_PATH)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void testParseFilterRules_emptyRules() throws Exception { + // Setup - empty denylist + String config = """ + { + "denylist_requests": [] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - no rules + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(0, filter.filterRules.size()); + } + + @Test + public void testParseFilterRules_singleRule() throws Exception { + // Setup - config with one rule + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - one rule + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(1, filter.filterRules.size()); + } + + @Test + public void testParseFilterRules_multipleRules() throws Exception { + // Setup - config with multiple rules + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + }, + { + "range": [1700010000, 1700013600], + "IPs": ["10.0.0.1", "10.0.0.2"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - two rules + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(2, filter.filterRules.size()); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_missingDenylistRequests() throws Exception { + // Setup - config without denylist_requests field + String config = """ + { + "other_field": "value" + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_invalidRange_startAfterEnd() throws Exception { + // Setup - range where start > end + String config = """ + { + "denylist_requests": [ + { + "range": [1700003600, 1700000000], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_invalidRange_startEqualsEnd() throws Exception { + // Setup - range where start == end + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700000000], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_rangeExceeds24Hours() throws Exception { + // Setup - range longer than 24 hours (86400 seconds) + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700086401], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_emptyIPs() throws Exception { + // Setup - rule with empty IP list + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": [] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_missingIPs() throws Exception { + // Setup - rule without IPs field + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test + public void testIsDenylisted_matchingIPAndTimestamp() throws Exception { + // Setup - filter with denylist rule + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1", "10.0.0.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + SqsParsedMessage message = createTestMessage(1700001800, "192.168.1.1"); + + // Act & Assert - denylisted + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertTrue(filter.isDenylisted(message)); + } + + @Test + public void testIsDenylisted_matchingIPOutsideTimeRange() throws Exception { + // Setup - filter with denylist rule + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - message before range not denylisted + SqsParsedMessage messageBefore = createTestMessage(1699999999, "192.168.1.1"); + assertFalse(filter.isDenylisted(messageBefore)); + // Act & Assert - message after range not denylisted + SqsParsedMessage messageAfter = createTestMessage(1700003601, "192.168.1.1"); + assertFalse(filter.isDenylisted(messageAfter)); + } + + @Test + public void testIsDenylisted_nonMatchingIP() throws Exception { + // Setup - filter with denylist rule + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - non-matching IP not denylisted + SqsParsedMessage message = createTestMessage(1700001800, "10.0.0.1"); + assertFalse(filter.isDenylisted(message)); + } + + @Test + public void testIsDenylisted_atRangeBoundaries() throws Exception { + // Setup - filter with denylist rule + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - message at start boundary (inclusive) denylisted + SqsParsedMessage messageAtStart = createTestMessage(1700000000, "192.168.1.1"); + assertTrue(filter.isDenylisted(messageAtStart)); + + // Act & Assert - message at end boundary (inclusive) denylisted + SqsParsedMessage messageAtEnd = createTestMessage(1700003600, "192.168.1.1"); + assertTrue(filter.isDenylisted(messageAtEnd)); + } + + @Test + public void testIsDenylisted_multipleRules() throws Exception { + // Setup - multiple denylist rules + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + }, + { + "range": [1700010000, 1700013600], + "IPs": ["10.0.0.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - message matches first rule + SqsParsedMessage msg1 = createTestMessage(1700001800, "192.168.1.1"); + assertTrue(filter.isDenylisted(msg1)); + + // Act & Assert - message matches second rule + SqsParsedMessage msg2 = createTestMessage(1700011800, "10.0.0.1"); + assertTrue(filter.isDenylisted(msg2)); + + // Act & Assert - message matches neither rule + SqsParsedMessage msg3 = createTestMessage(1700005000, "172.16.0.1"); + assertFalse(filter.isDenylisted(msg3)); + } + + @Test + public void testIsDenylisted_nullClientIp() throws Exception { + // Setup - filter with denylist rule + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - message with null IP not denylisted + SqsParsedMessage message = createTestMessage(1700001800, null); + assertFalse(filter.isDenylisted(message)); + } + + @Test + public void testReloadTrafficFilterConfig_success() throws Exception { + // Setup - config with one rule + String initialConfig = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), initialConfig); + + // Act & Assert - one rule + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(1, filter.filterRules.size()); + + // Setup - update config + String updatedConfig = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + }, + { + "range": [1700010000, 1700013600], + "IPs": ["10.0.0.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), updatedConfig); + + // Act & Assert - two rules + filter.reloadTrafficFilterConfig(); + assertEquals(2, filter.filterRules.size()); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testReloadTrafficFilterConfig_fileNotFound() throws Exception { + // Setup, Act & Assert - try to create filter with non-existent config + new OptOutTrafficFilter("./non-existent-file.json"); + } + + @Test + public void testParseFilterRules_maxValidRange() throws Exception { + // Setup - range exactly 24 hours (86400 seconds) - should be valid + String config = """ + { + "denylist_requests": [ + { + "range": [1700000000, 1700086400], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - one rule + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(1, filter.filterRules.size()); + } + + /** + * Helper method to create test SqsParsedMessage + */ + private SqsParsedMessage createTestMessage(long timestamp, String clientIp) { + Message mockMessage = Message.builder().build(); + byte[] hash = new byte[32]; + byte[] id = new byte[32]; + return new SqsParsedMessage(mockMessage, hash, id, timestamp, null, null, clientIp, null); + } +} From 6922b0f200942f6db4644e8be9e37ed9ef561104 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 16:40:18 -0700 Subject: [PATCH 09/25] create empty --- .../java/com/uid2/optout/vertx/OptOutTrafficCalculator.java | 6 ++++++ .../java/com/uid2/optout/vertx/OptOutTrafficFilter.java | 6 ++++++ .../com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java | 6 ++++++ .../java/com/uid2/optout/vertx/OptOutTrafficFilter.java | 6 ++++++ 4 files changed, 24 insertions(+) create mode 100644 src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java create mode 100644 src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java create mode 100644 src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java create mode 100644 src/test/java/com/uid2/optout/vertx/OptOutTrafficFilter.java diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java new file mode 100644 index 0000000..822236c --- /dev/null +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -0,0 +1,6 @@ +package com.uid2.optout.vertx; + +// todo remove +public class OptOutTrafficCalculator { + +} diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java new file mode 100644 index 0000000..3e73f0e --- /dev/null +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -0,0 +1,6 @@ +package com.uid2.optout.vertx; + +// todo remove +public class OptOutTrafficFilter { + +} diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java new file mode 100644 index 0000000..15a1296 --- /dev/null +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -0,0 +1,6 @@ +package com.uid2.optout.vertx; + +//todo remove +public class OptOutTrafficCalculatorTest { + +} diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilter.java new file mode 100644 index 0000000..3e73f0e --- /dev/null +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -0,0 +1,6 @@ +package com.uid2.optout.vertx; + +// todo remove +public class OptOutTrafficFilter { + +} From 52e7818d9c484c53bf8b2b6e13e1f77c24e264d7 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 16:41:09 -0700 Subject: [PATCH 10/25] create empty --- .../{OptOutTrafficFilter.java => OptOutTrafficFilterTest.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/test/java/com/uid2/optout/vertx/{OptOutTrafficFilter.java => OptOutTrafficFilterTest.java} (58%) diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java similarity index 58% rename from src/test/java/com/uid2/optout/vertx/OptOutTrafficFilter.java rename to src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java index 3e73f0e..9d833fd 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilter.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java @@ -1,6 +1,6 @@ package com.uid2.optout.vertx; // todo remove -public class OptOutTrafficFilter { +public class OptOutTrafficFilterTest { } From 0eb06a27bdef9cf151df355ec63f4d4b6975027d Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 17:18:26 -0700 Subject: [PATCH 11/25] file rename --- ...Calculator.java => TrafficCalculator.java} | 6 +- ...tTrafficFilter.java => TrafficFilter.java} | 6 +- ...orTest.java => TrafficCalculatorTest.java} | 222 +++++++++--------- ...FilterTest.java => TrafficFilterTest.java} | 54 ++--- 4 files changed, 144 insertions(+), 144 deletions(-) rename src/main/java/com/uid2/optout/traffic/{OptOutTrafficCalculator.java => TrafficCalculator.java} (99%) rename src/main/java/com/uid2/optout/traffic/{OptOutTrafficFilter.java => TrafficFilter.java} (97%) rename src/test/java/com/uid2/optout/traffic/{OptOutTrafficCalculatorTest.java => TrafficCalculatorTest.java} (86%) rename src/test/java/com/uid2/optout/traffic/{OptOutTrafficFilterTest.java => TrafficFilterTest.java} (87%) diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java similarity index 99% rename from src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java rename to src/main/java/com/uid2/optout/traffic/TrafficCalculator.java index ab6a2c0..fec8d32 100644 --- a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java @@ -32,8 +32,8 @@ * * Returns DELAYED_PROCESSING if sumCurrent >= thresholdMultiplier * baselineTraffic, indicating abnormal traffic spike. */ -public class OptOutTrafficCalculator { - private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficCalculator.class); +public class TrafficCalculator { + private static final Logger LOGGER = LoggerFactory.getLogger(TrafficCalculator.class); private static final int HOURS_24 = 24 * 3600; // 24 hours in seconds @@ -83,7 +83,7 @@ public MalformedTrafficCalcConfigException(String message) { * @param s3DeltaPrefix S3 prefix for delta files * @param trafficCalcConfigS3Path S3 path for traffic calc config */ - public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigPath) throws MalformedTrafficCalcConfigException { + public TrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigPath) throws MalformedTrafficCalcConfigException { this.cloudStorage = cloudStorage; this.s3DeltaPrefix = s3DeltaPrefix; this.trafficCalcConfigPath = trafficCalcConfigPath; diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/traffic/TrafficFilter.java similarity index 97% rename from src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java rename to src/main/java/com/uid2/optout/traffic/TrafficFilter.java index 445f032..4733827 100644 --- a/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java +++ b/src/main/java/com/uid2/optout/traffic/TrafficFilter.java @@ -15,8 +15,8 @@ import io.vertx.core.json.JsonObject; import io.vertx.core.json.JsonArray; -public class OptOutTrafficFilter { - private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficFilter.class); +public class TrafficFilter { + private static final Logger LOGGER = LoggerFactory.getLogger(TrafficFilter.class); private final String trafficFilterConfigPath; List filterRules; @@ -56,7 +56,7 @@ public MalformedTrafficFilterConfigException(String message) { * @param trafficFilterConfigPath S3 path for traffic filter config * @throws MalformedTrafficFilterConfigException if the traffic filter config is invalid */ - public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTrafficFilterConfigException { + public TrafficFilter(String trafficFilterConfigPath) throws MalformedTrafficFilterConfigException { this.trafficFilterConfigPath = trafficFilterConfigPath; // Initial filter rules load this.filterRules = Collections.emptyList(); // start empty diff --git a/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/traffic/TrafficCalculatorTest.java similarity index 86% rename from src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java rename to src/test/java/com/uid2/optout/traffic/TrafficCalculatorTest.java index fd435a9..5a0747c 100644 --- a/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/traffic/TrafficCalculatorTest.java @@ -24,8 +24,8 @@ import software.amazon.awssdk.services.sqs.model.Message; import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; -import com.uid2.optout.traffic.OptOutTrafficCalculator; -import com.uid2.optout.traffic.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; +import com.uid2.optout.traffic.TrafficCalculator; +import com.uid2.optout.traffic.TrafficCalculator.MalformedTrafficCalcConfigException; import java.io.ByteArrayInputStream; import java.util.*; @@ -35,7 +35,7 @@ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) -public class OptOutTrafficCalculatorTest { +public class TrafficCalculatorTest { @Mock private ICloudStorage cloudStorage; @@ -122,29 +122,29 @@ private void createConfigWithThreshold(int threshold) { @Test void testConstructor_defaultThreshold() throws Exception { // Setup - default threshold of 5 - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); // 10 < 5*3 + TrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); + assertEquals(TrafficCalculator.TrafficStatus.DEFAULT, status); // 10 < 5*3 status = calculator.determineStatus(15, 3); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 15 >= 5*3 + assertEquals(TrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 15 >= 5*3 } @Test void testConstructor_customThreshold() throws Exception { // Setup - custom threshold of 10 createConfigWithThreshold(10); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(49, 5); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); // 49 < 10*5 + TrafficCalculator.TrafficStatus status = calculator.determineStatus(49, 5); + assertEquals(TrafficCalculator.TrafficStatus.DEFAULT, status); // 49 < 10*5 status = calculator.determineStatus(50, 5); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 50 >= 10*5 + assertEquals(TrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 50 >= 10*5 } @Test @@ -152,13 +152,13 @@ void testConstructor_trafficCalcConfigLoadFailure() throws Exception { // Setup - traffic calc config load failure createTrafficConfigFile("Invalid JSON"); assertThrows(MalformedTrafficCalcConfigException.class, () -> { - new OptOutTrafficCalculator( + new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); }); // Create valid config to test reload failure createConfigFromPartialJson("{}"); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); createTrafficConfigFile("Invalid JSON"); @@ -174,7 +174,7 @@ void testConstructor_trafficCalcConfigLoadFailure() throws Exception { @Test void testParseTrafficCalcConfigRanges_emptyConfig() throws Exception { // Setup - no config - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject emptyConfig = new JsonObject(); @@ -188,7 +188,7 @@ void testParseTrafficCalcConfigRanges_emptyConfig() throws Exception { @Test void testParseTrafficCalcConfigRanges_singleRange() throws Exception { // Setup - single range - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); @@ -208,7 +208,7 @@ void testParseTrafficCalcConfigRanges_singleRange() throws Exception { @Test void testParseTrafficCalcConfigRanges_multipleRanges() throws Exception { // Setup - multiple ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); @@ -231,7 +231,7 @@ void testParseTrafficCalcConfigRanges_multipleRanges() throws Exception { @Test void testParseTrafficCalcConfigRanges_misorderedRange() throws Exception { // Setup - range with end < start is malformed - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); @@ -248,7 +248,7 @@ void testParseTrafficCalcConfigRanges_misorderedRange() throws Exception { @Test void testParseTrafficCalcConfigRanges_rangeTooLong() throws Exception { // Setup - range longer than 24 hours is malformed - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); @@ -265,7 +265,7 @@ void testParseTrafficCalcConfigRanges_rangeTooLong() throws Exception { @Test void testParseTrafficCalcConfigRanges_sortsByStartTime() throws Exception { // Setup - ranges added out of order - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); @@ -288,7 +288,7 @@ void testParseTrafficCalcConfigRanges_sortsByStartTime() throws Exception { @Test void testParseTrafficCalcConfigRanges_invalidRangeTooFewElements() throws Exception { // Setup - invalid range with only 1 element; - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); @@ -308,7 +308,7 @@ void testParseTrafficCalcConfigRanges_invalidRangeTooFewElements() throws Except @Test void testParseTrafficCalcConfigRanges_nullArray() throws Exception { // Setup - null array - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); @@ -324,7 +324,7 @@ void testParseTrafficCalcConfigRanges_nullArray() throws Exception { @Test void testParseTrafficCalcConfigRanges_overlappingRanges() throws Exception { // Setup - overlapping ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); @@ -342,7 +342,7 @@ void testParseTrafficCalcConfigRanges_overlappingRanges() throws Exception { @Test void testParseTrafficCalcConfigRanges_adjacentRangesWithSameBoundary() throws Exception { // Setup - ranges where end of first equals start of second (touching but not overlapping semantically, but we treat as overlap) - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); @@ -360,7 +360,7 @@ void testParseTrafficCalcConfigRanges_adjacentRangesWithSameBoundary() throws Ex @Test void testParseTrafficCalcConfigRanges_nonOverlappingRanges() throws Exception { // Setup - ranges that don't overlap (with gap between them) - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); @@ -392,7 +392,7 @@ void testIsInTrafficCalcConfig_withinSingleRange() throws Exception { """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - true when within range @@ -411,7 +411,7 @@ void testIsInTrafficCalcConfig_exactlyAtStart() throws Exception { """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - true when exactly at start of range @@ -430,7 +430,7 @@ void testIsInTrafficCalcConfig_exactlyAtEnd() throws Exception { """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - true when exactly at end of range @@ -449,7 +449,7 @@ void testIsInTrafficCalcConfig_beforeRange() throws Exception { """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when before range @@ -468,7 +468,7 @@ void testIsInTrafficCalcConfig_afterRange() throws Exception { """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when after range @@ -488,7 +488,7 @@ void testIsInTrafficCalcConfig_betweenRanges() throws Exception { """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when between ranges @@ -498,7 +498,7 @@ void testIsInTrafficCalcConfig_betweenRanges() throws Exception { @Test void testIsInTrafficCalcConfig_emptyRanges() throws Exception { // Setup uses default config from setUp() which has empty traffic calc config ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when empty ranges @@ -515,7 +515,7 @@ void testIsInTrafficCalcConfig_nullRanges() throws Exception { """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when null/empty ranges @@ -535,7 +535,7 @@ void testIsInTrafficCalcConfig_invalidRangeSize() throws Exception { """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert @@ -557,7 +557,7 @@ void testIsInTrafficCalcConfig_multipleRanges() throws Exception { """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert @@ -574,7 +574,7 @@ void testIsInTrafficCalcConfig_multipleRanges() throws Exception { @Test void testGetTrafficCalcConfigDuration_noRanges() throws Exception { // Setup - no ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert @@ -593,7 +593,7 @@ void testGetTrafficCalcConfigDuration_rangeFullyWithinWindow() throws Exception """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [6000, 7000] @@ -615,7 +615,7 @@ void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsStart() throws Excep """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [3000, 7000] @@ -637,7 +637,7 @@ void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsEnd() throws Excepti """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [8000, 12000] @@ -659,7 +659,7 @@ void testGetTrafficCalcConfigDuration_rangeCompletelyOutsideWindow() throws Exce """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [1000, 2000] @@ -682,7 +682,7 @@ void testGetTrafficCalcConfigDuration_multipleRanges() throws Exception { """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], ranges [6000, 7000] and [8000, 9000] @@ -704,7 +704,7 @@ void testGetTrafficCalcConfigDuration_rangeSpansEntireWindow() throws Exception """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [3000, 12000] @@ -721,7 +721,7 @@ void testGetTrafficCalcConfigDuration_rangeSpansEntireWindow() throws Exception @Test void testCalculateWindowStartWithAllowlist_noAllowlist() throws Exception { // Setup - no allowlist ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window should be [3, 8] with no extension @@ -743,7 +743,7 @@ void testCalculateWindowStartWithAllowlist_allowlistInOriginalWindowOnly() throw """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - newestDeltaTs=8, evaluationWindow=5 @@ -769,7 +769,7 @@ void testCalculateWindowStartWithAllowlist_allowlistInExtendedPortion() throws E """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act @@ -796,7 +796,7 @@ void testCalculateWindowStartWithAllowlist_allowlistBeforeWindow() throws Except """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act @@ -816,46 +816,46 @@ void testCalculateWindowStartWithAllowlist_allowlistBeforeWindow() throws Except @Test void testDetermineStatus_belowThreshold() throws Exception { // Setup - below threshold - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - 10 < 5 * 3 - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); + TrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); // Assert - DEFAULT when below threshold - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + assertEquals(TrafficCalculator.TrafficStatus.DEFAULT, status); } @Test void testDetermineStatus_atThreshold() throws Exception { // Setup - at threshold - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - 15 == 5 * 3 - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(15, 3); + TrafficCalculator.TrafficStatus status = calculator.determineStatus(15, 3); // Assert - DELAYED_PROCESSING when at threshold - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + assertEquals(TrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } @Test void testDetermineStatus_aboveThreshold() throws Exception { // Setup - above threshold - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - 20 > 5 * 3 - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(20, 3); + TrafficCalculator.TrafficStatus status = calculator.determineStatus(20, 3); // Assert - DELAYED_PROCESSING when above threshold - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + assertEquals(TrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } @Test void testDetermineStatus_sumPastZero() throws Exception { // Setup - sumPast is 0 - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act & Assert - should throw exception for invalid config @@ -865,7 +865,7 @@ void testDetermineStatus_sumPastZero() throws Exception { @Test void testDetermineStatus_bothZero() throws Exception { // Setup - both sumCurrent and sumPast are 0; - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act & Assert - should throw exception for invalid config @@ -875,14 +875,14 @@ void testDetermineStatus_bothZero() throws Exception { @Test void testDetermineStatus_sumCurrentZero() throws Exception { // Setup - sumCurrent is 0 - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - 0 < 5 * 10 - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 10); + TrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 10); // Assert - DEFAULT when sumCurrent is 0 - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + assertEquals(TrafficCalculator.TrafficStatus.DEFAULT, status); } @ParameterizedTest @@ -897,27 +897,27 @@ void testDetermineStatus_sumCurrentZero() throws Exception { void testDetermineStatus_variousThresholds(int threshold, int sumCurrent, int sumPast, String expectedStatus) throws Exception { // Setup - various thresholds createConfigWithThreshold(threshold); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(sumCurrent, sumPast); + TrafficCalculator.TrafficStatus status = calculator.determineStatus(sumCurrent, sumPast); // Assert - assertEquals(OptOutTrafficCalculator.TrafficStatus.valueOf(expectedStatus), status); + assertEquals(TrafficCalculator.TrafficStatus.valueOf(expectedStatus), status); } @Test void testDetermineStatus_largeNumbers() throws Exception { // Setup - test with large numbers - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(1_000_000, 200_000); + TrafficCalculator.TrafficStatus status = calculator.determineStatus(1_000_000, 200_000); // Assert - 1M >= 5 * 200K = 1M - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + assertEquals(TrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } // ============================================================================ @@ -937,7 +937,7 @@ void testReloadTrafficCalcConfig_success() throws Exception { """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Change the traffic calc config to a new range @@ -969,7 +969,7 @@ void testReloadTrafficCalcConfig_failure() throws Exception { """; createConfigFromPartialJson(trafficCalcConfigJson); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Now make it fail @@ -985,7 +985,7 @@ void testReloadTrafficCalcConfig_failure() throws Exception { @Test public void testReloadTrafficCalcConfig_failure_missingKeys() throws Exception { // Setup - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act & Assert missing threshold multiplier @@ -1016,7 +1016,7 @@ public void testReloadTrafficCalcConfig_failure_missingKeys() throws Exception { @Test public void testReloadTrafficCalcConfig_failure_misorderedRanges() throws Exception { // Setup - misordered ranges - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); createConfigFromPartialJson("{\"traffic_calc_allowlist_ranges\": [ [2000, 1000] ]}"); @@ -1029,7 +1029,7 @@ public void testReloadTrafficCalcConfig_failure_misorderedRanges() throws Except @Test public void testReloadTrafficCalcConfig_failure_rangeTooLong() throws Exception { // Setup - range greater than 24 hours - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); createConfigFromPartialJson("{\"traffic_calc_allowlist_ranges\": [ [1000, 200000] ]}"); @@ -1046,7 +1046,7 @@ public void testReloadTrafficCalcConfig_failure_rangeTooLong() throws Exception @Test void testGetCacheStats_emptyCache() throws Exception { // Setup - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act @@ -1113,7 +1113,7 @@ void testCalculateStatus_noDeltaFiles() throws Exception { // Setup - no delta files when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Collections.emptyList()); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act & Assert - should throw exception when no delta files @@ -1140,15 +1140,15 @@ void testCalculateStatus_normalTraffic() throws Exception { when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - 100+1 < 5 * 50 = 250, so should be DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + assertEquals(TrafficCalculator.TrafficStatus.DEFAULT, status); } @Test @@ -1171,15 +1171,15 @@ void testCalculateStatus_delayedProcessing() throws Exception { when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - 100+1 >= 5 * 10 = 50, DELAYED_PROCESSING - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + assertEquals(TrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } @Test @@ -1195,14 +1195,14 @@ void testCalculateStatus_noSqsMessages() throws Exception { when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - null SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null, null, 0, 0); + TrafficCalculator.TrafficStatus status = calculator.calculateStatus(null, null, 0, 0); // Assert - should still calculate based on delta files, DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + assertEquals(TrafficCalculator.TrafficStatus.DEFAULT, status); } @Test @@ -1218,14 +1218,14 @@ void testCalculateStatus_emptySqsMessages() throws Exception { when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - empty SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0); + TrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0); // Assert - should still calculate based on delta files, DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + assertEquals(TrafficCalculator.TrafficStatus.DEFAULT, status); } @Test @@ -1246,7 +1246,7 @@ void testCalculateStatus_multipleSqsMessages() throws Exception { when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Add 30 SQS entries in [t, t+5min] @@ -1254,10 +1254,10 @@ void testCalculateStatus_multipleSqsMessages() throws Exception { for (int i = 0; i < 30; i++) { sqsMessages.add(createSqsMessage(t - i * 10)); } - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - DELAYED_PROCESSING - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + assertEquals(TrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } @Test @@ -1292,17 +1292,17 @@ void testCalculateStatus_withTrafficCalcConfig() throws Exception { when(cloudStorage.download("optout-v2/delta/optout-delta-001_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - should filter out entries in traffic calc config ranges // Only 300 from window count (not in traffic calc config ranges) + 1 SQS = 301 // 301 < 5*100, so DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + assertEquals(TrafficCalculator.TrafficStatus.DEFAULT, status); } @Test @@ -1318,7 +1318,7 @@ void testCalculateStatus_cacheUtilization() throws Exception { when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - first call should populate cache @@ -1347,7 +1347,7 @@ void testCalculateStatus_s3Exception() throws Exception { // Setup - S3 list error when(cloudStorage.list(S3_DELTA_PREFIX)).thenThrow(new RuntimeException("S3 error")); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act & Assert - should throw exception on S3 error @@ -1361,7 +1361,7 @@ void testCalculateStatus_deltaFileReadException() throws Exception { when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenThrow(new CloudStorageException("Failed to download")); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act & Assert - should throw exception on S3 download error @@ -1381,15 +1381,15 @@ void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception { when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - SQS message without timestamp (should use current time) List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp()); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + assertEquals(TrafficCalculator.TrafficStatus.DEFAULT, status); } @Test @@ -1421,15 +1421,15 @@ void testCalculateStatus_multipleDeltaFiles() throws Exception { when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T01.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes2)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + assertEquals(TrafficCalculator.TrafficStatus.DEFAULT, status); // Verify cache has both files Map stats = calculator.getCacheStats(); @@ -1455,15 +1455,15 @@ void testCalculateStatus_windowBoundaryTimestamp() throws Exception { when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + assertEquals(TrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } @Test @@ -1479,15 +1479,15 @@ void testCalculateStatus_timestampsCached() throws Exception { when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); + TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + assertEquals(TrafficCalculator.TrafficStatus.DEFAULT, status); // Cache should contain the timestamps Map stats = calculator.getCacheStats(); @@ -1516,7 +1516,7 @@ void testCalculateStatus_delayedProcessingFromQueueAttributesOnly() throws Excep when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - 1 message read by us, but 600 invisible messages from other consumers @@ -1528,10 +1528,10 @@ void testCalculateStatus_delayedProcessingFromQueueAttributesOnly() throws Excep SqsMessageOperations.QueueAttributes queueAttributes = new SqsMessageOperations.QueueAttributes(0, 600, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0); + TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0); // Assert - DELAYED_PROCESSING due to high invisible message count from other consumers - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + assertEquals(TrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } @Test @@ -1552,7 +1552,7 @@ void testCalculateStatus_delayedProcessingFromBothQueueAndMessages() throws Exce when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + TrafficCalculator calculator = new TrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - 200 messages read by us + 450 invisible (200 are ours + 250 from others) @@ -1568,10 +1568,10 @@ void testCalculateStatus_delayedProcessingFromBothQueueAndMessages() throws Exce SqsMessageOperations.QueueAttributes queueAttributes = new SqsMessageOperations.QueueAttributes(0, 450, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0); + TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0); // Assert - DELAYED_PROCESSING due to combined count exceeding threshold - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + assertEquals(TrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } } \ No newline at end of file diff --git a/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/traffic/TrafficFilterTest.java similarity index 87% rename from src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java rename to src/test/java/com/uid2/optout/traffic/TrafficFilterTest.java index 127cd04..3e435e1 100644 --- a/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java +++ b/src/test/java/com/uid2/optout/traffic/TrafficFilterTest.java @@ -5,7 +5,7 @@ import org.junit.Test; import com.uid2.optout.sqs.SqsParsedMessage; -import com.uid2.optout.traffic.OptOutTrafficFilter; +import com.uid2.optout.traffic.TrafficFilter; import software.amazon.awssdk.services.sqs.model.Message; @@ -14,7 +14,7 @@ import static org.junit.Assert.*; -public class OptOutTrafficFilterTest { +public class TrafficFilterTest { private static final String TEST_CONFIG_PATH = "./traffic-config.json"; @@ -47,7 +47,7 @@ public void testParseFilterRules_emptyRules() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - no rules - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + TrafficFilter filter = new TrafficFilter(TEST_CONFIG_PATH); assertEquals(0, filter.filterRules.size()); } @@ -67,7 +67,7 @@ public void testParseFilterRules_singleRule() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - one rule - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + TrafficFilter filter = new TrafficFilter(TEST_CONFIG_PATH); assertEquals(1, filter.filterRules.size()); } @@ -91,11 +91,11 @@ public void testParseFilterRules_multipleRules() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - two rules - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + TrafficFilter filter = new TrafficFilter(TEST_CONFIG_PATH); assertEquals(2, filter.filterRules.size()); } - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) public void testParseFilterRules_missingDenylistRequests() throws Exception { // Setup - config without denylist_requests field String config = """ @@ -106,10 +106,10 @@ public void testParseFilterRules_missingDenylistRequests() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); + new TrafficFilter(TEST_CONFIG_PATH); } - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) public void testParseFilterRules_invalidRange_startAfterEnd() throws Exception { // Setup - range where start > end String config = """ @@ -125,10 +125,10 @@ public void testParseFilterRules_invalidRange_startAfterEnd() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); + new TrafficFilter(TEST_CONFIG_PATH); } - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) public void testParseFilterRules_invalidRange_startEqualsEnd() throws Exception { // Setup - range where start == end String config = """ @@ -144,10 +144,10 @@ public void testParseFilterRules_invalidRange_startEqualsEnd() throws Exception Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); + new TrafficFilter(TEST_CONFIG_PATH); } - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) public void testParseFilterRules_rangeExceeds24Hours() throws Exception { // Setup - range longer than 24 hours (86400 seconds) String config = """ @@ -163,10 +163,10 @@ public void testParseFilterRules_rangeExceeds24Hours() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); + new TrafficFilter(TEST_CONFIG_PATH); } - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) public void testParseFilterRules_emptyIPs() throws Exception { // Setup - rule with empty IP list String config = """ @@ -182,10 +182,10 @@ public void testParseFilterRules_emptyIPs() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); + new TrafficFilter(TEST_CONFIG_PATH); } - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) public void testParseFilterRules_missingIPs() throws Exception { // Setup - rule without IPs field String config = """ @@ -200,7 +200,7 @@ public void testParseFilterRules_missingIPs() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - throws exception - new OptOutTrafficFilter(TEST_CONFIG_PATH); + new TrafficFilter(TEST_CONFIG_PATH); } @Test @@ -220,7 +220,7 @@ public void testIsDenylisted_matchingIPAndTimestamp() throws Exception { SqsParsedMessage message = createTestMessage(1700001800, "192.168.1.1"); // Act & Assert - denylisted - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + TrafficFilter filter = new TrafficFilter(TEST_CONFIG_PATH); assertTrue(filter.isDenylisted(message)); } @@ -238,7 +238,7 @@ public void testIsDenylisted_matchingIPOutsideTimeRange() throws Exception { } """; Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + TrafficFilter filter = new TrafficFilter(TEST_CONFIG_PATH); // Act & Assert - message before range not denylisted SqsParsedMessage messageBefore = createTestMessage(1699999999, "192.168.1.1"); @@ -262,7 +262,7 @@ public void testIsDenylisted_nonMatchingIP() throws Exception { } """; Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + TrafficFilter filter = new TrafficFilter(TEST_CONFIG_PATH); // Act & Assert - non-matching IP not denylisted SqsParsedMessage message = createTestMessage(1700001800, "10.0.0.1"); @@ -283,7 +283,7 @@ public void testIsDenylisted_atRangeBoundaries() throws Exception { } """; Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + TrafficFilter filter = new TrafficFilter(TEST_CONFIG_PATH); // Act & Assert - message at start boundary (inclusive) denylisted SqsParsedMessage messageAtStart = createTestMessage(1700000000, "192.168.1.1"); @@ -313,7 +313,7 @@ public void testIsDenylisted_multipleRules() throws Exception { """; Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + TrafficFilter filter = new TrafficFilter(TEST_CONFIG_PATH); // Act & Assert - message matches first rule SqsParsedMessage msg1 = createTestMessage(1700001800, "192.168.1.1"); @@ -342,7 +342,7 @@ public void testIsDenylisted_nullClientIp() throws Exception { } """; Files.writeString(Path.of(TEST_CONFIG_PATH), config); - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + TrafficFilter filter = new TrafficFilter(TEST_CONFIG_PATH); // Act & Assert - message with null IP not denylisted SqsParsedMessage message = createTestMessage(1700001800, null); @@ -365,7 +365,7 @@ public void testReloadTrafficFilterConfig_success() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), initialConfig); // Act & Assert - one rule - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + TrafficFilter filter = new TrafficFilter(TEST_CONFIG_PATH); assertEquals(1, filter.filterRules.size()); // Setup - update config @@ -390,10 +390,10 @@ public void testReloadTrafficFilterConfig_success() throws Exception { assertEquals(2, filter.filterRules.size()); } - @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) public void testReloadTrafficFilterConfig_fileNotFound() throws Exception { // Setup, Act & Assert - try to create filter with non-existent config - new OptOutTrafficFilter("./non-existent-file.json"); + new TrafficFilter("./non-existent-file.json"); } @Test @@ -412,7 +412,7 @@ public void testParseFilterRules_maxValidRange() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - one rule - OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + TrafficFilter filter = new TrafficFilter(TEST_CONFIG_PATH); assertEquals(1, filter.filterRules.size()); } From 952dcf3dd1c53cd59880d1f7f336fdc57724776b Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 17:19:33 -0700 Subject: [PATCH 12/25] file update --- .../java/com/uid2/optout/vertx/OptOutTrafficCalculator.java | 6 ------ .../java/com/uid2/optout/vertx/OptOutTrafficFilter.java | 6 ------ .../com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java | 6 ------ .../java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java | 6 ------ 4 files changed, 24 deletions(-) delete mode 100644 src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java delete mode 100644 src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java delete mode 100644 src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java delete mode 100644 src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java deleted file mode 100644 index 822236c..0000000 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.uid2.optout.vertx; - -// todo remove -public class OptOutTrafficCalculator { - -} diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java deleted file mode 100644 index 3e73f0e..0000000 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.uid2.optout.vertx; - -// todo remove -public class OptOutTrafficFilter { - -} diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java deleted file mode 100644 index 15a1296..0000000 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.uid2.optout.vertx; - -//todo remove -public class OptOutTrafficCalculatorTest { - -} diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java deleted file mode 100644 index 9d833fd..0000000 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.uid2.optout.vertx; - -// todo remove -public class OptOutTrafficFilterTest { - -} From caf3d0efe48969b7546557156c22d596d901800f Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 17:21:13 -0700 Subject: [PATCH 13/25] file update --- .../java/com/uid2/optout/vertx/OptOutTrafficCalculator.java | 6 ++++++ .../java/com/uid2/optout/vertx/OptOutTrafficFilter.java | 6 ++++++ .../com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java | 6 ++++++ .../java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java | 6 ++++++ 4 files changed, 24 insertions(+) create mode 100644 src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java create mode 100644 src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java create mode 100644 src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java create mode 100644 src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java new file mode 100644 index 0000000..822236c --- /dev/null +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -0,0 +1,6 @@ +package com.uid2.optout.vertx; + +// todo remove +public class OptOutTrafficCalculator { + +} diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java new file mode 100644 index 0000000..3e73f0e --- /dev/null +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -0,0 +1,6 @@ +package com.uid2.optout.vertx; + +// todo remove +public class OptOutTrafficFilter { + +} diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java new file mode 100644 index 0000000..15a1296 --- /dev/null +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -0,0 +1,6 @@ +package com.uid2.optout.vertx; + +//todo remove +public class OptOutTrafficCalculatorTest { + +} diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java new file mode 100644 index 0000000..9d833fd --- /dev/null +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java @@ -0,0 +1,6 @@ +package com.uid2.optout.vertx; + +// todo remove +public class OptOutTrafficFilterTest { + +} From 813cd06d74f58a4b17d8338d54f92c26a1a3b7d0 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 17:22:14 -0700 Subject: [PATCH 14/25] file updates --- .../java/com/uid2/optout/vertx/OptOutTrafficCalculator.java | 6 ------ .../java/com/uid2/optout/vertx/OptOutTrafficFilter.java | 6 ------ .../com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java | 6 ------ .../java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java | 6 ------ 4 files changed, 24 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 822236c..e69de29 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -1,6 +0,0 @@ -package com.uid2.optout.vertx; - -// todo remove -public class OptOutTrafficCalculator { - -} diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java index 3e73f0e..e69de29 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -1,6 +0,0 @@ -package com.uid2.optout.vertx; - -// todo remove -public class OptOutTrafficFilter { - -} diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index 15a1296..e69de29 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -1,6 +0,0 @@ -package com.uid2.optout.vertx; - -//todo remove -public class OptOutTrafficCalculatorTest { - -} diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java index 9d833fd..e69de29 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java @@ -1,6 +0,0 @@ -package com.uid2.optout.vertx; - -// todo remove -public class OptOutTrafficFilterTest { - -} From e02e4f281cc18a7dac2d03e592061978476f248d Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 17:35:13 -0700 Subject: [PATCH 15/25] update comments --- .../optout/traffic/TrafficCalculator.java | 86 +++++++++---------- 1 file changed, 42 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java index fec8d32..f1dba04 100644 --- a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java @@ -35,11 +35,9 @@ public class TrafficCalculator { private static final Logger LOGGER = LoggerFactory.getLogger(TrafficCalculator.class); - private static final int HOURS_24 = 24 * 3600; // 24 hours in seconds - private final Map deltaFileCache = new ConcurrentHashMap<>(); private final ICloudStorage cloudStorage; - private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") + private final String s3DeltaPrefix; private final String trafficCalcConfigPath; private int baselineTraffic; private int thresholdMultiplier; @@ -58,8 +56,8 @@ public enum TrafficStatus { * 1GB of memory can store ~130 million timestamps (1024^3)/8 */ private static class FileRecordCache { - final List timestamps; // All non-sentinel record timestamps - final long newestTimestamp; // evict delta from cache based on oldest record timestamp + final List timestamps; // all non-sentinel record timestamps + final long newestTimestamp; // evict delta from cache based on newest record timestamp FileRecordCache(List timestamps) { this.timestamps = timestamps; @@ -77,17 +75,17 @@ public MalformedTrafficCalcConfigException(String message) { } /** - * Constructor for OptOutTrafficCalculator + * Constructor for TrafficCalculator * * @param cloudStorage Cloud storage for reading delta files * @param s3DeltaPrefix S3 prefix for delta files - * @param trafficCalcConfigS3Path S3 path for traffic calc config + * @param trafficCalcConfigPath mount path for traffic calc config */ public TrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigPath) throws MalformedTrafficCalcConfigException { this.cloudStorage = cloudStorage; this.s3DeltaPrefix = s3DeltaPrefix; this.trafficCalcConfigPath = trafficCalcConfigPath; - reloadTrafficCalcConfig(); // Load ConfigMap + reloadTrafficCalcConfig(); LOGGER.info("initialized: s3DeltaPrefix={}, threshold={}x", s3DeltaPrefix, thresholdMultiplier); @@ -181,7 +179,7 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic ranges.sort(Comparator.comparing(range -> range.get(0))); - // Validate no overlapping ranges + // validate that there are no overlapping ranges for (int i = 0; i < ranges.size() - 1; i++) { long currentEnd = ranges.get(i).get(1); long nextStart = ranges.get(i + 1).get(0); @@ -223,7 +221,7 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes, int denylistedCount, int filteredAsTooRecentCount) { try { - // Get list of delta files from S3 (sorted newest to oldest) + // get list of delta files from s3 (sorted newest to oldest) List deltaS3Paths = listDeltaFiles(); if (deltaS3Paths.isEmpty()) { @@ -231,25 +229,25 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat throw new RuntimeException("no delta files found in s3 at prefix=" + s3DeltaPrefix); } - // Find newest delta file timestamp for delta traffic window + // find newest delta file timestamp for delta traffic window long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths); LOGGER.info("traffic calculation: newestDeltaTs={}", newestDeltaTs); - // Find oldest SQS queue message timestamp for queue window + // find oldest sqs queue message timestamp for queue window long oldestQueueTs = findOldestQueueTimestamp(sqsMessages); LOGGER.info("traffic calculation: oldestQueueTs={}", oldestQueueTs); - // Define start time of the delta evaluation window - // We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend + // define start time of the delta evaluation window + // we need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend // the window to account for any allowlist ranges in the extended portion long deltaWindowStart = calculateWindowStartWithAllowlist(newestDeltaTs, this.evaluationWindowSeconds); - // Evict old cache entries (older than delta window start) + // evict old cache entries (older than delta window start) evictOldCacheEntries(deltaWindowStart); - // Process delta files and count records in [deltaWindowStart, newestDeltaTs] - // Files are sorted newest to oldest, records within files are sorted newest to oldest - // Stop when the newest record in a file is older than the window + // process delta files and count records in [deltaWindowStart, newestDeltaTs] + // files are sorted newest to oldest, records within files are sorted newest to oldest + // stop when the newest record in a file is older than the window int sum = 0; int deltaRecordsCount = 0; int deltaAllowlistedCount = 0; @@ -268,25 +266,25 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat List timestamps = getTimestampsFromFile(s3Path); filesProcessed++; - // Check newest record in file - if older than window, stop processing remaining files + // check newest record in file - if older than window, stop processing remaining files long newestRecordTs = timestamps.get(0); if (newestRecordTs < deltaWindowStart) { break; } for (long ts : timestamps) { - // Stop condition: record is older than our window + // stop condition: record is older than our window if (ts < deltaWindowStart) { break; } - // skip records in allowlisted ranges + // skip records that are in allowlisted ranges if (isInAllowlist(ts)) { deltaAllowlistedCount++; continue; } - // increment sum if record is in delta window + // increment sum if record is within the delta window if (ts >= deltaWindowStart) { deltaRecordsCount++; sum++; @@ -297,16 +295,16 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat LOGGER.info("delta files: processed={}, deltaRecords={}, allowlisted={}, cache hits={}, misses={}, cacheSize={}", filesProcessed, deltaRecordsCount, deltaAllowlistedCount, cacheHits, cacheMisses, deltaFileCache.size()); - // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] with allowlist filtering + // count sqs messages in [oldestQueueTs, oldestQueueTs + 5m] with allowlist filtering int sqsCount = 0; if (sqsMessages != null && !sqsMessages.isEmpty()) { sqsCount = countSqsMessages(sqsMessages, oldestQueueTs); sum += sqsCount; } - // Add invisible messages being processed by OTHER consumers + // add invisible messages being processed by other consumers // (notVisible count includes our messages, so subtract what we've read to avoid double counting) - // ourMessages = delta messages + denylisted messages + filtered "too recent" messages + // ourMessages = delta messages + denylisted messages + filtered as "too recent" messages int otherConsumersMessages = 0; if (queueAttributes != null) { int totalInvisible = queueAttributes.getApproximateNumberOfMessagesNotVisible(); @@ -317,7 +315,7 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat otherConsumersMessages, totalInvisible, ourMessages); } - // Determine status + // determine status TrafficStatus status = determineStatus(sum, this.baselineTraffic); LOGGER.info("traffic calculation complete: sum={} (deltaRecords={} + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}", @@ -332,15 +330,15 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat } /** - * Find the newest timestamp from delta files. - * Reads the newest delta file and returns its maximum timestamp. + * find the newest timestamp from delta files. + * reads the newest delta file and returns its maximum timestamp. */ private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOException { if (deltaS3Paths == null || deltaS3Paths.isEmpty()) { return System.currentTimeMillis() / 1000; } - // Delta files are sorted (ISO 8601 format, lexicographically sortable) so first file is newest + // delta files are sorted (ISO 8601 format, lexicographically sortable) so first file is newest String newestDeltaPath = deltaS3Paths.get(0); List timestamps = getTimestampsFromFile(newestDeltaPath); @@ -359,10 +357,10 @@ private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOExcept */ private List listDeltaFiles() { try { - // List all objects with the delta prefix + // list all objects with the delta prefix List allFiles = cloudStorage.list(s3DeltaPrefix); - // Filter to only .dat delta files and sort newest to oldest + // filter to only .dat delta files and sort newest to oldest List deltaFiles = allFiles.stream() .filter(OptOutUtils::isDeltaFile) .sorted(OptOutUtils.DeltaFilenameComparatorDescending) @@ -389,19 +387,19 @@ private boolean isCached(String s3Path) { * Get timestamps from a delta file (S3 path), using cache if available */ private List getTimestampsFromFile(String s3Path) throws IOException { - // Extract filename from S3 path for cache key + // extract filename from s3 path for cache key String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); - // Check cache first + // check cache first FileRecordCache cached = deltaFileCache.get(filename); if (cached != null) { return cached.timestamps; } - // Cache miss - download from S3 + // cache miss - download from s3 List timestamps = readTimestampsFromS3(s3Path); - // Store in cache + // store in cache deltaFileCache.put(filename, new FileRecordCache(timestamps)); return timestamps; @@ -419,7 +417,7 @@ private List readTimestampsFromS3(String s3Path) throws IOException { for (int i = 0; i < collection.size(); i++) { OptOutEntry entry = collection.get(i); - // Skip sentinel entries + // skip sentinel entries if (entry.isSpecialHash()) { continue; } @@ -443,7 +441,7 @@ long getAllowlistDuration(long t, long windowStart) { long start = range.get(0); long end = range.get(1); - // Clip range to window boundaries + // clip range to window boundaries if (start < windowStart) { start = windowStart; } @@ -451,7 +449,7 @@ long getAllowlistDuration(long t, long windowStart) { end = t; } - // Only add duration if there's actual overlap (start < end) + // only add duration if there's actual overlap (start < end) if (start < end) { totalDuration += end - start; } @@ -466,7 +464,7 @@ long getAllowlistDuration(long t, long windowStart) { long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowSeconds) { long allowlistDuration = getAllowlistDuration(newestDeltaTs, newestDeltaTs - evaluationWindowSeconds); - // Each iteration discovers at least one new allowlist range, so max iterations = number of ranges + // each iteration discovers at least one new allowlist range, so max iterations = number of ranges int maxIterations = this.allowlistRanges.size() + 1; for (int i = 0; i < maxIterations && allowlistDuration > 0; i++) { @@ -474,7 +472,7 @@ long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowS long newAllowlistDuration = getAllowlistDuration(newestDeltaTs, newWindowStart); if (newAllowlistDuration == allowlistDuration) { - // No new allowlist time in extended portion, we've converged + // no new allowlist time in extended portion, we've converged break; } @@ -506,17 +504,17 @@ private long findOldestQueueTimestamp(List sqsMessages) throws IOExcept * Extract timestamp from SQS message (from SentTimestamp attribute) */ private Long extractTimestampFromMessage(Message msg) { - // Get SentTimestamp attribute (milliseconds) + // get SentTimestamp attribute (milliseconds) String sentTimestamp = msg.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); if (sentTimestamp != null) { try { - return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds + return Long.parseLong(sentTimestamp) / 1000; // convert ms to seconds } catch (NumberFormatException e) { LOGGER.error("sqs_error: invalid sentTimestamp, messageId={}, sentTimestamp={}", msg.messageId(), sentTimestamp); } } - // Fallback: use current time + // fallback: use current time return System.currentTimeMillis() / 1000; } @@ -602,7 +600,7 @@ TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { int threshold = thresholdMultiplier * baselineTraffic; double thresholdPercent = (double) sumCurrent / threshold * 100; - // Log warnings at increasing thresholds before circuit breaker triggers + // log warnings at increasing thresholds before circuit breaker triggers if (thresholdPercent >= 90.0) { LOGGER.warn("high_message_volume: 90% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); From 49660be9bb2ca723bc01a5ac50a47cd3326f4761 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Wed, 10 Dec 2025 16:28:29 -0700 Subject: [PATCH 16/25] rename sum to totalRecords --- .../com/uid2/optout/traffic/TrafficCalculator.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java index f1dba04..ba5fe1e 100644 --- a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java @@ -248,7 +248,7 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat // process delta files and count records in [deltaWindowStart, newestDeltaTs] // files are sorted newest to oldest, records within files are sorted newest to oldest // stop when the newest record in a file is older than the window - int sum = 0; + int totalRecords = 0; int deltaRecordsCount = 0; int deltaAllowlistedCount = 0; int filesProcessed = 0; @@ -287,7 +287,7 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat // increment sum if record is within the delta window if (ts >= deltaWindowStart) { deltaRecordsCount++; - sum++; + totalRecords++; } } } @@ -299,7 +299,7 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat int sqsCount = 0; if (sqsMessages != null && !sqsMessages.isEmpty()) { sqsCount = countSqsMessages(sqsMessages, oldestQueueTs); - sum += sqsCount; + totalRecords += sqsCount; } // add invisible messages being processed by other consumers @@ -310,16 +310,16 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat int totalInvisible = queueAttributes.getApproximateNumberOfMessagesNotVisible(); int ourMessages = (sqsMessages != null ? sqsMessages.size() : 0) + denylistedCount + filteredAsTooRecentCount; otherConsumersMessages = Math.max(0, totalInvisible - ourMessages); - sum += otherConsumersMessages; + totalRecords += otherConsumersMessages; LOGGER.info("traffic calculation: adding {} invisible messages from other consumers (totalInvisible={}, ourMessages={})", otherConsumersMessages, totalInvisible, ourMessages); } // determine status - TrafficStatus status = determineStatus(sum, this.baselineTraffic); + TrafficStatus status = determineStatus(totalRecords, this.baselineTraffic); LOGGER.info("traffic calculation complete: sum={} (deltaRecords={} + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}", - sum, deltaRecordsCount, sqsCount, otherConsumersMessages, this.baselineTraffic, this.thresholdMultiplier, status); + totalRecords, deltaRecordsCount, sqsCount, otherConsumersMessages, this.baselineTraffic, this.thresholdMultiplier, status); return status; From cecc3e29e524d296836ca871916dd90b39e5dd0f Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Wed, 10 Dec 2025 16:58:27 -0700 Subject: [PATCH 17/25] refactor traffic calculator to use Parsed message to avoid duplicate timestamp parsing effort --- .../com/uid2/optout/sqs/SqsMessageParser.java | 11 ++- .../optout/traffic/TrafficCalculator.java | 44 +++-------- .../optout/traffic/TrafficCalculatorTest.java | 75 ++++++------------- 3 files changed, 41 insertions(+), 89 deletions(-) diff --git a/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java b/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java index ac948b9..10841dd 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java +++ b/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java @@ -74,13 +74,18 @@ public static List parseAndSortMessages(List messages * @param message The SQS message * @return Timestamp in seconds */ - private static long extractTimestamp(Message message, String traceId) { + public static long extractTimestamp(Message message, String traceId) { String sentTimestampStr = message.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); if (sentTimestampStr == null) { - LOGGER.info("message missing SentTimestamp, using current time instead, messageId={}, traceId={}", message.messageId(), traceId); + LOGGER.error("sqs_error: message missing SentTimestamp, using current time instead, messageId={}, traceId={}", message.messageId(), traceId); + return OptOutUtils.nowEpochSeconds(); + } + try { + return Long.parseLong(sentTimestampStr) / 1000; + } catch (NumberFormatException e) { + LOGGER.error("sqs_error: invalid SentTimestamp, using current time instead, messageId={}, traceId={}, sentTimestamp={}", message.messageId(), traceId, sentTimestampStr); return OptOutUtils.nowEpochSeconds(); } - return Long.parseLong(sentTimestampStr) / 1000; // ms to seconds } } diff --git a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java index ba5fe1e..3f0fe6f 100644 --- a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java @@ -5,13 +5,11 @@ import com.uid2.shared.optout.OptOutEntry; import com.uid2.shared.optout.OptOutUtils; import com.uid2.optout.Const; -import com.uid2.optout.sqs.SqsMessageOperations; - +import com.uid2.optout.sqs.SqsMessageOperations.QueueAttributes; +import com.uid2.optout.sqs.SqsParsedMessage; import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; import java.nio.charset.StandardCharsets; @@ -212,13 +210,13 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic * - SQS messages passed in (with allowlist filtering) * - Invisible messages from other consumers (from queue attributes, avoiding double count) * - * @param sqsMessages List of SQS messages this consumer has read (non-denylisted) + * @param sqsMessages List of parsed SQS messages this consumer has read (non-denylisted) * @param queueAttributes Queue attributes including invisible message count (can be null) * @param denylistedCount Number of denylisted messages read by this consumer * @param filteredAsTooRecentCount Number of messages filtered as "too recent" by window reader * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) */ - public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes, int denylistedCount, int filteredAsTooRecentCount) { + public TrafficStatus calculateStatus(List sqsMessages, QueueAttributes queueAttributes, int denylistedCount, int filteredAsTooRecentCount) { try { // get list of delta files from s3 (sorted newest to oldest) @@ -485,13 +483,13 @@ long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowS /** * Find the oldest SQS queue message timestamp */ - private long findOldestQueueTimestamp(List sqsMessages) throws IOException { + private long findOldestQueueTimestamp(List sqsMessages) throws IOException { long oldest = System.currentTimeMillis() / 1000; if (sqsMessages != null && !sqsMessages.isEmpty()) { - for (Message msg : sqsMessages) { - Long ts = extractTimestampFromMessage(msg); - if (ts != null && ts < oldest) { + for (SqsParsedMessage msg : sqsMessages) { + long ts = msg.timestamp(); + if (ts < oldest) { oldest = ts; } } @@ -501,34 +499,16 @@ private long findOldestQueueTimestamp(List sqsMessages) throws IOExcept } /** - * Extract timestamp from SQS message (from SentTimestamp attribute) - */ - private Long extractTimestampFromMessage(Message msg) { - // get SentTimestamp attribute (milliseconds) - String sentTimestamp = msg.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); - if (sentTimestamp != null) { - try { - return Long.parseLong(sentTimestamp) / 1000; // convert ms to seconds - } catch (NumberFormatException e) { - LOGGER.error("sqs_error: invalid sentTimestamp, messageId={}, sentTimestamp={}", msg.messageId(), sentTimestamp); - } - } - - // fallback: use current time - return System.currentTimeMillis() / 1000; - } - - /** - * Count SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes + * Count non-allowlisted SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes */ - private int countSqsMessages(List sqsMessages, long oldestQueueTs) { + private int countSqsMessages(List sqsMessages, long oldestQueueTs) { int count = 0; int allowlistedCount = 0; long windowEnd = oldestQueueTs + 5 * 60; - for (Message msg : sqsMessages) { - Long ts = extractTimestampFromMessage(msg); + for (SqsParsedMessage msg : sqsMessages) { + long ts = msg.timestamp(); if (ts < oldestQueueTs || ts > windowEnd) { continue; diff --git a/src/test/java/com/uid2/optout/traffic/TrafficCalculatorTest.java b/src/test/java/com/uid2/optout/traffic/TrafficCalculatorTest.java index 5a0747c..784416c 100644 --- a/src/test/java/com/uid2/optout/traffic/TrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/traffic/TrafficCalculatorTest.java @@ -5,6 +5,7 @@ import com.uid2.shared.optout.OptOutCollection; import com.uid2.shared.optout.OptOutEntry; import com.uid2.optout.sqs.SqsMessageOperations; +import com.uid2.optout.sqs.SqsParsedMessage; import com.uid2.optout.Const; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; @@ -1064,26 +1065,16 @@ void testGetCacheStats_emptyCache() throws Exception { /** * Create a mock SQS message with specified timestamp */ - private Message createSqsMessage(long timestampSeconds) { - Map attributes = new HashMap<>(); - attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampSeconds * 1000)); - - return Message.builder() - .messageId("test-msg-" + timestampSeconds) - .body("{\"test\": \"data\"}") - .attributes(attributes) - .build(); - } - - /** - * Create a mock SQS message without timestamp - */ - private Message createSqsMessageWithoutTimestamp() { - return Message.builder() - .messageId("test-msg-no-timestamp") - .body("{\"test\": \"data\"}") - .attributes(new HashMap<>()) - .build(); + private SqsParsedMessage createSqsMessage(long timestampSeconds) { + return new SqsParsedMessage( + Message.builder().build(), + new byte[32], + new byte[32], + timestampSeconds, + "test@test.test", + null, + "127.0.0.1", + "test-trace-id"); } /** @@ -1144,7 +1135,7 @@ void testCalculateStatus_normalTraffic() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); + List sqsMessages = Arrays.asList(createSqsMessage(t)); TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - 100+1 < 5 * 50 = 250, so should be DEFAULT @@ -1175,7 +1166,7 @@ void testCalculateStatus_delayedProcessing() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); + List sqsMessages = Arrays.asList(createSqsMessage(t)); TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - 100+1 >= 5 * 10 = 50, DELAYED_PROCESSING @@ -1250,7 +1241,7 @@ void testCalculateStatus_multipleSqsMessages() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Add 30 SQS entries in [t, t+5min] - List sqsMessages = new ArrayList<>(); + List sqsMessages = new ArrayList<>(); for (int i = 0; i < 30; i++) { sqsMessages.add(createSqsMessage(t - i * 10)); } @@ -1296,7 +1287,7 @@ void testCalculateStatus_withTrafficCalcConfig() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); + List sqsMessages = Arrays.asList(createSqsMessage(t)); TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - should filter out entries in traffic calc config ranges @@ -1322,7 +1313,7 @@ void testCalculateStatus_cacheUtilization() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - first call should populate cache - List sqsMessages = Arrays.asList(createSqsMessage(t)); + List sqsMessages = Arrays.asList(createSqsMessage(t)); calculator.calculateStatus(sqsMessages, null, 0, 0); Map stats = calculator.getCacheStats(); @@ -1368,30 +1359,6 @@ void testCalculateStatus_deltaFileReadException() throws Exception { assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); } - @Test - void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600); - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - TrafficCalculator calculator = new TrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - SQS message without timestamp (should use current time) - List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp()); - TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); - - // Assert - DEFAULT - assertEquals(TrafficCalculator.TrafficStatus.DEFAULT, status); - } - @Test void testCalculateStatus_multipleDeltaFiles() throws Exception { // Setup - create delta files with some entries @@ -1425,7 +1392,7 @@ void testCalculateStatus_multipleDeltaFiles() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); + List sqsMessages = Arrays.asList(createSqsMessage(t)); TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - DEFAULT @@ -1459,7 +1426,7 @@ void testCalculateStatus_windowBoundaryTimestamp() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); + List sqsMessages = Arrays.asList(createSqsMessage(t)); TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - DEFAULT @@ -1483,7 +1450,7 @@ void testCalculateStatus_timestampsCached() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); + List sqsMessages = Arrays.asList(createSqsMessage(t)); TrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert @@ -1520,7 +1487,7 @@ void testCalculateStatus_delayedProcessingFromQueueAttributesOnly() throws Excep cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - 1 message read by us, but 600 invisible messages from other consumers - List sqsMessages = Arrays.asList(createSqsMessage(t)); + List sqsMessages = Arrays.asList(createSqsMessage(t)); // QueueAttributes: 0 visible, 600 invisible (other consumers), 0 delayed // Since we read 1 message, otherConsumers = 600 - 1 = 599 @@ -1557,7 +1524,7 @@ void testCalculateStatus_delayedProcessingFromBothQueueAndMessages() throws Exce // Act - 200 messages read by us + 450 invisible (200 are ours + 250 from others) // Messages must be within 5-minute window to be counted, so use 1-second spacing - List sqsMessages = new ArrayList<>(); + List sqsMessages = new ArrayList<>(); for (int i = 0; i < 200; i++) { sqsMessages.add(createSqsMessage(t - i)); // 1 second apart, all within 5-minute window } From 844332d90bbc54de86f256d6ae0e460fa501b18e Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Wed, 10 Dec 2025 17:06:56 -0700 Subject: [PATCH 18/25] use stream to find oldest queue timestamp --- .../uid2/optout/traffic/TrafficCalculator.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java index 3f0fe6f..26ade94 100644 --- a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java @@ -483,19 +483,11 @@ long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowS /** * Find the oldest SQS queue message timestamp */ - private long findOldestQueueTimestamp(List sqsMessages) throws IOException { - long oldest = System.currentTimeMillis() / 1000; - - if (sqsMessages != null && !sqsMessages.isEmpty()) { - for (SqsParsedMessage msg : sqsMessages) { - long ts = msg.timestamp(); - if (ts < oldest) { - oldest = ts; - } - } - } - - return oldest; + private long findOldestQueueTimestamp(List sqsMessages) { + return sqsMessages.stream() + .mapToLong(SqsParsedMessage::timestamp) + .min() + .orElse(System.currentTimeMillis() / 1000); } /** From f8a3d03ed6de9d910e5e689a95ba3067e2363c31 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Wed, 10 Dec 2025 17:10:45 -0700 Subject: [PATCH 19/25] rename sumCurrent to recentTrafficTotal, remove dead if condition --- .../uid2/optout/traffic/TrafficCalculator.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java index 26ade94..857c534 100644 --- a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java @@ -502,7 +502,7 @@ private int countSqsMessages(List sqsMessages, long oldestQueu for (SqsParsedMessage msg : sqsMessages) { long ts = msg.timestamp(); - if (ts < oldestQueueTs || ts > windowEnd) { + if (ts > windowEnd) { continue; } @@ -563,35 +563,35 @@ private void evictOldCacheEntries(long cutoffTimestamp) { * Determine traffic status based on current vs baseline traffic. * Logs warnings at 50%, 75%, and 90% of the circuit breaker threshold. */ - TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { + TrafficStatus determineStatus(int recentTrafficTotal, int baselineTraffic) { if (baselineTraffic == 0 || thresholdMultiplier == 0) { LOGGER.error("circuit_breaker_config_error: baselineTraffic is 0 or thresholdMultiplier is 0"); throw new RuntimeException("invalid circuit breaker config: baselineTraffic=" + baselineTraffic + ", thresholdMultiplier=" + thresholdMultiplier); } int threshold = thresholdMultiplier * baselineTraffic; - double thresholdPercent = (double) sumCurrent / threshold * 100; + double thresholdPercent = (double) recentTrafficTotal / threshold * 100; // log warnings at increasing thresholds before circuit breaker triggers if (thresholdPercent >= 90.0) { LOGGER.warn("high_message_volume: 90% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", - sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + recentTrafficTotal, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); } else if (thresholdPercent >= 75.0) { LOGGER.warn("high_message_volume: 75% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", - sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + recentTrafficTotal, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); } else if (thresholdPercent >= 50.0) { LOGGER.warn("high_message_volume: 50% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", - sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + recentTrafficTotal, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); } - if (sumCurrent >= threshold) { + if (recentTrafficTotal >= threshold) { LOGGER.error("circuit_breaker_triggered: traffic threshold breached, sumCurrent={}, threshold={} ({}x{})", - sumCurrent, threshold, thresholdMultiplier, baselineTraffic); + recentTrafficTotal, threshold, thresholdMultiplier, baselineTraffic); return TrafficStatus.DELAYED_PROCESSING; } LOGGER.info("traffic within normal range: sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", - sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + recentTrafficTotal, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); return TrafficStatus.DEFAULT; } From 4e22c341e556b0ccd6b93c09d62043329321387f Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Wed, 10 Dec 2025 17:16:50 -0700 Subject: [PATCH 20/25] consolidate warning logs for high message volume, add null check for oldestqueuetimestamp back --- .../uid2/optout/traffic/TrafficCalculator.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java index 857c534..07e3104 100644 --- a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java @@ -484,6 +484,9 @@ long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowS * Find the oldest SQS queue message timestamp */ private long findOldestQueueTimestamp(List sqsMessages) { + if (sqsMessages == null || sqsMessages.isEmpty()) { + return System.currentTimeMillis() / 1000; + } return sqsMessages.stream() .mapToLong(SqsParsedMessage::timestamp) .min() @@ -572,16 +575,10 @@ TrafficStatus determineStatus(int recentTrafficTotal, int baselineTraffic) { int threshold = thresholdMultiplier * baselineTraffic; double thresholdPercent = (double) recentTrafficTotal / threshold * 100; - // log warnings at increasing thresholds before circuit breaker triggers - if (thresholdPercent >= 90.0) { - LOGGER.warn("high_message_volume: 90% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", - recentTrafficTotal, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); - } else if (thresholdPercent >= 75.0) { - LOGGER.warn("high_message_volume: 75% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", - recentTrafficTotal, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); - } else if (thresholdPercent >= 50.0) { - LOGGER.warn("high_message_volume: 50% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", - recentTrafficTotal, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + // log warning if we reach 50 percent of the threshold + if (thresholdPercent >= 50.0) { + LOGGER.warn("high_message_volume: {}% of threshold reached, recentTrafficTotal={}, threshold={} ({}x{})", + String.format("%.1f", thresholdPercent), recentTrafficTotal, threshold, thresholdMultiplier, baselineTraffic); } if (recentTrafficTotal >= threshold) { From 11af1ed87be6b1b3f940e43d1f2d92de44fdcc0f Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Wed, 10 Dec 2025 17:20:15 -0700 Subject: [PATCH 21/25] update log level for circuit_breaker_triggered. Change ipAddresses from list to set --- .../com/uid2/optout/traffic/TrafficCalculator.java | 2 +- .../java/com/uid2/optout/traffic/TrafficFilter.java | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java index 07e3104..74438f3 100644 --- a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java @@ -582,7 +582,7 @@ TrafficStatus determineStatus(int recentTrafficTotal, int baselineTraffic) { } if (recentTrafficTotal >= threshold) { - LOGGER.error("circuit_breaker_triggered: traffic threshold breached, sumCurrent={}, threshold={} ({}x{})", + LOGGER.warn("circuit_breaker_triggered: traffic threshold breached, sumCurrent={}, threshold={} ({}x{})", recentTrafficTotal, threshold, thresholdMultiplier, baselineTraffic); return TrafficStatus.DELAYED_PROCESSING; } diff --git a/src/main/java/com/uid2/optout/traffic/TrafficFilter.java b/src/main/java/com/uid2/optout/traffic/TrafficFilter.java index 4733827..4b74f89 100644 --- a/src/main/java/com/uid2/optout/traffic/TrafficFilter.java +++ b/src/main/java/com/uid2/optout/traffic/TrafficFilter.java @@ -8,6 +8,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Collections; +import java.util.Set; +import java.util.HashSet; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Paths; @@ -26,9 +28,9 @@ public class TrafficFilter { */ private static class TrafficFilterRule { private final List range; - private final List ipAddresses; + private final Set ipAddresses; - TrafficFilterRule(List range, List ipAddresses) { + TrafficFilterRule(List range, Set ipAddresses) { this.range = range; this.ipAddresses = ipAddresses; } @@ -39,7 +41,7 @@ public long getRangeStart() { public long getRangeEnd() { return range.get(1); } - public List getIpAddresses() { + public Set getIpAddresses() { return ipAddresses; } } @@ -131,7 +133,7 @@ List parseFilterRules(JsonObject config) throws MalformedTraf // parse IPs var ipAddressesJson = ruleJson.getJsonArray("IPs"); - List ipAddresses = new ArrayList<>(); + Set ipAddresses = new HashSet<>(); if (ipAddressesJson != null) { for (int j = 0; j < ipAddressesJson.size(); j++) { ipAddresses.add(ipAddressesJson.getString(j)); From 8d6cd71208ec97a5da03accd570f3d847ca88143 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Wed, 10 Dec 2025 17:26:59 -0700 Subject: [PATCH 22/25] update trafficfiltertest to use Junit5 not Junit4 --- .../optout/traffic/TrafficFilterTest.java | 88 ++++++++++--------- 1 file changed, 47 insertions(+), 41 deletions(-) diff --git a/src/test/java/com/uid2/optout/traffic/TrafficFilterTest.java b/src/test/java/com/uid2/optout/traffic/TrafficFilterTest.java index 3e435e1..6117d0c 100644 --- a/src/test/java/com/uid2/optout/traffic/TrafficFilterTest.java +++ b/src/test/java/com/uid2/optout/traffic/TrafficFilterTest.java @@ -1,25 +1,24 @@ package com.uid2.optout.traffic; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import com.uid2.optout.sqs.SqsParsedMessage; -import com.uid2.optout.traffic.TrafficFilter; import software.amazon.awssdk.services.sqs.model.Message; import java.nio.file.Files; import java.nio.file.Path; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; public class TrafficFilterTest { private static final String TEST_CONFIG_PATH = "./traffic-config.json"; - @Before - public void setUp() { + @BeforeEach + void setUp() { try { Files.deleteIfExists(Path.of(TEST_CONFIG_PATH)); } catch (Exception e) { @@ -27,8 +26,8 @@ public void setUp() { } } - @After - public void tearDown() { + @AfterEach + void tearDown() { try { Files.deleteIfExists(Path.of(TEST_CONFIG_PATH)); } catch (Exception e) { @@ -37,7 +36,7 @@ public void tearDown() { } @Test - public void testParseFilterRules_emptyRules() throws Exception { + void testParseFilterRules_emptyRules() throws Exception { // Setup - empty denylist String config = """ { @@ -52,7 +51,7 @@ public void testParseFilterRules_emptyRules() throws Exception { } @Test - public void testParseFilterRules_singleRule() throws Exception { + void testParseFilterRules_singleRule() throws Exception { // Setup - config with one rule String config = """ { @@ -72,7 +71,7 @@ public void testParseFilterRules_singleRule() throws Exception { } @Test - public void testParseFilterRules_multipleRules() throws Exception { + void testParseFilterRules_multipleRules() throws Exception { // Setup - config with multiple rules String config = """ { @@ -95,8 +94,8 @@ public void testParseFilterRules_multipleRules() throws Exception { assertEquals(2, filter.filterRules.size()); } - @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_missingDenylistRequests() throws Exception { + @Test + void testParseFilterRules_missingDenylistRequests() throws Exception { // Setup - config without denylist_requests field String config = """ { @@ -106,11 +105,12 @@ public void testParseFilterRules_missingDenylistRequests() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - throws exception - new TrafficFilter(TEST_CONFIG_PATH); + assertThrows(TrafficFilter.MalformedTrafficFilterConfigException.class, + () -> new TrafficFilter(TEST_CONFIG_PATH)); } - @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_invalidRange_startAfterEnd() throws Exception { + @Test + void testParseFilterRules_invalidRange_startAfterEnd() throws Exception { // Setup - range where start > end String config = """ { @@ -125,11 +125,12 @@ public void testParseFilterRules_invalidRange_startAfterEnd() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - throws exception - new TrafficFilter(TEST_CONFIG_PATH); + assertThrows(TrafficFilter.MalformedTrafficFilterConfigException.class, + () -> new TrafficFilter(TEST_CONFIG_PATH)); } - @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_invalidRange_startEqualsEnd() throws Exception { + @Test + void testParseFilterRules_invalidRange_startEqualsEnd() throws Exception { // Setup - range where start == end String config = """ { @@ -144,11 +145,12 @@ public void testParseFilterRules_invalidRange_startEqualsEnd() throws Exception Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - throws exception - new TrafficFilter(TEST_CONFIG_PATH); + assertThrows(TrafficFilter.MalformedTrafficFilterConfigException.class, + () -> new TrafficFilter(TEST_CONFIG_PATH)); } - @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_rangeExceeds24Hours() throws Exception { + @Test + void testParseFilterRules_rangeExceeds24Hours() throws Exception { // Setup - range longer than 24 hours (86400 seconds) String config = """ { @@ -163,11 +165,12 @@ public void testParseFilterRules_rangeExceeds24Hours() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - throws exception - new TrafficFilter(TEST_CONFIG_PATH); + assertThrows(TrafficFilter.MalformedTrafficFilterConfigException.class, + () -> new TrafficFilter(TEST_CONFIG_PATH)); } - @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_emptyIPs() throws Exception { + @Test + void testParseFilterRules_emptyIPs() throws Exception { // Setup - rule with empty IP list String config = """ { @@ -182,11 +185,12 @@ public void testParseFilterRules_emptyIPs() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - throws exception - new TrafficFilter(TEST_CONFIG_PATH); + assertThrows(TrafficFilter.MalformedTrafficFilterConfigException.class, + () -> new TrafficFilter(TEST_CONFIG_PATH)); } - @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_missingIPs() throws Exception { + @Test + void testParseFilterRules_missingIPs() throws Exception { // Setup - rule without IPs field String config = """ { @@ -200,11 +204,12 @@ public void testParseFilterRules_missingIPs() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); // Act & Assert - throws exception - new TrafficFilter(TEST_CONFIG_PATH); + assertThrows(TrafficFilter.MalformedTrafficFilterConfigException.class, + () -> new TrafficFilter(TEST_CONFIG_PATH)); } @Test - public void testIsDenylisted_matchingIPAndTimestamp() throws Exception { + void testIsDenylisted_matchingIPAndTimestamp() throws Exception { // Setup - filter with denylist rule String config = """ { @@ -225,7 +230,7 @@ public void testIsDenylisted_matchingIPAndTimestamp() throws Exception { } @Test - public void testIsDenylisted_matchingIPOutsideTimeRange() throws Exception { + void testIsDenylisted_matchingIPOutsideTimeRange() throws Exception { // Setup - filter with denylist rule String config = """ { @@ -249,7 +254,7 @@ public void testIsDenylisted_matchingIPOutsideTimeRange() throws Exception { } @Test - public void testIsDenylisted_nonMatchingIP() throws Exception { + void testIsDenylisted_nonMatchingIP() throws Exception { // Setup - filter with denylist rule String config = """ { @@ -270,7 +275,7 @@ public void testIsDenylisted_nonMatchingIP() throws Exception { } @Test - public void testIsDenylisted_atRangeBoundaries() throws Exception { + void testIsDenylisted_atRangeBoundaries() throws Exception { // Setup - filter with denylist rule String config = """ { @@ -295,7 +300,7 @@ public void testIsDenylisted_atRangeBoundaries() throws Exception { } @Test - public void testIsDenylisted_multipleRules() throws Exception { + void testIsDenylisted_multipleRules() throws Exception { // Setup - multiple denylist rules String config = """ { @@ -329,7 +334,7 @@ public void testIsDenylisted_multipleRules() throws Exception { } @Test - public void testIsDenylisted_nullClientIp() throws Exception { + void testIsDenylisted_nullClientIp() throws Exception { // Setup - filter with denylist rule String config = """ { @@ -350,7 +355,7 @@ public void testIsDenylisted_nullClientIp() throws Exception { } @Test - public void testReloadTrafficFilterConfig_success() throws Exception { + void testReloadTrafficFilterConfig_success() throws Exception { // Setup - config with one rule String initialConfig = """ { @@ -390,14 +395,15 @@ public void testReloadTrafficFilterConfig_success() throws Exception { assertEquals(2, filter.filterRules.size()); } - @Test(expected = TrafficFilter.MalformedTrafficFilterConfigException.class) - public void testReloadTrafficFilterConfig_fileNotFound() throws Exception { + @Test + void testReloadTrafficFilterConfig_fileNotFound() { // Setup, Act & Assert - try to create filter with non-existent config - new TrafficFilter("./non-existent-file.json"); + assertThrows(TrafficFilter.MalformedTrafficFilterConfigException.class, + () -> new TrafficFilter("./non-existent-file.json")); } @Test - public void testParseFilterRules_maxValidRange() throws Exception { + void testParseFilterRules_maxValidRange() throws Exception { // Setup - range exactly 24 hours (86400 seconds) - should be valid String config = """ { From 836cd993512fb38656df053a47a2d40e77e8f2f9 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Wed, 10 Dec 2025 17:28:12 -0700 Subject: [PATCH 23/25] remove unused imports --- .../java/com/uid2/optout/traffic/TrafficCalculatorTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/com/uid2/optout/traffic/TrafficCalculatorTest.java b/src/test/java/com/uid2/optout/traffic/TrafficCalculatorTest.java index 784416c..4e3d346 100644 --- a/src/test/java/com/uid2/optout/traffic/TrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/traffic/TrafficCalculatorTest.java @@ -23,9 +23,7 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; -import com.uid2.optout.traffic.TrafficCalculator; import com.uid2.optout.traffic.TrafficCalculator.MalformedTrafficCalcConfigException; import java.io.ByteArrayInputStream; import java.util.*; From ac022fb4ed10ef07846e6af8b396570c9b489f95 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Wed, 10 Dec 2025 17:38:00 -0700 Subject: [PATCH 24/25] TrafficFilter minor efficiency improvements, usee stream to parse IPs, use two longs for time ranges instead of List --- .../uid2/optout/traffic/TrafficFilter.java | 69 ++++++++----------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/uid2/optout/traffic/TrafficFilter.java b/src/main/java/com/uid2/optout/traffic/TrafficFilter.java index 4b74f89..585aec6 100644 --- a/src/main/java/com/uid2/optout/traffic/TrafficFilter.java +++ b/src/main/java/com/uid2/optout/traffic/TrafficFilter.java @@ -6,10 +6,10 @@ import com.uid2.optout.sqs.SqsParsedMessage; import java.util.ArrayList; -import java.util.List; import java.util.Collections; +import java.util.List; import java.util.Set; -import java.util.HashSet; +import java.util.stream.Collectors; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Paths; @@ -27,19 +27,21 @@ public class TrafficFilter { * Traffic filter rule defining a time range and a list of IP addresses to exclude */ private static class TrafficFilterRule { - private final List range; + private final long rangeStart; + private final long rangeEnd; private final Set ipAddresses; - TrafficFilterRule(List range, Set ipAddresses) { - this.range = range; + TrafficFilterRule(long rangeStart, long rangeEnd, Set ipAddresses) { + this.rangeStart = rangeStart; + this.rangeEnd = rangeEnd; this.ipAddresses = ipAddresses; } public long getRangeStart() { - return range.get(0); + return rangeStart; } public long getRangeEnd() { - return range.get(1); + return rangeEnd; } public Set getIpAddresses() { return ipAddresses; @@ -110,49 +112,38 @@ List parseFilterRules(JsonObject config) throws MalformedTraf for (int i = 0; i < denylistRequests.size(); i++) { JsonObject ruleJson = denylistRequests.getJsonObject(i); - // parse range + // parse and validate range var rangeJson = ruleJson.getJsonArray("range"); - List range = new ArrayList<>(); - if (rangeJson != null && rangeJson.size() == 2) { - long start = rangeJson.getLong(0); - long end = rangeJson.getLong(1); - - if (start >= end) { - LOGGER.error("circuit_breaker_config_error: rule range start must be less than end, rule={}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range start must be less than end"); - } - range.add(start); - range.add(end); - } - - // log error and throw exception if range is not 2 elements - if (range.size() != 2) { + if (rangeJson == null || rangeJson.size() != 2) { LOGGER.error("circuit_breaker_config_error: rule range is not 2 elements, rule={}", ruleJson.encode()); throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range is not 2 elements"); } - // parse IPs - var ipAddressesJson = ruleJson.getJsonArray("IPs"); - Set ipAddresses = new HashSet<>(); - if (ipAddressesJson != null) { - for (int j = 0; j < ipAddressesJson.size(); j++) { - ipAddresses.add(ipAddressesJson.getString(j)); - } - } + long start = rangeJson.getLong(0); + long end = rangeJson.getLong(1); - // log error and throw exception if IPs is empty - if (ipAddresses.size() == 0) { - LOGGER.error("circuit_breaker_config_error: rule IPs is empty, rule={}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: IPs is empty"); + if (start >= end) { + LOGGER.error("circuit_breaker_config_error: rule range start must be less than end, rule={}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range start must be less than end"); } - // log error and throw exception if rule is invalid - if (range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less + if (end - start > 86400) { LOGGER.error("circuit_breaker_config_error: rule range must be 24 hours or less, rule={}", ruleJson.encode()); throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range must be 24 hours or less"); } - TrafficFilterRule rule = new TrafficFilterRule(range, ipAddresses); + // parse IPs using stream + var ipAddressesJson = ruleJson.getJsonArray("IPs"); + if (ipAddressesJson == null || ipAddressesJson.isEmpty()) { + LOGGER.error("circuit_breaker_config_error: rule IPs is empty, rule={}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: IPs is empty"); + } + + Set ipAddresses = ipAddressesJson.stream() + .map(Object::toString) + .collect(Collectors.toSet()); + + TrafficFilterRule rule = new TrafficFilterRule(start, end, ipAddresses); LOGGER.info("loaded traffic filter rule: range=[{}, {}], IPs={}", rule.getRangeStart(), rule.getRangeEnd(), rule.getIpAddresses()); rules.add(rule); @@ -178,7 +169,7 @@ public boolean isDenylisted(SqsParsedMessage message) { if(rule.getIpAddresses().contains(clientIp)) { return true; } - }; + } } return false; } From 7725d44c40c2fa19df2ad304f055e680a3b7b1cc Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Wed, 10 Dec 2025 17:48:14 -0700 Subject: [PATCH 25/25] TrafficCalculator: remove redundant checks --- .../uid2/optout/traffic/TrafficCalculator.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java index 74438f3..6fd5149 100644 --- a/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/TrafficCalculator.java @@ -265,8 +265,7 @@ public TrafficStatus calculateStatus(List sqsMessages, QueueAt filesProcessed++; // check newest record in file - if older than window, stop processing remaining files - long newestRecordTs = timestamps.get(0); - if (newestRecordTs < deltaWindowStart) { + if (timestamps.isEmpty() || timestamps.get(0) < deltaWindowStart) { break; } @@ -282,11 +281,8 @@ public TrafficStatus calculateStatus(List sqsMessages, QueueAt continue; } - // increment sum if record is within the delta window - if (ts >= deltaWindowStart) { - deltaRecordsCount++; - totalRecords++; - } + deltaRecordsCount++; + totalRecords++; } } @@ -490,7 +486,7 @@ private long findOldestQueueTimestamp(List sqsMessages) { return sqsMessages.stream() .mapToLong(SqsParsedMessage::timestamp) .min() - .orElse(System.currentTimeMillis() / 1000); + .getAsLong(); } /** @@ -529,11 +525,7 @@ boolean isInAllowlist(long timestamp) { return false; } - for (List range : allowlistRanges) { - if (range.size() < 2) { - continue; - } - + for (List range : allowlistRanges) { long start = range.get(0); long end = range.get(1);