diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java index 30ed00090c..b037545195 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java @@ -97,15 +97,15 @@ private void searchForNewTicketsAndAddToQueue(JiraSourceConfig configuration, In Queue itemInfoQueue) { log.trace("Looking for Add/Modified tickets with a Search API call"); StringBuilder jql = createIssueFilterCriteria(configuration, timestamp); - int total; - int startAt = 0; + int total = 0; + String nextPageToken = ""; do { - SearchResults searchIssues = jiraRestClient.getAllIssues(jql, startAt); + SearchResults searchIssues = jiraRestClient.getAllIssues(jql, nextPageToken); List issueList = new ArrayList<>(searchIssues.getIssues()); - total = searchIssues.getTotal(); - startAt += searchIssues.getIssues().size(); + total = total + issueList.size(); + nextPageToken = searchIssues.getNextPageToken(); addItemsToQueue(issueList, itemInfoQueue); - } while (startAt < total); + } while (nextPageToken != null && !nextPageToken.isEmpty() && !nextPageToken.isBlank()); searchResultsFoundCounter.increment(total); log.info("Number of tickets found in search api call: {}", total); } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResults.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResults.java index 677967a340..cad4d4350a 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResults.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResults.java @@ -24,16 +24,16 @@ public class SearchResults { @JsonProperty("expand") private String expand = null; - @JsonProperty("startAt") - private Integer startAt = null; - @JsonProperty("maxResults") private Integer maxResults = null; - @JsonProperty("total") - private Integer total = null; - @JsonProperty("issues") private List issues = null; + @JsonProperty("nextPageToken") + private String nextPageToken = null; + + @JsonProperty("isLast") + private Boolean isLast = false; + } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java index 329fe28026..96141964f8 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -26,16 +26,18 @@ import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.EXPAND_FIELD; import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.EXPAND_VALUE; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.FIELDS_FIELD; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.FIELDS_VALUE; import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.JQL_FIELD; @Slf4j @Named public class JiraRestClient extends AtlassianRestClient { - public static final String REST_API_SEARCH = "rest/api/3/search"; + public static final String REST_API_SEARCH = "rest/api/3/search/jql"; public static final String REST_API_FETCH_ISSUE = "rest/api/3/issue"; public static final String FIFTY = "50"; - public static final String START_AT = "startAt"; + public static final String NEXT_PAGE_TOKEN = "nextPageToken"; public static final String MAX_RESULT = "maxResults"; private static final String TICKET_FETCH_LATENCY_TIMER = "ticketFetchLatency"; private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; @@ -72,16 +74,30 @@ public JiraRestClient(RestTemplate restTemplate, AtlassianAuthConfig authConfig, * @param startAt the start at * @return InputStream input stream */ - public SearchResults getAllIssues(StringBuilder jql, int startAt) { + public SearchResults getAllIssues(StringBuilder jql, String nextPageToken) { String url = authConfig.getUrl() + REST_API_SEARCH; - URI uri = UriComponentsBuilder.fromHttpUrl(url) - .queryParam(MAX_RESULT, FIFTY) - .queryParam(START_AT, startAt) - .queryParam(JQL_FIELD, jql) - .queryParam(EXPAND_FIELD, EXPAND_VALUE) - .buildAndExpand().toUri(); + URI uri; + + if(nextPageToken!= null && !nextPageToken.isBlank() && !nextPageToken.isEmpty()){ + uri = UriComponentsBuilder.fromHttpUrl(url) + .queryParam(MAX_RESULT, FIFTY) + .queryParam(NEXT_PAGE_TOKEN, nextPageToken) + .queryParam(JQL_FIELD, jql) + .queryParam(EXPAND_FIELD, EXPAND_VALUE) + .queryParam(FIELDS_FIELD, FIELDS_VALUE) + .buildAndExpand().toUri(); + } + else{ + uri = UriComponentsBuilder.fromHttpUrl(url) + .queryParam(MAX_RESULT, FIFTY) + .queryParam(JQL_FIELD, jql) + .queryParam(EXPAND_FIELD, EXPAND_VALUE) + .queryParam(FIELDS_FIELD, FIELDS_VALUE) + .buildAndExpand().toUri(); + } + return searchCallLatencyTimer.record( () -> { try { diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JqlConstants.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JqlConstants.java index e142a23140..bbb2745339 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JqlConstants.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JqlConstants.java @@ -27,4 +27,6 @@ public class JqlConstants { public static final String JQL_FIELD = "jql"; public static final String EXPAND_FIELD = "expand"; public static final String EXPAND_VALUE = "all"; + public static final String FIELDS_FIELD = "fields"; + public static final String FIELDS_VALUE = "*all"; } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java index bd73e3e966..dae1a5eba9 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java @@ -35,7 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -72,8 +72,7 @@ void testInitialization() { assertNotNull(jiraIterator); jiraIterator.initialize(Instant.ofEpochSecond(0)); when(mockSearchResults.getIssues()).thenReturn(new ArrayList<>()); - when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyString()); assertFalse(jiraIterator.hasNext()); } @@ -103,8 +102,7 @@ void testItemInfoQueueNotEmpty() { IssueBean issue1 = createIssueBean(false); mockIssues.add(issue1); when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyString()); jiraIterator.initialize(Instant.ofEpochSecond(0)); jiraIterator.setCrawlerQWaitTimeMillis(1); @@ -132,8 +130,7 @@ void testFuturesCompleted() throws InterruptedException { IssueBean issue3 = createIssueBean(false); mockIssues.add(issue3); when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyString()); jiraIterator.initialize(Instant.ofEpochSecond(0)); jiraIterator.setCrawlerQWaitTimeMillis(1); @@ -149,8 +146,7 @@ void testItemInfoQueueEmpty() { jiraIterator = createObjectUnderTest(); List mockIssues = new ArrayList<>(); when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyString()); jiraIterator.initialize(Instant.ofEpochSecond(0)); jiraIterator.setCrawlerQWaitTimeMillis(1); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java index 6ccaa03b91..ffd1d77286 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java @@ -48,7 +48,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; @@ -204,9 +203,8 @@ public void testGetJiraEntities() throws JsonProcessingException { SearchResults mockSearchResults = mock(SearchResults.class); when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(mockIssues.size()); - doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyString()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); @@ -223,16 +221,16 @@ public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingExcep JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient, pluginMetrics)); List mockIssues = new ArrayList<>(); - for (int i = 0; i < 50; i++) { + Integer mockIssuesCount = 100; + for (int i = 0; i < mockIssuesCount; i++) { IssueBean issue1 = createIssueBean(false, false); mockIssues.add(issue1); } SearchResults mockSearchResults = mock(SearchResults.class); when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(100); - doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyString()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); @@ -271,7 +269,7 @@ public void testGetJiraEntitiesException() throws JsonProcessingException { JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient, pluginMetrics)); - doThrow(RuntimeException.class).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); + doThrow(RuntimeException.class).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyString()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResultsTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResultsTest.java index bd0d54e2bf..975584a550 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResultsTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResultsTest.java @@ -44,18 +44,18 @@ public void testConstructor() { assertNotNull(searchResults); assertNull(searchResults.getExpand()); - assertNull(searchResults.getStartAt()); assertNull(searchResults.getMaxResults()); - assertNull(searchResults.getTotal()); + assertEquals(searchResults.getIsLast(), false); + assertNull(searchResults.getNextPageToken()); assertNull(searchResults.getIssues()); } @Test public void testGetters() throws JsonProcessingException { String expand = "expandTest"; - Integer startAt = 1; + String nextPageToken = "tokenTest"; Integer maxResults = 100; - Integer total = 10; + Boolean isLast = true; List testIssues = new ArrayList<>(); IssueBean issue1 = new IssueBean(); IssueBean issue2 = new IssueBean(); @@ -67,19 +67,20 @@ public void testGetters() throws JsonProcessingException { Map searchResultsMap = new HashMap<>(); searchResultsMap.put("expand", expand); - searchResultsMap.put("startAt", startAt); searchResultsMap.put("maxResults", maxResults); - searchResultsMap.put("total", total); + searchResultsMap.put("nextPageToken", nextPageToken); + searchResultsMap.put("isLast", isLast); searchResultsMap.put("issues", testIssues); + String jsonString = objectMapper.writeValueAsString(searchResultsMap); searchResults = objectMapper.readValue(jsonString, SearchResults.class); assertEquals(searchResults.getExpand(), expand); - assertEquals(searchResults.getStartAt(), startAt); assertEquals(searchResults.getMaxResults(), maxResults); - assertEquals(searchResults.getTotal(), total); + assertEquals(searchResults.getNextPageToken(), nextPageToken); + assertEquals(searchResults.getIsLast(), isLast); List returnedIssues = searchResults.getIssues(); assertNotNull(returnedIssues); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java index ada6d8b059..741ddcb07f 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java @@ -118,7 +118,7 @@ public void testGetAllIssuesOauth2() { SearchResults mockSearchResults = mock(SearchResults.class); doReturn("http://mock-service.jira.com/").when(authConfig).getUrl(); doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); - SearchResults results = jiraRestClient.getAllIssues(jql, 0); + SearchResults results = jiraRestClient.getAllIssues(jql, null); assertNotNull(results); } @@ -130,7 +130,7 @@ public void testGetAllIssuesBasic() { SearchResults mockSearchResults = mock(SearchResults.class); when(authConfig.getUrl()).thenReturn("https://example.com/"); doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); - SearchResults results = jiraRestClient.getAllIssues(jql, 0); + SearchResults results = jiraRestClient.getAllIssues(jql, null); assertNotNull(results); }