Improving Workspace Importer Pagination (#707)

AzBuilder · Feb 14, 2024 · f1acba5 · f1acba5
1 parent ec6f318
commit f1acba5
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 15 deletions.
diff --git a/api/src/main/java/org/terrakube/api/plugin/importer/tfcloud/WorkspaceListResponse.java b/api/src/main/java/org/terrakube/api/plugin/importer/tfcloud/WorkspaceListResponse.java
@@ -4,8 +4,36 @@
 import lombok.Setter;
 import java.util.List;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 @Getter
 @Setter
 public class WorkspaceListResponse {
     private List<WorkspaceImport.WorkspaceData> data;
+    private WorkspaceMeta meta;
+
+    @Getter
+    @Setter
+    public static class WorkspaceMeta {
+        private Pagination pagination;
+
+        @Getter
+        @Setter
+        public static class Pagination {
+            @JsonProperty("current-page")
+            private int currentPage;
+
+            @JsonProperty("prev-page")
+            private Integer prevPage;
+
+            @JsonProperty("next-page")
+            private Integer nextPage;
+
+            @JsonProperty("total-pages")
+            private int totalPages;
+
+            @JsonProperty("total-count")
+            private int totalCount;
+        }
+    }
 }
diff --git a/api/src/main/java/org/terrakube/api/plugin/importer/tfcloud/services/WorkspaceService.java b/api/src/main/java/org/terrakube/api/plugin/importer/tfcloud/services/WorkspaceService.java
@@ -28,6 +28,7 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
@@ -81,19 +82,33 @@ private <T> T makeRequest(String apiToken, String url, Class<T> responseType) {
     }
 
     public List<WorkspaceImport.WorkspaceData> getWorkspaces(String apiToken, String apiUrl, String organization) {
-        UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(apiUrl)
-                .pathSegment("organizations")
-                .pathSegment(organization)
-                .pathSegment("workspaces")
-                .queryParam("page[size]", 100);
+        List<WorkspaceImport.WorkspaceData> allData = new ArrayList<>();
+        int currentPage = 1;
 
-        String url = builder.toUriString();
-        WorkspaceListResponse response = makeRequest(apiToken, url, WorkspaceListResponse.class);
-        if (response == null || response.getData() == null) {
-            return Collections.emptyList();
-        } else {
-            return response.getData();
+        while (true) {
+            UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(apiUrl)
+                    .pathSegment("organizations")
+                    .pathSegment(organization)
+                    .pathSegment("workspaces");
+
+            String url = builder.toUriString() + "?page[size]=50&page[number]=" + currentPage;
+            log.info("url: {}", url);
+            WorkspaceListResponse response = makeRequest(apiToken, url, WorkspaceListResponse.class);
+
+            if (response == null || response.getData() == null) {
+                break;
+            } else {
+                allData.addAll(response.getData());
+            }
+
+            if (response.getMeta().getPagination().getNextPage() == null) {
+                break;
+            }
+
+            currentPage++;
         }
+
+        return allData;
     }
 
     public List<VariableAttributes> getVariables(String apiToken, String apiUrl, String organizationName,
@@ -156,6 +171,7 @@ public boolean importWorkspace(String apiToken, String apiUrl, WorkspaceImportRe
         workspace.setExecutionMode("remote");
 
         // If the workspace has a VCS, set it
+        log.info("VCS ID: {}", workspaceImportRequest.getVcsId());
         if (workspaceImportRequest.getVcsId() != null && !workspaceImportRequest.getVcsId().isEmpty()) {
             UUID vcsId = UUID.fromString(workspaceImportRequest.getVcsId());
             vcsRepository.findById(vcsId).ifPresent(workspace::setVcs);
@@ -173,7 +189,6 @@ public boolean importWorkspace(String apiToken, String apiUrl, WorkspaceImportRe
         organizationRepository.findById(UUID.fromString(workspaceImportRequest.getOrganizationId()))
                 .ifPresent(workspace::setOrganization);
         workspace.setIacType("terraform");
-        workspace.setSource(workspaceImportRequest.getSource());
         workspace = workspaceRepository.save(workspace);
 
         List<VariableAttributes> variablesImporter = getVariables(

diff --git a/ui/src/domain/Workspaces/Import.jsx b/ui/src/domain/Workspaces/Import.jsx
@@ -285,11 +285,11 @@ export const ImportWorkspace = () => {
           vcsId: vcsId,
           id: workspace.id,
           organization: form.getFieldValue("organization"),
-          branch: workspace.attributes["vcs-repo"].branch,
-          folder: workspace.attributes["vcs-repo"].directory,
+          branch: workspace.attributes["vcs-repo"]?.branch,
+          folder: workspace.attributes["vcs-repo"]?.directory,
           name: workspace.attributes.name,
           terraformVersion: workspace.attributes["terraform-version"],
-          source: workspace.attributes["vcs-repo"]["repository-http-url"],
+          source: workspace.attributes["vcs-repo"]?.["repository-http-url"],
           description: workspace.attributes.description,
         },
         {