Implement Remote Storage outside of GAE

appinventor/appengine/src/com/google/appinventor/client/actions/BarcodeAction.java

@@ -1,6 +1,6 @@
 // -*- mode: java; c-basic-offset: 2; -*-
 // Copyright 2009-2011 Google, All Rights reserved
-// Copyright 2011-2023 MIT, All rights reserved
+// Copyright 2011-2025 MIT, All rights reserved
 // Released under the Apache License, Version 2.0
 // http://www.apache.org/licenses/LICENSE-2.0
 
@@ -16,8 +16,8 @@
 import com.google.appinventor.client.explorer.commands.WaitForBuildResultCommand;
 import com.google.appinventor.client.explorer.commands.WarningDialogCommand;
 import com.google.appinventor.client.tracking.Tracking;
+import com.google.appinventor.shared.util.BuildOutputFiles;
 import com.google.appinventor.shared.rpc.project.ProjectRootNode;
-import com.google.appinventor.shared.rpc.project.youngandroid.YoungAndroidProjectNode;
 import com.google.gwt.user.client.Command;
 
 public class BarcodeAction implements Command {
@@ -50,7 +50,7 @@ public void setIsAab(boolean isAab_p) {
   public void execute() {
     ProjectRootNode projectRootNode = Ode.getInstance().getCurrentYoungAndroidProjectRootNode();
     if (projectRootNode != null) {
-      String target = YoungAndroidProjectNode.YOUNG_ANDROID_TARGET_ANDROID;
+      String target = BuildOutputFiles.getTargetName();
       ChainableCommand cmd = new SaveAllEditorsCommand(
           new GenerateYailCommand(
               new BuildCommand(target, secondBuildserver, isAab,

appinventor/appengine/src/com/google/appinventor/server/BuildOutputServlet.java

@@ -1,29 +1,30 @@
 // -*- mode: java; c-basic-offset: 2; -*-
 // Copyright 2009-2011 Google, All Rights reserved
-// Copyright 2011-2019 MIT, All rights reserved
+// Copyright 2011-2025 MIT, All rights reserved
 // Released under the Apache License, Version 2.0
 // http://www.apache.org/licenses/LICENSE-2.0
 
 package com.google.appinventor.server;
 
-import com.google.appinventor.common.utils.StringUtils;
 import com.google.appinventor.server.storage.StorageIo;
 import com.google.appinventor.server.storage.StorageIoInstanceHolder;
+import com.google.appinventor.server.storage.remote.RemoteStorage;
+import com.google.appinventor.server.storage.remote.RemoteStorageInstanceHolder;
+import com.google.appinventor.shared.util.BuildOutputFiles;
 import com.google.appinventor.server.util.CacheHeaders;
 import com.google.appinventor.server.util.CacheHeadersImpl;
 import com.google.appinventor.shared.rpc.Nonce;
-import com.google.appinventor.shared.rpc.ServerLayout;
-import com.google.appinventor.shared.rpc.project.ProjectSourceZip;
 import com.google.appinventor.shared.rpc.project.RawFile;
 import com.google.appinventor.shared.storage.StorageUtil;
 
 import javax.servlet.ServletOutputStream;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Date;
-import java.util.NoSuchElementException;
+import java.util.List;
 import java.util.logging.Logger;
 
 /**
@@ -68,7 +69,6 @@ public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOExc
 
     RawFile downloadableFile;
 
-    String userId = null;
     String nonceValue = null;
 
     try {
@@ -95,6 +95,37 @@ public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOExc
         resp.sendError(resp.SC_NOT_FOUND, "Link has timed out");
         return;
       }
+
+      final String userId = nonce.getUserId();
+      final long projectId = nonce.getProjectId();
+
+      if (RemoteStorageInstanceHolder.isRemoteConfigured(RemoteStorageInstanceHolder.Usage.BUILD)) {
+        final StorageIo storageIo = StorageIoInstanceHolder.getInstance();
+        // Given all the output files, try to find the one that matches an output extension
+        final List<String> files = storageIo.getProjectOutputFiles(userId, projectId);
+        String outputFileName = null;
+        for (String fileName : files) {
+          if (BuildOutputFiles.isOutputFile(fileName)) {
+            outputFileName = new File(fileName).getName();
+            break;
+          }
+        }
+
+        if (outputFileName == null) {
+          throw new FileNotFoundException("No target file found!");
+        }
+
+        final RemoteStorage remoteStorage = RemoteStorageInstanceHolder.getInstance(RemoteStorageInstanceHolder.Usage.BUILD);
+        // For now, we use Android always. If we eventually support iOS, we should decide here
+        //   whether we are going for Android or Apple, as well as in the BarcodeAction.
+        final String target = BuildOutputFiles.getTargetName();
+        final String objectKey = remoteStorage.getBuildOutputObjectKey(target, userId, projectId, outputFileName);
+        final String remoteUrl = remoteStorage.generateRetrieveUrl(objectKey);
+
+        resp.sendRedirect(remoteUrl);
+        return;
+      }
+
       downloadableFile = fileExporter.exportProjectOutputFile(nonce.getUserId(), nonce.getProjectId(), null);
 
     } catch (FileNotFoundException e) {

appinventor/appengine/src/com/google/appinventor/server/DownloadServlet.java

@@ -1,6 +1,6 @@
 // -*- mode: java; c-basic-offset: 2; -*-
 // Copyright 2009-2011 Google, All Rights reserved
-// Copyright 2011-2019 MIT, All rights reserved
+// Copyright 2011-2025 MIT, All rights reserved
 // Released under the Apache License, Version 2.0
 // http://www.apache.org/licenses/LICENSE-2.0
 
@@ -11,6 +11,8 @@
 import com.google.appinventor.server.storage.StorageIo;
 import com.google.appinventor.server.storage.StorageIoInstanceHolder;
 
+import com.google.appinventor.server.storage.remote.RemoteStorage;
+import com.google.appinventor.server.storage.remote.RemoteStorageInstanceHolder;
 import com.google.appinventor.server.util.CacheHeaders;
 import com.google.appinventor.server.util.CacheHeadersImpl;
 
@@ -19,20 +21,23 @@
 import com.google.appinventor.shared.rpc.project.RawFile;
 
 import com.google.appinventor.shared.storage.StorageUtil;
+import com.google.common.io.ByteStreams;
 
-import java.io.File;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 
-import java.nio.file.Files;
-import java.nio.file.Path;
-
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 
 import java.util.ArrayList;
 import java.util.Formatter;
 import java.util.List;
 import java.util.NoSuchElementException;
+import java.util.Set;
 import java.util.logging.Logger;
 
 import javax.servlet.ServletOutputStream;
@@ -94,6 +99,21 @@ public class DownloadServlet extends OdeServlet {
   private static final int USERFILE_PATH_INDEX = 4;
   private static final int SPLIT_LIMIT_USERFILE = 5;
 
+  // If any file we try to export exceeds 20MB, then use the remote storage solution if
+  //   configured.
+  // If unconfigured, still use GAE, but it may fail for large files due to response
+  //   payload limit.
+  private static final int DIRECT_DOWNLOAD_MAX_FILE_SIZE = 20_000_000;
+
+  // Only use remote downloads for specific kind of downloads (and avoid using for other
+  //   files like assets, even if "larger", although impossible).
+  private static final Set<String> REMOTE_DOWNLOAD_KINDS = Set.of(
+      ServerLayout.DOWNLOAD_PROJECT_OUTPUT,
+      ServerLayout.DOWNLOAD_PROJECT_SOURCE,
+      ServerLayout.DOWNLOAD_USER_PROJECT_SOURCE,
+      ServerLayout.DOWNLOAD_SELECTED_PROJECTS_SOURCE,
+      ServerLayout.DOWNLOAD_ALL_PROJECTS_SOURCE
+  );
 
   // Logging support
   private static final Logger LOG = Logger.getLogger(DownloadServlet.class.getName());
@@ -111,8 +131,9 @@ public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOExc
     // Set a default http header to avoid security vulnerabilities.
     CACHE_HEADERS.setNotCacheable(resp);
     resp.setContentType(CONTENT_TYPE);
+    final boolean isInline = req.getParameter("inline") != null;
 
-    RawFile downloadableFile = null;
+    RawFile downloadableFile;
 
     String userId = null;
 
@@ -270,6 +291,15 @@ public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOExc
       } else {
         throw new IllegalArgumentException("Unknown download kind: " + downloadKind);
       }
+
+      if (!isInline) {
+        final String downloadObjectUrl = shouldUseRemoteStorageDownload(downloadKind, userId, downloadableFile);
+        if (downloadObjectUrl != null) {
+          LOG.info("File sent to Remote Storage: " + downloadableFile.getFileName());
+          resp.sendRedirect(downloadObjectUrl);
+          return;
+        }
+      }
     } catch (IllegalArgumentException e) {
       throw CrashReport.createAndLogError(LOG, req, "user=" + userId, e);
     } catch (SecurityException e) {
@@ -294,9 +324,13 @@ public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOExc
       String fileName = downloadableFile.getFileName();
       byte[] content = downloadableFile.getContent();
       // Set http response information
-      resp.setHeader(
-        "content-disposition",
-        req.getParameter("inline") != null ? "inline" : "attachment" + "; filename=\"" + fileName + "\"");
+      final String contentDispositionHeaderValue;
+      if (isInline) {
+        contentDispositionHeaderValue = "inline";
+      } else {
+        contentDispositionHeaderValue = "attachment; filename=\"" + fileName + "\"";
+      }
+      resp.setHeader("content-disposition", contentDispositionHeaderValue);
       resp.setContentType(StorageUtil.getContentTypeForFilePath(fileName));
       resp.setContentLength(content.length);
 
@@ -312,6 +346,56 @@ public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOExc
     }
   }
 
+  private String shouldUseRemoteStorageDownload(final String downloadKind, final String userId, final RawFile file)
+      throws IOException {
+    if (!REMOTE_DOWNLOAD_KINDS.contains(downloadKind)) {
+      // Skip non-project related downloads
+      return null;
+    }
+
+    if (!RemoteStorageInstanceHolder.isRemoteConfigured(RemoteStorageInstanceHolder.Usage.EXPORT)) {
+      // Skip any further checks if unconfigured
+      return null;
+    }
+
+    if (file.getContent().length <= DIRECT_DOWNLOAD_MAX_FILE_SIZE) {
+      // File is within the GAE limits, hence no need to use the remote option
+      return null;
+    }
+
+    final String fileName = file.getFileName();
+    LOG.info("Sending file to Remote Storage: " + fileName);
+
+    final RemoteStorage remoteStorage = RemoteStorageInstanceHolder.getInstance(RemoteStorageInstanceHolder.Usage.EXPORT);
+    final String objectKey = remoteStorage.getProjectExportObjectKey(downloadKind, userId, fileName);
+
+    final String uploadUrlStr = remoteStorage.generateUploadUrl(objectKey);
+
+    HttpURLConnection connection = (HttpURLConnection) new URL(uploadUrlStr).openConnection();
+    connection.setDoOutput(true);
+    connection.setRequestMethod("PUT");
+    connection.addRequestProperty("Content-Type", StorageUtil.getContentTypeForFilePath(fileName));
+    // Ensure that, when downloading the file from remote, it preserves the same name
+    // If this is not set, and Content-Type is set to ZIP for AIA, browsers will rewrite .aia to .zip
+    connection.addRequestProperty("Content-Disposition", "attachment; filename=\"" + fileName + "\"");
+    connection.setConnectTimeout(60000);
+    connection.setReadTimeout(60000);
+    try (BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(connection.getOutputStream())) {
+      try (BufferedInputStream bufferedInputStream = new BufferedInputStream(new ByteArrayInputStream(file.getContent()))) {
+        ByteStreams.copy(bufferedInputStream, bufferedOutputStream);
+        bufferedOutputStream.flush();
+      }
+    }
+
+    if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
+      LOG.severe("Failed to upload output: "+ connection.getResponseCode());
+      // If we fail to upload the output, return null so we try to fallback to GAE
+      return null;
+    }
+
+    return remoteStorage.generateRetrieveUrl(objectKey);
+  }
+
   private static String byteArray2Hex(final byte[] hash) {
     Formatter formatter = new Formatter();
     for (byte b : hash) {

appinventor/appengine/src/com/google/appinventor/server/FileExporterImpl.java

@@ -1,13 +1,14 @@
 // -*- mode: java; c-basic-offset: 2; -*-
 // Copyright 2009-2011 Google, All Rights reserved
-// Copyright 2011-2019 MIT, All rights reserved
+// Copyright 2011-2025 MIT, All rights reserved
 // Released under the Apache License, Version 2.0
 // http://www.apache.org/licenses/LICENSE-2.0
 
 package com.google.appinventor.server;
 
 import com.google.appinventor.server.storage.StorageIo;
 import com.google.appinventor.server.storage.StorageIoInstanceHolder;
+import com.google.appinventor.shared.util.BuildOutputFiles;
 import com.google.appinventor.shared.rpc.project.ProjectSourceZip;
 import com.google.appinventor.shared.rpc.project.RawFile;
 import com.google.appinventor.shared.storage.StorageUtil;
@@ -47,7 +48,7 @@ public RawFile exportProjectOutputFile(String userId, long projectId, @Nullable
     // There should never be more than one .apk file.
 
     for (String fileName : files) {
-      if (fileName.endsWith(".apk") || fileName.endsWith(".aab")) {
+      if (BuildOutputFiles.isOutputFile(fileName)) {
         byte[] content = storageIo.downloadRawFile(userId, projectId, fileName);
         return new RawFile(StorageUtil.basename(fileName), content);
       }

appinventor/appengine/src/com/google/appinventor/server/ReceiveBuildServlet.java

@@ -1,6 +1,6 @@
 // -*- mode: java; c-basic-offset: 2; -*-
 // Copyright 2009-2011 Google, All Rights reserved
-// Copyright 2011-2019 MIT, All rights reserved
+// Copyright 2011-2025 MIT, All rights reserved
 // Released under the Apache License, Version 2.0
 // http://www.apache.org/licenses/LICENSE-2.0
 
@@ -58,6 +58,8 @@ public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOEx
       throw CrashReport.createAndLogError(LOG, req, null, e);
     }
 
+    boolean isFinalStatusReport = false;
+
     // Set the user in the OdeFilter, which is used everywhere as the UserInfoProvider.
     odeFilter.setUserFromUserId(userId, false, false);
     try {
@@ -82,10 +84,18 @@ public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOEx
           String filePath = buildFileDirPath + "/" + fileName;
           LOG.info("Saving build output files: " + filePath);
           storageIo.addOutputFilesToProject(userId, projectId, filePath);
-          storageIo.uploadRawFileForce(projectId, filePath, userId, fileBytes);
-          storageIo.storeBuildStatus(userId, projectId, 0); // Reset for the next build
+          isFinalStatusReport = true;
+          if (fileBytes.length > 0) {
+            // We only "upload" the file if it actually has something. This is because in the remote build,
+            //   we will issue an empty file back to GAE for the APK/AAB files.
+            storageIo.uploadRawFileForce(projectId, filePath, userId, fileBytes);
+          }
         }
       }
+
+      if (isFinalStatusReport) {
+        storageIo.storeBuildStatus(userId, projectId, 0); // Reset for the next build
+      }
     } finally {
       odeFilter.removeUser();
     }