diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index e5d366e..e911e53 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -3,6 +3,7 @@ ferload-client { abbreviate=30 config-name=".ferload-client.properties" manifest-header="url" + manifest-size="size" manifest-separator="\t" download-files-pool=10 download-agreement="yes" diff --git a/src/main/scala/ca/ferlab/ferload/client/commands/Download.scala b/src/main/scala/ca/ferlab/ferload/client/commands/Download.scala index f77a599..3f5817d 100644 --- a/src/main/scala/ca/ferlab/ferload/client/commands/Download.scala +++ b/src/main/scala/ca/ferlab/ferload/client/commands/Download.scala @@ -12,6 +12,7 @@ import picocli.CommandLine.{Command, IExitCodeGenerator, Option} import java.io.{File, FileReader} import java.util.Optional +import scala.collection.mutable import scala.util.{Failure, Success, Try, Using} @Command(name = "download", mixinStandardHelpOptions = true, description = Array("Download files based on provided manifest."), @@ -31,6 +32,8 @@ class Download(userConfig: UserConfig, @Option(names = Array("-p", "--password"), description = Array("password")) var password: Optional[String] = Optional.empty + + private val NO_SIZE = 0L; override def run(): Unit = { @@ -58,7 +61,7 @@ class Download(userConfig: UserConfig, val padding = appConfig.getInt("padding") - val manifestContent: String = CommandBlock("Checking manifest file", successEmoji, padding) { + val manifestContent: ManifestContent = CommandBlock("Checking manifest file", successEmoji, padding) { extractManifestContent } @@ -78,7 +81,7 @@ class Download(userConfig: UserConfig, } val links: Map[String, String] = CommandBlock("Retrieve Ferload download link(s)", successEmoji, padding) { - Try(ferload.getDownloadLinks(token, manifestContent)) match { + Try(ferload.getDownloadLinks(token, manifestContent.urls)) match { case Success(links) => links case Failure(e) => { // always refresh token if failed @@ -90,16 +93,17 @@ class Download(userConfig: UserConfig, } val totalExpectedDownloadSize = CommandBlock("Compute total average expected download size", successEmoji, padding) { - Try(s3.getTotalExpectedDownloadSize(links, appConfig.getLong("size-estimation-timeout"))) match { - case Success(size) => size - case Failure(e) => { - println() - println() - println(s"Failed to compute total average expected download size, reason: ${e.getMessage}") - print(s"You can still proceed with the download, verify you have remaining disk-space available.") - 0L - } - } + manifestContent.totalSize.getOrElse( + Try(s3.getTotalExpectedDownloadSize(links, appConfig.getLong("size-estimation-timeout"))) match { + case Success(size) => size + case Failure(e) => { + println() + println() + println(s"Failed to compute total average expected download size, reason: ${e.getMessage}") + print(s"You can still proceed with the download, verify you have remaining disk-space available.") + NO_SIZE + } + }) } val totalExpectedDownloadSizeStr = if(totalExpectedDownloadSize > 0) FileUtils.byteCountToDisplaySize(totalExpectedDownloadSize) else "-" @@ -127,8 +131,9 @@ class Download(userConfig: UserConfig, } } - private def extractManifestContent: String = { + private def extractManifestContent: ManifestContent = { val manifestHeader = appConfig.getString("manifest-header") + val manifestSize = appConfig.getString("manifest-size") val manifestSeparator = appConfig.getString("manifest-separator").charAt(0) if (!manifest.exists()) { @@ -142,8 +147,10 @@ class Download(userConfig: UserConfig, .withTrim() .withFirstRecordAsHeader() .parse(reader) - val builder = new StringBuilder + val urls = new mutable.StringBuilder + var totalSize = NO_SIZE val fileIdColumnIndex = parser.getHeaderMap.getOrDefault(manifestHeader, -1) + val sizeColumnIndex = parser.getHeaderMap.getOrDefault(manifestSize, -1) if (fileIdColumnIndex == -1) { throw new IllegalStateException("Missing column: " + manifestHeader) @@ -152,21 +159,30 @@ class Download(userConfig: UserConfig, parser.getRecords.stream().forEach(record => { val url = record.get(fileIdColumnIndex) if (StringUtils.isNotBlank(url)) { - builder.append(s"$url\n") + urls.append(s"$url\n") + } + if (sizeColumnIndex != -1) { // size column exist (optional) + val size = record.get(sizeColumnIndex) + if (StringUtils.isNotBlank(size)) { + totalSize += size.toLong + } } }) - if (builder.isEmpty) { + if (urls.isEmpty) { throw new IllegalStateException("Empty content") } - builder.toString() + + ManifestContent(urls.toString(), if(totalSize == NO_SIZE) None else Some(totalSize)) } match { case Success(value) => value case Failure(e) => throw new IllegalStateException(s"Invalid manifest file: " + e.getMessage, e) } } + + case class ManifestContent(urls: String, totalSize: scala.Option[Long]) override def getExitCode: Int = 1 } \ No newline at end of file diff --git a/src/test/resources/manifest-valid.tsv b/src/test/resources/manifest-valid.tsv index b63afca..4975dab 100644 --- a/src/test/resources/manifest-valid.tsv +++ b/src/test/resources/manifest-valid.tsv @@ -1,4 +1,4 @@ -file_id -file1 -file2 -file3 \ No newline at end of file +file_id size +file1 1 +file2 1 +file3 1 \ No newline at end of file diff --git a/src/test/scala/ca/ferlab/ferload/client/commands/DownloadTest.scala b/src/test/scala/ca/ferlab/ferload/client/commands/DownloadTest.scala index 2b370be..c88da01 100644 --- a/src/test/scala/ca/ferlab/ferload/client/commands/DownloadTest.scala +++ b/src/test/scala/ca/ferlab/ferload/client/commands/DownloadTest.scala @@ -152,7 +152,7 @@ class DownloadTest extends AnyFunSuite with BeforeAndAfter { Set(new File("f1"), new File("f2")) } - override def getTotalExpectedDownloadSize(links: Map[String, String], timeout: Long): Long = 2L + override def getTotalExpectedDownloadSize(links: Map[String, String], timeout: Long): Long = ??? override def getTotalAvailableDiskSpaceAt(manifest: File): Long = 1L }