eckit::geo: Download error handling (fixes ecmwf/eckit#370)

src/eckit/geo/Exceptions.cc

@@ -12,6 +12,10 @@ AreaError::AreaError(const std::string& what, const CodeLocation& location) :
     Exception("AreaError: [" + what + "]", location) {}
 
 
+DownloadError::DownloadError(const std::string& what, const CodeLocation& location) :
+    Exception("DownloadError: [" + what + "]", location) {}
+
+
 FigureError::FigureError(const std::string& what, const CodeLocation& location) :
     Exception("FigureError: [" + what + "]", location) {}
 

src/eckit/geo/Exceptions.h

@@ -38,6 +38,13 @@ class AreaError : public geo::Exception {
 };
 
 
+class DownloadError : public geo::Exception {
+public:
+
+    explicit DownloadError(const std::string&, const CodeLocation&);
+};
+
+
 class FigureError : public geo::Exception {
 public:
 

src/eckit/geo/cache/Download.cc

@@ -4,23 +4,25 @@
 
 #include "eckit/geo/cache/Download.h"
 
-#include <fstream>
-#include <iterator>
 #include <ostream>
 
 #include "eckit/eckit_config.h"
 #include "eckit/geo/Exceptions.h"
 #include "eckit/geo/cache/MemoryCache.h"
 #include "eckit/geo/util/mutex.h"
-#include "eckit/io/FileLock.h"
-#include "eckit/io/Length.h"
 #include "eckit/log/Log.h"
-#include "eckit/log/Timer.h"
-#include "eckit/os/AutoUmask.h"
 #include "eckit/utils/MD5.h"
+#include "eckit/utils/StringTools.h"
 
 #if eckit_HAVE_CURL
-#include "eckit/io/URLHandle.h"
+#include <fstream>
+#include <vector>
+
+#include "eckit/io/EasyCURL.h"
+#include "eckit/io/FileLock.h"
+#include "eckit/log/Timer.h"
+#include "eckit/os/AutoUmask.h"
+#include "eckit/utils/Translator.h"
 #endif
 
 
@@ -41,6 +43,7 @@ class lock_type {
 };
 
 
+#if eckit_HAVE_CURL
 class file_lock_type {
     struct flock_type {
         explicit flock_type(const PathName& path) :
@@ -70,6 +73,7 @@ class file_lock_type {
 
     explicit file_lock_type(const PathName& path) : flock_(path), lock_guard_(flock_) {}
 };
+#endif
 
 
 }  // namespace
@@ -101,14 +105,38 @@ std::string Download::url_file_extension(const url_type& url) {
 }
 
 
+std::string Download::validate_response(const url_type& url, long code, const headers_type& headers, bool html) {
+    // non-HTTP schemes (eg. file://) have no status nor headers, rely on transport errors only
+    if (const auto scheme = StringTools::lower(url.substr(0, url.find(':'))); scheme != "http" && scheme != "https") {
+        return {};
+    }
+
+    // require positive evidence of success: a 2xx status (0 means no response was received)
+    if (code < 200 || code >= 300) {
+        return "HTTP status " + std::to_string(code);
+    }
+
+    // a 2xx HTML page is not data (eg. captive portal, proxy error page, login page after redirects)
+    if (auto it = headers.find("content-type"); !html && it != headers.end()) {
+        if (const auto type = StringTools::lower(StringTools::trim(it->second));
+            StringTools::startsWith(type, "text/html") || StringTools::startsWith(type, "application/xhtml")) {
+            return "unexpected HTML response (Content-Type: " + type + ")";
+        }
+    }
+
+    return {};
+}
+
+
+#if eckit_HAVE_CURL
 Download::info_type Download::to_path(const url_type& url, const PathName& path, bool html) {
     // control concurrent download
     lock_type lock;
 
-    Length length = 0;
     Timer timer;
+    std::string error;
+    unsigned long long bytes = 0;
 
-#if eckit_HAVE_CURL  // for eckit::URLHandle
     file_lock_type flock(path);
 
     auto tmp = path + ".part";
@@ -118,42 +146,69 @@ Download::info_type Download::to_path(const url_type& url, const PathName& path,
     ASSERT(dir.exists());
 
     try {
-        length = URLHandle{url}.saveInto(tmp);
-    }
-    catch (...) {
-        length = 0;
-    }
+        EasyCURL curl;
+        curl.useSSL(true);
 
-    // no empty files
-    if (length <= 0) {
-        if (tmp.exists()) {
-            tmp.unlink(true);
-        }
+        // follows redirects
+        auto response       = curl.GET(url, true);
+        const auto& headers = response.headers();
 
-        throw UserError("Download error: '" + url + "' to '" + path + "'", Here());
-    }
+        // fail early on a known status (eg. 404)
+        if (response.code() != 0) {
+            error = validate_response(url, response.code(), headers, html);
+        }
 
-    // no html response (eg. http standard response codes) in arbitrarily small files
-    if (!html) {
-        constexpr long long HTML_MAX_SIZE(32748);
+        if (error.empty()) {
+            std::ofstream out(tmp.asString(), std::ios::binary);
+            ASSERT(out);
 
-        if (length < Length{HTML_MAX_SIZE}) {
-            ASSERT(tmp.exists());
-            std::string contents{std::istreambuf_iterator<char>(std::ifstream(tmp.asString()).rdbuf()), {}};
+            std::vector<char> buffer(64 * 1024);  // 64k buffering
+            for (size_t n = 0; (n = response.read(buffer.data(), buffer.size())) > 0;) {
+                out.write(buffer.data(), static_cast<std::streamsize>(n));
+                ASSERT(out);
+                bytes += n;
+            }
 
-            if (contents.find("<!DOCTYPE html>") != std::string::npos) {
-                tmp.unlink(true);
+            // validate the final response
+            error = validate_response(url, response.code(), headers, html);
 
-                throw UserError("Download error: '" + url + "' to '" + path + "'", Here());
+            // protect against truncated transfer
+            if (auto it = headers.find("content-length"); error.empty() && it != headers.end()) {
+                if (const auto expected = Translator<std::string, unsigned long long>{}(it->second);
+                    expected != bytes) {
+                    error =
+                        "incomplete transfer (" + std::to_string(bytes) + " of " + std::to_string(expected) + " bytes)";
+                }
             }
         }
     }
+    catch (const std::exception& e) {
+        // transport errors (eg. DNS resolution, connection, SSL, partial file)
+        error = e.what();
+    }
+
+    if (error.empty() && bytes == 0) {
+        error = "empty response";
+    }
+
+    if (!error.empty()) {
+        if (tmp.exists()) {
+            tmp.unlink(true);
+        }
+
+        throw exception::DownloadError("'" + url + "' to '" + path.asString() + "': " + error, Here());
+    }
 
     PathName::rename(tmp, path);
-#endif
 
-    return {static_cast<long long>(length), timer.elapsed()};
+    return {static_cast<long long>(bytes), timer.elapsed()};
 }
+#else
+Download::info_type Download::to_path(const url_type& url, const PathName& path, bool /*html*/) {
+    throw exception::DownloadError("'" + url + "' to '" + path.asString() + "': eckit built without CURL support",
+                                   Here());
+}
+#endif
 
 
 PathName Download::to_cached_path(const url_type& url, const std::string& prefix, const std::string& suffix) const {

src/eckit/geo/cache/Download.h

@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include <map>
 #include <string>
 
 #include "eckit/filesystem/PathName.h"
@@ -19,7 +20,8 @@ class Download final : public DiskCache {
 
     // -- Types
 
-    using url_type = std::string;
+    using url_type     = std::string;
+    using headers_type = std::map<std::string, std::string>;  // keys in lowercase
 
     struct info_type {
         Bytes bytes;
@@ -43,6 +45,8 @@ class Download final : public DiskCache {
 
     static info_type to_path(const url_type&, const PathName&, bool html = false);
 
+    static std::string validate_response(const url_type&, long code, const headers_type&, bool html = false);
+
     static int version() { return VERSION; }
 
     static std::string url_file_basename(const url_type&, bool ext = true);

tests/geo/CMakeLists.txt

@@ -17,6 +17,7 @@ foreach(_test
         area_boundingbox
         area_polygon
         cache
+        download
         figure
         figure_earth
         figure_sun

tests/geo/cache.cc

@@ -8,27 +8,18 @@
 #include <string>
 
 #include "eckit/eckit_config.h"
-#include "eckit/filesystem/PathName.h"
 #include "eckit/geo/Exceptions.h"
 #include "eckit/geo/Grid.h"
-#include "eckit/geo/cache/Download.h"
 #include "eckit/geo/cache/MemoryCache.h"
 #include "eckit/geo/util.h"
 #include "eckit/log/Log.h"
 #include "eckit/spec/Custom.h"
 #include "eckit/testing/Test.h"
-#include "eckit/utils/StringTools.h"
 
 
 namespace eckit::geo::test {
 
 
-const std::string URL             = "https://www.ecmwf.int/robots.txt";
-const std::string URL_NOT_FOUND_1 = "https://does.not/exist";
-const std::string URL_NOT_FOUND_2 = "https://sites.ecmwf.int/repository/does/not/exist";
-const std::string URL_BAD_SSL     = "https://expired.badssl.com/robots.txt";
-
-
 CASE("eckit::geo::util") {
     using Cache = cache::MemoryCache;
 
@@ -115,72 +106,6 @@ CASE("eckit::geo::util") {
 
 
 #if eckit_HAVE_CURL
-CASE("download: error handling") {
-    const PathName path("test.download");
-    if (path.exists()) {
-        path.unlink();
-        ASSERT(!path.exists());
-    }
-
-    SECTION("not found") {
-        EXPECT_THROWS_AS(cache::Download::to_path(URL_NOT_FOUND_1, path), UserError);
-        EXPECT(!path.exists());
-
-        EXPECT_THROWS_AS(cache::Download::to_path(URL_NOT_FOUND_2, path), UserError);
-        EXPECT(!path.exists());
-    }
-
-    SECTION("bad ssl") {
-        EXPECT_THROWS_AS(cache::Download::to_path(URL_BAD_SSL, path), UserError);
-        EXPECT(!path.exists());
-    }
-}
-
-
-CASE("download: non-cached") {
-    const PathName path("test.download");
-    if (path.exists()) {
-        path.unlink();
-        ASSERT(!path.exists());
-    }
-
-    auto info = cache::Download::to_path(URL, path);
-
-    EXPECT(info.bytes.value() > 0.);
-    EXPECT(path.exists());
-
-    path.unlink();
-    ASSERT(!path.exists());
-}
-
-
-CASE("download: cached") {
-    const std::string prefix = "prefix-";
-    const std::string suffix = ".suffix";
-
-    const PathName root("test.download.dir", true);
-
-    cache::Download download(root);
-    EXPECT(root == download.cache_root());
-
-    download.rm_cache_root();
-    EXPECT(!root.exists());
-
-    auto path = download.to_cached_path(URL, prefix, suffix);
-
-    EXPECT(root.exists() && root.isDir());
-    EXPECT(path.exists());
-
-    std::string basename = path.baseName();
-    EXPECT(StringTools::startsWith(basename, prefix));
-    EXPECT(StringTools::endsWith(basename, suffix));
-    EXPECT(path.dirName() == root);
-
-    download.rm_cache_root();
-    EXPECT(!root.exists());
-}
-
-
 CASE("grid") {
     using Cache = cache::MemoryCache;