BinaryCacheStore: Allow using NARs from the local NAR cache

src/libstore/binary-cache-store.cc

@@ -13,6 +13,7 @@
 #include "nix/util/callback.hh"
 #include "nix/util/signals.hh"
 #include "nix/util/archive.hh"
+#include "nix/store/nar-cache.hh"
 
 #include <chrono>
 #include <future>
@@ -26,6 +27,7 @@ namespace nix {
 
 BinaryCacheStore::BinaryCacheStore(Config & config)
     : config{config}
+    , narCache{config.localNarCache.get() ? std::make_shared<NarCache>(*config.localNarCache.get()) : nullptr}
 {
     if (config.secretKeyFile != "")
         signers.push_back(std::make_unique<LocalSigner>(SecretKey{readFile(config.secretKeyFile)}));
@@ -406,11 +408,27 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
 {
     auto info = queryPathInfo(storePath).cast<const NarInfo>();
 
+    if (narCache) {
+        if (auto nar = narCache->getNar(info->narHash)) {
+            notice("substituted '%s' from local NAR cache", printStorePath(storePath));
+            sink(*nar);
+            stats.narRead++;
+            stats.narReadBytes += nar->size();
+            return;
+        }
+    }
+
+    std::unique_ptr<Sink> narCacheSink;
+    if (narCache)
+        narCacheSink = sourceToSink([&](Source & source) { narCache->upsertNar(info->narHash, source); });
+
     uint64_t narSize = 0;
 
     LambdaSink uncompressedSink{
         [&](std::string_view data) {
             narSize += data.size();
+            if (narCacheSink)
+                (*narCacheSink)(data);
             sink(data);
         },
         [&]() {
@@ -552,7 +570,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info)
 
 ref<RemoteFSAccessor> BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath)
 {
-    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, config.localNarCache);
+    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, narCache);
 }
 
 ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)

src/libstore/include/nix/store/binary-cache-store.hh

@@ -12,6 +12,7 @@
 namespace nix {
 
 struct NarInfo;
+class NarCache;
 class RemoteFSAccessor;
 
 struct BinaryCacheStoreConfig : virtual StoreConfig
@@ -38,9 +39,9 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
     const Setting<std::string> secretKeyFiles{
         this, "", "secret-keys", "List of comma-separated paths to the secret keys used to sign the binary cache."};
 
-    const Setting<Path> localNarCache{
+    const Setting<std::optional<std::filesystem::path>> localNarCache{
         this,
-        "",
+        std::nullopt,
         "local-nar-cache",
         "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
 
@@ -89,6 +90,8 @@ protected:
 
     constexpr const static std::string cacheInfoFile = "nix-cache-info";
 
+    std::shared_ptr<NarCache> narCache;
+
     BinaryCacheStore(Config &);
 
     /**

src/libstore/include/nix/store/meson.build

@@ -57,6 +57,7 @@ headers = [ config_pub_h ] + files(
   'machines.hh',
   'make-content-addressed.hh',
   'names.hh',
+  'nar-cache.hh',
   'nar-info-disk-cache.hh',
   'nar-info.hh',
   'outputs-spec.hh',

src/libstore/include/nix/store/nar-cache.hh

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "nix/util/hash.hh"
+#include "nix/util/nar-accessor.hh"
+
+#include <filesystem>
+
+namespace nix {
+
+class NarCache
+{
+
+    const std::filesystem::path cacheDir;
+
+    std::filesystem::path makeCacheFile(const Hash & narHash, const std::string & ext);
+
+public:
+
+    NarCache(std::filesystem::path cacheDir);
+
+    void upsertNar(const Hash & narHash, Source & source);
+
+    void upsertNarListing(const Hash & narHash, std::string_view narListingData);
+
+    // FIXME: use a sink.
+    std::optional<std::string> getNar(const Hash & narHash);
+
+    // FIXME: use a sink.
+    GetNarBytes getNarBytes(const Hash & narHash);
+
+    std::optional<std::string> getNarListing(const Hash & narHash);
+};
+
+} // namespace nix

src/libstore/include/nix/store/remote-fs-accessor.hh

@@ -7,6 +7,8 @@
 
 namespace nix {
 
+struct NarCache;
+
 class RemoteFSAccessor : public SourceAccessor
 {
     ref<Store> store;
@@ -15,25 +17,20 @@ class RemoteFSAccessor : public SourceAccessor
 
     bool requireValidPath;
 
-    Path cacheDir;
+    std::shared_ptr<NarCache> narCache;
 
     std::pair<ref<SourceAccessor>, CanonPath> fetch(const CanonPath & path);
 
     friend struct BinaryCacheStore;
 
-    Path makeCacheFile(std::string_view hashPart, const std::string & ext);
-
-    ref<SourceAccessor> addToCache(std::string_view hashPart, std::string && nar);
-
 public:
 
     /**
      * @return nullptr if the store does not contain any object at that path.
      */
     std::shared_ptr<SourceAccessor> accessObject(const StorePath & path);
 
-    RemoteFSAccessor(
-        ref<Store> store, bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = "");
+    RemoteFSAccessor(ref<Store> store, bool requireValidPath = true, std::shared_ptr<NarCache> narCache = {});
 
     std::optional<Stat> maybeLstat(const CanonPath & path) override;
 

src/libstore/meson.build

@@ -319,6 +319,7 @@ sources = files(
   'make-content-addressed.cc',
   'misc.cc',
   'names.cc',
+  'nar-cache.cc',
   'nar-info-disk-cache.cc',
   'nar-info.cc',
   'optimise-store.cc',

src/libstore/nar-cache.cc

@@ -0,0 +1,64 @@
+#include "nix/store/nar-cache.hh"
+#include "nix/util/file-system.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+namespace nix {
+
+NarCache::NarCache(std::filesystem::path cacheDir_)
+    : cacheDir(std::move(cacheDir_))
+{
+    assert(!cacheDir.empty());
+    createDirs(cacheDir);
+}
+
+std::filesystem::path NarCache::makeCacheFile(const Hash & narHash, const std::string & ext)
+{
+    return (cacheDir / narHash.to_string(HashFormat::Nix32, false)) + "." + ext;
+}
+
+void NarCache::upsertNar(const Hash & narHash, Source & source)
+{
+    try {
+        /* FIXME: do this asynchronously. */
+        writeFile(makeCacheFile(narHash, "nar"), source);
+    } catch (SystemError &) {
+        ignoreExceptionExceptInterrupt();
+    }
+}
+
+void NarCache::upsertNarListing(const Hash & narHash, std::string_view narListingData)
+{
+    try {
+        writeFile(makeCacheFile(narHash, "ls"), narListingData);
+    } catch (SystemError &) {
+        ignoreExceptionExceptInterrupt();
+    }
+}
+
+std::optional<std::string> NarCache::getNar(const Hash & narHash)
+{
+    try {
+        return nix::readFile(makeCacheFile(narHash, "nar"));
+    } catch (SystemError &) {
+        return std::nullopt;
+    }
+}
+
+GetNarBytes NarCache::getNarBytes(const Hash & narHash)
+{
+    return seekableGetNarBytes(makeCacheFile(narHash, "nar"));
+}
+
+std::optional<std::string> NarCache::getNarListing(const Hash & narHash)
+{
+    try {
+        return nix::readFile(makeCacheFile(narHash, "ls"));
+    } catch (SystemError &) {
+        return std::nullopt;
+    }
+}
+
+} // namespace nix

src/libstore/remote-fs-accessor.cc

@@ -1,52 +1,14 @@
 #include <nlohmann/json.hpp>
 #include "nix/store/remote-fs-accessor.hh"
-#include "nix/util/nar-accessor.hh"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
+#include "nix/store/nar-cache.hh"
 
 namespace nix {
 
-RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, bool requireValidPath, const Path & cacheDir)
+RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, bool requireValidPath, std::shared_ptr<NarCache> narCache)
     : store(store)
     , requireValidPath(requireValidPath)
-    , cacheDir(cacheDir)
-{
-    if (cacheDir != "")
-        createDirs(cacheDir);
-}
-
-Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
-{
-    assert(cacheDir != "");
-    return fmt("%s/%s.%s", cacheDir, hashPart, ext);
-}
-
-ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)
+    , narCache(std::move(narCache))
 {
-    if (cacheDir != "") {
-        try {
-            /* FIXME: do this asynchronously. */
-            writeFile(makeCacheFile(hashPart, "nar"), nar);
-        } catch (...) {
-            ignoreExceptionExceptInterrupt();
-        }
-    }
-
-    auto narAccessor = makeNarAccessor(std::move(nar));
-    nars.emplace(hashPart, narAccessor);
-
-    if (cacheDir != "") {
-        try {
-            nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
-            writeFile(makeCacheFile(hashPart, "ls"), j.dump());
-        } catch (...) {
-            ignoreExceptionExceptInterrupt();
-        }
-    }
-
-    return narAccessor;
 }
 
 std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path)
@@ -63,33 +25,43 @@ std::shared_ptr<SourceAccessor> RemoteFSAccessor::accessObject(const StorePath &
     if (i != nars.end())
         return i->second;
 
-    std::string listing;
-    Path cacheFile;
+    Hash narHash{HashAlgorithm::SHA256};
 
-    if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
-
-        try {
-            listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
-            auto listingJson = nlohmann::json::parse(listing);
-            auto narAccessor = makeLazyNarAccessor(listingJson, seekableGetNarBytes(cacheFile));
+    if (narCache) {
+        auto info = store->queryPathInfo(storePath);
+        narHash = info->narHash;
 
+        if (auto listingData = narCache->getNarListing(narHash)) {
+            auto listingJson = nlohmann::json::parse(*listingData);
+            auto narAccessor = makeLazyNarAccessor(listingJson, narCache->getNarBytes(narHash));
             nars.emplace(storePath.hashPart(), narAccessor);
             return narAccessor;
-
-        } catch (SystemError &) {
         }
 
-        try {
-            auto narAccessor = makeNarAccessor(nix::readFile(cacheFile));
+        if (auto nar = narCache->getNar(narHash)) {
+            auto narAccessor = makeNarAccessor(std::move(*nar));
             nars.emplace(storePath.hashPart(), narAccessor);
             return narAccessor;
-        } catch (SystemError &) {
         }
     }
 
     StringSink sink;
     store->narFromPath(storePath, sink);
-    return addToCache(storePath.hashPart(), std::move(sink.s));
+
+    if (narCache) {
+        StringSource source{sink.s};
+        narCache->upsertNar(narHash, source);
+    }
+
+    auto narAccessor = makeNarAccessor(std::move(sink.s));
+    nars.emplace(storePath.hashPart(), narAccessor);
+
+    if (narCache) {
+        nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
+        narCache->upsertNarListing(narHash, j.dump());
+    }
+
+    return narAccessor;
 }
 
 std::optional<SourceAccessor::Stat> RemoteFSAccessor::maybeLstat(const CanonPath & path)

tests/functional/binary-cache.sh

@@ -233,13 +233,23 @@ mkdir "$narCache"
 
 [[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]]
 
-rm -rfv "$cacheDir/nar"
+mv "$cacheDir/nar" "$cacheDir/nar2"
 
 [[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]]
 
 (! nix store cat --store "file://$cacheDir" "$outPath/foobar")
 
 
+# Check substitution from the local NAR cache.
+clearStore
+rm -rf "$narCache" "$cacheDir/nar"
+mv "$cacheDir/nar2" "$cacheDir/nar"
+nix-store -r --substituters "file://$cacheDir?local-nar-cache=$narCache" --no-require-sigs "$outPath"
+mv "$cacheDir/nar" "$cacheDir/nar2"
+clearStore
+nix-store -r --substituters "file://$cacheDir?local-nar-cache=$narCache" --no-require-sigs "$outPath"
+
+
 # Test NAR listing generation.
 clearCache