diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 314c5d0d5af..87d46f84b81 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -156,7 +156,7 @@ ref BinaryCacheStore::addToStoreCommon( config.compression, teeSinkCompressed, config.parallelCompression, config.compressionLevel); TeeSink teeSinkUncompressed{*compressionSink, narHashSink}; TeeSource teeSource{narSource, teeSinkUncompressed}; - narAccessor = makeNarAccessor(teeSource); + narAccessor = makeNarAccessor(parseNarListing(teeSource)); compressionSink->finish(); fileSink.flush(); } diff --git a/src/libstore/include/nix/store/remote-fs-accessor.hh b/src/libstore/include/nix/store/remote-fs-accessor.hh index ef2882e6d99..a8c5c8a0a23 100644 --- a/src/libstore/include/nix/store/remote-fs-accessor.hh +++ b/src/libstore/include/nix/store/remote-fs-accessor.hh @@ -3,6 +3,7 @@ #include "nix/util/source-accessor.hh" #include "nix/util/ref.hh" +#include "nix/util/nar-cache.hh" #include "nix/store/store-api.hh" namespace nix { @@ -13,20 +14,15 @@ class RemoteFSAccessor : public SourceAccessor /** * Map from store path hash part to NAR hash. Used to then look up - * in `nars`. The indirection allows avoiding opening multiple + * in the NAR cache. The indirection allows avoiding opening multiple * redundant NAR accessors for the same NAR. */ std::map> narHashes; - /** - * Map from NAR hash to NAR accessor. - */ - std::map> nars; + NarCache narCache; bool requireValidPath; - std::optional cacheDir; - std::pair, CanonPath> fetch(const CanonPath & path); friend struct BinaryCacheStore; diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 5c46912121d..b52baadd0d4 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -1,21 +1,13 @@ -#include #include "nix/store/remote-fs-accessor.hh" -#include "nix/util/nar-accessor.hh" - -#include -#include -#include namespace nix { RemoteFSAccessor::RemoteFSAccessor( - ref store, bool requireValidPath, std::optional cacheDir_) + ref store, bool requireValidPath, std::optional cacheDir) : store(store) + , narCache(cacheDir) , requireValidPath(requireValidPath) - , cacheDir(std::move(cacheDir_)) { - if (cacheDir) - createDirs(*cacheDir); } std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) @@ -28,72 +20,18 @@ std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPat std::shared_ptr RemoteFSAccessor::accessObject(const StorePath & storePath) { - if (auto * narHash = get(narHashes, storePath.hashPart())) { - if (auto * accessor = get(nars, *narHash)) - return *accessor; - } + // Check if we already have the NAR hash for this store path + if (auto * narHash = get(narHashes, storePath.hashPart())) + return narCache.getOrInsert(*narHash, [&](Sink & sink) { store->narFromPath(storePath, sink); }); + // Query the path info to get the NAR hash auto info = store->queryPathInfo(storePath); - auto cacheAccessor = [&](ref accessor) { - narHashes.emplace(storePath.hashPart(), info->narHash); - nars.emplace(info->narHash, accessor); - return accessor; - }; - - auto getNar = [&]() { - StringSink sink; - store->narFromPath(storePath, sink); - return std::move(sink.s); - }; - - if (cacheDir) { - auto makeCacheFile = [&](const std::string & ext) { - auto res = *cacheDir / info->narHash.to_string(HashFormat::Nix32, false); - res += "."; - res += ext; - return res; - }; - - auto cacheFile = makeCacheFile("nar"); - auto listingFile = makeCacheFile("ls"); - - if (nix::pathExists(cacheFile)) { - try { - return cacheAccessor(makeLazyNarAccessor( - nlohmann::json::parse(nix::readFile(listingFile)).template get(), - seekableGetNarBytes(cacheFile))); - } catch (SystemError &) { - } - - try { - return cacheAccessor(makeNarAccessor(nix::readFile(cacheFile))); - } catch (SystemError &) { - } - } - - auto nar = getNar(); - - try { - /* FIXME: do this asynchronously. */ - writeFile(cacheFile, nar); - } catch (...) { - ignoreExceptionExceptInterrupt(); - } - - auto narAccessor = makeNarAccessor(std::move(nar)); - - try { - nlohmann::json j = narAccessor->getListing(); - writeFile(listingFile, j.dump()); - } catch (...) { - ignoreExceptionExceptInterrupt(); - } - - return cacheAccessor(narAccessor); - } + // Cache the mapping from store path to NAR hash + narHashes.emplace(storePath.hashPart(), info->narHash); - return cacheAccessor(makeNarAccessor(getNar())); + // Get or create the NAR accessor + return narCache.getOrInsert(info->narHash, [&](Sink & sink) { store->narFromPath(storePath, sink); }); } std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 1ab2c47e47d..be8e603ca57 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -53,6 +53,7 @@ headers = files( 'mounted-source-accessor.hh', 'muxable-pipe.hh', 'nar-accessor.hh', + 'nar-cache.hh', 'nar-listing.hh', 'os-string.hh', 'pool.hh', diff --git a/src/libutil/include/nix/util/nar-accessor.hh b/src/libutil/include/nix/util/nar-accessor.hh index d01302e901e..60520066642 100644 --- a/src/libutil/include/nix/util/nar-accessor.hh +++ b/src/libutil/include/nix/util/nar-accessor.hh @@ -27,7 +27,12 @@ struct NarAccessor : SourceAccessor */ ref makeNarAccessor(std::string && nar); -ref makeNarAccessor(Source & source); +/** + * This NAR accessor doesn't actually access a NAR, and thus cannot read + * the contents of files. It just conveys the information which is + * gotten from `listing`. + */ +ref makeNarAccessor(NarListing listing); /** * Create a NAR accessor from a NAR listing (in the format produced by @@ -44,12 +49,9 @@ GetNarBytes seekableGetNarBytes(const std::filesystem::path & path); GetNarBytes seekableGetNarBytes(Descriptor fd); -ref makeLazyNarAccessor(NarListing listing, GetNarBytes getNarBytes); - /** - * Creates a NAR accessor from a given stream and a GetNarBytes getter. - * @param source Consumed eagerly. References to it are not persisted in the resulting SourceAccessor. + * Creates a NAR accessor from a given listing and a `GetNarBytes` getter. */ -ref makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes); +ref makeLazyNarAccessor(NarListing listing, GetNarBytes getNarBytes); } // namespace nix diff --git a/src/libutil/include/nix/util/nar-cache.hh b/src/libutil/include/nix/util/nar-cache.hh new file mode 100644 index 00000000000..0f5ac2d530f --- /dev/null +++ b/src/libutil/include/nix/util/nar-cache.hh @@ -0,0 +1,47 @@ +#pragma once + +#include "nix/util/hash.hh" +#include "nix/util/nar-accessor.hh" +#include "nix/util/ref.hh" +#include "nix/util/source-accessor.hh" + +#include +#include +#include +#include + +namespace nix { + +/** + * A cache for NAR accessors with optional disk caching. + */ +class NarCache +{ + /** + * Optional directory for caching NARs and listings on disk. + */ + std::optional cacheDir; + + /** + * Map from NAR hash to NAR accessor. + */ + std::map> nars; + +public: + + /** + * Create a NAR cache with an optional cache directory for disk storage. + */ + NarCache(std::optional cacheDir = {}); + + /** + * Lookup or create a NAR accessor, optionally using disk cache. + * + * @param narHash The NAR hash to use as cache key + * @param populate Function called with a Sink to populate the NAR if not cached + * @return The cached or newly created accessor + */ + ref getOrInsert(const Hash & narHash, std::function populate); +}; + +} // namespace nix diff --git a/src/libutil/meson.build b/src/libutil/meson.build index fdd310ad8e4..f91605de932 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -151,6 +151,7 @@ sources = [ config_priv_h ] + files( 'memory-source-accessor/json.cc', 'mounted-source-accessor.cc', 'nar-accessor.cc', + 'nar-cache.cc', 'nar-listing.cc', 'pos-table.cc', 'position.cc', diff --git a/src/libutil/nar-accessor.cc b/src/libutil/nar-accessor.cc index d67c5ecd13c..fb319dc879c 100644 --- a/src/libutil/nar-accessor.cc +++ b/src/libutil/nar-accessor.cc @@ -34,14 +34,8 @@ struct NarAccessorImpl : NarAccessor { } - NarAccessorImpl(Source & source) - : root{parseNarListing(source)} - { - } - - NarAccessorImpl(Source & source, GetNarBytes getNarBytes) - : root{parseNarListing(source)} - , getNarBytes{std::move(getNarBytes)} + NarAccessorImpl(NarListing && listing) + : root{std::move(listing)} { } @@ -147,9 +141,9 @@ ref makeNarAccessor(std::string && nar) return make_ref(std::move(nar)); } -ref makeNarAccessor(Source & source) +ref makeNarAccessor(NarListing listing) { - return make_ref(source); + return make_ref(std::move(listing)); } ref makeLazyNarAccessor(NarListing listing, GetNarBytes getNarBytes) @@ -157,11 +151,6 @@ ref makeLazyNarAccessor(NarListing listing, GetNarBytes getNarBytes return make_ref(std::move(listing), getNarBytes); } -ref makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes) -{ - return make_ref(source, getNarBytes); -} - GetNarBytes seekableGetNarBytes(const std::filesystem::path & path) { AutoCloseFD fd = openFileReadonly(path); diff --git a/src/libutil/nar-cache.cc b/src/libutil/nar-cache.cc new file mode 100644 index 00000000000..ba5e60ced2f --- /dev/null +++ b/src/libutil/nar-cache.cc @@ -0,0 +1,84 @@ +#include "nix/util/nar-cache.hh" +#include "nix/util/file-system.hh" + +#include +#include +#include +#include + +namespace nix { + +NarCache::NarCache(std::optional cacheDir_) + : cacheDir(std::move(cacheDir_)) +{ + if (cacheDir) + createDirs(*cacheDir); +} + +ref NarCache::getOrInsert(const Hash & narHash, std::function populate) +{ + // Check in-memory cache first + if (auto * accessor = get(nars, narHash)) + return *accessor; + + auto cacheAccessor = [&](ref accessor) { + nars.emplace(narHash, accessor); + return accessor; + }; + + auto getNar = [&]() { + StringSink sink; + populate(sink); + return std::move(sink.s); + }; + + if (cacheDir) { + auto makeCacheFile = [&](const std::string & ext) { + auto res = *cacheDir / narHash.to_string(HashFormat::Nix32, false); + res += "."; + res += ext; + return res; + }; + + auto cacheFile = makeCacheFile("nar"); + auto listingFile = makeCacheFile("ls"); + + if (nix::pathExists(cacheFile)) { + try { + return cacheAccessor(makeLazyNarAccessor( + nlohmann::json::parse(nix::readFile(listingFile)).template get(), + seekableGetNarBytes(cacheFile))); + } catch (SystemError &) { + } + + try { + return cacheAccessor(makeNarAccessor(nix::readFile(cacheFile))); + } catch (SystemError &) { + } + } + + auto nar = getNar(); + + try { + /* FIXME: do this asynchronously. */ + writeFile(cacheFile, nar); + } catch (...) { + ignoreExceptionExceptInterrupt(); + } + + auto narAccessor = makeNarAccessor(std::move(nar)); + + try { + nlohmann::json j = narAccessor->getListing(); + writeFile(listingFile, j.dump()); + } catch (...) { + ignoreExceptionExceptInterrupt(); + } + + return cacheAccessor(narAccessor); + } + + return cacheAccessor(makeNarAccessor(getNar())); +} + +} // namespace nix diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 0199feda7c4..bf24e285ed0 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -154,7 +154,7 @@ struct CmdLsNar : Command, MixLs if (!fd) throw NativeSysError("opening NAR file %s", PathFmt(narPath)); auto source = FdSource{fd.get()}; - list(makeLazyNarAccessor(source, seekableGetNarBytes(fd.get())), CanonPath{path}); + list(makeLazyNarAccessor(parseNarListing(source), seekableGetNarBytes(fd.get())), CanonPath{path}); } };