Skip to content

Commit f11d278

Browse files
edolstraedolstra
committed
Factor out the local NAR cache into its own class
1 parent 6f8b214 commit f11d278

8 files changed

Lines changed: 142 additions & 80 deletions

File tree

‎src/libstore/binary-cache-store.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "nix/util/callback.hh"
1414
#include "nix/util/signals.hh"
1515
#include "nix/util/archive.hh"
16+
#include "nix/store/nar-cache.hh"
1617

1718
#include <chrono>
1819
#include <future>
@@ -26,6 +27,7 @@ namespace nix {
2627

2728
BinaryCacheStore::BinaryCacheStore(Config & config)
2829
: config{config}
30+
, narCache{config.localNarCache.get() ? std::make_shared<NarCache>(*config.localNarCache.get()) : nullptr}
2931
{
3032
if (config.secretKeyFile != "")
3133
signers.push_back(std::make_unique<LocalSigner>(SecretKey{readFile(config.secretKeyFile)}));
@@ -552,7 +554,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info)
552554

553555
ref<RemoteFSAccessor> BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath)
554556
{
555-
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, config.localNarCache);
557+
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, narCache);
556558
}
557559

558560
ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)

‎src/libstore/include/nix/store/binary-cache-store.hh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
namespace nix {
1313

1414
struct NarInfo;
15+
class NarCache;
1516
class RemoteFSAccessor;
1617

1718
struct BinaryCacheStoreConfig : virtual StoreConfig
@@ -89,6 +90,8 @@ protected:
8990

9091
constexpr const static std::string cacheInfoFile = "nix-cache-info";
9192

93+
std::shared_ptr<NarCache> narCache;
94+
9295
BinaryCacheStore(Config &);
9396

9497
/**

‎src/libstore/include/nix/store/meson.build‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ headers = [ config_pub_h ] + files(
5757
'machines.hh',
5858
'make-content-addressed.hh',
5959
'names.hh',
60+
'nar-cache.hh',
6061
'nar-info-disk-cache.hh',
6162
'nar-info.hh',
6263
'outputs-spec.hh',

‎src/libstore/include/nix/store/nar-cache.hh‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#pragma once
2+
3+
#include "nix/util/hash.hh"
4+
#include "nix/util/nar-accessor.hh"
5+
6+
#include <filesystem>
7+
8+
namespace nix {
9+
10+
class NarCache
11+
{
12+
13+
const std::filesystem::path cacheDir;
14+
15+
std::filesystem::path makeCacheFile(const Hash & narHash, const std::string & ext);
16+
17+
public:
18+
19+
NarCache(std::filesystem::path cacheDir);
20+
21+
void upsertNar(const Hash & narHash, Source & source);
22+
23+
void upsertNarListing(const Hash & narHash, std::string_view narListingData);
24+
25+
// FIXME: use a sink.
26+
std::optional<std::string> getNar(const Hash & narHash);
27+
28+
// FIXME: use a sink.
29+
GetNarBytes getNarBytes(const Hash & narHash);
30+
31+
std::optional<std::string> getNarListing(const Hash & narHash);
32+
};
33+
34+
} // namespace nix

‎src/libstore/include/nix/store/remote-fs-accessor.hh‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77

88
namespace nix {
99

10+
struct NarCache;
11+
1012
class RemoteFSAccessor : public SourceAccessor
1113
{
1214
ref<Store> store;
@@ -15,29 +17,20 @@ class RemoteFSAccessor : public SourceAccessor
1517

1618
bool requireValidPath;
1719

18-
std::optional<std::filesystem::path> cacheDir;
20+
std::shared_ptr<NarCache> narCache;
1921

2022
std::pair<ref<SourceAccessor>, CanonPath> fetch(const CanonPath & path);
2123

2224
friend struct BinaryCacheStore;
2325

24-
std::filesystem::path makeCacheFile(const Hash & narHash, const std::string & ext);
25-
26-
ref<SourceAccessor> addToCache(
27-
std::string_view hashPart,
28-
const std::filesystem::path & cacheFile,
29-
const std::filesystem::path & listingFile,
30-
std::string && nar);
31-
3226
public:
3327

3428
/**
3529
* @return nullptr if the store does not contain any object at that path.
3630
*/
3731
std::shared_ptr<SourceAccessor> accessObject(const StorePath & path);
3832

39-
RemoteFSAccessor(
40-
ref<Store> store, bool requireValidPath = true, std::optional<std::filesystem::path> cacheDir = {});
33+
RemoteFSAccessor(ref<Store> store, bool requireValidPath = true, std::shared_ptr<NarCache> narCache = {});
4134

4235
std::optional<Stat> maybeLstat(const CanonPath & path) override;
4336

‎src/libstore/meson.build‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,7 @@ sources = files(
319319
'make-content-addressed.cc',
320320
'misc.cc',
321321
'names.cc',
322+
'nar-cache.cc',
322323
'nar-info-disk-cache.cc',
323324
'nar-info.cc',
324325
'optimise-store.cc',

‎src/libstore/nar-cache.cc‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#include "nix/store/nar-cache.hh"
2+
#include "nix/util/file-system.hh"
3+
4+
#include <sys/types.h>
5+
#include <sys/stat.h>
6+
#include <fcntl.h>
7+
8+
namespace nix {
9+
10+
NarCache::NarCache(std::filesystem::path cacheDir_)
11+
: cacheDir(std::move(cacheDir_))
12+
{
13+
assert(!cacheDir.empty());
14+
createDirs(cacheDir);
15+
}
16+
17+
std::filesystem::path NarCache::makeCacheFile(const Hash & narHash, const std::string & ext)
18+
{
19+
return (cacheDir / narHash.to_string(HashFormat::Nix32, false)) + "." + ext;
20+
}
21+
22+
void NarCache::upsertNar(const Hash & narHash, Source & source)
23+
{
24+
try {
25+
/* FIXME: do this asynchronously. */
26+
writeFile(makeCacheFile(narHash, "nar"), source);
27+
} catch (SystemError &) {
28+
ignoreExceptionExceptInterrupt();
29+
}
30+
}
31+
32+
void NarCache::upsertNarListing(const Hash & narHash, std::string_view narListingData)
33+
{
34+
try {
35+
writeFile(makeCacheFile(narHash, "ls"), narListingData);
36+
} catch (SystemError &) {
37+
ignoreExceptionExceptInterrupt();
38+
}
39+
}
40+
41+
std::optional<std::string> NarCache::getNar(const Hash & narHash)
42+
{
43+
try {
44+
return nix::readFile(makeCacheFile(narHash, "nar"));
45+
} catch (SystemError &) {
46+
return std::nullopt;
47+
}
48+
}
49+
50+
GetNarBytes NarCache::getNarBytes(const Hash & narHash)
51+
{
52+
return seekableGetNarBytes(makeCacheFile(narHash, "nar"));
53+
}
54+
55+
std::optional<std::string> NarCache::getNarListing(const Hash & narHash)
56+
{
57+
try {
58+
return nix::readFile(makeCacheFile(narHash, "ls"));
59+
} catch (SystemError &) {
60+
return std::nullopt;
61+
}
62+
}
63+
64+
} // namespace nix

‎src/libstore/remote-fs-accessor.cc‎

Lines changed: 32 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,14 @@
11
#include <nlohmann/json.hpp>
22
#include "nix/store/remote-fs-accessor.hh"
3-
#include "nix/util/nar-accessor.hh"
4-
5-
#include <sys/types.h>
6-
#include <sys/stat.h>
7-
#include <fcntl.h>
3+
#include "nix/store/nar-cache.hh"
84

95
namespace nix {
106

11-
RemoteFSAccessor::RemoteFSAccessor(
12-
ref<Store> store, bool requireValidPath, std::optional<std::filesystem::path> cacheDir_)
7+
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, bool requireValidPath, std::shared_ptr<NarCache> narCache)
138
: store(store)
149
, requireValidPath(requireValidPath)
15-
, cacheDir(std::move(cacheDir_))
16-
{
17-
if (cacheDir)
18-
createDirs(*cacheDir);
19-
}
20-
21-
std::filesystem::path RemoteFSAccessor::makeCacheFile(const Hash & narHash, const std::string & ext)
22-
{
23-
assert(cacheDir);
24-
return (*cacheDir / narHash.to_string(HashFormat::Nix32, false)) + "." + ext;
25-
}
26-
27-
ref<SourceAccessor> RemoteFSAccessor::addToCache(
28-
std::string_view hashPart,
29-
const std::filesystem::path & cacheFile,
30-
const std::filesystem::path & listingFile,
31-
std::string && nar)
10+
, narCache(std::move(narCache))
3211
{
33-
if (!cacheFile.empty()) {
34-
try {
35-
/* FIXME: do this asynchronously. */
36-
writeFile(cacheFile, nar);
37-
} catch (...) {
38-
ignoreExceptionExceptInterrupt();
39-
}
40-
}
41-
42-
auto narAccessor = makeNarAccessor(std::move(nar));
43-
nars.emplace(hashPart, narAccessor);
44-
45-
if (!listingFile.empty()) {
46-
try {
47-
nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
48-
writeFile(listingFile, j.dump());
49-
} catch (...) {
50-
ignoreExceptionExceptInterrupt();
51-
}
52-
}
53-
54-
return narAccessor;
5512
}
5613

5714
std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path)
@@ -68,36 +25,43 @@ std::shared_ptr<SourceAccessor> RemoteFSAccessor::accessObject(const StorePath &
6825
if (i != nars.end())
6926
return i->second;
7027

71-
std::filesystem::path cacheFile, listingFile;
28+
Hash narHash{HashAlgorithm::SHA256};
7229

73-
if (cacheDir) {
30+
if (narCache) {
7431
auto info = store->queryPathInfo(storePath);
32+
narHash = info->narHash;
7533

76-
cacheFile = makeCacheFile(info->narHash, "nar");
77-
listingFile = makeCacheFile(info->narHash, "ls");
78-
79-
if (nix::pathExists(cacheFile)) {
80-
try {
81-
auto listing = nix::readFile(listingFile);
82-
auto listingJson = nlohmann::json::parse(listing);
83-
auto narAccessor = makeLazyNarAccessor(listingJson, seekableGetNarBytes(cacheFile));
84-
nars.emplace(storePath.hashPart(), narAccessor);
85-
return narAccessor;
86-
} catch (SystemError &) {
87-
}
88-
89-
try {
90-
auto narAccessor = makeNarAccessor(nix::readFile(cacheFile));
91-
nars.emplace(storePath.hashPart(), narAccessor);
92-
return narAccessor;
93-
} catch (SystemError &) {
94-
}
34+
if (auto listingData = narCache->getNarListing(narHash)) {
35+
auto listingJson = nlohmann::json::parse(*listingData);
36+
auto narAccessor = makeLazyNarAccessor(listingJson, narCache->getNarBytes(narHash));
37+
nars.emplace(storePath.hashPart(), narAccessor);
38+
return narAccessor;
39+
}
40+
41+
if (auto nar = narCache->getNar(narHash)) {
42+
auto narAccessor = makeNarAccessor(std::move(*nar));
43+
nars.emplace(storePath.hashPart(), narAccessor);
44+
return narAccessor;
9545
}
9646
}
9747

9848
StringSink sink;
9949
store->narFromPath(storePath, sink);
100-
return addToCache(storePath.hashPart(), cacheFile, listingFile, std::move(sink.s));
50+
51+
if (narCache) {
52+
StringSource source{sink.s};
53+
narCache->upsertNar(narHash, source);
54+
}
55+
56+
auto narAccessor = makeNarAccessor(std::move(sink.s));
57+
nars.emplace(storePath.hashPart(), narAccessor);
58+
59+
if (narCache) {
60+
nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
61+
narCache->upsertNarListing(narHash, j.dump());
62+
}
63+
64+
return narAccessor;
10165
}
10266

10367
std::optional<SourceAccessor::Stat> RemoteFSAccessor::maybeLstat(const CanonPath & path)

0 commit comments

Comments
 (0)