aboutsummaryrefslogtreecommitdiff
path: root/libbuild2/file-cache.hxx
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2021-03-16 11:36:10 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2021-03-16 11:36:10 +0200
commit7a4f1ae238b918fe21671d0d1a5549aeb1fe6425 (patch)
tree6501d18854cc85644a3e73a857fda82b17b73c45 /libbuild2/file-cache.hxx
parent99046d23d78d341674bbad3414567f362ffc60cd (diff)
Define intermediate build results file cache interface
Diffstat (limited to 'libbuild2/file-cache.hxx')
-rw-r--r--libbuild2/file-cache.hxx209
1 files changed, 209 insertions, 0 deletions
diff --git a/libbuild2/file-cache.hxx b/libbuild2/file-cache.hxx
new file mode 100644
index 0000000..4486307
--- /dev/null
+++ b/libbuild2/file-cache.hxx
@@ -0,0 +1,209 @@
+// file : libbuild2/file-cache.hxx -*- C++ -*-
+// license : MIT; see accompanying LICENSE file
+
+#ifndef LIBBUILD2_FILE_CACHE_HXX
+#define LIBBUILD2_FILE_CACHE_HXX
+
+#include <libbuild2/types.hxx>
+#include <libbuild2/forward.hxx>
+#include <libbuild2/utility.hxx>
+
+#include <libbuild2/export.hxx>
+
+namespace build2
+{
+ // We sometimes have intermediate build results that must be stored and
+ // accessed as files (for example, partially-preprocessed C/C++ translation
+ // units; those .i/.ii files). These files can be quite large which can lead
+ // to excessive disk usage (for example, the .ii files can be several MB
+ // each and can end up dominating object file sizes in a build with debug
+ // information). These files are also often temporary which means writing
+ // them to disk is really a waste.
+ //
+ // The file cache attempts to address this by still presenting a file-like
+ // entry (which can be a real file or a named pipe) but potentially storing
+ // the file contents in memory and/or compressed.
+ //
+ // Each cache entry is identified by the filesystem entry path that will be
+ // written to or read from. The file cache reserves a filesystem entry path
+ // that is derived by adding a compression extension to the main entry path
+ // (for example, .ii.lz4). When cleaning intermediate build results that are
+ // managed by the cache, the rule must clean such a reserved path in
+ // addition to the main entry path (see compressed_extension() below).
+ //
+ // While the cache is MT-safe (that is, we can insert multiple entries
+ // concurrently), each entry is expected to be accessed serially by a single
+ // thread. Furthermore, each entry can either be written to or read from at
+ // any give time and it can only be read from by a single reader at a time.
+ // In other words, there meant to be a single cache entry for any given path
+ // and it is not meant to be shared.
+ //
+ // The underlying filesystem entry can be either temporary or permanent. A
+ // temporary entry only exists during the build, normally between the match
+ // and execute phases. A permanent entry exists across builds. Note,
+ // however, that a permanent entry is often removed in cases of an error and
+ // sometimes a temporary entry is left behind for diagnostics. It is also
+ // possible that the distinction only becomes known some time after the
+ // entry has been created. As a result, all entries by default start as
+ // temporary and can later be made permanent if desired.
+ //
+ // A cache entry can be pinned or unpinned. A cache entry is created pinned.
+ // A cache entry being written to or read from remains pinned.
+ //
+ // An unpinned entry can be preempted. Preempting a cache entry can mean any
+ // of the following:
+ //
+ // - An in-memory content is compressed (but stays in memory).
+ //
+ // - An in-memory content (compressed or not) is flushed to disk (with or
+ // without compression).
+ //
+ // - An uncompressed on-disk content is compressed.
+ //
+ // Naturally, any of the above degrees of preemption make accessing the
+ // contents of a cache entry slower. Note also that pinned/unpinned and
+ // temporary/permanent are independent and a temporary entry does not need
+ // to be unpinned to be removed.
+ //
+ // After creation, a cache entry must be initialized by either writing new
+ // contents to the filesystem entry or by using an existing (permanent)
+ // filesystem entry. Once initialized, an entry can be opened for reading,
+ // potentially multiple times.
+ //
+ // Note also that a noop implementation of this caching semantics (that is,
+ // one that simply saves the file on disk) is file_cache::entry that is just
+ // auto_rmfile.
+ //
+ class /*LIBBUILD2_SYMEXPORT*/ file_cache
+ {
+ public:
+
+ // A cache entry write handle. During the lifetime of this object the
+ // filesystem entry can be opened for writing and written to.
+ //
+ // A successful write must be terminated with an explicit call to close()
+ // (similar semantics to ofdstream). A write handle that is destroyed
+ // without a close() call is treated as an unsuccessful write and the
+ // initialization can be attempted again.
+ //
+ struct write
+ {
+ void close () {}
+ };
+
+ // A cache entry read handle. During the lifetime of this object the
+ // filesystem entry can be opened for reading and read from.
+ //
+ struct read
+ {
+ };
+
+ // A cache entry handle. When it is destroyed, a temporary entry is
+ // automatically removed from the filesystem.
+ //
+ struct entry
+ {
+ using path_type = build2::path;
+
+ bool temporary;
+
+ // The returned reference is valid and stable for the lifetime of the
+ // entry handle.
+ //
+ const path_type&
+ path () const {return path_;}
+
+ // Initialization.
+ //
+ write
+ init_new ()
+ {
+ return write ();
+ }
+
+ void
+ init_existing () {}
+
+ // Reading.
+ //
+ read
+ open ()
+ {
+ return read ();
+ }
+
+ // Pinning.
+ //
+ void
+ pin () {}
+
+ void
+ unpin () {}
+
+ // NULL entry handle.
+ //
+ entry () = default;
+
+ explicit operator bool () const
+ {
+ return !path_.empty ();
+ }
+
+ // Move-to-NULL-entry-only type.
+ //
+ entry (entry&&);
+ entry (const entry&) = delete;
+ entry& operator= (entry&&);
+ entry& operator= (const entry&) = delete;
+
+ // Implementation details.
+ //
+ entry (path_type, bool);
+ ~entry ();
+
+ path_type path_;
+ };
+
+ // Create a cache entry corresponding to the specified filesystem path.
+ // The path must be absolute and normalized. The temporary argument may be
+ // used to hint whether the entry is likely to be temporary or permanent.
+ //
+ entry
+ create (path f, optional<bool> /*temporary*/)
+ {
+ return entry (move (f), true /* temporary */);
+ };
+
+ // A shortcut for creating and initializing an existing permanent entry.
+ //
+ entry
+ create_existing (path f)
+ {
+ entry e (move (f), false /* temporary */);
+ e.init_existing ();
+ return e;
+ };
+
+ // Return the compressed filesystem entry extension (with the leading dot)
+ // or empty string if no compression is used by this cache implementation.
+ //
+ // If the passed extension is not NULL, then it is included as a first-
+ // level extension into the returned value (useful to form extensions for
+ // clean_extra()).
+ //
+ string
+ compressed_extension (const char* = nullptr)
+ {
+ return string ();
+ }
+
+ explicit
+ file_cache (scheduler&)
+ {
+ }
+ };
+}
+
+#include <libbuild2/file-cache.ixx>
+
+#endif // LIBBUILD2_FILE_CACHE_HXX