// file : libbuild2/file-cache.hxx -*- C++ -*- // license : MIT; see accompanying LICENSE file #ifndef LIBBUILD2_FILE_CACHE_HXX #define LIBBUILD2_FILE_CACHE_HXX #include #include #include #include namespace build2 { // We sometimes have intermediate build results that must be stored and // accessed as files (for example, partially-preprocessed C/C++ translation // units; those .i/.ii files). These files can be quite large which can lead // to excessive disk usage (for example, the .ii files can be several MB // each and can end up dominating object file sizes in a build with debug // information). These files are also often temporary which means writing // them to disk is really a waste. // // The file cache attempts to address this by still presenting a file-like // entry (which can be a real file or a named pipe) but potentially storing // the file contents in memory and/or compressed. // // Each cache entry is identified by the filesystem entry path that will be // written to or read from. The file cache reserves a filesystem entry path // that is derived by adding a compression extension to the main entry path // (for example, .ii.lz4). When cleaning intermediate build results that are // managed by the cache, the rule must clean such a reserved path in // addition to the main entry path (see compressed_extension() below). // // While the cache is MT-safe (that is, we can insert multiple entries // concurrently), each entry is expected to be accessed serially by a single // thread. Furthermore, each entry can either be written to or read from at // any give time and it can only be read from by a single reader at a time. // In other words, there meant to be a single cache entry for any given path // and it is not meant to be shared. // // The underlying filesystem entry can be either temporary or permanent. A // temporary entry only exists during the build, normally between the match // and execute phases. A permanent entry exists across builds. Note, // however, that a permanent entry is often removed in cases of an error and // sometimes a temporary entry is left behind for diagnostics. It is also // possible that the distinction only becomes known some time after the // entry has been created. As a result, all entries by default start as // temporary and can later be made permanent if desired. // // A cache entry can be pinned or unpinned. A cache entry is created pinned. // A cache entry being written to or read from remains pinned. // // An unpinned entry can be preempted. Preempting a cache entry can mean any // of the following: // // - An in-memory content is compressed (but stays in memory). // // - An in-memory content (compressed or not) is flushed to disk (with or // without compression). // // - An uncompressed on-disk content is compressed. // // Naturally, any of the above degrees of preemption make accessing the // contents of a cache entry slower. Note also that pinned/unpinned and // temporary/permanent are independent and a temporary entry does not need // to be unpinned to be removed. // // After creation, a cache entry must be initialized by either writing new // contents to the filesystem entry or by using an existing (permanent) // filesystem entry. Once initialized, an entry can be opened for reading, // potentially multiple times. // // Note also that a noop implementation of this caching semantics (that is, // one that simply saves the file on disk) is file_cache::entry that is just // auto_rmfile. // class /*LIBBUILD2_SYMEXPORT*/ file_cache { public: // A cache entry write handle. During the lifetime of this object the // filesystem entry can be opened for writing and written to. // // A successful write must be terminated with an explicit call to close() // (similar semantics to ofdstream). A write handle that is destroyed // without a close() call is treated as an unsuccessful write and the // initialization can be attempted again. // struct write { void close () {} }; // A cache entry read handle. During the lifetime of this object the // filesystem entry can be opened for reading and read from. // struct read { }; // A cache entry handle. When it is destroyed, a temporary entry is // automatically removed from the filesystem. // struct entry { using path_type = build2::path; bool temporary; // The returned reference is valid and stable for the lifetime of the // entry handle. // const path_type& path () const {return path_;} // Initialization. // write init_new () { return write (); } void init_existing () {} // Reading. // read open () { return read (); } // Pinning. // void pin () {} void unpin () {} // NULL entry handle. // entry () = default; explicit operator bool () const { return !path_.empty (); } // Move-to-NULL-entry-only type. // entry (entry&&); entry (const entry&) = delete; entry& operator= (entry&&); entry& operator= (const entry&) = delete; // Implementation details. // entry (path_type, bool); ~entry (); path_type path_; }; // Create a cache entry corresponding to the specified filesystem path. // The path must be absolute and normalized. The temporary argument may be // used to hint whether the entry is likely to be temporary or permanent. // entry create (path f, optional /*temporary*/) { return entry (move (f), true /* temporary */); }; // A shortcut for creating and initializing an existing permanent entry. // entry create_existing (path f) { entry e (move (f), false /* temporary */); e.init_existing (); return e; }; // Return the compressed filesystem entry extension (with the leading dot) // or empty string if no compression is used by this cache implementation. // // If the passed extension is not NULL, then it is included as a first- // level extension into the returned value (useful to form extensions for // clean_extra()). // string compressed_extension (const char* = nullptr) { return string (); } explicit file_cache (scheduler&) { } }; } #include #endif // LIBBUILD2_FILE_CACHE_HXX