From 722cf9d345e38b6f5ff4ed538d1f68bc75b2ab51 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Fri, 10 Jul 2015 15:29:42 +0200 Subject: Implement automatic subproject discovery Currently we only capture their directories without the project names. We will need project names when we hook import search into this. --- build/b.cxx | 7 + build/bootstrap.build | 2 +- build/config/operation.cxx | 27 ++- build/context | 3 + build/context.cxx | 24 +++ build/file | 7 + build/file.cxx | 246 ++++++++++++++++++++++++++-- build/variable | 8 + build/variable.cxx | 29 +++- tests/amalgam/config/build/bootstrap.build | 1 - tests/amalgam/simple/build/bootstrap.build | 2 - tests/amalgam/simple/buildfile | 9 - tests/amalgam/simple/driver.cxx | 4 - tests/amalgam/simple1/build/bootstrap.build | 2 + tests/amalgam/simple1/buildfile | 9 + tests/amalgam/simple1/driver.cxx | 4 + tests/amalgam/simple2/build/bootstrap.build | 2 + tests/amalgam/simple2/buildfile | 9 + tests/amalgam/simple2/driver.cxx | 4 + 19 files changed, 357 insertions(+), 42 deletions(-) delete mode 100644 tests/amalgam/simple/build/bootstrap.build delete mode 100644 tests/amalgam/simple/buildfile delete mode 100644 tests/amalgam/simple/driver.cxx create mode 100644 tests/amalgam/simple1/build/bootstrap.build create mode 100644 tests/amalgam/simple1/buildfile create mode 100644 tests/amalgam/simple1/driver.cxx create mode 100644 tests/amalgam/simple2/build/bootstrap.build create mode 100644 tests/amalgam/simple2/buildfile create mode 100644 tests/amalgam/simple2/driver.cxx diff --git a/build/b.cxx b/build/b.cxx index 1650a73..593fbc1 100644 --- a/build/b.cxx +++ b/build/b.cxx @@ -296,6 +296,13 @@ main (int argc, char* argv[]) if (!src_base.empty ()) { + // Make sure it exists. While we will fail further down + // if it doesn't, the diagnostics could be confusing (e.g., + // unknown operation because we don't load bootstrap.build). + // + if (!dir_exists (src_base)) + fail << "src_base directory " << src_base << " does not exist"; + if (src_base.relative ()) src_base = work / src_base; diff --git a/build/bootstrap.build b/build/bootstrap.build index 3dce047..381b45e 100644 --- a/build/bootstrap.build +++ b/build/bootstrap.build @@ -1,6 +1,6 @@ # file : build/bootstrap.build # copyright : Copyright (c) 2014-2015 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file - project = build2 +subprojects = # No subprojects. using config diff --git a/build/config/operation.cxx b/build/config/operation.cxx index 300dd22..ea338f5 100644 --- a/build/config/operation.cxx +++ b/build/config/operation.cxx @@ -21,11 +21,7 @@ namespace build { namespace config { - static const dir_path build_dir ("build"); - static const dir_path bootstrap_dir ("build/bootstrap"); - static const path config_file ("build/config.build"); - static const path src_root_file ("build/bootstrap/src-root.build"); // configure // @@ -149,7 +145,7 @@ namespace build // if (out_root != src_root) { - mkdir (out_root); + mkdir_p (out_root); mkdir (out_root / build_dir); mkdir (out_root / bootstrap_dir); } @@ -278,7 +274,7 @@ namespace build // dir_path out_nroot (out_root / n.dir); - // The same logic to src_root as in create_bootstrap_inner(). + // The same logic for src_root as in create_bootstrap_inner(). // scope& nroot (create_root (out_nroot, dir_path ())); bootstrap_out (nroot); @@ -295,6 +291,25 @@ namespace build bootstrap_src (nroot); m = disfigure_project (a, nroot) || m; + + // We use mkdir_p() to create the out_root of a subproject + // which means there could be empty parent directories left + // behind. Clean them up. + // + if (!n.dir.simple () && out_root != src_root) + { + for (dir_path d (n.dir.directory ()); + !d.empty (); + d = d.directory ()) + { + rmdir_status s (rmdir (out_root / d)); + + if (s == rmdir_status::not_empty) + break; // No use trying do remove parent ones. + + m = (s == rmdir_status::success) || m; + } + } } } diff --git a/build/context b/build/context index a52232a..8cf42dc 100644 --- a/build/context +++ b/build/context @@ -56,6 +56,9 @@ namespace build fs_status mkdir (const dir_path&); + fs_status + mkdir_p (const dir_path&); + // Remove the file and print the standard diagnostics. The second // argument is only used in diagnostics, to print the target name. // Passing the path for target will result in the relative path diff --git a/build/context.cxx b/build/context.cxx index d62be61..dce6967 100644 --- a/build/context.cxx +++ b/build/context.cxx @@ -105,6 +105,30 @@ namespace build return ms; } + fs_status + mkdir_p (const dir_path& d) + { + // We don't want to print the command if the directory already + // exists. This makes the below code a bit ugly. + // + mkdir_status ms; + + try + { + ms = try_mkdir_p (d); + } + catch (const system_error& e) + { + text << "mkdir -p " << d; + fail << "unable to create directory " << d << ": " << e.what (); + } + + if (ms == mkdir_status::success) + text << "mkdir -p " << d; + + return ms; + } + dir_path src_out (const dir_path& out, scope& s) { diff --git a/build/file b/build/file index 8e96f4a..b786cf7 100644 --- a/build/file +++ b/build/file @@ -13,6 +13,13 @@ namespace build class scope; class location; + extern const dir_path build_dir; // build + extern const dir_path bootstrap_dir; // build/bootstrap + + extern const path root_file; // build/root.build + extern const path bootstrap_file; // build/bootstrap.build + extern const path src_root_file; // build/bootstrap/src-root.build + bool is_src_root (const dir_path&); diff --git a/build/file.cxx b/build/file.cxx index e65d775..3c4fc1c 100644 --- a/build/file.cxx +++ b/build/file.cxx @@ -4,7 +4,9 @@ #include +#include #include +#include // move() #include @@ -13,22 +15,33 @@ #include #include +#include +#include + using namespace std; using namespace butl; namespace build { + const dir_path build_dir ("build"); + const dir_path bootstrap_dir ("build/bootstrap"); + + const path root_file ("build/root.build"); + const path bootstrap_file ("build/bootstrap.build"); + const path src_root_file ("build/bootstrap/src-root.build"); + bool is_src_root (const dir_path& d) { - return file_exists (d / path ("build/bootstrap.build")) || - file_exists (d / path ("build/root.build")); + // @@ Can we have root without bootstrap? I don't think so. + // + return file_exists (d / bootstrap_file) || file_exists (d / root_file); } bool is_out_root (const dir_path& d) { - return file_exists (d / path ("build/bootstrap/src-root.build")); + return file_exists (d / src_root_file); } dir_path @@ -172,6 +185,128 @@ namespace build source_once (bf, root, root); } + // Extract the specified variable value from a buildfile. It is + // expected to be the first non-comment line and not to rely on + // any variable expansion other than those from the global scope. + // + static value_ptr + extract_variable (const path& bf, const char* var) + { + ifstream ifs (bf.string ()); + if (!ifs.is_open ()) + fail << "unable to open " << bf; + + ifs.exceptions (ifstream::failbit | ifstream::badbit); + + try + { + path rbf (diag_relative (bf)); + + lexer l (ifs, rbf.string ()); + token t (l.next ()); + token_type tt; + + if (t.type () != token_type::name || t.name () != var || + ((tt = l.next ().type ()) != token_type::equal && + tt != token_type::plus_equal)) + fail << "variable '" << var << "' expected as first line in " << rbf; + + parser p; + temp_scope tmp (*global_scope); + p.parse_variable (l, tmp, t.name (), tt); + + auto val (tmp.vars[var]); + assert (val.defined ()); + value_ptr& vp (val); + return move (vp); // Steal the value, the scope is going away. + } + catch (const std::ios_base::failure&) + { + fail << "failed to read from " << bf; + } + + return nullptr; + } + + using subprojects = map; + + // Scan the specified directory for any subprojects. If a subdirectory + // is a subproject, then enter it into the map, handling the duplicates. + // Otherwise, scan the subdirectory recursively. + // + static void + find_subprojects (subprojects& sps, + const dir_path& d, + const dir_path& root, + bool out) + { + tracer trace ("find_subprojects"); + + for (const dir_entry& de: dir_iterator (d)) + { + if (de.ltype () != entry_type::directory) + continue; + + dir_path sd (d / path_cast (de.path ())); + + bool src (false); + if (!((out && is_out_root (sd)) || (src = is_src_root (sd)))) + { + find_subprojects (sps, sd, root, out); + continue; + } + + // Calculate relative subdirectory for this subproject. + // + dir_path dir (sd.leaf (root)); + level4 ([&]{trace << "subproject " << sd << " as " << dir;}); + + // Load the project name. If this subdirectory is the subproject's + // src_root, then we can get directly to that. Otherwise, we first + // have to discover its src_root. + // + dir_path src_sd; + if (src) + src_sd = move (sd); + else + { + value_ptr vp (extract_variable (sd / src_root_file, "src_root")); + value_proxy v (&vp, nullptr); // Read-only. + src_sd = move (v.as ()); + level4 ([&]{trace << "extracted src_root " << src_sd << " for " + << sd;}); + } + + string name; + { + value_ptr vp (extract_variable (src_sd / bootstrap_file, "project")); + value_proxy v (&vp, nullptr); // Read-only. + name = move (v.as ()); + level4 ([&]{trace << "extracted project name " << name << " for " + << src_sd;}); + } + + // @@ Can't use move() because we may need the values in diagnostics + // below. Looks like C++17 try_emplace() is what we need. + // + auto rp (sps.emplace (name, dir)); + + // Handle duplicates. + // + if (!rp.second) + { + const dir_path& dir1 (rp.first->second); + + if (dir != dir1) + fail << "inconsistent subproject directories for " << name << + info << "first alternative: " << dir1 << + info << "second alternative: " << dir; + + level5 ([&]{trace << "skipping duplicate";}); + } + } + } + bool bootstrap_src (scope& root) { @@ -179,7 +314,10 @@ namespace build bool r (false); - path bf (root.src_path () / path ("build/bootstrap.build")); + const dir_path& out_root (root.path ()); + const dir_path& src_root (root.src_path ()); + + path bf (src_root / path ("build/bootstrap.build")); if (file_exists (bf)) { @@ -199,27 +337,30 @@ namespace build // user (in bootstrap.build) or by an earlier call to this // function for the same scope. When set by the user, the // empty special value means that the project shall not be - // amalgamated. When calculated, the NULL value indicates - // that we are not amalgamated. + // amalgamated (and which we convert to NULL below). When + // calculated, the NULL value indicates that we are not + // amalgamated. // { auto rp (root.vars.assign("amalgamation")); // Set NULL by default. auto& val (rp.first); - const dir_path& d (root.path ()); + + if (!val.null () && val.empty ()) // Convert empty to NULL. + val = nullptr; if (scope* aroot = root.parent_scope ()->root_scope ()) { const dir_path& ad (aroot->path ()); - dir_path rd (ad.relative (d)); + dir_path rd (ad.relative (out_root)); // If we already have the amalgamation variable set, verify // that aroot matches its value. // if (!rp.second) { - if (val.null () || val.empty ()) + if (val.null ()) { - fail << d << " cannot be amalgamated" << + fail << out_root << " cannot be amalgamated" << info << "amalgamated by " << ad; } else @@ -228,7 +369,7 @@ namespace build if (vd != rd) { - fail << "inconsistent amalgamation of " << d << + fail << "inconsistent amalgamation of " << out_root << info << "specified: " << vd << info << "actual: " << rd << " by " << ad; } @@ -238,7 +379,7 @@ namespace build { // Otherwise, use the outer root as our amalgamation. // - level4 ([&]{trace << d << " amalgamated as " << rd;}); + level4 ([&]{trace << out_root << " amalgamated as " << rd;}); val = move (rd); } } @@ -249,18 +390,89 @@ namespace build // outer directories is a project's out_root. If so, then // that's our amalgamation. // - const dir_path& d (root.path ()); - const dir_path& ad (find_out_root (d.directory ())); + const dir_path& ad (find_out_root (out_root.directory ())); if (!ad.empty ()) { - dir_path rd (ad.relative (d)); - level4 ([&]{trace << d << " amalgamated as " << rd;}); + dir_path rd (ad.relative (out_root)); + level4 ([&]{trace << out_root << " amalgamated as " << rd;}); val = move (rd); } } } + // See if we have any subprojects. In a sense, this is the other + // side/direction of the amalgamation logic above. Here, the + // subprojects variable may or may not be set by the user (in + // bootstrap.build) or by an earlier call to this function for + // the same scope. When set by the user, the empty special value + // means that there are no subproject and none should be searched + // for (and which we convert to NULL below). Otherwise, it is a + // list of directory[=project] pairs. The directory must be + // relative to our out_root. If the project name is not specified, + // then we have to figure it out. When subprojects are calculated, + // the NULL value indicates that we found no subprojects. + // + { + auto rp (root.vars.assign("subprojects")); // Set NULL by default. + auto& val (rp.first); + + if (rp.second) + { + // No subprojects set so we need to figure out if there are any. + // + // First we are going to scan our out_root and find all the + // pre-configured subprojects. Then, if out_root != src_root, + // we are going to do the same for src_root. Here, however, + // we need to watch out for duplicates. + // + subprojects sps; + + if (dir_exists (out_root)) + find_subprojects (sps, out_root, out_root, true); + + if (out_root != src_root) + find_subprojects (sps, src_root, src_root, false); + + // Transform our map to list_value. + // + if (!sps.empty ()) + { + list_value_ptr vp (new list_value); + for (auto& p: sps) + vp->emplace_back (move (p.second)); + val = move (vp); + } + } + else if (!val.null ()) + { + // Convert empty to NULL. + // + if (val.empty ()) + val = nullptr; + else + { + // Scan the value and convert it to the "canonical" form, + // that is, a list of dir=simple pairs. + // + list_value& lv (val.as ()); + + for (name& n: lv) + { + if (n.simple ()) + { + n.dir = dir_path (move (n.value)); + n.value.clear (); + } + + if (!n.directory ()) + fail << "expected directory instead of '" << n << "' in the " + << "subprojects variable"; + } + } + } + } + return r; } @@ -269,7 +481,7 @@ namespace build { auto v (root.vars["amalgamation"]); - if (!v || v.empty ()) + if (!v) return; const dir_path& d (v.as ()); diff --git a/build/variable b/build/variable index 064a10a..9c6d3bc 100644 --- a/build/variable +++ b/build/variable @@ -189,10 +189,18 @@ namespace build as () const {return as ();} template <> + std::string& value_proxy:: + as () const; + + template <> const std::string& value_proxy:: as () const; template <> + dir_path& value_proxy:: + as () const; + + template <> const dir_path& value_proxy:: as () const; } diff --git a/build/variable.cxx b/build/variable.cxx index 6a40bc7..d00fcf6 100644 --- a/build/variable.cxx +++ b/build/variable.cxx @@ -15,6 +15,17 @@ namespace build // value_proxy // template <> + string& value_proxy:: + as () const + { + list_value& lv (as ()); + assert (lv.size () == 1); + name& n (lv.front ()); + assert (n.simple ()); + return n.value; + } + + template <> const string& value_proxy:: as () const { @@ -26,11 +37,22 @@ namespace build const name& n (lv.front ()); - assert (n.type.empty () && n.dir.empty ()); + assert (n.simple ()); return n.value; } template <> + dir_path& value_proxy:: + as () const + { + list_value& lv (as ()); + assert (lv.size () == 1); + name& n (lv.front ()); + assert (n.directory ()); + return n.dir; + } + + template <> const dir_path& value_proxy:: as () const { @@ -42,7 +64,10 @@ namespace build const name& n (lv.front ()); - assert (n.type.empty () && n.value.empty ()); + if (n.empty ()) + return empty_dir_path; + + assert (n.directory ()); return n.dir; } } diff --git a/tests/amalgam/config/build/bootstrap.build b/tests/amalgam/config/build/bootstrap.build index 5a01046..52d415a 100644 --- a/tests/amalgam/config/build/bootstrap.build +++ b/tests/amalgam/config/build/bootstrap.build @@ -1,4 +1,3 @@ project = amalgam-config amalgamation = # Shall not be amalgamated. -subprojects = 1/ 2/ using config diff --git a/tests/amalgam/simple/build/bootstrap.build b/tests/amalgam/simple/build/bootstrap.build deleted file mode 100644 index 698a248..0000000 --- a/tests/amalgam/simple/build/bootstrap.build +++ /dev/null @@ -1,2 +0,0 @@ -project = amalgam-simple -using config diff --git a/tests/amalgam/simple/buildfile b/tests/amalgam/simple/buildfile deleted file mode 100644 index ec6a4e1..0000000 --- a/tests/amalgam/simple/buildfile +++ /dev/null @@ -1,9 +0,0 @@ -using cxx - -hxx.ext = hxx -cxx.ext = cxx -ixx.ext = ixx - -cxx.poptions += -I$out_root - -exe{driver}: cxx{driver} diff --git a/tests/amalgam/simple/driver.cxx b/tests/amalgam/simple/driver.cxx deleted file mode 100644 index 70b4146..0000000 --- a/tests/amalgam/simple/driver.cxx +++ /dev/null @@ -1,4 +0,0 @@ -int -main () -{ -} diff --git a/tests/amalgam/simple1/build/bootstrap.build b/tests/amalgam/simple1/build/bootstrap.build new file mode 100644 index 0000000..e4502f8 --- /dev/null +++ b/tests/amalgam/simple1/build/bootstrap.build @@ -0,0 +1,2 @@ +project = amalgam-simple1 +using config diff --git a/tests/amalgam/simple1/buildfile b/tests/amalgam/simple1/buildfile new file mode 100644 index 0000000..ec6a4e1 --- /dev/null +++ b/tests/amalgam/simple1/buildfile @@ -0,0 +1,9 @@ +using cxx + +hxx.ext = hxx +cxx.ext = cxx +ixx.ext = ixx + +cxx.poptions += -I$out_root + +exe{driver}: cxx{driver} diff --git a/tests/amalgam/simple1/driver.cxx b/tests/amalgam/simple1/driver.cxx new file mode 100644 index 0000000..70b4146 --- /dev/null +++ b/tests/amalgam/simple1/driver.cxx @@ -0,0 +1,4 @@ +int +main () +{ +} diff --git a/tests/amalgam/simple2/build/bootstrap.build b/tests/amalgam/simple2/build/bootstrap.build new file mode 100644 index 0000000..599ec1d --- /dev/null +++ b/tests/amalgam/simple2/build/bootstrap.build @@ -0,0 +1,2 @@ +project = amalgam-simple2 +using config diff --git a/tests/amalgam/simple2/buildfile b/tests/amalgam/simple2/buildfile new file mode 100644 index 0000000..ec6a4e1 --- /dev/null +++ b/tests/amalgam/simple2/buildfile @@ -0,0 +1,9 @@ +using cxx + +hxx.ext = hxx +cxx.ext = cxx +ixx.ext = ixx + +cxx.poptions += -I$out_root + +exe{driver}: cxx{driver} diff --git a/tests/amalgam/simple2/driver.cxx b/tests/amalgam/simple2/driver.cxx new file mode 100644 index 0000000..70b4146 --- /dev/null +++ b/tests/amalgam/simple2/driver.cxx @@ -0,0 +1,4 @@ +int +main () +{ +} -- cgit v1.1