From 934f2a9a90c5cad3cdc8a66b50c17827a3ddbcee Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Sat, 20 Jan 2018 13:46:11 +0200 Subject: Get rid of action rule override semantics Instead we now have two more or less separate match states for outer and inner parts of an action. --- build2/algorithm.cxx | 688 +++--- build2/algorithm.hxx | 87 +- build2/algorithm.ixx | 144 +- build2/bin/init.cxx | 25 +- build2/bin/rule.cxx | 24 +- build2/bin/rule.hxx | 9 +- build2/bin/target.cxx | 4 +- build2/bin/target.hxx | 2 +- build2/cc/common.cxx | 28 +- build2/cc/common.hxx | 8 +- build2/cc/compile-rule.cxx | 4631 +++++++++++++++++++++++++++++++++++++ build2/cc/compile-rule.hxx | 156 ++ build2/cc/compile.cxx | 4627 ------------------------------------ build2/cc/compile.hxx | 153 -- build2/cc/install-rule.cxx | 269 +++ build2/cc/install-rule.hxx | 77 + build2/cc/install.cxx | 246 -- build2/cc/install.hxx | 67 - build2/cc/link-rule.cxx | 2104 +++++++++++++++++ build2/cc/link-rule.hxx | 166 ++ build2/cc/link.cxx | 2108 ----------------- build2/cc/link.hxx | 137 -- build2/cc/module.cxx | 33 +- build2/cc/module.hxx | 22 +- build2/cc/pkgconfig.cxx | 37 +- build2/cc/windows-manifest.cxx | 6 +- build2/cc/windows-rpath.cxx | 42 +- build2/cli/init.cxx | 10 +- build2/cli/rule.cxx | 8 +- build2/cli/rule.hxx | 6 +- build2/cli/target.cxx | 2 +- build2/cli/target.hxx | 2 +- build2/context.hxx | 2 +- build2/dist/rule.cxx | 2 +- build2/dist/rule.hxx | 3 +- build2/dump.cxx | 6 +- build2/install/init.cxx | 2 +- build2/install/rule.cxx | 257 +- build2/install/rule.hxx | 73 +- build2/operation.cxx | 19 +- build2/operation.hxx | 35 +- build2/parser.cxx | 13 +- build2/rule-map.hxx | 2 + build2/rule.cxx | 14 +- build2/rule.hxx | 44 +- build2/target.cxx | 98 +- build2/target.hxx | 166 +- build2/target.ixx | 104 +- build2/test/common.hxx | 22 +- build2/test/init.cxx | 111 +- build2/test/module.hxx | 13 +- build2/test/rule.cxx | 716 +++--- build2/test/rule.hxx | 38 +- build2/test/script/runner.cxx | 4 + build2/test/script/script.cxx | 8 +- build2/variable.cxx | 15 +- build2/variable.ixx | 10 + build2/version/init.cxx | 16 +- build2/version/rule.cxx | 22 +- build2/version/rule.hxx | 12 +- doc/testscript.cli | 4 +- tests/cc/preprocessed/testscript | 2 +- tests/test/config-test/testscript | 3 +- 63 files changed, 8900 insertions(+), 8864 deletions(-) create mode 100644 build2/cc/compile-rule.cxx create mode 100644 build2/cc/compile-rule.hxx delete mode 100644 build2/cc/compile.cxx delete mode 100644 build2/cc/compile.hxx create mode 100644 build2/cc/install-rule.cxx create mode 100644 build2/cc/install-rule.hxx delete mode 100644 build2/cc/install.cxx delete mode 100644 build2/cc/install.hxx create mode 100644 build2/cc/link-rule.cxx create mode 100644 build2/cc/link-rule.hxx delete mode 100644 build2/cc/link.cxx delete mode 100644 build2/cc/link.hxx diff --git a/build2/algorithm.cxx b/build2/algorithm.cxx index 7ef5267..0e57c08 100644 --- a/build2/algorithm.cxx +++ b/build2/algorithm.cxx @@ -100,7 +100,7 @@ namespace build2 return q ? import_existing (pk) : search_existing_target (pk); } - // If the work_queue is not present, then we don't wait. + // If the work_queue is absent, then we don't wait. // target_lock lock_impl (action a, const target& ct, optional wq) @@ -113,27 +113,25 @@ namespace build2 size_t b (target::count_base ()); size_t e (b + target::offset_touched - 1); - size_t lock (b + target::offset_locked); + size_t appl (b + target::offset_applied); size_t busy (b + target::offset_busy); - for (;;) + atomic_count& task_count (ct[a].task_count); + + while (!task_count.compare_exchange_strong ( + e, + busy, + memory_order_acq_rel, // Synchronize on success. + memory_order_acquire)) // Synchronize on failure. { - // First try to grab the spin lock which we later may upgrade to busy. + // Wait for the count to drop below busy if someone is already working + // on this target. // - if (ct.task_count.compare_exchange_strong ( - e, - lock, - memory_order_acq_rel, // Synchronize on success. - memory_order_acquire)) // Synchronize on failure. + if (e >= busy) { - break; - } + if (!wq) + return target_lock {a, nullptr, e - b}; - while (e == lock || e >= busy) - { - // Wait for the count to drop below busy if someone is already working - // on this target. - // // We also unlock the phase for the duration of the wait. Why? // Consider this scenario: we are trying to match a dir{} target whose // buildfile still needs to be loaded. Let's say someone else started @@ -141,118 +139,54 @@ namespace build2 // to switch the phase to load. Which would result in a deadlock // unless we release the phase. // - if (e >= busy) - { - if (!wq) - return target_lock {nullptr, e - b}; - - phase_unlock ul; - e = sched.wait (busy - 1, ct.task_count, *wq); - } - - // Spin if it is locked. - // - for (; e == lock; e = ct.task_count.load (memory_order_acquire)) - this_thread::yield (); + phase_unlock ul; + e = sched.wait (busy - 1, task_count, *wq); } + + // We don't lock already applied or executed targets. + // + if (e >= appl) + return target_lock {a, nullptr, e - b}; } - // We now have the sping lock. Analyze the old value and decide what to - // do. + // We now have the lock. Analyze the old value and decide what to do. // target& t (const_cast (ct)); + target::opstate& s (t[a]); size_t offset; if (e <= b) { // First lock for this operation. // - t.action = a; - t.rule = nullptr; - t.dependents.store (0, memory_order_release); + s.rule = nullptr; + s.dependents.store (0, memory_order_release); + offset = target::offset_touched; } else { offset = e - b; - - switch (offset) - { - case target::offset_executed: - { - if (t.action >= a) - { - // We don't lock already executed targets. - // - t.task_count.store (e, memory_order_release); - return target_lock {nullptr, target::offset_executed}; - } - - // Override, fall through. - // - assert (a > t.action); - } - case target::offset_touched: - case target::offset_tried: - case target::offset_matched: - case target::offset_applied: - { - if (a > t.action) - { - // Only noop_recipe can be overridden. - // - if (offset >= target::offset_applied) - { - recipe_function** f (t.recipe_.target ()); - assert (f != nullptr && *f == &noop_action); - } - - t.action = a; - t.rule = nullptr; - offset = target::offset_touched; // Back to just touched. - } - else - { - assert (t.action >= a); - - // Release the lock if already applied for this action. This is - // necessary no to confuse execute since otherwise it might see - // that the target is busy and assume that someone is already - // executing it. Note that we cannot handle this case in the loop - // above since we need to lock target::action. - // - if (offset == target::offset_applied || t.action > a) - { - // Release the spin lock. - // - t.task_count.store (e, memory_order_release); - return target_lock {nullptr, offset}; - } - } - - break; - } - default: - assert (false); - } + assert (offset == target::offset_touched || + offset == target::offset_tried || + offset == target::offset_matched); } - // We are keeping it so upgrade to busy. - // - t.task_count.store (busy, memory_order_release); - return target_lock (&t, offset); + return target_lock (a, &t, offset); } void - unlock_impl (target& t, size_t offset) + unlock_impl (action a, target& t, size_t offset) { assert (phase == run_phase::match); + atomic_count& task_count (t[a].task_count); + // Set the task count and wake up any threads that might be waiting for // this target. // - t.task_count.store (offset + target::count_base (), memory_order_release); - sched.resume (t.task_count); + task_count.store (offset + target::count_base (), memory_order_release); + sched.resume (task_count); } target_lock @@ -283,160 +217,137 @@ namespace build2 return l; }; - // Return the matching rule and the recipe action. + // Return the matching rule or NULL if no match and try_match is true. // - pair>*, action> + const rule_match* match_impl (action a, target& t, const rule* skip, bool try_match) { - // Clear the resolved targets list before calling match(). The rule is - // free to modify this list in match() (provided that it matches) in order - // to, for example, prepare it for apply(). + // If this is an outer operation (Y_for_X), then we look for rules + // registered for the outer id (X). Note that we still pass the original + // action to the rule's match() function so that it can distinguish + // between a pre/post operation (Y_for_X) and the actual operation (X). // - t.clear_data (); - t.prerequisite_targets.clear (); + meta_operation_id mo (a.meta_operation ()); + operation_id o (a.inner () ? a.operation () : a.outer_operation ()); - // If this is a nested operation, first try the outer operation. - // This allows a rule to implement a "precise match", that is, - // both inner and outer operations match. - // - for (operation_id oo (a.outer_operation ()), io (a.operation ()), - o (oo != 0 ? oo : io); - o != 0; - o = (oo != 0 && o != io ? io : 0)) + const scope& bs (t.base_scope ()); + + for (auto tt (&t.type ()); tt != nullptr; tt = tt->base) { - // Adjust action for recipe: on the first iteration we want it - // {inner, outer} (which is the same as 'a') while on the second - // -- {inner, 0}. Note that {inner, 0} is the same or "stronger" - // (i.e., overrides; see action::operator<()) than 'a'. This - // allows "unconditional inner" to override "inner for outer" - // recipes. + // Search scopes outwards, stopping at the project root. // - action ra (a.meta_operation (), io, o != oo ? 0 : oo); + for (const scope* s (&bs); + s != nullptr; + s = s->root () ? global_scope : s->parent_scope ()) + { + const operation_rule_map* om (s->rules[mo]); - const scope& bs (t.base_scope ()); + if (om == nullptr) + continue; // No entry for this meta-operation id. - for (auto tt (&t.type ()); tt != nullptr; tt = tt->base) - { - // Search scopes outwards, stopping at the project root. + // First try the map for the actual operation. If that doesn't yeld + // anything, try the wildcard map. // - for (const scope* s (&bs); - s != nullptr; - s = s->root () ? global_scope : s->parent_scope ()) + for (operation_id oi (o), oip (o); oip != 0; oip = oi, oi = 0) { - const operation_rule_map* om (s->rules[a.meta_operation ()]); + const target_type_rule_map* ttm ((*om)[oi]); - if (om == nullptr) - continue; // No entry for this meta-operation id. + if (ttm == nullptr) + continue; // No entry for this operation id. - // First try the map for the actual operation. If that doesn't yeld - // anything, try the wildcard map. - // - for (operation_id oi (o), oip (o); oip != 0; oip = oi, oi = 0) - { - const target_type_rule_map* ttm ((*om)[oi]); + if (ttm->empty ()) + continue; // Empty map for this operation id. - if (ttm == nullptr) - continue; // No entry for this operation id. + auto i (ttm->find (tt)); - if (ttm->empty ()) - continue; // Empty map for this operation id. + if (i == ttm->end () || i->second.empty ()) + continue; // No rules registered for this target type. - auto i (ttm->find (tt)); + const auto& rules (i->second); // Hint map. - if (i == ttm->end () || i->second.empty ()) - continue; // No rules registered for this target type. + // @@ TODO + // + // Different rules can be used for different operations (update vs + // test is a good example). So, at some point, we will probably have + // to support a list of hints or even an operation-hint map (e.g., + // 'hint=cxx test=foo' if cxx supports the test operation but we + // want the foo rule instead). This is also the place where the + // '{build clean}=cxx' construct (which we currently do not support) + // can come handy. + // + // Also, ignore the hint (that is most likely ment for a different + // operation) if this is a unique match. + // + string hint; + auto rs (rules.size () == 1 + ? make_pair (rules.begin (), rules.end ()) + : rules.find_sub (hint)); - const auto& rules (i->second); // Hint map. + for (auto i (rs.first); i != rs.second; ++i) + { + const auto& r (*i); + const string& n (r.first); + const rule& ru (r.second); - // @@ TODO - // - // Different rules can be used for different operations (update - // vs test is a good example). So, at some point, we will probably - // have to support a list of hints or even an operation-hint map - // (e.g., 'hint=cxx test=foo' if cxx supports the test operation - // but we want the foo rule instead). This is also the place where - // the '{build clean}=cxx' construct (which we currently do not - // support) can come handy. - // - // Also, ignore the hint (that is most likely ment for a different - // operation) if this is a unique match. - // - string hint; - auto rs (rules.size () == 1 - ? make_pair (rules.begin (), rules.end ()) - : rules.find_sub (hint)); + if (&ru == skip) + continue; - for (auto i (rs.first); i != rs.second; ++i) { - const auto& r (*i); - const string& n (r.first); - const rule& ru (r.second); + auto df = make_diag_frame ( + [a, &t, &n](const diag_record& dr) + { + if (verb != 0) + dr << info << "while matching rule " << n << " to " + << diag_do (a, t); + }); - if (&ru == skip) + if (!ru.match (a, t, hint)) continue; + } + + // Do the ambiguity test. + // + bool ambig (false); + + diag_record dr; + for (++i; i != rs.second; ++i) + { + const string& n1 (i->first); + const rule& ru1 (i->second); - match_result m (false); { auto df = make_diag_frame ( - [ra, &t, &n](const diag_record& dr) + [a, &t, &n1](const diag_record& dr) { if (verb != 0) - dr << info << "while matching rule " << n << " to " - << diag_do (ra, t); + dr << info << "while matching rule " << n1 << " to " + << diag_do (a, t); }); - if (!(m = ru.match (ra, t, hint))) + // @@ TODO: this makes target state in match() undetermined + // so need to fortify rules that modify anything in match + // to clear things. + // + // @@ Can't we temporarily swap things out in target? + // + if (!ru1.match (a, t, hint)) continue; - - if (m.recipe_action.valid ()) - assert (m.recipe_action > ra); - else - m.recipe_action = ra; // Default, if not set. } - // Do the ambiguity test. - // - bool ambig (false); - - diag_record dr; - for (++i; i != rs.second; ++i) + if (!ambig) { - const string& n1 (i->first); - const rule& ru1 (i->second); - - { - auto df = make_diag_frame ( - [ra, &t, &n1](const diag_record& dr) - { - if (verb != 0) - dr << info << "while matching rule " << n1 << " to " - << diag_do (ra, t); - }); - - // @@ TODO: this makes target state in match() undetermined - // so need to fortify rules that modify anything in match - // to clear things. - // - if (!ru1.match (ra, t, hint)) - continue; - } - - if (!ambig) - { - dr << fail << "multiple rules matching " - << diag_doing (ra, t) - << info << "rule " << n << " matches"; - ambig = true; - } - - dr << info << "rule " << n1 << " also matches"; + dr << fail << "multiple rules matching " << diag_doing (a, t) + << info << "rule " << n << " matches"; + ambig = true; } - if (!ambig) - return make_pair (&r, m.recipe_action); - else - dr << info << "use rule hint to disambiguate this match"; + dr << info << "rule " << n1 << " also matches"; } + + if (!ambig) + return &r; + else + dr << info << "use rule hint to disambiguate this match"; } } } @@ -451,14 +362,13 @@ namespace build2 dr << info << "re-run with --verbose 4 for more information"; } - return pair>*, - action> {nullptr, a}; + return nullptr; } recipe - apply_impl (target& t, - const pair>& r, - action a) + apply_impl (action a, + target& t, + const pair>& r) { auto df = make_diag_frame ( [a, &t, &r](const diag_record& dr) @@ -468,9 +378,6 @@ namespace build2 << diag_do (a, t); }); - // @@ We could also allow the rule to change the recipe action in - // apply(). Could be useful with delegates. - // return r.second.get ().apply (a, t); } @@ -480,13 +387,15 @@ namespace build2 // the first half of the result. // static pair - match_impl (action a, - target_lock& l, + match_impl (target_lock& l, bool step = false, bool try_match = false) { assert (l.target != nullptr); + + action a (l.action); target& t (*l.target); + target::opstate& s (t[a]); try { @@ -506,29 +415,38 @@ namespace build2 { // Match. // - auto mr (match_impl (a, t, nullptr, try_match)); - if (mr.first == nullptr) // Not found (try_match == true). + // Clear the resolved targets list and the data pad before calling + // match(). The rule is free to modify these in its match() + // (provided that it matches) in order to, for example, convey some + // information to apply(). + // + t.prerequisite_targets[a].clear (); + if (a.inner ()) t.clear_data (); + + const rule_match* r (match_impl (a, t, nullptr, try_match)); + + if (r == nullptr) // Not found (try_match == true). { l.offset = target::offset_tried; return make_pair (false, target_state::unknown); } - t.rule = mr.first; - t.action = mr.second; // In case overriden. + s.rule = r; l.offset = target::offset_matched; if (step) - // t.state_ is not yet set. - // + // Note: s.state is still undetermined. return make_pair (true, target_state::unknown); + + // Otherwise ... } // Fall through. case target::offset_matched: { // Apply. // - t.recipe (apply_impl (t, *t.rule, t.action)); + set_recipe (l, apply_impl (a, t, *s.rule)); l.offset = target::offset_applied; break; } @@ -541,14 +459,14 @@ namespace build2 // As a sanity measure clear the target data since it can be incomplete // or invalid (mark()/unmark() should give you some ideas). // - t.clear_data (); - t.prerequisite_targets.clear (); + t.prerequisite_targets[a].clear (); + if (a.inner ()) t.clear_data (); - t.state_ = target_state::failed; + s.state = target_state::failed; l.offset = target::offset_applied; } - return make_pair (true, t.state_); + return make_pair (true, s.state); } // If try_match is true, then indicate whether there is a rule match with @@ -599,7 +517,7 @@ namespace build2 return make_pair (false, target_state::unknown); if (task_count == nullptr) - return match_impl (a, l, false /* step */, try_match); + return match_impl (l, false /* step */, try_match); // Pass "disassembled" lock since the scheduler queue doesn't support // task destruction. Also pass our diagnostics stack (this is safe since @@ -617,8 +535,8 @@ namespace build2 { phase_lock pl (run_phase::match); // Can throw. { - target_lock l {&t, offset}; // Reassemble. - match_impl (a, l, false /* step */, try_match); + target_lock l {a, &t, offset}; // Reassemble. + match_impl (l, false /* step */, try_match); // Unlock withing the match phase. } } @@ -651,7 +569,7 @@ namespace build2 { // Match (locked). // - if (match_impl (a, l, true).second == target_state::failed) + if (match_impl (l, true).second == target_state::failed) throw failed (); if ((r = g.group_members (a)).members != nullptr) @@ -666,10 +584,14 @@ namespace build2 // not seem like it will be easy to fix (we don't know whether // someone else will execute this target). // + // @@ What if we always do match & execute together? After all, + // if a group can be resolved in apply(), then it can be + // resolved in match()! + // // Apply (locked). // - if (match_impl (a, l, true).second == target_state::failed) + if (match_impl (l, true).second == target_state::failed) throw failed (); if ((r = g.group_members (a)).members != nullptr) @@ -707,12 +629,12 @@ namespace build2 static void match_prerequisite_range (action a, target& t, R&& r, const scope* s) { - auto& pts (t.prerequisite_targets); + auto& pts (t.prerequisite_targets[a]); // Start asynchronous matching of prerequisites. Wait with unlocked phase // to allow phase switching. // - wait_guard wg (target::count_busy (), t.task_count, true); + wait_guard wg (target::count_busy (), t[a].task_count, true); size_t i (pts.size ()); // Index of the first to be added. for (auto&& p: forward (r)) @@ -722,7 +644,7 @@ namespace build2 if (s != nullptr && !pt.in (*s)) continue; - match_async (a, pt, target::count_busy (), t.task_count); + match_async (a, pt, target::count_busy (), t[a].task_count); pts.push_back (&pt); } @@ -751,12 +673,12 @@ namespace build2 template void - match_members (action a, target& t, T ts[], size_t n) + match_members (action a, target& t, T const* ts, size_t n) { // Pretty much identical to match_prerequisite_range() except we don't // search. // - wait_guard wg (target::count_busy (), t.task_count, true); + wait_guard wg (target::count_busy (), t[a].task_count, true); for (size_t i (0); i != n; ++i) { @@ -765,7 +687,7 @@ namespace build2 if (m == nullptr || marked (m)) continue; - match_async (a, *m, target::count_busy (), t.task_count); + match_async (a, *m, target::count_busy (), t[a].task_count); } wg.wait (); @@ -786,11 +708,11 @@ namespace build2 // Instantiate only for what we need. // template void - match_members (action, target&, const target*[], size_t); + match_members (action, target&, const target* const*, size_t); template void match_members ( - action, target&, prerequisite_target[], size_t); + action, target&, prerequisite_target const*, size_t); const fsdir* inject_fsdir (action a, target& t, bool parent) @@ -844,7 +766,7 @@ namespace build2 if (r != nullptr) { match (a, *r); - t.prerequisite_targets.emplace_back (r); + t.prerequisite_targets[a].emplace_back (r); } return r; @@ -922,16 +844,16 @@ namespace build2 // Note that if the result is group, then the group's state can be // failed. // - switch (t.state_ = ts) + switch (t[a].state = ts) { case target_state::changed: case target_state::unchanged: break; case target_state::postponed: - ts = t.state_ = target_state::unchanged; + ts = t[a].state = target_state::unchanged; break; case target_state::group: - ts = t.group->state_; + ts = (*t.group)[a].state; break; default: assert (false); @@ -939,7 +861,7 @@ namespace build2 } catch (const failed&) { - ts = t.state_ = target_state::failed; + ts = t[a].state = target_state::failed; } return ts; @@ -948,15 +870,17 @@ namespace build2 static target_state execute_impl (action a, target& t) { - assert (t.task_count.load (memory_order_consume) == target::count_busy () - && t.state_ == target_state::unknown); + target::opstate& s (t[a]); - target_state ts (execute_recipe (a, t, t.recipe_)); + assert (s.task_count.load (memory_order_consume) == target::count_busy () + && s.state == target_state::unknown); - // Decrement the target count (see target::recipe() for details). + target_state ts (execute_recipe (a, t, s.recipe)); + + // Decrement the target count (see set_recipe() for details). // { - recipe_function** f (t.recipe_.target ()); + recipe_function** f (s.recipe.target ()); if (f == nullptr || *f != &group_action) target_count.fetch_sub (1, memory_order_relaxed); } @@ -964,11 +888,11 @@ namespace build2 // Decrement the task count (to count_executed) and wake up any threads // that might be waiting for this target. // - size_t tc (t.task_count.fetch_sub ( + size_t tc (s.task_count.fetch_sub ( target::offset_busy - target::offset_executed, memory_order_release)); assert (tc == target::count_busy ()); - sched.resume (t.task_count); + sched.resume (s.task_count); return ts; } @@ -980,11 +904,12 @@ namespace build2 atomic_count* task_count) { target& t (const_cast (ct)); // MT-aware. + target::opstate& s (t[a]); // Update dependency counts and make sure they are not skew. // size_t gd (dependency_count.fetch_sub (1, memory_order_relaxed)); - size_t td (t.dependents.fetch_sub (1, memory_order_release)); + size_t td (s.dependents.fetch_sub (1, memory_order_release)); assert (td != 0 && gd != 0); td--; @@ -1012,133 +937,103 @@ namespace build2 if (current_mode == execution_mode::last && td != 0) return target_state::postponed; - // Try to atomically change applied to busy. Note that we are in the - // execution phase so the target shall not be spin-locked. + // Try to atomically change applied to busy. // - size_t touc (target::count_touched ()); - size_t matc (target::count_matched ()); - size_t appc (target::count_applied ()); + size_t tc (target::count_applied ()); + size_t exec (target::count_executed ()); size_t busy (target::count_busy ()); - for (size_t tc (appc);;) + if (s.task_count.compare_exchange_strong ( + tc, + busy, + memory_order_acq_rel, // Synchronize on success. + memory_order_acquire)) // Synchronize on failure. { - if (t.task_count.compare_exchange_strong ( - tc, - busy, - memory_order_acq_rel, // Synchronize on success. - memory_order_acquire)) // Synchronize on failure. + // Handle the noop recipe. + // + if (s.state == target_state::unchanged) { - // Overriden touch/tried/match-only or noop recipe. + // There could still be scope operations. // - if ((tc >= touc && tc <= matc) || t.state_ == target_state::unchanged) - { - // If we have a noop recipe, there could still be scope operations. - // - if (tc == appc && t.is_a ()) - execute_recipe (a, t, nullptr /* recipe */); - else - t.state_ = target_state::unchanged; - - t.task_count.store (exec, memory_order_release); - sched.resume (t.task_count); - } - else - { - if (task_count == nullptr) - return execute_impl (a, t); + if (t.is_a ()) + execute_recipe (a, t, nullptr /* recipe */); - // Pass our diagnostics stack (this is safe since we expect the - // caller to wait for completion before unwinding its diag stack). - // - if (sched.async (start_count, - *task_count, - [a] (target& t, const diag_frame* ds) - { - diag_frame df (ds); - execute_impl (a, t); - }, - ref (t), - diag_frame::stack)) - return target_state::unknown; // Queued. - - // Executed synchronously, fall through. - } + s.task_count.store (exec, memory_order_release); + sched.resume (s.task_count); } else { - // Normally either busy or already executed. + if (task_count == nullptr) + return execute_impl (a, t); + + // Pass our diagnostics stack (this is safe since we expect the + // caller to wait for completion before unwinding its diag stack). // - if (tc >= busy) return target_state::busy; - else if (tc != exec) - { - // This can happen if we touched/tried/matched (a noop) recipe which - // then got overridden as part of group resolution but not all the - // way to applied. In this case we treat it as noop. - // - assert ((tc >= touc && tc <= matc) && t.action > a); - continue; - } + if (sched.async (start_count, + *task_count, + [a] (target& t, const diag_frame* ds) + { + diag_frame df (ds); + execute_impl (a, t); + }, + ref (t), + diag_frame::stack)) + return target_state::unknown; // Queued. + + // Executed synchronously, fall through. } - - break; + } + else + { + // Either busy or already executed. + // + if (tc >= busy) return target_state::busy; + else assert (tc == exec); } - return t.executed_state (false); + return t.executed_state (a, false); } target_state execute_direct (action a, const target& ct) { target& t (const_cast (ct)); // MT-aware. + target::opstate& s (t[a]); - // Similar logic to match() above. + // Similar logic to match() above except we execute synchronously. // - size_t touc (target::count_touched ()); - size_t matc (target::count_matched ()); - size_t appc (target::count_applied ()); + size_t tc (target::count_applied ()); + size_t exec (target::count_executed ()); size_t busy (target::count_busy ()); - for (size_t tc (appc);;) + if (s.task_count.compare_exchange_strong ( + tc, + busy, + memory_order_acq_rel, // Synchronize on success. + memory_order_acquire)) // Synchronize on failure. { - if (t.task_count.compare_exchange_strong ( - tc, - busy, - memory_order_acq_rel, // Synchronize on success. - memory_order_acquire)) // Synchronize on failure. + if (s.state == target_state::unchanged) { - if ((tc >= touc && tc <= matc) || t.state_ == target_state::unchanged) - { - // If we have a noop recipe, there could still be scope operations. - // - if (tc == appc && t.is_a ()) - execute_recipe (a, t, nullptr /* recipe */); - else - t.state_ = target_state::unchanged; + if (t.is_a ()) + execute_recipe (a, t, nullptr /* recipe */); - t.task_count.store (exec, memory_order_release); - sched.resume (t.task_count); - } - else - execute_impl (a, t); + s.task_count.store (exec, memory_order_release); + sched.resume (s.task_count); } else - { + execute_impl (a, t); + } + else + { // If the target is busy, wait for it. // - if (tc >= busy) sched.wait (exec, t.task_count, scheduler::work_none); - else if (tc != exec) - { - assert ((tc >= touc && tc <= matc) && t.action > a); - continue; - } - } - - break; + if (tc >= busy) sched.wait (exec, s.task_count, scheduler::work_none); + else assert (tc == exec); } - return t.executed_state (); + return t.executed_state (a); } template @@ -1149,7 +1044,7 @@ namespace build2 // Start asynchronous execution of prerequisites. // - wait_guard wg (target::count_busy (), t.task_count); + wait_guard wg (target::count_busy (), t[a].task_count); for (size_t i (0); i != n; ++i) { @@ -1160,7 +1055,7 @@ namespace build2 target_state s ( execute_async ( - a, *mt, target::count_busy (), t.task_count)); + a, *mt, target::count_busy (), t[a].task_count)); if (s == target_state::postponed) { @@ -1177,18 +1072,18 @@ namespace build2 // for (size_t i (0); i != n; ++i) { - const target*& mt (ts[i]); - - if (mt == nullptr) + if (ts[i] == nullptr) continue; + const target& mt (*ts[i]); + // If the target is still busy, wait for its completion. // - if (mt->task_count.load (memory_order_acquire) >= target::count_busy ()) - sched.wait ( - target::count_executed (), mt->task_count, scheduler::work_none); + const auto& tc (mt[a].task_count); + if (tc.load (memory_order_acquire) >= target::count_busy ()) + sched.wait (target::count_executed (), tc, scheduler::work_none); - r |= mt->executed_state (); + r |= mt.executed_state (a); } return r; @@ -1202,18 +1097,18 @@ namespace build2 // target_state r (target_state::unchanged); - wait_guard wg (target::count_busy (), t.task_count); + wait_guard wg (target::count_busy (), t[a].task_count); - for (size_t i (n); i != 0; --i) + for (size_t i (n); i != 0; ) { - const target*& mt (ts[i - 1]); + const target*& mt (ts[--i]); if (mt == nullptr) continue; target_state s ( execute_async ( - a, *mt, target::count_busy (), t.task_count)); + a, *mt, target::count_busy (), t[a].task_count)); if (s == target_state::postponed) { @@ -1224,18 +1119,18 @@ namespace build2 wg.wait (); - for (size_t i (n); i != 0; --i) + for (size_t i (n); i != 0; ) { - const target*& mt (ts[i - 1]); - - if (mt == nullptr) + if (ts[--i] == nullptr) continue; - if (mt->task_count.load (memory_order_acquire) >= target::count_busy ()) - sched.wait ( - target::count_executed (), mt->task_count, scheduler::work_none); + const target& mt (*ts[i]); + + const auto& tc (mt[a].task_count); + if (tc.load (memory_order_acquire) >= target::count_busy ()) + sched.wait (target::count_executed (), tc, scheduler::work_none); - r |= mt->executed_state (); + r |= mt.executed_state (a); } return r; @@ -1267,7 +1162,7 @@ namespace build2 { assert (current_mode == execution_mode::first); - auto& pts (const_cast (t).prerequisite_targets); // MT-aware. + auto& pts (t.prerequisite_targets[a]); if (n == 0) n = pts.size (); @@ -1276,7 +1171,7 @@ namespace build2 // target_state rs (target_state::unchanged); - wait_guard wg (target::count_busy (), t.task_count); + wait_guard wg (target::count_busy (), t[a].task_count); for (size_t i (0); i != n; ++i) { @@ -1287,7 +1182,7 @@ namespace build2 target_state s ( execute_async ( - a, *pt, target::count_busy (), t.task_count)); + a, *pt, target::count_busy (), t[a].task_count)); if (s == target_state::postponed) { @@ -1303,25 +1198,25 @@ namespace build2 for (size_t i (0); i != n; ++i) { - const target*& pt (pts[i]); - - if (pt == nullptr) + if (pts[i] == nullptr) continue; - if (pt->task_count.load (memory_order_acquire) >= target::count_busy ()) - sched.wait ( - target::count_executed (), pt->task_count, scheduler::work_none); + const target& pt (*pts[i]); + + const auto& tc (pt[a].task_count); + if (tc.load (memory_order_acquire) >= target::count_busy ()) + sched.wait (target::count_executed (), tc, scheduler::work_none); - target_state s (pt->executed_state ()); + target_state s (pt.executed_state (a)); rs |= s; // Should we compare the timestamp to this target's? // - if (!e && (!pf || pf (*pt, i))) + if (!e && (!pf || pf (pt, i))) { // If this is an mtime-based target, then compare timestamps. // - if (auto mpt = dynamic_cast (pt)) + if (const mtime_target* mpt = pt.is_a ()) { timestamp mp (mpt->mtime ()); @@ -1340,8 +1235,8 @@ namespace build2 } } - if (rt == nullptr && pt->is_a (*tt)) - rt = pt; + if (rt == nullptr && pt.is_a (*tt)) + rt = &pt; } assert (rt != nullptr); @@ -1355,7 +1250,7 @@ namespace build2 noop_action (action a, const target& t) { text << "noop action triggered for " << diag_doing (a, t); - assert (false); // We shouldn't be called (see target::recipe()). + assert (false); // We shouldn't be called (see set_recipe()). return target_state::unchanged; } @@ -1367,8 +1262,9 @@ namespace build2 const target& g (*t.group); if (execute (a, g) == target_state::busy) - sched.wait ( - target::count_executed (), g.task_count, scheduler::work_none); + sched.wait (target::count_executed (), + g[a].task_count, + scheduler::work_none); // Indicate to execute() that this target's state comes from the group // (which, BTW, can be failed). @@ -1488,7 +1384,7 @@ namespace build2 // for (const target* m (ft.member); m != nullptr; m = m->member) { - const file* fm (dynamic_cast (m)); + const file* fm (m->is_a ()); const path* fp (fm != nullptr ? &fm->path () : nullptr); if (fm == nullptr || fp->empty ()) diff --git a/build2/algorithm.hxx b/build2/algorithm.hxx index ff70b21..8978eba 100644 --- a/build2/algorithm.hxx +++ b/build2/algorithm.hxx @@ -102,33 +102,39 @@ namespace build2 const scope&, const dir_path& out = dir_path ()); - // Target match lock: a non-const target reference as well as the - // target::offset_* state that has already been "achieved". + // Target match lock: a non-const target reference and the target::offset_* + // state that has already been "achieved". Note that target::task_count + // itself is set to busy for the duration or the lock. // struct target_lock { + using action_type = build2::action; using target_type = build2::target; + action_type action; target_type* target = nullptr; - size_t offset = 0; + size_t offset = 0; explicit operator bool () const {return target != nullptr;} - void unlock (); - target_type* release (); + void + unlock (); target_lock () = default; - target_lock (target_lock&&); target_lock& operator= (target_lock&&); - // Implementation details. - // target_lock (const target_lock&) = delete; target_lock& operator= (const target_lock&) = delete; - target_lock (target_type* t, size_t o): target (t), offset (o) {} + // Implementation details. + // ~target_lock (); + target_lock (action_type a, target_type* t, size_t o) + : action (a), target (t), offset (o) {} + + target_type* + release () {auto r (target); target = nullptr; return r;} }; // If the target is already applied (for this action ) or executed, then no @@ -222,13 +228,23 @@ namespace build2 // Match a "delegate rule" from withing another rules' apply() function // avoiding recursive matches (thus the third argument). Unless try_match is - // true, fail if not rule is found. Otherwise return empty recipe. Note that - // unlike match(), this function does not increment the dependents - // count. See also the companion execute_delegate(). + // true, fail if no rule is found. Otherwise return empty recipe. Note that + // unlike match(), this function does not increment the dependents count and + // the two rules must coordinate who is using the target's data pad and/or + // prerequisite_targets. See also the companion execute_delegate(). // recipe match_delegate (action, target&, const rule&, bool try_match = false); + // Match a rule for the inner operation from withing the outer rule's + // apply() function. See also the companion execute_inner(). + // + target_state + match_inner (action, const target&); + + bool + match_inner (action, const target&, unmatch); + // The standard prerequisite search and match implementations. They call // search() and then match() for each prerequisite in a loop omitting out of // project prerequisites for the clean operation. If this target is a member @@ -259,7 +275,7 @@ namespace build2 // template void - match_members (action, target&, T[], size_t); + match_members (action, target&, T const*, size_t); template inline void @@ -269,7 +285,10 @@ namespace build2 } inline void - match_members (action a, target& t, prerequisite_targets& ts, size_t start) + match_members (action a, + target& t, + prerequisite_targets& ts, + size_t start = 0) { match_members (a, t, ts.data () + start, ts.size () - start); } @@ -279,6 +298,13 @@ namespace build2 // member's list might still not be available (e.g., if some wildcard/ // fallback rule matched). // + // If the action is is for an outer operation, then it is changed to inner + // which means the members are always resolved by the inner (e.g., update) + // rule. This feels right since this is the rule that will normally do the + // work (e.g., update) and therefore knows what it will produce (and if we + // don't do this, then the group resolution will be racy since we will use + // two different task_count instances for synchronization). + // group_view resolve_group_members (action, const target&); @@ -307,6 +333,12 @@ namespace build2 target_state execute (action, const target&); + // As above but wait for completion if the target is busy and translate + // target_state::failed to the failed exception. + // + target_state + execute_wait (action, const target&); + // As above but start asynchronous execution. Return target_state::unknown // if the asynchrounous execution has been started and target_state::busy if // the target has already been busy. @@ -328,6 +360,18 @@ namespace build2 target_state execute_delegate (const recipe&, action, const target&); + // Execute the inner operation matched with match_inner(). Note that the + // returned target state is for the inner operation. The appropriate usage + // is to call this function from the outer operation's recipe and to factor + // the obtained state into the one returned (similar to how we do it for + // prerequisites). + // + // Note: waits for the completion if the target is busy and translates + // target_state::failed to the failed exception. + // + target_state + execute_inner (action, const target&); + // A special version of the above that should be used for "direct" and "now" // execution, that is, side-stepping the normal target-prerequisite // relationship (so no dependents count is decremented) and execution order @@ -344,30 +388,29 @@ namespace build2 // for their completion. Return target_state::changed if any of them were // changed and target_state::unchanged otherwise. If a prerequisite's // execution is postponed, then set its pointer in prerequisite_targets to - // NULL (since its state cannot be queried MT-safely). - // - // Note that this function can be used as a recipe. + // NULL (since its state cannot be queried MT-safely). If count is not 0, + // then only the first count prerequisites are executed. // target_state - straight_execute_prerequisites (action, const target&); + straight_execute_prerequisites (action, const target&, size_t count = 0); // As above but iterates over the prerequisites in reverse. // target_state - reverse_execute_prerequisites (action, const target&); + reverse_execute_prerequisites (action, const target&, size_t count = 0); // Call straight or reverse depending on the current mode. // target_state - execute_prerequisites (action, const target&); + execute_prerequisites (action, const target&, size_t count = 0); // A version of the above that also determines whether the action needs to // be executed on the target based on the passed timestamp and filter. // // The filter is passed each prerequisite target and is expected to signal // which ones should be used for timestamp comparison. If the filter is - // NULL, then all the prerequisites are used. If the count is not 0, then - // only the first count prerequisites are executed. + // NULL, then all the prerequisites are used. If count is not 0, then only + // the first count prerequisites are executed. // // Note that the return value is an optional target state. If the target // needs updating, then the value absent. Otherwise it is the state that diff --git a/build2/algorithm.ixx b/build2/algorithm.ixx index 565414a..9baa650 100644 --- a/build2/algorithm.ixx +++ b/build2/algorithm.ixx @@ -129,26 +129,18 @@ namespace build2 lock_impl (action, const target&, optional); void - unlock_impl (target&, size_t); + unlock_impl (action, target&, size_t); inline void target_lock:: unlock () { if (target != nullptr) { - unlock_impl (*target, offset); + unlock_impl (action, *target, offset); target = nullptr; } } - inline target* target_lock:: - release () - { - target_type* r (target); - target = nullptr; - return r; - } - inline target_lock:: ~target_lock () { @@ -157,8 +149,9 @@ namespace build2 inline target_lock:: target_lock (target_lock&& x) - : target (x.release ()), offset (x.offset) + : action (x.action), target (x.target), offset (x.offset) { + x.target = nullptr; } inline target_lock& target_lock:: @@ -167,8 +160,10 @@ namespace build2 if (this != &x) { unlock (); - target = x.release (); + action = x.action; + target = x.target; offset = x.offset; + x.target = nullptr; } return *this; } @@ -185,13 +180,11 @@ namespace build2 return r; } - pair>*, action> + const rule_match* match_impl (action, target&, const rule* skip, bool try_match = false); recipe - apply_impl (target&, - const pair>&, - action); + apply_impl (action, target&, const rule_match&); pair match (action, const target&, size_t, atomic_count*, bool try_match = false); @@ -206,7 +199,7 @@ namespace build2 if (r != target_state::failed) { dependency_count.fetch_add (1, memory_order_relaxed); - t.dependents.fetch_add (1, memory_order_release); + t[a].dependents.fetch_add (1, memory_order_release); } else if (fail) throw failed (); @@ -227,7 +220,7 @@ namespace build2 if (r.second != target_state::failed) { dependency_count.fetch_add (1, memory_order_relaxed); - t.dependents.fetch_add (1, memory_order_release); + t[a].dependents.fetch_add (1, memory_order_release); } else if (fail) throw failed (); @@ -236,7 +229,6 @@ namespace build2 return r; } - inline bool match (action a, const target& t, unmatch um) { @@ -261,8 +253,8 @@ namespace build2 { // Safe if unchanged or someone else is also a dependent. // - if (s == target_state::unchanged || - t.dependents.load (memory_order_consume) != 0) + if (s == target_state::unchanged || + t[a].dependents.load (memory_order_consume) != 0) return true; break; @@ -270,7 +262,7 @@ namespace build2 } dependency_count.fetch_add (1, memory_order_relaxed); - t.dependents.fetch_add (1, memory_order_release); + t[a].dependents.fetch_add (1, memory_order_release); return false; } @@ -290,24 +282,76 @@ namespace build2 } inline void + set_recipe (target_lock& l, recipe&& r) + { + target::opstate& s ((*l.target)[l.action]); + + s.recipe = move (r); + + // If this is a noop recipe, then mark the target unchanged to allow for + // some optimizations. + // + recipe_function** f (s.recipe.target ()); + + if (f != nullptr && *f == &noop_action) + s.state = target_state::unchanged; + else + { + s.state = target_state::unknown; + + // This gets tricky when we start considering direct execution, etc. So + // here seems like the best place to do it. + // + // We also ignore the group recipe since it is used for ad hoc groups + // (which are not executed). Plus, group action means real recipe is in + // the group so this also feels right conceptually. + // + // Note that we will increment this count twice for the same target if + // we have non-noop recipes for both inner and outer operations. While + // not ideal, the alternative (trying to "merge" the count keeping track + // whether inner and/or outer is noop) gets hairy rather quickly. + // + if (f == nullptr || *f != &group_action) + target_count.fetch_add (1, memory_order_relaxed); + } + } + + inline void match_recipe (target_lock& l, recipe r) { assert (phase == run_phase::match && l.target != nullptr); - target& t (*l.target); - t.rule = nullptr; // No rule. - t.recipe (move (r)); + (*l.target)[l.action].rule = nullptr; // No rule. + set_recipe (l, move (r)); l.offset = target::offset_applied; } inline recipe - match_delegate (action a, target& t, const rule& r, bool try_match) + match_delegate (action a, target& t, const rule& dr, bool try_match) { assert (phase == run_phase::match); - auto mr (match_impl (a, t, &r, try_match)); - return mr.first != nullptr - ? apply_impl (t, *mr.first, mr.second) - : empty_recipe; + + // Note: we don't touch any of the t[a] state since that was/will be set + // for the delegating rule. + // + const rule_match* r (match_impl (a, t, &dr, try_match)); + return r != nullptr ? apply_impl (a, t, *r) : empty_recipe; + } + + inline target_state + match_inner (action a, const target& t) + { + // In a sense this is like any other dependency. + // + assert (a.outer ()); + return match (action (a.meta_operation (), a.operation ()), t); + } + + inline bool + match_inner (action a, const target& t, unmatch um) + { + assert (a.outer ()); + return match (action (a.meta_operation (), a.operation ()), t, um); } group_view @@ -318,6 +362,9 @@ namespace build2 { group_view r; + if (a.outer ()) + a = action (a.meta_operation (), a.operation ()); + // We can be called during execute though everything should have been // already resolved. // @@ -395,6 +442,17 @@ namespace build2 } inline target_state + execute_wait (action a, const target& t) + { + if (execute (a, t) == target_state::busy) + sched.wait (target::count_executed (), + t[a].task_count, + scheduler::work_none); + + return t.executed_state (a); + } + + inline target_state execute_async (action a, const target& t, size_t sc, atomic_count& tc, bool fail) @@ -414,26 +472,32 @@ namespace build2 } inline target_state - straight_execute_prerequisites (action a, const target& t) + execute_inner (action a, const target& t) + { + assert (a.outer ()); + return execute_wait (action (a.meta_operation (), a.operation ()), t); + } + + inline target_state + straight_execute_prerequisites (action a, const target& t, size_t c) { - auto& p (const_cast (t).prerequisite_targets); // MT-aware. - return straight_execute_members (a, t, p.data (), p.size ()); + auto& p (t.prerequisite_targets[a]); + return straight_execute_members (a, t, p.data (), c == 0 ? p.size () : c); } inline target_state - reverse_execute_prerequisites (action a, const target& t) + reverse_execute_prerequisites (action a, const target& t, size_t c) { - auto& p (const_cast (t).prerequisite_targets); // MT-aware. - return reverse_execute_members (a, t, p.data (), p.size ()); + auto& p (t.prerequisite_targets[a]); + return reverse_execute_members (a, t, p.data (), c == 0 ? p.size () : c); } inline target_state - execute_prerequisites (action a, const target& t) + execute_prerequisites (action a, const target& t, size_t c) { - auto& p (const_cast (t).prerequisite_targets); // MT-aware. return current_mode == execution_mode::first - ? straight_execute_members (a, t, p.data (), p.size ()) - : reverse_execute_members (a, t, p.data (), p.size ()); + ? straight_execute_prerequisites (a, t, c) + : reverse_execute_prerequisites (a, t, c); } // If the first argument is NULL, then the result is treated as a boolean diff --git a/build2/bin/init.cxx b/build2/bin/init.cxx index d9f3c0e..565936f 100644 --- a/build2/bin/init.cxx +++ b/build2/bin/init.cxx @@ -11,6 +11,10 @@ #include #include + +#include + +#include #include #include @@ -456,17 +460,24 @@ namespace build2 r.insert (perform_update_id, "bin.libu", fail_); r.insert (perform_clean_id, "bin.libu", fail_); - r.insert (perform_update_id, "bin.lib", lib_); - r.insert (perform_clean_id, "bin.lib", lib_); - - // Configure members. + // Similar to alias. // - r.insert (configure_update_id, "bin.lib", lib_); + r.insert (perform_id, 0, "bin.lib", lib_); + r.insert (configure_id, 0, "bin.lib", lib_); + // Treat as a see through group for install and test. + // if (install_loaded) { - r.insert (perform_install_id, "bin.lib", lib_); - r.insert (perform_uninstall_id, "bin.lib", lib_); + auto& gr (install::group_rule::instance); + + r.insert (perform_install_id, "bin.lib", gr); + r.insert (perform_uninstall_id, "bin.lib", gr); + } + + if (const test::module* m = rs.modules.lookup ("test")) + { + r.insert (perform_test_id, "bin.lib", m->group_rule ()); } } diff --git a/build2/bin/rule.cxx b/build2/bin/rule.cxx index bb9036b..79270c3 100644 --- a/build2/bin/rule.cxx +++ b/build2/bin/rule.cxx @@ -19,7 +19,7 @@ namespace build2 { // fail_rule // - match_result fail_rule:: + bool fail_rule:: match (action a, target& t, const string&) const { const char* n (t.dynamic_type ().name); // Ignore derived type. @@ -37,8 +37,8 @@ namespace build2 // The whole logic is pretty much as if we had our two group members as // our prerequisites. // - match_result lib_rule:: - match (action act, target& xt, const string&) const + bool lib_rule:: + match (action, target& xt, const string&) const { lib& t (xt.as ()); @@ -57,35 +57,27 @@ namespace build2 t.a = a ? &search (t, t.dir, t.out, t.name) : nullptr; t.s = s ? &search (t, t.dir, t.out, t.name) : nullptr; - match_result mr (true); - - // If there is an outer operation, indicate that we match - // unconditionally so that we don't override ourselves. - // - if (act.outer_operation () != 0) - mr.recipe_action = action (act.meta_operation (), act.operation ()); - - return mr; + return true; } recipe lib_rule:: - apply (action act, target& xt) const + apply (action a, target& xt) const { lib& t (xt.as ()); const target* m[] = {t.a, t.s}; - match_members (act, t, m); + match_members (a, t, m); return &perform; } target_state lib_rule:: - perform (action act, const target& xt) + perform (action a, const target& xt) { const lib& t (xt.as ()); const target* m[] = {t.a, t.s}; - return execute_members (act, t, m); + return execute_members (a, t, m); } } } diff --git a/build2/bin/rule.hxx b/build2/bin/rule.hxx index b4835dc..6385830 100644 --- a/build2/bin/rule.hxx +++ b/build2/bin/rule.hxx @@ -14,26 +14,29 @@ namespace build2 { namespace bin { - // Fail rule for obj{}, bmi{}, and libu{}. + // "Fail rule" for obj{}, bmi{}, and libu{} that issues diagnostics if + // someone tries to build any of these groups directly. // class fail_rule: public rule { public: fail_rule () {} - virtual match_result + virtual bool match (action, target&, const string&) const override; virtual recipe apply (action, target&) const override; }; + // Pass-through to group members rule, similar to alias. + // class lib_rule: public rule { public: lib_rule () {} - virtual match_result + virtual bool match (action, target&, const string&) const override; virtual recipe diff --git a/build2/bin/target.cxx b/build2/bin/target.cxx index 533da43..2bcb8bc 100644 --- a/build2/bin/target.cxx +++ b/build2/bin/target.cxx @@ -281,7 +281,7 @@ namespace build2 // lib // group_view lib:: - group_members (action_type) const + group_members (action) const { static_assert (sizeof (lib_members) == sizeof (const target*) * 2, "member layout incompatible with array"); @@ -321,7 +321,7 @@ namespace build2 nullptr, nullptr, &target_search, - false + false // Note: not see-through ("alternatives" group). }; // libi diff --git a/build2/bin/target.hxx b/build2/bin/target.hxx index 790d1f0..329b4a9 100644 --- a/build2/bin/target.hxx +++ b/build2/bin/target.hxx @@ -226,7 +226,7 @@ namespace build2 using libx::libx; virtual group_view - group_members (action_type) const override; + group_members (action) const override; public: static const target_type static_type; diff --git a/build2/cc/common.cxx b/build2/cc/common.cxx index 3daada1..e4dbfe8 100644 --- a/build2/cc/common.cxx +++ b/build2/cc/common.cxx @@ -46,7 +46,7 @@ namespace build2 // void common:: process_libraries ( - action act, + action a, const scope& top_bs, linfo top_li, const dir_paths& top_sysd, @@ -222,23 +222,23 @@ namespace build2 // if (impl && !c_e_libs.defined () && !x_e_libs.defined ()) { - for (auto pt: l.prerequisite_targets) + for (const prerequisite_target& pt: l.prerequisite_targets[a]) { if (pt == nullptr) continue; - bool a; + bool la; const file* f; - if ((a = (f = pt->is_a ())) || - (a = (f = pt->is_a ())) || - ( f = pt->is_a ())) + if ((la = (f = pt->is_a ())) || + (la = (f = pt->is_a ())) || + ( f = pt->is_a ())) { if (sysd == nullptr) find_sysd (); if (!li) find_linfo (); - process_libraries (act, bs, *li, *sysd, - *f, a, pt.data, + process_libraries (a, bs, *li, *sysd, + *f, la, pt.data, proc_impl, proc_lib, proc_opt, true); } } @@ -275,7 +275,7 @@ namespace build2 &proc_impl, &proc_lib, &proc_opt, &sysd, &usrd, &find_sysd, &find_linfo, &sys_simple, - &bs, act, &li, this] (const lookup& lu) + &bs, a, &li, this] (const lookup& lu) { const vector* ns (cast_null> (lu)); if (ns == nullptr || ns->empty ()) @@ -300,7 +300,7 @@ namespace build2 if (sysd == nullptr) find_sysd (); if (!li) find_linfo (); - const file& t (resolve_library (act, bs, n, *li, *sysd, usrd)); + const file& t (resolve_library (a, bs, n, *li, *sysd, usrd)); if (proc_lib) { @@ -324,7 +324,7 @@ namespace build2 // @@ Where can we get the link flags? Should we try to find them // in the library's prerequisites? What about installed stuff? // - process_libraries (act, bs, *li, *sysd, + process_libraries (a, bs, *li, *sysd, t, t.is_a () || t.is_a (), 0, proc_impl, proc_lib, proc_opt, true); } @@ -402,7 +402,7 @@ namespace build2 // that's the only way to guarantee it will be up-to-date. // const file& common:: - resolve_library (action act, + resolve_library (action a, const scope& s, name n, linfo li, @@ -439,7 +439,7 @@ namespace build2 // dir_path out; prerequisite_key pk {n.proj, {tt, &n.dir, &out, &n.value, ext}, &s}; - xt = search_library_existing (act, sysd, usrd, pk); + xt = search_library_existing (a, sysd, usrd, pk); if (xt == nullptr) { @@ -454,7 +454,7 @@ namespace build2 // If this is lib{}/libu{}, pick appropriate member. // if (const libx* l = xt->is_a ()) - xt = &link_member (*l, act, li); // Pick lib*{e,a,s}{}. + xt = &link_member (*l, a, li); // Pick lib*{e,a,s}{}. return xt->as (); } diff --git a/build2/cc/common.hxx b/build2/cc/common.hxx index 5ed7173..5952df6 100644 --- a/build2/cc/common.hxx +++ b/build2/cc/common.hxx @@ -225,7 +225,7 @@ namespace build2 bool = false) const; const target* - search_library (action act, + search_library (action a, const dir_paths& sysd, optional& usrd, const prerequisite& p) const @@ -234,7 +234,7 @@ namespace build2 if (r == nullptr) { - if ((r = search_library (act, sysd, usrd, p.key ())) != nullptr) + if ((r = search_library (a, sysd, usrd, p.key ())) != nullptr) { const target* e (nullptr); if (!p.target.compare_exchange_strong ( @@ -274,12 +274,12 @@ namespace build2 bool existing = false) const; const target* - search_library_existing (action act, + search_library_existing (action a, const dir_paths& sysd, optional& usrd, const prerequisite_key& pk) const { - return search_library (act, sysd, usrd, pk, true); + return search_library (a, sysd, usrd, pk, true); } dir_paths diff --git a/build2/cc/compile-rule.cxx b/build2/cc/compile-rule.cxx new file mode 100644 index 0000000..df84547 --- /dev/null +++ b/build2/cc/compile-rule.cxx @@ -0,0 +1,4631 @@ +// file : build2/cc/compile-rule.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include // exit() +#include // strlen() + +#include +#include +#include +#include +#include +#include +#include + +#include +#include // create_project() + +#include +#include // h +#include +#include + +using namespace std; +using namespace butl; + +namespace build2 +{ + namespace cc + { + using namespace bin; + + // module_info string serialization. + // + // The string representation is a space-separated list of module names + // with the following rules: + // + // 1. If this is a module interface unit, then the first name is the + // module name intself following by either '!' for an interface unit or + // by '+' for an implementation unit. + // + // 2. If an imported module is re-exported, then the module name is + // followed by '*'. + // + // For example: + // + // foo! foo.core* foo.base* foo.impl + // foo.base+ foo.impl + // foo.base foo.impl + // + static string + to_string (const module_info& m) + { + string s; + + if (!m.name.empty ()) + { + s += m.name; + s += m.iface ? '!' : '+'; + } + + for (const module_import& i: m.imports) + { + if (!s.empty ()) + s += ' '; + + s += i.name; + + if (i.exported) + s += '*'; + } + + return s; + } + + static module_info + to_module_info (const string& s) + { + module_info m; + + for (size_t b (0), e (0), n; (n = next_word (s, b, e, ' ')) != 0; ) + { + char c (s[e - 1]); + switch (c) + { + case '!': + case '+': + case '*': break; + default: c = '\0'; + } + + string w (s, b, n - (c == '\0' ? 0 : 1)); + + if (c == '!' || c == '+') + { + m.name = move (w); + m.iface = (c == '!'); + } + else + m.imports.push_back (module_import {move (w), c == '*', 0}); + } + + return m; + } + + // preprocessed + // + template + inline bool + operator< (preprocessed l, T r) // Template because of VC14 bug. + { + return static_cast (l) < static_cast (r); + } + + preprocessed + to_preprocessed (const string& s) + { + if (s == "none") return preprocessed::none; + if (s == "includes") return preprocessed::includes; + if (s == "modules") return preprocessed::modules; + if (s == "all") return preprocessed::all; + throw invalid_argument ("invalid preprocessed value '" + s + "'"); + } + + struct compile_rule::match_data + { + explicit + match_data (translation_type t, const prerequisite_member& s) + : type (t), src (s) {} + + translation_type type; + preprocessed pp = preprocessed::none; + bool symexport = false; // Target uses __symexport. + bool touch = false; // Target needs to be touched. + timestamp mt = timestamp_unknown; // Target timestamp. + prerequisite_member src; + auto_rmfile psrc; // Preprocessed source, if any. + path dd; // Dependency database path. + module_positions mods = {0, 0, 0}; + }; + + compile_rule:: + compile_rule (data&& d) + : common (move (d)), + rule_id (string (x) += ".compile 4") + { + static_assert (sizeof (match_data) <= target::data_size, + "insufficient space"); + } + + const char* compile_rule:: + langopt (const match_data& md) const + { + bool m (md.type == translation_type::module_iface); + //preprocessed p (md.pp); + + switch (cid) + { + case compiler_id::gcc: + { + // Ignore the preprocessed value since for GCC it is handled via + // -fpreprocessed -fdirectives-only. + // + switch (x_lang) + { + case lang::c: return "c"; + case lang::cxx: return "c++"; + } + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + // Clang has *-cpp-output (but not c++-module-cpp-output) and they + // handle comments and line continuations. However, currently this + // is only by accident since these modes are essentially equivalent + // to their cpp-output-less versions. + // + switch (x_lang) + { + case lang::c: return "c"; + case lang::cxx: return m ? "c++-module" : "c++"; + } + } + case compiler_id::msvc: + { + switch (x_lang) + { + case lang::c: return "/TC"; + case lang::cxx: return "/TP"; + } + } + case compiler_id::icc: + { + switch (x_lang) + { + case lang::c: return "c"; + case lang::cxx: return "c++"; + } + } + } + + return nullptr; + } + + inline void compile_rule:: + append_symexport_options (cstrings& args, const target& t) const + { + // With VC if a BMI is compiled with dllexport, then when such BMI is + // imported, it is auto-magically treated as dllimport. Let's hope + // other compilers follow suit. + // + args.push_back (t.is_a () && tclass == "windows" + ? "-D__symexport=__declspec(dllexport)" + : "-D__symexport="); + } + + bool compile_rule:: + match (action a, target& t, const string&) const + { + tracer trace (x, "compile_rule::match"); + + bool mod (t.is_a () || t.is_a () || t.is_a ()); + + // Link-up to our group (this is the obj/bmi{} target group protocol + // which means this can be done whether we match or not). + // + if (t.group == nullptr) + t.group = &search (t, + mod ? bmi::static_type : obj::static_type, + t.dir, t.out, t.name); + + // See if we have a source file. Iterate in reverse so that a source + // file specified for a member overrides the one specified for the + // group. Also "see through" groups. + // + for (prerequisite_member p: reverse_group_prerequisite_members (a, t)) + { + if (p.is_a (mod ? *x_mod : x_src)) + { + // Save in the target's auxiliary storage. Translation type will + // be refined in apply(). + // + t.data (match_data (mod + ? translation_type::module_iface + : translation_type::plain, + p)); + return true; + } + } + + l4 ([&]{trace << "no " << x_lang << " source file for target " << t;}); + return false; + } + + // Append or hash library options from a pair of *.export.* variables + // (first one is cc.export.*) recursively, prerequisite libraries first. + // + void compile_rule:: + append_lib_options (const scope& bs, + cstrings& args, + action a, + const target& t, + linfo li) const + { + // See through utility libraries. + // + auto imp = [] (const file& l, bool la) {return la && l.is_a ();}; + + auto opt = [&args, this] ( + const file& l, const string& t, bool com, bool exp) + { + // Note that in our model *.export.poptions are always "interface", + // even if set on liba{}/libs{}, unlike loptions. + // + if (!exp) // Ignore libux. + return; + + const variable& var ( + com + ? c_export_poptions + : (t == x ? x_export_poptions : var_pool[t + ".export.poptions"])); + + append_options (args, l, var); + }; + + // In case we don't have the "small function object" optimization. + // + const function impf (imp); + const function optf (opt); + + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + // Should be already searched and matched for libraries. + // + if (const target* pt = p.load ()) + { + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, li); + + bool la; + if (!((la = pt->is_a ()) || + (la = pt->is_a ()) || + pt->is_a ())) + continue; + + process_libraries (a, bs, li, sys_lib_dirs, + pt->as (), la, 0, // Hack: lflags unused. + impf, nullptr, optf); + } + } + } + + void compile_rule:: + hash_lib_options (const scope& bs, + sha256& cs, + action a, + const target& t, + linfo li) const + { + auto imp = [] (const file& l, bool la) {return la && l.is_a ();}; + + auto opt = [&cs, this] ( + const file& l, const string& t, bool com, bool exp) + { + if (!exp) + return; + + const variable& var ( + com + ? c_export_poptions + : (t == x ? x_export_poptions : var_pool[t + ".export.poptions"])); + + hash_options (cs, l, var); + }; + + // The same logic as in append_lib_options(). + // + const function impf (imp); + const function optf (opt); + + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + if (const target* pt = p.load ()) + { + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, li); + + bool la; + if (!((la = pt->is_a ()) || + (la = pt->is_a ()) || + pt->is_a ())) + continue; + + process_libraries (a, bs, li, sys_lib_dirs, + pt->as (), la, 0, // Hack: lflags unused. + impf, nullptr, optf); + } + } + } + + // Append library prefixes based on the *.export.poptions variables + // recursively, prerequisite libraries first. + // + void compile_rule:: + append_lib_prefixes (const scope& bs, + prefix_map& m, + action a, + target& t, + linfo li) const + { + auto imp = [] (const file& l, bool la) {return la && l.is_a ();}; + + auto opt = [&m, this] ( + const file& l, const string& t, bool com, bool exp) + { + if (!exp) + return; + + const variable& var ( + com + ? c_export_poptions + : (t == x ? x_export_poptions : var_pool[t + ".export.poptions"])); + + append_prefixes (m, l, var); + }; + + // The same logic as in append_lib_options(). + // + const function impf (imp); + const function optf (opt); + + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + if (const target* pt = p.load ()) + { + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, li); + + bool la; + if (!((la = pt->is_a ()) || + (la = pt->is_a ()) || + pt->is_a ())) + continue; + + process_libraries (a, bs, li, sys_lib_dirs, + pt->as (), la, 0, // Hack: lflags unused. + impf, nullptr, optf); + } + } + } + + // Update the target during the match phase. Return true if it has changed + // or if the passed timestamp is not timestamp_unknown and is older than + // the target. + // + // This function is used to make sure header dependencies are up to date. + // + // There would normally be a lot of headers for every source file (think + // all the system headers) and just calling execute_direct() on all of + // them can get expensive. At the same time, most of these headers are + // existing files that we will never be updating (again, system headers, + // for example) and the rule that will match them is the fallback + // file_rule. That rule has an optimization: it returns noop_recipe (which + // causes the target state to be automatically set to unchanged) if the + // file is known to be up to date. So we do the update "smartly". + // + static bool + update (tracer& trace, action a, const target& t, timestamp ts) + { + const path_target* pt (t.is_a ()); + + if (pt == nullptr) + ts = timestamp_unknown; + + target_state os (t.matched_state (a)); + + if (os == target_state::unchanged) + { + if (ts == timestamp_unknown) + return false; + else + { + // We expect the timestamp to be known (i.e., existing file). + // + timestamp mt (pt->mtime ()); + assert (mt != timestamp_unknown); + return mt > ts; + } + } + else + { + // We only want to return true if our call to execute() actually + // caused an update. In particular, the target could already have been + // in target_state::changed because of a dependency extraction run for + // some other source file. + // + // @@ MT perf: so we are going to switch the phase and execute for + // any generated header. + // + phase_switch ps (run_phase::execute); + target_state ns (execute_direct (a, t)); + + if (ns != os && ns != target_state::unchanged) + { + l6 ([&]{trace << "updated " << t + << "; old state " << os + << "; new state " << ns;}); + return true; + } + else + return ts != timestamp_unknown ? pt->newer (ts) : false; + } + } + + recipe compile_rule:: + apply (action a, target& xt) const + { + tracer trace (x, "compile_rule::apply"); + + file& t (xt.as ()); // Either obj*{} or bmi*{}. + + match_data& md (t.data ()); + bool mod (md.type == translation_type::module_iface); + + const scope& bs (t.base_scope ()); + const scope& rs (*bs.root_scope ()); + + otype ot (compile_type (t, mod)); + linfo li (link_info (bs, ot)); // Link info for selecting libraries. + compile_target_types tt (compile_types (ot)); + + // Derive file name from target name. + // + string e; // Primary target extension (module or object). + { + const char* o ("o"); // Object extension (.o or .obj). + + if (tsys == "win32-msvc") + { + switch (ot) + { + case otype::e: e = "exe."; break; + case otype::a: e = "lib."; break; + case otype::s: e = "dll."; break; + } + o = "obj"; + } + else if (tsys == "mingw32") + { + switch (ot) + { + case otype::e: e = "exe."; break; + case otype::a: e = "a."; break; + case otype::s: e = "dll."; break; + } + } + else if (tsys == "darwin") + { + switch (ot) + { + case otype::e: e = ""; break; + case otype::a: e = "a."; break; + case otype::s: e = "dylib."; break; + } + } + else + { + switch (ot) + { + case otype::e: e = ""; break; + case otype::a: e = "a."; break; + case otype::s: e = "so."; break; + } + } + + switch (cid) + { + case compiler_id::gcc: + { + e += mod ? "nms" : o; + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + e += mod ? "pcm" : o; + break; + } + case compiler_id::msvc: + { + e += mod ? "ifc" : o; + break; + } + case compiler_id::icc: + { + assert (!mod); + e += o; + } + } + + // If we are compiling a module, then the obj*{} is an ad hoc member + // of bmi*{}. + // + if (mod) + { + // The module interface unit can be the same as an implementation + // (e.g., foo.mxx and foo.cxx) which means obj*{} targets could + // collide. So we add the module extension to the target name. + // + target_lock obj (add_adhoc_member (a, t, tt.obj, e.c_str ())); + obj.target->as ().derive_path (o); + match_recipe (obj, group_recipe); // Set recipe and unlock. + } + } + + const path& tp (t.derive_path (e.c_str ())); + + // Inject dependency on the output directory. + // + const fsdir* dir (inject_fsdir (a, t)); + + // Match all the existing prerequisites. The injection code takes care + // of the ones it is adding. + // + // When cleaning, ignore prerequisites that are not in the same or a + // subdirectory of our project root. + // + auto& pts (t.prerequisite_targets[a]); + optional usr_lib_dirs; // Extract lazily. + + // Start asynchronous matching of prerequisites. Wait with unlocked + // phase to allow phase switching. + // + wait_guard wg (target::count_busy (), t[a].task_count, true); + + size_t start (pts.size ()); // Index of the first to be added. + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + const target* pt (nullptr); + + // A dependency on a library is there so that we can get its + // *.export.poptions, modules, etc. This is the "library + // meta-information protocol". See also append_lib_options(). + // + if (p.is_a () || + p.is_a () || + p.is_a () || + p.is_a ()) + { + if (a.operation () == update_id) + { + // Handle (phase two) imported libraries. We know that for such + // libraries we don't need to do match() in order to get options + // (if any, they would be set by search_library()). + // + if (p.proj ()) + { + if (search_library (a, + sys_lib_dirs, + usr_lib_dirs, + p.prerequisite) != nullptr) + continue; + } + + pt = &p.search (t); + + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, li); + } + else + continue; + } + // + // For modules we pick only what we import which is done below so + // skip it here. One corner case is clean: we assume that someone + // else (normally library/executable) also depends on it and will + // clean it up. + // + else if (p.is_a () || p.is_a (tt.bmi)) + continue; + else + { + pt = &p.search (t); + + if (a.operation () == clean_id && !pt->dir.sub (rs.out_path ())) + continue; + } + + match_async (a, *pt, target::count_busy (), t[a].task_count); + pts.push_back (pt); + } + + wg.wait (); + + // Finish matching all the targets that we have started. + // + for (size_t i (start), n (pts.size ()); i != n; ++i) + { + const target*& pt (pts[i]); + + // Making sure a library is updated before us will only restrict + // parallelism. But we do need to match it in order to get its imports + // resolved and prerequisite_targets populated. So we match it but + // then unmatch if it is safe. And thanks to the two-pass prerequisite + // match in link::apply() it will be safe unless someone is building + // an obj?{} target directory. + // + if (build2::match ( + a, + *pt, + pt->is_a () || pt->is_a () || pt->is_a () + ? unmatch::safe + : unmatch::none)) + pt = nullptr; // Ignore in execute. + } + + // Inject additional prerequisites. We only do it when performing update + // since chances are we will have to update some of our prerequisites in + // the process (auto-generated source code). + // + if (a == perform_update_id) + { + // The cached prerequisite target should be the same as what is in + // t.prerequisite_targets since we used standard search() and match() + // above. + // + const file& src (*md.src.search (t).is_a ()); + + // Figure out if __symexport is used. While normally it is specified + // on the project root (which we cached), it can be overridden with + // a target-specific value for installed modules (which we sidebuild + // as part of our project). + // + if (modules && src.is_a (*x_mod)) + { + lookup l (src.vars[x_symexport]); + md.symexport = l ? cast (l) : symexport; + } + + // Make sure the output directory exists. + // + // Is this the right thing to do? It does smell a bit, but then we do + // worse things in inject_prerequisites() below. There is also no way + // to postpone this until update since we need to extract and inject + // header dependencies now (we don't want to be calling search() and + // match() in update), which means we need to cache them now as well. + // So the only alternative, it seems, is to cache the updates to the + // database until later which will sure complicate (and slow down) + // things. + // + if (dir != nullptr) + { + // We can do it properly by using execute_direct(). But this means + // we will be switching to the execute phase with all the associated + // overheads. At the same time, in case of update, creation of a + // directory is not going to change the external state in any way + // that would affect any parallel efforts in building the internal + // state. So we are just going to create the directory directly. + // Note, however, that we cannot modify the fsdir{} target since + // this can very well be happening in parallel. But that's not a + // problem since fsdir{}'s update is idempotent. + // + fsdir_rule::perform_update_direct (a, t); + } + + // Note: the leading '@' is reserved for the module map prefix (see + // extract_modules()) and no other line must start with it. + // + md.dd = tp + ".d"; + depdb dd (md.dd); + + // First should come the rule name/version. + // + if (dd.expect (rule_id) != nullptr) + l4 ([&]{trace << "rule mismatch forcing update of " << t;}); + + // Then the compiler checksum. Note that here we assume it + // incorporates the (default) target so that if the compiler changes + // but only in what it targets, then the checksum will still change. + // + if (dd.expect (cast (rs[x_checksum])) != nullptr) + l4 ([&]{trace << "compiler mismatch forcing update of " << t;}); + + // Then the options checksum. + // + // The idea is to keep them exactly as they are passed to the compiler + // since the order may be significant. + // + { + sha256 cs; + + // These flags affect how we compile the source and/or the format of + // depdb so factor them in. + // + cs.append (&md.pp, sizeof (md.pp)); + cs.append (&md.symexport, sizeof (md.symexport)); + + if (md.pp != preprocessed::all) + { + hash_options (cs, t, c_poptions); + hash_options (cs, t, x_poptions); + + // Hash *.export.poptions from prerequisite libraries. + // + hash_lib_options (bs, cs, a, t, li); + + // Extra system header dirs (last). + // + assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); + hash_option_values ( + cs, "-I", + sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), + [] (const dir_path& d) {return d.string ();}); + } + + hash_options (cs, t, c_coptions); + hash_options (cs, t, x_coptions); + hash_options (cs, tstd); + + if (ot == otype::s) + { + // On Darwin, Win32 -fPIC is the default. + // + if (tclass == "linux" || tclass == "bsd") + cs.append ("-fPIC"); + } + + if (dd.expect (cs.string ()) != nullptr) + l4 ([&]{trace << "options mismatch forcing update of " << t;}); + } + + // Finally the source file. + // + if (dd.expect (src.path ()) != nullptr) + l4 ([&]{trace << "source file mismatch forcing update of " << t;}); + + // If any of the above checks resulted in a mismatch (different + // compiler, options, or source file) or if the depdb is newer than + // the target (interrupted update), then do unconditional update. + // + timestamp mt; + bool u (dd.writing () || dd.mtime () > (mt = file_mtime (tp))); + if (u) + mt = timestamp_nonexistent; // Treat as if it doesn't exist. + + // Update prerequisite targets (normally just the source file). + // + // This is an unusual place and time to do it. But we have to do it + // before extracting dependencies. The reasoning for source file is + // pretty clear. What other prerequisites could we have? While + // normally they will be some other sources (as in, static content + // from src_root), it's possible they are some auto-generated stuff. + // And it's possible they affect the preprocessor result. Say some ad + // hoc/out-of-band compiler input file that is passed via the command + // line. So, to be safe, we make sure everything is up to date. + // + for (const target* pt: pts) + { + if (pt == nullptr || pt == dir) + continue; + + u = update (trace, a, *pt, u ? timestamp_unknown : mt) || u; + } + + // Check if the source is already preprocessed to a certain degree. + // This determines which of the following steps we perform and on + // what source (original or preprocessed). + // + // Note: must be set on the src target. + // + if (const string* v = cast_null (src[x_preprocessed])) + try + { + md.pp = to_preprocessed (*v); + } + catch (const invalid_argument& e) + { + fail << "invalid " << x_preprocessed.name << " variable value " + << "for target " << src << ": " << e; + } + + // If we have no #include directives, then skip header dependency + // extraction. + // + pair psrc (auto_rmfile (), false); + if (md.pp < preprocessed::includes) + psrc = extract_headers (a, bs, t, li, src, md, dd, u, mt); + + // Next we "obtain" the translation unit information. What exactly + // "obtain" entails is tricky: If things changed, then we re-parse the + // translation unit. Otherwise, we re-create this information from + // depdb. We, however, have to do it here and now in case the database + // is invalid and we still have to fallback to re-parse. + // + // Store a translation unit's checksum to detect ignorable changes + // (whitespaces, comments, etc). + // + { + string cs; + if (string* l = dd.read ()) + cs = move (*l); + else + u = true; // Database is invalid, force re-parse. + + translation_unit tu; + for (bool f (true);; f = false) + { + if (u) + { + auto p (parse_unit (a, t, li, src, psrc.first, md)); + + if (cs != p.second) + { + assert (f); // Unchanged TU has a different checksum? + dd.write (p.second); + } + else if (f) // Don't clear if it was forced. + { + // Clear the update flag and set the touch flag. Unless there + // is no object file, of course. See also the md.mt logic + // below. + // + if (mt != timestamp_nonexistent) + { + u = false; + md.touch = true; + } + } + + tu = move (p.first); + } + + if (modules) + { + if (u || !f) + { + string s (to_string (tu.mod)); + + if (f) + dd.expect (s); + else + dd.write (s); + } + else + { + if (string* l = dd.read ()) + tu.mod = to_module_info (*l); + else + { + u = true; // Database is invalid, force re-parse. + continue; + } + } + } + + break; + } + + // Make sure the translation unit type matches the resulting target + // type. + // + switch (tu.type ()) + { + case translation_type::plain: + case translation_type::module_impl: + { + if (mod) + fail << "translation unit " << src << " is not a module interface" << + info << "consider using " << x_src.name << "{} instead"; + break; + } + case translation_type::module_iface: + { + if (!mod) + fail << "translation unit " << src << " is a module interface" << + info << "consider using " << x_mod->name << "{} instead"; + break; + } + } + + md.type = tu.type (); + + // Extract the module dependency information in addition to header + // dependencies. + // + // NOTE: assumes that no further targets will be added into + // t.prerequisite_targets! + // + extract_modules (a, bs, t, li, tt, src, md, move (tu.mod), dd, u); + } + + // If anything got updated, then we didn't rely on the cache. However, + // the cached data could actually have been valid and the compiler run + // in extract_headers() as well as the code above merely validated it. + // + // We do need to update the database timestamp, however. Failed that, + // we will keep re-validating the cached data over and over again. + // + if (u && dd.reading ()) + dd.touch (); + + dd.close (); + + // If the preprocessed output is suitable for compilation and is not + // disabled, then pass it along. + // + if (psrc.second && !cast_false (t[c_reprocess])) + { + md.psrc = move (psrc.first); + + // Without modules keeping the (partially) preprocessed output + // around doesn't buy us much: if the source/headers haven't changed + // then neither will the object file. Modules make things more + // interesting: now we may have to recompile an otherwise unchanged + // translation unit because a BMI it depends on has changed. In this + // case re-processing the translation unit would be a waste and + // compiling the original source would break distributed + // compilation. + // + // Note also that the long term trend will (hopefully) be for + // modularized projects to get rid of #include's which means the + // need for producing this partially preprocessed output will + // (hopefully) gradually disappear. + // + if (modules) + md.psrc.active = false; // Keep. + } + + // Above we may have ignored changes to the translation unit. The + // problem is, unless we also update the target's timestamp, we will + // keep re-checking this on subsequent runs and it is not cheap. + // Updating the target's timestamp is not without problems either: it + // will cause a re-link on a subsequent run. So, essentially, we + // somehow need to remember two timestamps: one for checking + // "preprocessor prerequisites" above and one for checking other + // prerequisites (like modules) below. So what we are going to do is + // store the first in the target file (so we do touch it) and the + // second in depdb (which is never newer that the target). + // + md.mt = u ? timestamp_nonexistent : dd.mtime (); + } + + switch (a) + { + case perform_update_id: return [this] (action a, const target& t) + { + return perform_update (a, t); + }; + case perform_clean_id: return [this] (action a, const target& t) + { + return perform_clean (a, t); + }; + default: return noop_recipe; // Configure update. + } + } + + // Reverse-lookup target type from extension. + // + const target_type* compile_rule:: + map_extension (const scope& s, const string& n, const string& e) const + { + // We will just have to try all of the possible ones, in the "most + // likely to match" order. + // + auto test = [&s, &n, &e] (const target_type& tt) -> bool + { + // Call the extension derivation function. Here we know that it will + // only use the target type and name from the target key so we can + // pass bogus values for the rest. + // + target_key tk {&tt, nullptr, nullptr, &n, nullopt}; + + // This is like prerequisite search. + // + if (optional de = tt.default_extension (tk, s, true)) + if (*de == e) + return true; + + return false; + }; + + for (const target_type* const* p (x_inc); *p != nullptr; ++p) + if (test (**p)) return *p; + + return nullptr; + } + + void compile_rule:: + append_prefixes (prefix_map& m, const target& t, const variable& var) const + { + tracer trace (x, "compile_rule::append_prefixes"); + + // If this target does not belong to any project (e.g, an "imported as + // installed" library), then it can't possibly generate any headers for + // us. + // + const scope& bs (t.base_scope ()); + const scope* rs (bs.root_scope ()); + if (rs == nullptr) + return; + + const dir_path& out_base (t.dir); + const dir_path& out_root (rs->out_path ()); + + if (auto l = t[var]) + { + const auto& v (cast (l)); + + for (auto i (v.begin ()), e (v.end ()); i != e; ++i) + { + // -I can either be in the "-Ifoo" or "-I foo" form. For VC it can + // also be /I. + // + const string& o (*i); + + if (o.size () < 2 || (o[0] != '-' && o[0] != '/') || o[1] != 'I') + continue; + + dir_path d; + if (o.size () == 2) + { + if (++i == e) + break; // Let the compiler complain. + + d = dir_path (*i); + } + else + d = dir_path (*i, 2, string::npos); + + l6 ([&]{trace << "-I " << d;}); + + if (d.relative ()) + fail << "relative -I directory " << d + << " in variable " << var.name + << " for target " << t; + + // If we are not inside our project root, then ignore. + // + if (!d.sub (out_root)) + continue; + + // If the target directory is a sub-directory of the include + // directory, then the prefix is the difference between the + // two. Otherwise, leave it empty. + // + // The idea here is to make this "canonical" setup work auto- + // magically: + // + // 1. We include all files with a prefix, e.g., . + // 2. The library target is in the foo/ sub-directory, e.g., + // /tmp/foo/. + // 3. The poptions variable contains -I/tmp. + // + dir_path p (out_base.sub (d) ? out_base.leaf (d) : dir_path ()); + + // We use the target's directory as out_base but that doesn't work + // well for targets that are stashed in subdirectories. So as a + // heuristics we are going to also enter the outer directories of + // the original prefix. It is, however, possible, that another -I + // option after this one will produce one of these outer prefixes as + // its original prefix in which case we should override it. + // + // So we are going to assign the original prefix priority value 0 + // (highest) and then increment it for each outer prefix. + // + auto enter = [&trace, &m] (dir_path p, dir_path d, size_t prio) + { + auto j (m.find (p)); + + if (j != m.end ()) + { + prefix_value& v (j->second); + + // We used to reject duplicates but it seems this can be + // reasonably expected to work according to the order of the + // -I options. + // + // Seeing that we normally have more "specific" -I paths first, + // (so that we don't pick up installed headers, etc), we ignore + // it. + // + if (v.directory == d) + { + if (v.priority > prio) + v.priority = prio; + } + else if (v.priority <= prio) + { + if (verb >= 4) + trace << "ignoring dependency prefix " << p << '\n' + << " existing mapping to " << v.directory + << " priority " << v.priority << '\n' + << " another mapping to " << d + << " priority " << prio; + } + else + { + if (verb >= 4) + trace << "overriding dependency prefix " << p << '\n' + << " existing mapping to " << v.directory + << " priority " << v.priority << '\n' + << " new mapping to " << d + << " priority " << prio; + + v.directory = move (d); + v.priority = prio; + } + } + else + { + l6 ([&]{trace << p << " -> " << d << " priority " << prio;}); + m.emplace (move (p), prefix_value {move (d), prio}); + } + }; + + size_t prio (0); + for (bool e (false); !e; ++prio) + { + dir_path n (p.directory ()); + e = n.empty (); + enter ((e ? move (p) : p), (e ? move (d) : d), prio); + p = move (n); + } + } + } + } + + auto compile_rule:: + build_prefix_map (const scope& bs, + action a, + target& t, + linfo li) const -> prefix_map + { + prefix_map m; + + // First process our own. + // + append_prefixes (m, t, c_poptions); + append_prefixes (m, t, x_poptions); + + // Then process the include directories from prerequisite libraries. + // + append_lib_prefixes (bs, m, a, t, li); + + return m; + } + + // Return the next make prerequisite starting from the specified + // position and update position to point to the start of the + // following prerequisite or l.size() if there are none left. + // + static string + next_make (const string& l, size_t& p) + { + size_t n (l.size ()); + + // Skip leading spaces. + // + for (; p != n && l[p] == ' '; p++) ; + + // Lines containing multiple prerequisites are 80 characters max. + // + string r; + r.reserve (n); + + // Scan the next prerequisite while watching out for escape sequences. + // + for (; p != n && l[p] != ' '; p++) + { + char c (l[p]); + + if (p + 1 != n) + { + if (c == '$') + { + // Got to be another (escaped) '$'. + // + if (l[p + 1] == '$') + ++p; + } + else if (c == '\\') + { + // This may or may not be an escape sequence depending on whether + // what follows is "escapable". + // + switch (c = l[++p]) + { + case '\\': break; + case ' ': break; + default: c = '\\'; --p; // Restore. + } + } + } + + r += c; + } + + // Skip trailing spaces. + // + for (; p != n && l[p] == ' '; p++) ; + + // Skip final '\'. + // + if (p == n - 1 && l[p] == '\\') + p++; + + return r; + } + + // VC /showIncludes output. The first line is the file being compiled + // (handled by our caller). Then we have the list of headers, one per + // line, in this form (text can presumably be translated): + // + // Note: including file: C:\Program Files (x86)\[...]\iostream + // + // Finally, if we hit a non-existent header, then we end with an error + // line in this form: + // + // x.cpp(3): fatal error C1083: Cannot open include file: 'd/h.hpp': + // No such file or directory + // + // Distinguishing between the include note and the include error is + // easy: we can just check for C1083. Distinguising between the note and + // other errors/warnings is harder: an error could very well end with + // what looks like a path so we cannot look for the note but rather have + // to look for an error. Here we assume that a line containing ' CNNNN:' + // is an error. Should be robust enough in the face of language + // translation, etc. + // + // It turns out C1083 is also used when we are unable to open the main + // source file and the error line looks like this: + // + // c1xx: fatal error C1083: Cannot open source file: 's.cpp': No such + // file or directory + + // Sense whether this is an include note (return npos) or a diagnostics + // line (return postion of the NNNN code in CNNNN). + // + static inline size_t + next_show_sense (const string& l) + { + size_t p (l.find (':')); + + for (size_t n (l.size ()); + p != string::npos; + p = ++p != n ? l.find (':', p) : string::npos) + { + auto isnum = [](char c) {return c >= '0' && c <= '9';}; + + if (p > 5 && + l[p - 6] == ' ' && + l[p - 5] == 'C' && + isnum (l[p - 4]) && + isnum (l[p - 3]) && + isnum (l[p - 2]) && + isnum (l[p - 1])) + { + p -= 4; // Start of the error code. + break; + } + } + + return p; + } + + // Extract the include path from the VC /showIncludes output line. Return + // empty string if the line is not an include note or include error. Set + // the good_error flag if it is an include error (which means the process + // will terminate with the error status that needs to be ignored). + // + static string + next_show (const string& l, bool& good_error) + { + // The include error should be the last line that we handle. + // + assert (!good_error); + + size_t p (next_show_sense (l)); + if (p == string::npos) + { + // Include note. We assume the path is always at the end but need to + // handle both absolute Windows and POSIX ones. + // + // Note that VC appears to always write the absolute path to the + // included file even if it is ""-included and the source path is + // relative. Aren't we lucky today? + // + p = l.rfind (':'); + + if (p != string::npos) + { + // See if this one is part of the Windows drive letter. + // + if (p > 1 && p + 1 < l.size () && // 2 chars before, 1 after. + l[p - 2] == ' ' && + alpha (l[p - 1]) && + path::traits::is_separator (l[p + 1])) + p = l.rfind (':', p - 2); + } + + if (p != string::npos) + { + // VC uses indentation to indicate the include nesting so there + // could be any number of spaces after ':'. Skip them. + // + p = l.find_first_not_of (' ', p + 1); + } + + if (p == string::npos) + fail << "unable to parse /showIncludes include note line"; + + return string (l, p); + } + else if (l.compare (p, 4, "1083") == 0 && + l.compare (0, 5, "c1xx:") != 0 /* Not the main source file. */ ) + { + // Include error. The path is conveniently quoted with ''. + // + size_t p2 (l.rfind ('\'')); + + if (p2 != string::npos && p2 != 0) + { + size_t p1 (l.rfind ('\'', p2 - 1)); + + if (p1 != string::npos) + { + good_error = true; + return string (l, p1 + 1 , p2 - p1 - 1); + } + } + + fail << "unable to parse /showIncludes include error line" << endf; + } + else + { + // Some other error. + // + return string (); + } + } + + // Extract and inject header dependencies. Return the preprocessed source + // file as well as an indication if it is usable for compilation (see + // below for details). + // + pair compile_rule:: + extract_headers (action a, + const scope& bs, + file& t, + linfo li, + const file& src, + const match_data& md, + depdb& dd, + bool& updating, + timestamp mt) const + { + tracer trace (x, "compile_rule::extract_headers"); + + l5 ([&]{trace << "target: " << t;}); + + auto_rmfile psrc; + bool puse (true); + + // If things go wrong (and they often do in this area), give the user a + // bit extra context. + // + auto df = make_diag_frame ( + [&src](const diag_record& dr) + { + if (verb != 0) + dr << info << "while extracting header dependencies from " << src; + }); + + const scope& rs (*bs.root_scope ()); + + // Preprocess mode that preserves as much information as possible while + // still performing inclusions. Also serves as a flag indicating whether + // this compiler uses the separate preprocess and compile setup. + // + const char* pp (nullptr); + + switch (cid) + { + case compiler_id::gcc: + { + // -fdirectives-only is available since GCC 4.3.0. + // + if (cmaj > 4 || (cmaj == 4 && cmin >= 3)) + pp = "-fdirectives-only"; + + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + // -frewrite-includes is available since vanilla Clang 3.2.0. + // + // Apple Clang 5.0 is based on LLVM 3.3svn so it should have this + // option (4.2 is based on 3.2svc so it may or may not have it and, + // no, we are not going to try to find out). + // + if (cid == compiler_id::clang_apple + ? (cmaj >= 5) + : (cmaj > 3 || (cmaj == 3 && cmin >= 2))) + pp = "-frewrite-includes"; + + break; + } + case compiler_id::msvc: + { + pp = "/C"; + break; + } + case compiler_id::icc: + break; + } + + // Initialize lazily, only if required. + // + environment env; + cstrings args; + string out; // Storage. + + // Some compilers in certain modes (e.g., when also producing the + // preprocessed output) are incapable of writing the dependecy + // information to stdout. In this case we use a temporary file. + // + auto_rmfile drm; + + // Here is the problem: neither GCC nor Clang allow -MG (treat missing + // header as generated) when we produce any kind of other output (-MD). + // And that's probably for the best since otherwise the semantics gets + // pretty hairy (e.g., what is the exit code and state of the output)? + // + // One thing to note about generated headers: if we detect one, then, + // after generating it, we re-run the compiler since we need to get + // this header's dependencies. + // + // So this is how we are going to work around this problem: we first run + // with -E but without -MG. If there are any errors (maybe because of + // generated headers maybe not), we restart with -MG and without -E. If + // this fixes the error (so it was a generated header after all), then + // we have to restart at which point we go back to -E and no -MG. And we + // keep yo-yoing like this. Missing generated headers will probably be + // fairly rare occurrence so this shouldn't be too expensive. + // + // Actually, there is another error case we would like to handle: an + // outdated generated header that is now causing an error (e.g., because + // of a check that is now triggering #error or some such). So there are + // actually three error cases: outdated generated header, missing + // generated header, and some other error. To handle the outdated case + // we need the compiler to produce the dependency information even in + // case of an error. Clang does it, for VC we parse diagnostics + // ourselves, but GCC does not (but a patch has been submitted). + // + // So the final plan is then as follows: + // + // 1. Start wothout -MG and with suppressed diagnostics. + // 2. If error but we've updated a header, then repeat step 1. + // 3. Otherwise, restart with -MG and diagnostics. + // + // Note that below we don't even check if the compiler supports the + // dependency info on error. We just try to use it and if it's not + // there we ignore the io error since the compiler has failed. + // + bool args_gen; // Current state of args. + size_t args_i; // Start of the -M/-MD "tail". + + // Ok, all good then? Not so fast, the rabbit hole is deeper than it + // seems: When we run with -E we have to discard diagnostics. This is + // not a problem for errors since they will be shown on the re-run but + // it is for (preprocessor) warnings. + // + // Clang's -frewrite-includes is nice in that it preserves the warnings + // so they will be shown during the compilation of the preprocessed + // source. They are also shown during -E but that we discard. And unlike + // GCC, in Clang -M does not imply -w (disable warnings) so it would + // have been shown in -M -MG re-runs but we suppress that with explicit + // -w. All is good in the Clang land then (even -Werror works nicely). + // + // GCC's -fdirective-only, on the other hand, processes all the + // directives so they are gone from the preprocessed source. Here is + // what we are going to do to work around this: we will detect if any + // diagnostics has been written to stderr on the -E run. If that's the + // case (but the compiler indicated success) then we assume they are + // warnings and disable the use of the preprocessed output for + // compilation. This in turn will result in compilation from source + // which will display the warnings. Note that we may still use the + // preprocessed output for other things (e.g., C++ module dependency + // discovery). BTW, another option would be to collect all the + // diagnostics and then dump it if the run is successful, similar to + // the VC semantics (and drawbacks) described below. + // + // Finally, for VC, things are completely different: there is no -MG + // equivalent and we handle generated headers by analyzing the + // diagnostics. This means that unlike in the above two cases, the + // preprocessor warnings are shown during dependency extraction, not + // compilation. Not ideal but that's the best we can do. Or is it -- we + // could implement ad hoc diagnostics sensing... It appears warnings are + // in the C4000-C4999 code range though there can also be note lines + // which don't have any C-code. + // + // BTW, triggering a warning in the VC preprocessor is not easy; there + // is no #warning and pragmas are passed through to the compiler. One + // way to do it is to redefine a macro, for example: + // + // hello.cxx(4): warning C4005: 'FOO': macro redefinition + // hello.cxx(3): note: see previous definition of 'FOO' + // + // So seeing that it is hard to trigger a legitimate VC preprocessor + // warning, for now, we will just treat them as errors by adding /WX. + // + // Note: diagnostics sensing is currently only supported if dependency + // info is written to a file (see above). + // + bool sense_diag (false); + + // And here is another problem: if we have an already generated header + // in src and the one in out does not yet exist, then the compiler will + // pick the one in src and we won't even notice. Note that this is not + // only an issue with mixing in- and out-of-tree builds (which does feel + // wrong but is oh so convenient): this is also a problem with + // pre-generated headers, a technique we use to make installing the + // generator by end-users optional by shipping pre-generated headers. + // + // This is a nasty problem that doesn't seem to have a perfect solution + // (except, perhaps, C++ modules). So what we are going to do is try to + // rectify the situation by detecting and automatically remapping such + // mis-inclusions. It works as follows. + // + // First we will build a map of src/out pairs that were specified with + // -I. Here, for performance and simplicity, we will assume that they + // always come in pairs with out first and src second. We build this + // map lazily only if we are running the preprocessor and reuse it + // between restarts. + // + // With the map in hand we can then check each included header for + // potentially having a doppelganger in the out tree. If this is the + // case, then we calculate a corresponding header in the out tree and, + // (this is the most important part), check if there is a target for + // this header in the out tree. This should be fairly accurate and not + // require anything explicit from the user except perhaps for a case + // where the header is generated out of nothing (so there is no need to + // explicitly mention its target in the buildfile). But this probably + // won't be very common. + // + // One tricky area in this setup are target groups: if the generated + // sources are mentioned in the buildfile as a group, then there might + // be no header target (yet). The way we solve this is by requiring code + // generator rules to cooperate and create at least the header target as + // part of the group creation. While not all members of the group may be + // generated depending on the options (e.g., inline files might be + // suppressed), headers are usually non-optional. + // + // Note that we use path_map instead of dir_path_map to allow searching + // using path (file path). + // + using srcout_map = path_map; + srcout_map so_map; + + // The gen argument to init_args() is in/out. The caller signals whether + // to force the generated header support and on return it signals + // whether this support is enabled. The first call to init_args is + // expected to have gen false. + // + // Return NULL if the dependency information goes to stdout and a + // pointer to the temporary file path otherwise. + // + auto init_args = [&t, a, li, + &src, &md, &psrc, &sense_diag, + &rs, &bs, + pp, &env, &args, &args_gen, &args_i, &out, &drm, + &so_map, this] + (bool& gen) -> const path* + { + const path* r (nullptr); + + if (args.empty ()) // First call. + { + assert (!gen); + + // We use absolute/relative paths in the dependency output to + // distinguish existing headers from (missing) generated. Which + // means we have to (a) use absolute paths in -I and (b) pass + // absolute source path (for ""-includes). That (b) is a problem: + // if we use an absolute path, then all the #line directives will be + // absolute and all the diagnostics will have long, noisy paths + // (actually, we will still have long paths for diagnostics in + // headers). + // + // To work around this we used to pass a relative path to the source + // file and then check every relative path in the dependency output + // for existence in the source file's directory. This is not without + // issues: it is theoretically possible for a generated header that + // is <>-included and found via -I to exist in the source file's + // directory. Note, however, that this is a lot more likely to + // happen with prefix-less inclusion (e.g., ) and in this case + // we assume the file is in the project anyway. And if there is a + // conflict with a prefixed include (e.g., ), then, well, + // we will just have to get rid of quoted includes (which are + // generally a bad idea, anyway). + // + // But then this approach (relative path) fell apart further when we + // tried to implement precise changed detection: the preprocessed + // output would change depending from where it was compiled because + // of #line (which we could work around) and __FILE__/assert() + // (which we can't really do anything about). So it looks like using + // the absolute path is the lesser of all the evils (and there are + // many). + // + // Note that we detect and diagnose relative -I directories lazily + // when building the include prefix map. + // + args.push_back (cpath.recall_string ()); + + // Add *.export.poptions from prerequisite libraries. + // + append_lib_options (bs, args, a, t, li); + + append_options (args, t, c_poptions); + append_options (args, t, x_poptions); + + // Populate the src-out with the -I$out_base -I$src_base pairs. + // + { + // Try to be fast and efficient by reusing buffers as much as + // possible. + // + string ds; + + // Previous -I innermost scope if out_base plus the difference + // between the scope path and the -I path (normally empty). + // + const scope* s (nullptr); + dir_path p; + + for (auto i (args.begin ()), e (args.end ()); i != e; ++i) + { + // -I can either be in the "-Ifoo" or "-I foo" form. For VC it + // can also be /I. + // + const char* o (*i); + size_t n (strlen (o)); + + if (n < 2 || (o[0] != '-' && o[0] != '/') || o[1] != 'I') + { + s = nullptr; + continue; + } + + if (n == 2) + { + if (++i == e) + break; // Let the compiler complain. + + ds = *i; + } + else + ds.assign (o + 2, n - 2); + + if (!ds.empty ()) + { + // Note that we don't normalize the paths since it would be + // quite expensive and normally the pairs we are inerested in + // are already normalized (since they are usually specified as + // -I$src/out_*). We just need to add a trailing directory + // separator if it's not already there. + // + if (!dir_path::traits::is_separator (ds.back ())) + ds += dir_path::traits::directory_separator; + + dir_path d (move (ds), dir_path::exact); // Move the buffer in. + + // Ignore invalid paths (buffer is not moved). + // + if (!d.empty ()) + { + // Ignore any paths containing '.', '..' components. Allow + // any directory separators thought (think -I$src_root/foo + // on Windows). + // + if (d.absolute () && d.normalized (false)) + { + // If we have a candidate out_base, see if this is its + // src_base. + // + if (s != nullptr) + { + const dir_path& bp (s->src_path ()); + + if (d.sub (bp)) + { + if (p.empty () || d.leaf (bp) == p) + { + // We've got a pair. + // + so_map.emplace (move (d), s->out_path () / p); + s = nullptr; // Taken. + continue; + } + } + + // Not a pair. Fall through to consider as out_base. + // + s = nullptr; + } + + // See if this path is inside a project with an out-of- + // tree build and is in the out directory tree. + // + const scope& bs (scopes.find (d)); + if (bs.root_scope () != nullptr) + { + const dir_path& bp (bs.out_path ()); + if (bp != bs.src_path ()) + { + bool e; + if ((e = (d == bp)) || d.sub (bp)) + { + s = &bs; + if (e) + p.clear (); + else + p = d.leaf (bp); + } + } + } + } + else + s = nullptr; + + ds = move (d).string (); // Move the buffer out. + } + else + s = nullptr; + } + else + s = nullptr; + } + } + + // Extra system header dirs (last). + // + assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); + append_option_values ( + args, "-I", + sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), + [] (const dir_path& d) {return d.string ().c_str ();}); + + if (md.symexport) + append_symexport_options (args, t); + + // Some compile options (e.g., -std, -m) affect the preprocessor. + // + // Currently Clang supports importing "header modules" even when in + // the TS mode. And "header modules" support macros which means + // imports have to be resolved during preprocessing. Which poses a + // bit of a chicken and egg problem for us. For now, the workaround + // is to remove the -fmodules-ts option when preprocessing. Hopefully + // there will be a "pure modules" mode at some point. + // + + // Don't treat warnings as errors. + // + const char* werror (nullptr); + switch (cclass) + { + case compiler_class::gcc: werror = "-Werror"; break; + case compiler_class::msvc: werror = "/WX"; break; + } + + bool clang (cid == compiler_id::clang || + cid == compiler_id::clang_apple); + + append_options (args, t, c_coptions, werror); + append_options (args, t, x_coptions, werror); + append_options (args, tstd, + tstd.size () - (modules && clang ? 1 : 0)); + + switch (cclass) + { + case compiler_class::msvc: + { + assert (pp != nullptr); + + args.push_back ("/nologo"); + + // See perform_update() for details on overriding the default + // exceptions and runtime. + // + if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) + args.push_back ("/EHsc"); + + if (!find_option_prefixes ({"/MD", "/MT"}, args)) + args.push_back ("/MD"); + + args.push_back ("/P"); // Preprocess to file. + args.push_back ("/showIncludes"); // Goes to stdout (with diag). + args.push_back (pp); // /C (preserve comments). + args.push_back ("/WX"); // Warning as error (see above). + + psrc = auto_rmfile (t.path () + x_pext); + + if (cast (rs[x_version_major]) >= 18) + { + args.push_back ("/Fi:"); + args.push_back (psrc.path.string ().c_str ()); + } + else + { + out = "/Fi" + psrc.path.string (); + args.push_back (out.c_str ()); + } + + args.push_back (langopt (md)); // Compile as. + gen = args_gen = true; + break; + } + case compiler_class::gcc: + { + if (t.is_a ()) + { + // On Darwin, Win32 -fPIC is the default. + // + if (tclass == "linux" || tclass == "bsd") + args.push_back ("-fPIC"); + } + + // Depending on the compiler, decide whether (and how) we can + // produce preprocessed output as a side effect of dependency + // extraction. + // + // Note: -MM -MG skips missing <>-included. + + // Clang's -M does not imply -w (disable warnings). We also + // don't need them in the -MD case (see above) so disable for + // both. + // + if (clang) + args.push_back ("-w"); + + // Previously we used '*' as a target name but it gets expanded + // to the current directory file names by GCC (4.9) that comes + // with MSYS2 (2.4). Yes, this is the (bizarre) behavior of GCC + // being executed in the shell with -MQ '*' option and not just + // -MQ *. + // + args.push_back ("-MQ"); // Quoted target name. + args.push_back ("^"); // Old versions can't do empty target. + + args.push_back ("-x"); + args.push_back (langopt (md)); + + if (pp != nullptr) + { + // Note that the options are carefully laid out to be easy to + // override (see below). + // + args_i = args.size (); + + args.push_back ("-MD"); + args.push_back ("-E"); + args.push_back (pp); + + // Dependency output. + // + args.push_back ("-MF"); + + // GCC is not capable of writing the dependency info to + // stdout. We also need to sense the diagnostics on the -E + // runs. + // + if (cid == compiler_id::gcc) + { + // Use the .t extension (for "temporary"; .d is taken). + // + r = &(drm = auto_rmfile (t.path () + ".t")).path; + args.push_back (r->string ().c_str ()); + + sense_diag = true; + } + else + args.push_back ("-"); + + // Preprocessor output. + // + psrc = auto_rmfile (t.path () + x_pext); + args.push_back ("-o"); + args.push_back (psrc.path.string ().c_str ()); + } + else + { + args.push_back ("-M"); + args.push_back ("-MG"); // Treat missing headers as generated. + } + + gen = args_gen = (pp == nullptr); + break; + } + } + + args.push_back (src.path ().string ().c_str ()); + args.push_back (nullptr); + + // Note: only doing it here. + // + if (!env.empty ()) + env.push_back (nullptr); + } + else + { + assert (gen != args_gen); + + size_t i (args_i); + + if (gen) + { + // Overwrite. + // + args[i++] = "-M"; + args[i++] = "-MG"; + args[i++] = src.path ().string ().c_str (); + args[i] = nullptr; + + if (cid == compiler_id::gcc) + { + sense_diag = false; + } + } + else + { + // Restore. + // + args[i++] = "-MD"; + args[i++] = "-E"; + args[i++] = pp; + args[i] = "-MF"; + + if (cid == compiler_id::gcc) + { + r = &drm.path; + sense_diag = true; + } + } + + args_gen = gen; + } + + return r; + }; + + // Build the prefix map lazily only if we have non-existent files. + // Also reuse it over restarts since it doesn't change. + // + optional pfx_map; + + // If any prerequisites that we have extracted changed, then we have to + // redo the whole thing. The reason for this is auto-generated headers: + // the updated header may now include a yet-non-existent header. Unless + // we discover this and generate it (which, BTW, will trigger another + // restart since that header, in turn, can also include auto-generated + // headers), we will end up with an error during compilation proper. + // + // One complication with this restart logic is that we will see a + // "prefix" of prerequisites that we have already processed (i.e., they + // are already in our prerequisite_targets list) and we don't want to + // keep redoing this over and over again. One thing to note, however, is + // that the prefix that we have seen on the previous run must appear + // exactly the same in the subsequent run. The reason for this is that + // none of the files that it can possibly be based on have changed and + // thus it should be exactly the same. To put it another way, the + // presence or absence of a file in the dependency output can only + // depend on the previous files (assuming the compiler outputs them as + // it encounters them and it is hard to think of a reason why would + // someone do otherwise). And we have already made sure that all those + // files are up to date. And here is the way we are going to exploit + // this: we are going to keep track of how many prerequisites we have + // processed so far and on restart skip right to the next one. + // + // And one more thing: most of the time this list of headers would stay + // unchanged and extracting them by running the compiler every time is a + // bit wasteful. So we are going to cache them in the depdb. If the db + // hasn't been invalidated yet (e.g., because the compiler options have + // changed), then we start by reading from it. If anything is out of + // date then we use the same restart and skip logic to switch to the + // compiler run. + // + size_t skip_count (0); + + // Update and add a header file to the list of prerequisite targets. + // Depending on the cache flag, the file is assumed to either have come + // from the depdb cache or from the compiler run. Return whether the + // extraction process should be restarted. + // + auto add = [&trace, &pfx_map, &so_map, + a, &t, li, + &dd, &updating, &skip_count, + &bs, this] + (path f, bool cache, timestamp mt) -> bool + { + // Find or maybe insert the target. The directory is only moved + // from if insert is true. + // + auto find = [&trace, &t, this] + (dir_path&& d, path&& f, bool insert) -> const path_target* + { + // Split the file into its name part and extension. Here we can + // assume the name part is a valid filesystem name. + // + // Note that if the file has no extension, we record an empty + // extension rather than NULL (which would signify that the default + // extension should be added). + // + string e (f.extension ()); + string n (move (f).string ()); + + if (!e.empty ()) + n.resize (n.size () - e.size () - 1); // One for the dot. + + // Determine the target type. + // + const target_type* tt (nullptr); + + // See if this directory is part of any project out_root hierarchy. + // Note that this will miss all the headers that come from src_root + // (so they will be treated as generic C headers below). Generally, + // we don't have the ability to determine that some file belongs to + // src_root of some project. But that's not a problem for our + // purposes: it is only important for us to accurately determine + // target types for headers that could be auto-generated. + // + // While at it also try to determine if this target is from the src + // or out tree of said project. + // + dir_path out; + + const scope& bs (scopes.find (d)); + if (const scope* rs = bs.root_scope ()) + { + tt = map_extension (bs, n, e); + + if (bs.out_path () != bs.src_path () && d.sub (bs.src_path ())) + out = out_src (d, *rs); + } + + // If it is outside any project, or the project doesn't have such an + // extension, assume it is a plain old C header. + // + if (tt == nullptr) + { + // If the project doesn't "know" this extension then we won't + // possibly find an explicit target of this type. + // + if (!insert) + return nullptr; + + tt = &h::static_type; + } + + // Find or insert target. + // + // @@ OPT: move d, out, n + // + const target* r; + if (insert) + r = &search (t, *tt, d, out, n, &e, nullptr); + else + { + // Note that we skip any target type-specific searches (like for + // an existing file) and go straight for the target object since + // we need to find the target explicitly spelled out. + // + r = targets.find (*tt, d, out, n, e, trace); + } + + return static_cast (r); + }; + + // If it's not absolute then it either does not (yet) exist or is + // a relative ""-include (see init_args() for details). Reduce the + // second case to absolute. + // + // Note: we now always use absolute path to the translation unit so + // this no longer applies. + // +#if 0 + if (f.relative () && rels.relative ()) + { + // If the relative source path has a directory component, make sure + // it matches since ""-include will always start with that (none of + // the compilers we support try to normalize this path). Failed that + // we may end up searching for a generated header in a random + // (working) directory. + // + const string& fs (f.string ()); + const string& ss (rels.string ()); + + size_t p (path::traits::rfind_separator (ss)); + + if (p == string::npos || // No directory. + (fs.size () > p + 1 && + path::traits::compare (fs.c_str (), p, ss.c_str (), p) == 0)) + { + path t (work / f); // The rels path is relative to work. + + if (exists (t)) + f = move (t); + } + } +#endif + + const path_target* pt (nullptr); + + // If still relative then it does not exist. + // + if (f.relative ()) + { + f.normalize (); + + // This is probably as often an error as an auto-generated file, so + // trace at level 4. + // + l4 ([&]{trace << "non-existent header '" << f << "'";}); + + if (!pfx_map) + pfx_map = build_prefix_map (bs, a, t, li); + + // First try the whole file. Then just the directory. + // + // @@ Has to be a separate map since the prefix can be the same as + // the file name. + // + // auto i (pfx_map->find (f)); + + // Find the most qualified prefix of which we are a sub-path. + // + if (!pfx_map->empty ()) + { + dir_path d (f.directory ()); + auto i (pfx_map->find_sup (d)); + + if (i != pfx_map->end ()) + { + const dir_path& pd (i->second.directory); + + // If this is a prefixless mapping, then only use it if we can + // resolve it to an existing target (i.e., it is explicitly + // spelled out in a buildfile). + // + // Note that at some point we will probably have a list of + // directories. + // + pt = find (pd / d, f.leaf (), !i->first.empty ()); + if (pt != nullptr) + { + f = pd / f; + l4 ([&]{trace << "mapped as auto-generated " << f;}); + } + } + } + + if (pt == nullptr) + { + diag_record dr (fail); + dr << "header '" << f << "' not found and cannot be generated"; + //for (const auto& p: pm) + // dr << info << p.first.string () << " -> " << p.second.string (); + } + } + else + { + // We used to just normalize the path but that could result in an + // invalid path (e.g., on CentOS 7 with Clang 3.4) because of the + // symlinks. So now we realize (i.e., realpath(3)) it instead. + // Unless it comes from the depdb, in which case we've already done + // that. This is also where we handle src-out remap (again, not + // needed if cached) + // + if (!cache) + { + // While we can reasonably expect this path to exit, things do + // go south from time to time (like compiling under wine with + // file wlantypes.h included as WlanTypes.h). + // + try + { + f.realize (); + } + catch (const invalid_path&) + { + fail << "invalid header path '" << f << "'"; + } + catch (const system_error& e) + { + fail << "invalid header path '" << f << "': " << e; + } + + if (!so_map.empty ()) + { + // Find the most qualified prefix of which we are a sub-path. + // + auto i (so_map.find_sup (f)); + if (i != so_map.end ()) + { + // Ok, there is an out tree for this headers. Remap to a path + // from the out tree and see if there is a target for it. + // + dir_path d (i->second); + d /= f.leaf (i->first).directory (); + pt = find (move (d), f.leaf (), false); // d is not moved from. + + if (pt != nullptr) + { + path p (d / f.leaf ()); + l4 ([&]{trace << "remapping " << f << " to " << p;}); + f = move (p); + } + } + } + } + + if (pt == nullptr) + { + l6 ([&]{trace << "injecting " << f;}); + pt = find (f.directory (), f.leaf (), true); + } + } + + // Cache the path. + // + const path& pp (pt->path (move (f))); + + // Match to a rule. + // + // If we are reading the cache, then it is possible the file has since + // been removed (think of a header in /usr/local/include that has been + // uninstalled and now we need to use one from /usr/include). This + // will lead to the match failure which we translate to a restart. + // + if (!cache) + build2::match (a, *pt); + else if (!build2::try_match (a, *pt).first) + { + dd.write (); // Invalidate this line. + updating = true; + return true; + } + + // Update. + // + bool restart (update (trace, a, *pt, mt)); + + // Verify/add it to the dependency database. We do it after update in + // order not to add bogus files (non-existent and without a way to + // update). + // + if (!cache) + dd.expect (pp); + + // Add to our prerequisite target list. + // + t.prerequisite_targets[a].push_back (pt); + skip_count++; + + updating = updating || restart; + return restart; + }; + + // If nothing so far has invalidated the dependency database, then try + // the cached data before running the compiler. + // + bool cache (!updating); + + // See init_args() above for details on generated header support. + // + bool gen (false); + optional force_gen; + optional force_gen_skip; // Skip count at last force_gen run. + + const path* drmp (nullptr); // Points to drm.path () if active. + + for (bool restart (true); restart; cache = false) + { + restart = false; + + if (cache) + { + // If any, this is always the first run. + // + assert (skip_count == 0); + + // We should always end with a blank line. + // + for (;;) + { + string* l (dd.read ()); + + // If the line is invalid, run the compiler. + // + if (l == nullptr) + { + restart = true; + break; + } + + if (l->empty ()) // Done, nothing changed. + { + // If modules are enabled, then we keep the preprocessed output + // around (see apply() for details). + // + return modules + ? make_pair (auto_rmfile (t.path () + x_pext, false), true) + : make_pair (auto_rmfile (), false); + } + + // If this header came from the depdb, make sure it is no older + // than the target (if it has changed since the target was + // updated, then the cached data is stale). + // + restart = add (path (move (*l)), true, mt); + + if (restart) + { + l6 ([&]{trace << "restarting (cache)";}); + break; + } + } + } + else + { + try + { + if (force_gen) + gen = *force_gen; + + if (args.empty () || gen != args_gen) + drmp = init_args (gen); + + if (verb >= 3) + print_process (args.data ()); // Disable pipe mode. + + process pr; + + try + { + // Assume the preprocessed output (if produced) is usable + // until proven otherwise. + // + puse = true; + + // Save the timestamp just before we start preprocessing. If + // we depend on any header that has been updated since, then + // we should assume we've "seen" the old copy and re-process. + // + timestamp pmt (system_clock::now ()); + + // If we have no generated header support, then suppress all + // diagnostics (if things go badly we will restart with this + // support). + // + if (drmp == nullptr) + { + // Dependency info goes to stdout. + // + assert (!sense_diag); + + // For VC with /P the dependency info and diagnostics all go + // to stderr so redirect it to stdout. + // + pr = process ( + cpath, + args.data (), + 0, + -1, + cclass == compiler_class::msvc ? 1 : gen ? 2 : -2, + nullptr, // CWD + env.empty () ? nullptr : env.data ()); + } + else + { + // Dependency info goes to a temporary file. + // + pr = process (cpath, + args.data (), + 0, + 2, // Send stdout to stderr. + gen ? 2 : sense_diag ? -1 : -2, + nullptr, // CWD + env.empty () ? nullptr : env.data ()); + + // If requested, monitor for diagnostics and if detected, mark + // the preprocessed output as unusable for compilation. + // + if (sense_diag) + { + ifdstream is (move (pr.in_efd), fdstream_mode::skip); + puse = puse && (is.peek () == ifdstream::traits_type::eof ()); + is.close (); + } + + // The idea is to reduce it to the stdout case. + // + pr.wait (); + pr.in_ofd = fdopen (*drmp, fdopen_mode::in); + } + + // We may not read all the output (e.g., due to a restart). + // Before we used to just close the file descriptor to signal to + // the other end that we are not interested in the rest. This + // works fine with GCC but Clang (3.7.0) finds this impolite and + // complains, loudly (broken pipe). So now we are going to skip + // until the end. + // + ifdstream is (move (pr.in_ofd), + fdstream_mode::text | fdstream_mode::skip, + ifdstream::badbit); + + // In some cases we may need to ignore the error return status. + // The good_error flag keeps track of that. Similarly we + // sometimes expect the error return status based on the output + // we see. The bad_error flag is for that. + // + bool good_error (false), bad_error (false); + + size_t skip (skip_count); + string l; // Reuse. + for (bool first (true), second (false); !restart; ) + { + if (eof (getline (is, l))) + break; + + l6 ([&]{trace << "header dependency line '" << l << "'";}); + + // Parse different dependency output formats. + // + switch (cclass) + { + case compiler_class::msvc: + { + if (first) + { + // The first line should be the file we are compiling. + // If it is not, then something went wrong even before + // we could compile anything (e.g., file does not + // exist). In this case the first line (and everything + // after it) is presumably diagnostics. + // + if (l != src.path ().leaf ().string ()) + { + text << l; + bad_error = true; + break; + } + + first = false; + continue; + } + + string f (next_show (l, good_error)); + + if (f.empty ()) // Some other diagnostics. + { + text << l; + bad_error = true; + break; + } + + // Skip until where we left off. + // + if (skip != 0) + { + // We can't be skipping over a non-existent header. + // + assert (!good_error); + skip--; + } + else + { + restart = add (path (move (f)), false, pmt); + + // If the header does not exist (good_error), then + // restart must be true. Except that it is possible that + // someone running in parallel has already updated it. + // In this case we must force a restart since we haven't + // yet seen what's after this at-that-time-non-existent + // header. + // + // We also need to force the target update (normally + // done by add()). + // + if (good_error) + restart = updating = true; + // + // And if we have updated the header (restart is true), + // then we may end up in this situation: an old header + // got included which caused the preprocessor to fail + // down the line. So if we are restarting, set the good + // error flag in case the process fails because of + // something like this (and if it is for a valid reason, + // then we will pick it up on the next round). + // + else if (restart) + good_error = true; + + if (restart) + l6 ([&]{trace << "restarting";}); + } + + break; + } + case compiler_class::gcc: + { + // Make dependency declaration. + // + size_t pos (0); + + if (first) + { + // Empty/invalid output should mean the wait() call + // below will return false. + // + if (l.empty () || + l[0] != '^' || l[1] != ':' || l[2] != ' ') + { + if (!l.empty ()) + text << l; + + bad_error = true; + break; + } + + first = false; + second = true; + + // While normally we would have the source file on the + // first line, if too long, it will be moved to the next + // line and all we will have on this line is "^: \". + // + if (l.size () == 4 && l[3] == '\\') + continue; + else + pos = 3; // Skip "^: ". + + // Fall through to the 'second' block. + } + + if (second) + { + second = false; + next_make (l, pos); // Skip the source file. + } + + while (pos != l.size ()) + { + string f (next_make (l, pos)); + + // Skip until where we left off. + // + if (skip != 0) + { + skip--; + continue; + } + + restart = add (path (move (f)), false, pmt); + + if (restart) + { + // The same "preprocessor may fail down the line" + // logic as above. + // + good_error = true; + + l6 ([&]{trace << "restarting";}); + break; + } + } + + break; + } + } + + if (bad_error) + break; + } + + // In case of VC, we are parsing stderr and if things go south, + // we need to copy the diagnostics for the user to see. + // + if (bad_error && cclass == compiler_class::msvc) + { + // We used to just dump the whole rdbuf but it turns out VC + // may continue writing include notes interleaved with the + // diagnostics. So we have to filter them out. + // + for (; !eof (getline (is, l)); ) + { + size_t p (next_show_sense (l)); + if (p != string::npos && l.compare (p, 4, "1083") != 0) + diag_stream_lock () << l << endl; + } + } + + is.close (); + + // This is tricky: it is possible that in parallel someone has + // generated all our missing headers and we wouldn't restart + // normally. + // + // In this case we also need to force the target update + // (normally done by add()). + // + if (force_gen && *force_gen) + { + restart = updating = true; + force_gen = false; + } + + if (pr.wait ()) + { + if (!bad_error) + continue; + + fail << "expected error exist status from " << x_lang + << " compiler"; + } + else if (pr.exit->normal ()) + { + if (good_error) // Ignore expected errors (restart). + continue; + } + + // Fall through. + } + catch (const io_error&) + { + if (pr.wait ()) + fail << "unable to read " << x_lang << " compiler header " + << "dependency output"; + + // Fall through. + } + + assert (pr.exit && !*pr.exit); + const process_exit& e (*pr.exit); + + // For normal exit we assume the child process issued some + // diagnostics. + // + if (e.normal ()) + { + // If this run was with the generated header support then we + // have issued diagnostics and it's time to give up. + // + if (gen) + throw failed (); + + // Just to recap, being here means something is wrong with the + // source: it can be a missing generated header, it can be an + // outdated generated header (e.g., some check triggered #error + // which will go away if only we updated the generated header), + // or it can be a real error that is not going away. + // + // So this is what we are going to do here: if anything got + // updated on this run (i.e., the compiler has produced valid + // dependency information even though there were errors and we + // managed to find and update a header based on this + // informaion), then we restart in the same mode hoping that + // this fixes things. Otherwise, we force the generated header + // support which will either uncover a missing generated header + // or will issue diagnostics. + // + if (restart) + l6 ([&]{trace << "trying again without generated headers";}); + else + { + // In some pathological situations (e.g., we are out of disk + // space) we may end up switching back and forth indefinitely + // without making any headway. So we use skip_count to track + // our progress. + // + if (force_gen_skip && *force_gen_skip == skip_count) + { + diag_record dr (fail); + + dr << "inconsistent " << x_lang << " compiler behavior"; + + // Show the yo-yo'ing command lines. + // + dr << info; + print_process (dr, args.data ()); // No pipes. + + init_args ((gen = true)); + dr << info << ""; + print_process (dr, args.data ()); // No pipes. + + dr << info << "perhaps you are running out of disk space?"; + } + + restart = true; + force_gen = true; + force_gen_skip = skip_count; + l6 ([&]{trace << "restarting with forced generated headers";}); + } + continue; + } + else + run_finish (args, pr); // Throws. + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + // In a multi-threaded program that fork()'ed but did not exec(), + // it is unwise to try to do any kind of cleanup (like unwinding + // the stack and running destructors). + // + if (e.child) + { + drm.cancel (); + exit (1); + } + + throw failed (); + } + } + } + + // Add the terminating blank line (we are updated depdb). + // + dd.expect (""); + + puse = puse && !psrc.path.empty (); + return make_pair (move (psrc), puse); + } + + pair compile_rule:: + parse_unit (action a, + file& t, + linfo lo, + const file& src, + auto_rmfile& psrc, + const match_data& md) const + { + tracer trace (x, "compile_rule::parse_unit"); + + // If things go wrong give the user a bit extra context. + // + auto df = make_diag_frame ( + [&src](const diag_record& dr) + { + if (verb != 0) + dr << info << "while parsing " << src; + }); + + // For some compilers (GCC, Clang) the preporcessed output is only + // partially preprocessed. For others (VC), it is already fully + // preprocessed (well, almost: it still has comments but we can handle + // that). Plus, the source file might already be (sufficiently) + // preprocessed. + // + // So the plan is to start the compiler process that writes the fully + // preprocessed output to stdout and reduce the already preprocessed + // case to it. + // + environment env; + cstrings args; + const path* sp; // Source path. + + bool ps; // True if extracting from psrc. + if (md.pp < preprocessed::modules) + { + ps = !psrc.path.empty (); + sp = &(ps ? psrc.path : src.path ()); + + // VC's preprocessed output, if present, is fully preprocessed. + // + if (cclass != compiler_class::msvc || !ps) + { + // This should match with how we setup preprocessing and is pretty + // similar to init_args() from extract_headers(). + // + args.push_back (cpath.recall_string ()); + + append_lib_options (t.base_scope (), args, a, t, lo); + + append_options (args, t, c_poptions); + append_options (args, t, x_poptions); + + assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); + append_option_values ( + args, "-I", + sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), + [] (const dir_path& d) {return d.string ().c_str ();}); + + if (md.symexport) + append_symexport_options (args, t); + + // Make sure we don't fail because of warnings. + // + // @@ Can be both -WX and /WX. + // + const char* werror (nullptr); + switch (cclass) + { + case compiler_class::gcc: werror = "-Werror"; break; + case compiler_class::msvc: werror = "/WX"; break; + } + + bool clang (cid == compiler_id::clang || + cid == compiler_id::clang_apple); + + append_options (args, t, c_coptions, werror); + append_options (args, t, x_coptions, werror); + append_options (args, tstd, + tstd.size () - (modules && clang ? 1 : 0)); + + switch (cclass) + { + case compiler_class::msvc: + { + args.push_back ("/nologo"); + + if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) + args.push_back ("/EHsc"); + + if (!find_option_prefixes ({"/MD", "/MT"}, args)) + args.push_back ("/MD"); + + args.push_back ("/E"); + args.push_back ("/C"); + args.push_back (langopt (md)); // Compile as. + + break; + } + case compiler_class::gcc: + { + if (t.is_a ()) + { + if (tclass == "linux" || tclass == "bsd") + args.push_back ("-fPIC"); + } + + // Options that trigger preprocessing of partially preprocessed + // output are a bit of a compiler-specific voodoo. + // + args.push_back ("-E"); + + if (ps) + { + args.push_back ("-x"); + args.push_back (langopt (md)); + + if (cid == compiler_id::gcc) + { + args.push_back ("-fpreprocessed"); + args.push_back ("-fdirectives-only"); + } + } + + break; + } + } + + args.push_back (sp->string ().c_str ()); + args.push_back (nullptr); + } + + if (!env.empty ()) + env.push_back (nullptr); + } + else + { + // Extracting directly from source. + // + ps = false; + sp = &src.path (); + } + + // Preprocess and parse. + // + for (;;) // Breakout loop. + try + { + // Disarm the removal of the preprocessed file in case of an error. + // We re-arm it below. + // + if (ps) + psrc.active = false; + + process pr; + + try + { + if (args.empty ()) + { + pr = process (process_exit (0)); // Successfully exited. + pr.in_ofd = fdopen (*sp, fdopen_mode::in); + } + else + { + if (verb >= 3) + print_process (args); + + // We don't want to see warnings multiple times so ignore all + // diagnostics. + // + pr = process (cpath, + args.data (), + 0, -1, -2, + nullptr, // CWD + env.empty () ? nullptr : env.data ()); + } + + // Use binary mode to obtain consistent positions. + // + ifdstream is (move (pr.in_ofd), + fdstream_mode::binary | fdstream_mode::skip); + + parser p; + translation_unit tu (p.parse (is, *sp)); + + is.close (); + + if (pr.wait ()) + { + if (ps) + psrc.active = true; // Re-arm. + + // Prior to 15u5 VC was not using the 'export module' syntax so we + // use the preprequisite type to distinguish between interface and + // implementation units. + // + if (cid == compiler_id::msvc && + cmaj == 19 && cmin <= 11 && + x_mod != nullptr && src.is_a (*x_mod)) + { + // It's quite painful to guard the export with an #if/#endif so + // if it is present, "fixup" the (temporary) preprocessed output + // by getting rid of the keyword. + // + // Note: when removing this also remember to remove the test. + // + if (tu.mod.iface) + { + // We can only fixup a temporary file. + // + if (!ps) + fail (relative (src)) << "fixup requires preprocessor"; + + // Stomp out the export keyword with spaces. We are using + // std::fstream since our fdstream does not yet support + // seeking. + // + fstream os (psrc.path.string (), fstream::out | fstream::in); + auto pos (static_cast (p.export_pos)); + + if (!os.is_open () || + !os.seekp (pos) || + !os.write (" ", 6)) + fail << "unable to overwrite preprocessor output"; + } + else + tu.mod.iface = true; + } + + return pair (move (tu), p.checksum); + } + + // Fall through. + } + catch (const io_error&) + { + if (pr.wait ()) + fail << "unable to read " << x_lang << " preprocessor output"; + + // Fall through. + } + + assert (pr.exit && !*pr.exit); + const process_exit& e (*pr.exit); + + // What should we do with a normal error exit? Remember we suppressed + // the compiler's diagnostics. We used to issue a warning and continue + // with the assumption that the compilation step will fail with + // diagnostics. The problem with this approach is that we may fail + // before that because the information we return (e.g., module name) + // is bogus. So looks like failing is the only option. + // + if (e.normal ()) + { + fail << "unable to preprocess " << src << + info << "re-run with -s -V to display failing command" << + info << "then run failing command to display compiler diagnostics"; + } + else + run_finish (args, pr); // Throws. + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + } + + throw failed (); + } + + // Extract and inject module dependencies. + // + void compile_rule:: + extract_modules (action a, + const scope& bs, + file& t, + linfo li, + const compile_target_types& tt, + const file& src, + match_data& md, + module_info&& mi, + depdb& dd, + bool& updating) const + { + tracer trace (x, "compile_rule::extract_modules"); + l5 ([&]{trace << "target: " << t;}); + + // If things go wrong, give the user a bit extra context. + // + auto df = make_diag_frame ( + [&src](const diag_record& dr) + { + if (verb != 0) + dr << info << "while extracting module dependencies from " << src; + }); + + if (!modules) + { + if (!mi.name.empty () || !mi.imports.empty ()) + fail (relative (src)) << "modules support not enabled/available"; + + return; + } + + // Sanity checks. + // + // If we are compiling a module interface unit, make sure it has the + // necessary declarations. + // + if (src.is_a (*x_mod) && (mi.name.empty () || !mi.iface)) + fail << src << " is not a module interface unit"; + + // Search and match all the modules we depend on. If this is a module + // implementation unit, then treat the module itself as if it was + // imported (we insert it first since for some compilers we have to + // differentiate between this special module and real imports). Note: + // move. + // + if (!mi.iface && !mi.name.empty ()) + mi.imports.insert (mi.imports.begin (), + module_import {move (mi.name), false, 0}); + + // The change to the set of imports would have required a change to + // source code (or options). Changes to the bmi{}s themselves will be + // detected via the normal prerequisite machinery. However, the same set + // of imports could be resolved to a different set of bmi{}s (in a sense + // similar to changing the source file). To detect this we calculate and + // store a hash of all (not just direct) bmi{}'s paths. + // + sha256 cs; + + if (!mi.imports.empty ()) + md.mods = search_modules (a, bs, t, li, tt.bmi, src, mi.imports, cs); + + if (dd.expect (cs.string ()) != nullptr) + updating = true; + +#if 0 + // Save the module map for compilers that use it. + // + if (md.mods.start != 0) + { + switch (cid) + { + case compiler_id::gcc: + case compiler_id::clang: + case compiler_id::clang_apple: + { + // We don't need to redo this if the above hash hasn't changed and + // the database is valid. + // + if (dd.writing () || !dd.skip ()) + { + const auto& pts (t.prerequisite_targets); + + for (size_t i (md.mods.start); i != pts.size (); ++i) + { + if (const target* m = pts[i]) + { + // Save a variable lookup by getting the module name from + // the import list (see search_modules()). + // + dd.write ('@', false); + dd.write (mi.imports[i - md.mods.start].name, false); + dd.write ('=', false); + dd.write (m->as ().path ()); + } + } + } + break; + } + default: + break; + } + } +#endif + + // Set the cc.module_name variable if this is an interface unit. Note + // that it may seem like a good idea to set it on the bmi{} group to + // avoid duplication. We, however, cannot do it MT-safely since we don't + // match the group. + // + if (mi.iface) + { + if (value& v = t.vars.assign (c_module_name)) + assert (cast (v) == mi.name); + else + v = move (mi.name); // Note: move. + } + } + + inline bool + std_module (const string& m) + { + size_t n (m.size ()); + return (n >= 3 && + m[0] == 's' && m[1] == 't' && m[2] == 'd' && + (n == 3 || m[3] == '.')); + }; + + // Resolve imported modules to bmi*{} targets. + // + module_positions compile_rule:: + search_modules (action a, + const scope& bs, + file& t, + linfo li, + const target_type& mtt, + const file& src, + module_imports& imports, + sha256& cs) const + { + tracer trace (x, "compile_rule::search_modules"); + + // So we have a list of imports and a list of "potential" module + // prerequisites. They are potential in the sense that they may or may + // not be required by this translation unit. In other words, they are + // the pool where we can resolve actual imports. + // + // Because we may not need all of these prerequisites, we cannot just go + // ahead and match all of them (and they can even have cycles; see rule + // synthesis). This poses a bit of a problem: the only way to discover + // the module's actual name (see cc.module_name) is by matching it. + // + // One way to solve this would be to make the user specify the module + // name for each mxx{} explicitly. This will be a major pain, however. + // Another would be to require encoding of the module name in the + // interface unit file name. For example, hello.core -> hello-core.mxx. + // This is better but still too restrictive: some will want to call it + // hello_core.mxx or HelloCore.mxx (because that's their file naming + // convention) or place it in a subdirectory, say, hello/core.mxx. + // + // In the above examples one common theme about all the file names is + // that they contain, in one form or another, the "tail" of the module + // name ('core'). So what we are going to do is require that the + // interface file names contain enough of the module name tail to + // unambiguously resolve all the module imports. On our side we are + // going to implement a "fuzzy" module name to file name match. This + // should be reliable enough since we will always verify our guesses + // once we match the target and extract the actual module name. Plus, + // the user will always have the option of resolving any impasses by + // specifying the module name explicitly. + // + // So, the fuzzy match: the idea is that each match gets a score, the + // number of characters in the module name that got matched. A match + // with the highest score is used. And we use the (length + 1) for a + // match against an actual module name. + // + // For std.* modules we only accept non-fuzzy matches (think std.core vs + // some core.mxx). And if such a module is unresolved, then we assume it + // is pre-built and will be found by some other means (e.g., VC's + // IFCPATH). + // + auto match = [] (const string& f, const string& m) -> size_t + { + size_t fi (f.size ()); + size_t mi (m.size ()); + + // Scan backwards for as long as we match. Keep track of the previous + // character for case change detection. + // + for (char fc, mc, fp ('\0'), mp ('\0'); + fi != 0 && mi != 0; + fp = fc, mp = mc, --fi, --mi) + { + fc = f[fi - 1]; + mc = m[mi - 1]; + + if (casecmp (fc, mc) == 0) + continue; + + // We consider all separators equal and character case change being + // a separators. Some examples of the latter: + // + // foo.bar + // fooBAR + // FOObar + // + bool fs (fc == '_' || fc == '-' || fc == '.' || + path::traits::is_separator (fc)); + bool ms (mc == '_' || mc == '.'); + + if (fs && ms) + continue; + + // Only if one is a real separator do we consider case change. + // + if (fs || ms) + { + auto cc = [] (char c1, char c2) -> bool + { + return (alpha (c1) && + alpha (c2) && + (ucase (c1) == c1) != (ucase (c2) == c2)); + }; + + bool fa (false), ma (false); + if ((fs || (fa = cc (fp, fc))) && (ms || (ma = cc (mp, mc)))) + { + // Stay on this character if imaginary punctuation (note: cannot + // be both true). + // + if (fa) ++fi; + if (ma) ++mi; + continue; + } + } + + break; // No match. + } + + // Return the number of characters matched in the module name and not + // in the file (this may not be the same because of the imaginary + // separators). + // + return m.size () - mi; + }; + + auto& pts (t.prerequisite_targets[a]); + size_t start (pts.size ()); // Index of the first to be added. + + // We have two parallel vectors: module names/scores in imports and + // targets in prerequisite_targets (offset with start). Pre-allocate + // NULL entries in the latter. + // + size_t n (imports.size ()); + pts.resize (start + n, nullptr); + + // Oh, yes, there is one "minor" complication. It's the last one, I + // promise. It has to do with module re-exporting (export import M;). + // In this case (currently) all implementations simply treat it as a + // shallow (from the BMI's point of view) reference to the module (or an + // implicit import, if you will). Do you see where it's going? Nowever + // good, that's right. This shallow reference means that the compiler + // should be able to find BMIs for all the re-exported modules, + // recursive. The good news is we are actually in a pretty good shape to + // handle this: after match all our prerequisite BMIs will have their + // prerequisite BMIs known, recursively. The only bit that is missing is + // the re-export flag of some sorts. As well as deciding where to handle + // it: here or in append_modules(). After some meditation it became + // clear handling it here will be simpler: We need to weed out + // duplicates for which we can re-use the imports vector. And we may + // also need to save this "flattened" list of modules in depdb. + // + // Ok, so, here is the plan: + // + // 1. There is no good place in prerequisite_targets to store the + // exported flag (no, using the marking facility across match/execute + // is a bad idea). So what we are going to do is put re-exported + // bmi{}s at the back and store (in the target's data pad) the start + // position. One bad aspect about this part is that we assume those + // bmi{}s have been matched by the same rule. But let's not kid + // ourselves, there will be no other rule that matches bmi{}s. + // + // 2. Once we have matched all the bmi{}s we are importing directly + // (with all the re-exported by us at the back), we will go over them + // and copy all of their re-exported bmi{}s (using the position we + // saved on step #1). The end result will be a recursively-explored + // list of imported bmi{}s that append_modules() can simply convert + // to the list of options. + // + // One issue with this approach is that these copied targets will be + // executed which means we need to adjust their dependent counts + // (which is normally done by match). While this seems conceptually + // correct (especially if you view re-exports as implicit imports), + // it's just extra overhead (we know they will be updated). So what + // we are going to do is save another position, that of the start of + // these copied-over targets, and will only execute up to this point. + // + // And after implementing this came the reality check: all the current + // implementations require access to all the imported BMIs, not only + // re-exported. Some (like Clang) store references to imported BMI files + // so we actually don't need to pass any extra options (unless things + // get moved) but they still need access to the BMIs (and things will + // most likely have to be done differenly for distributed compilation). + // + // So the revised plan: on the off chance that some implementation will + // do it differently we will continue maintaing the imported/re-exported + // split and how much to copy-over can be made compiler specific. + // + // As a first sub-step of step #1, move all the re-exported imports to + // the end of the vector. This will make sure they end up at the end + // of prerequisite_targets. Note: the special first import, if any, + // should be unaffected. + // + sort (imports.begin (), imports.end (), + [] (const module_import& x, const module_import& y) + { + return !x.exported && y.exported; + }); + + // Go over the prerequisites once. + // + // For (direct) library prerequisites, check their prerequisite bmi{}s + // (which should be searched and matched with module names discovered; + // see the library meta-information protocol for details). + // + // For our own bmi{} prerequisites, checking if each (better) matches + // any of the imports. + + // For fuzzy check if a file name (better) resolves any of our imports + // and if so make it the new selection. For exact the name is the actual + // module name and it can only resolve one import (there are no + // duplicates). + // + // Set done to true if all the imports have now been resolved to actual + // module names (which means we can stop searching). This will happens + // if all the modules come from libraries. Which will be fairly common + // (think of all the tests) so it's worth optimizing for. + // + bool done (false); + + auto check_fuzzy = [&trace, &imports, &pts, &match, start, n] + (const target* pt, const string& name) + { + for (size_t i (0); i != n; ++i) + { + module_import& m (imports[i]); + + if (std_module (m.name)) // No fuzzy std.* matches. + continue; + + size_t n (m.name.size ()); + + if (m.score > n) // Resolved to module name. + continue; + + size_t s (match (name, m.name)); + + l5 ([&]{trace << name << " ~ " << m.name << ": " << s;}); + + if (s > m.score) + { + pts[start + i] = pt; + m.score = s; + } + } + }; + + // If resolved, return the "slot" in pts (we don't want to create a + // side build until we know we match; see below for details). + // + auto check_exact = [&trace, &imports, &pts, start, n, &done] + (const string& name) -> const target** + { + const target** r (nullptr); + done = true; + + for (size_t i (0); i != n; ++i) + { + module_import& m (imports[i]); + + size_t n (m.name.size ()); + + if (m.score > n) // Resolved to module name (no effect on done). + continue; + + if (r == nullptr) + { + size_t s (name == m.name ? n + 1 : 0); + + l5 ([&]{trace << name << " ~ " << m.name << ": " << s;}); + + if (s > m.score) + { + r = &pts[start + i].target; + m.score = s; + continue; // Scan the rest to detect if all done. + } + } + + done = false; + } + + return r; + }; + + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + const target* pt (p.load ()); // Should be cached for libraries. + + if (pt != nullptr) + { + const target* lt (nullptr); + + if (const libx* l = pt->is_a ()) + lt = &link_member (*l, a, li); + else if (pt->is_a () || pt->is_a () || pt->is_a ()) + lt = pt; + + // If this is a library, check its bmi{}s and mxx{}s. + // + if (lt != nullptr) + { + for (const target* bt: lt->prerequisite_targets[a]) + { + if (bt == nullptr) + continue; + + // Note that here we (try) to use whatever flavor of bmi*{} is + // available. + // + // @@ MOD: BMI compatibility check. + // @@ UTL: we need to (recursively) see through libux{} (and + // also in pkgconfig_save()). + // + if (bt->is_a () || + bt->is_a () || + bt->is_a ()) + { + const string& n (cast (bt->vars[c_module_name])); + + if (const target** p = check_exact (n)) + *p = bt; + } + else if (bt->is_a (*x_mod)) + { + // This is an installed library with a list of module sources + // (the source are specified as prerequisites but the fallback + // file rule puts them into prerequisite_targets for us). + // + // The module names should be specified but if not assume + // something else is going on and ignore. + // + const string* n (cast_null (bt->vars[c_module_name])); + if (n == nullptr) + continue; + + if (const target** p = check_exact (*n)) + *p = &make_module_sidebuild (a, bs, *lt, *bt, *n); + } + else + continue; + + if (done) + break; + } + + if (done) + break; + + continue; + } + + // Fall through. + } + + // While it would have been even better not to search for a target, we + // need to get hold of the corresponding mxx{} (unlikely but possible + // for bmi{} to have a different name). + // + if (p.is_a ()) + pt = &search (t, mtt, p.key ()); // Same logic as in picking obj*{}. + else if (p.is_a (mtt)) + { + if (pt == nullptr) + pt = &p.search (t); + } + else + continue; + + // Find the mxx{} prerequisite and extract its "file name" for the + // fuzzy match unless the user specified the module name explicitly. + // + for (prerequisite_member p: group_prerequisite_members (a, *pt)) + { + if (p.is_a (*x_mod)) + { + // Check for an explicit module name. Only look for an existing + // target (which means the name can only be specified on the + // target itself, no target type/pattern-spec). + // + const target* t (p.search_existing ()); + const string* n (t != nullptr + ? cast_null (t->vars[c_module_name]) + : nullptr); + if (n != nullptr) + { + if (const target** p = check_exact (*n)) + *p = pt; + } + else + { + // Fuzzy match. + // + string f; + + // Add the directory part if it is relative. The idea is to + // include it into the module match, say hello.core vs + // hello/mxx{core}. + // + // @@ MOD: Why not for absolute? Good question. What if it + // contains special components, say, ../mxx{core}? + // + const dir_path& d (p.dir ()); + + if (!d.empty () && d.relative ()) + f = d.representation (); // Includes trailing slash. + + f += p.name (); + check_fuzzy (pt, f); + } + break; + } + } + + if (done) + break; + } + + // Diagnose unresolved modules. + // + if (!done) + { + for (size_t i (0); i != n; ++i) + { + if (pts[start + i] == nullptr && !std_module (imports[i].name)) + { + // It would have been nice to print the location of the import + // declaration. And we could save it during parsing at the expense + // of a few paths (that can be pooled). The question is what to do + // when we re-create this information from depdb? We could have + // saved the location information there but the relative paths + // (e.g., from the #line directives) could end up being wrong if + // the we re-run from a different working directory. + // + // It seems the only workable approach is to extract full location + // info during parse, not save it in depdb, when re-creating, + // fallback to just src path without any line/column information. + // This will probably cover the majority of case (most of the time + // it will be a misspelled module name, not a removal of module + // from buildfile). + // + // But at this stage this doesn't seem worth the trouble. + // + fail (relative (src)) << "unable to resolve module " + << imports[i].name; + } + } + } + + // Match in parallel and wait for completion. + // + match_members (a, t, pts, start); + + // Post-process the list of our (direct) imports. While at it, calculate + // the checksum of all (direct and indirect) bmi{} paths. + // + size_t exported (n); + size_t copied (pts.size ()); + + for (size_t i (0); i != n; ++i) + { + const module_import& m (imports[i]); + + // Determine the position of the first re-exported bmi{}. + // + if (m.exported && exported == n) + exported = i; + + const target* bt (pts[start + i]); + + if (bt == nullptr) + continue; // Unresolved (std.*). + + // Verify our guesses against extracted module names but don't waste + // time if it was a match against the actual module name. + // + const string& in (m.name); + + if (m.score <= in.size ()) + { + const string& mn (cast (bt->vars[c_module_name])); + + if (in != mn) + { + for (prerequisite_member p: group_prerequisite_members (a, *bt)) + { + if (p.is_a (*x_mod)) // Got to be there. + { + fail (relative (src)) + << "failed to correctly guess module name from " << p << + info << "guessed: " << in << + info << "actual: " << mn << + info << "consider adjusting module interface file names or" << + info << "consider specifying module name with " << x + << ".module_name"; + } + } + } + } + + // Hash (we know it's a file). + // + cs.append (static_cast (*bt).path ().string ()); + + // Copy over bmi{}s from our prerequisites weeding out duplicates. + // + if (size_t j = bt->data ().mods.start) + { + // Hard to say whether we should reserve or not. We will probably + // get quite a bit of duplications. + // + auto& bpts (bt->prerequisite_targets[a]); + for (size_t m (bpts.size ()); j != m; ++j) + { + const target* et (bpts[j]); + + if (et == nullptr) + continue; // Unresolved (std.*). + + const string& mn (cast (et->vars[c_module_name])); + + if (find_if (imports.begin (), imports.end (), + [&mn] (const module_import& i) + { + return i.name == mn; + }) == imports.end ()) + { + pts.push_back (et); + cs.append (static_cast (*et).path ().string ()); + + // Add to the list of imports for further duplicate suppression. + // We could have probably stored reference to the name (e.g., in + // score) but it's probably not worth it if we have a small + // string optimization. + // + imports.push_back (module_import {mn, true, 0}); + } + } + } + } + + if (copied == pts.size ()) // No copied tail. + copied = 0; + + if (exported == n) // No (own) re-exported imports. + exported = copied; + else + exported += start; // Rebase. + + return module_positions {start, exported, copied}; + } + + // Synthesize a dependency for building a module binary interface on + // the side. + // + const target& compile_rule:: + make_module_sidebuild (action a, + const scope& bs, + const target& lt, + const target& mt, + const string& mn) const + { + tracer trace (x, "compile_rule::make_module_sidebuild"); + + // First figure out where we are going to build. We want to avoid + // multiple sidebuilds so the outermost scope that has loaded the + // cc.config module and that is within our amalgmantion seems like a + // good place. + // + const scope& rs (*bs.root_scope ()); + const scope* as (&rs); + { + const scope* ws (as->weak_scope ()); + if (as != ws) + { + const scope* s (as); + do + { + s = s->parent_scope ()->root_scope (); + + // Use cc.core.vars as a proxy for {c,cxx}.config (a bit smelly). + // + // This is also the module that registers the scope operation + // callback that cleans up the subproject. + // + if (cast_false ((*s)["cc.core.vars.loaded"])) + as = s; + + } while (s != ws); + } + } + + // We build modules in a subproject (since there might be no full + // language support module loaded in the amalgamation, only *.config). + // So the first step is to check if the project has already been created + // and/or loaded and if not, then to go ahead and do so. + // + dir_path pd (as->out_path () / modules_sidebuild_dir /= x); + { + const scope* ps (&scopes.find (pd)); + + if (ps->out_path () != pd) + { + // Switch the phase to load then create and load the subproject. + // + phase_switch phs (run_phase::load); + + // Re-test again now that we are in exclusive phase (another thread + // could have already created and loaded the subproject). + // + ps = &scopes.find (pd); + + if (ps->out_path () != pd) + { + // The project might already be created in which case we just need + // to load it. + // + if (!is_src_root (pd)) + { + // Copy our standard and force modules. + // + string extra; + + if (const string* std = cast_null (rs[x_std])) + extra += string (x) + ".std = " + *std + '\n'; + + extra += string (x) + ".features.modules = true"; + + config::create_project ( + pd, + as->out_path ().relative (pd), /* amalgamation */ + {}, /* boot_modules */ + extra, /* root_pre */ + {string (x) + '.'}, /* root_modules */ + "", /* root_post */ + false, /* config */ + false, /* buildfile */ + "the cc module", + 2); /* verbosity */ + } + + ps = &load_project (as->rw () /* lock */, pd, pd); + } + } + + // Some sanity checks. + // +#ifndef NDEBUG + assert (ps->root ()); + const module* m (ps->modules.lookup (x)); + assert (m != nullptr && m->modules); +#endif + } + + // Next we need to come up with a file/target name that will be unique + // enough not to conflict with other modules. If we assume that within + // an amalgamation there is only one "version" of each module, then the + // module name itself seems like a good fit. We just replace '.' with + // '-'. + // + string mf; + transform (mn.begin (), mn.end (), + back_inserter (mf), + [] (char c) {return c == '.' ? '-' : c;}); + + // It seems natural to build a BMI type that corresponds to the library + // type. After all, this is where the object file part of the BMI is + // going to come from (though things will probably be different for + // module-only libraries). + // + const target_type* tt (nullptr); + switch (link_type (lt).type) + { + case otype::a: tt = &bmia::static_type; break; + case otype::s: tt = &bmis::static_type; break; + case otype::e: assert (false); + } + + // Store the BMI target in the subproject root. If the target already + // exists then we assume all this is already done (otherwise why would + // someone have created such a target). + // + if (const target* bt = targets.find ( + *tt, + pd, + dir_path (), // Always in the out tree. + mf, + nullopt, // Use default extension. + trace)) + return *bt; + + prerequisites ps; + ps.push_back (prerequisite (mt)); + + // We've added the mxx{} but it may import other modules from this + // library. Or from (direct) dependencies of this library. We add them + // all as prerequisites so that the standard module search logic can + // sort things out. This is pretty similar to what we do in link when + // synthesizing dependencies for bmi{}'s. + // + ps.push_back (prerequisite (lt)); + for (prerequisite_member p: group_prerequisite_members (a, lt)) + { + // @@ TODO: will probably need revision if using sidebuild for + // non-installed libraries (e.g., direct BMI dependencies + // will probably have to be translated to mxx{} or some such). + // + if (p.is_a () || + p.is_a () || p.is_a () || p.is_a ()) + { + ps.push_back (p.as_prerequisite ()); + } + } + + auto p (targets.insert_locked (*tt, + move (pd), + dir_path (), // Always in the out tree. + move (mf), + nullopt, // Use default extension. + true, // Implied. + trace)); + const target& bt (p.first); + + // Note that this is racy and someone might have created this target + // while we were preparing the prerequisite list. + // + if (p.second.owns_lock ()) + bt.prerequisites (move (ps)); + + return bt; + } + + // Filter cl.exe noise (msvc.cxx). + // + void + msvc_filter_cl (ifdstream&, const path& src); + + void compile_rule:: + append_modules (environment& env, + cstrings& args, + strings& stor, + action a, + const file& t, + const match_data& md) const + { + const module_positions& ms (md.mods); + assert (ms.start != 0); + + dir_path stdifc; // See the VC case below. + + auto& pts (t.prerequisite_targets[a]); + +#if 0 + switch (cid) + { + case compiler_id::gcc: + { + // Use the module map stored in depdb. + // + string s (relative (md.dd).string ()); + s.insert (0, "-fmodule-file-map=@="); + stor.push_back (move (s)); + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + // In Clang the module implementation's unit .pcm is special and + // must be "loaded". + // + if (md.type == translation_type::module_impl) + { + const file& f (pts[ms.start]->as ()); + string s (relative (f.path ()).string ()); + s.insert (0, "-fmodule-file="); + stor.push_back (move (s)); + } + + // Use the module map stored in depdb for others. + // + string s (relative (md.dd).string ()); + s.insert (0, "-fmodule-file-map=@="); + stor.push_back (move (s)); + break; + } + case compiler_id::msvc: + { + for (size_t i (ms.start), n (pts.size ()); + i != n; + ++i) + { + const target* pt (pts[i]); + + if (pt == nullptr) + continue; + + // Here we use whatever bmi type has been added. And we know all + // of these are bmi's. + // + const file& f (pt->as ()); + + // In VC std.* modules can only come from a single directory + // specified with the IFCPATH environment variable or the + // /module:stdIfcDir option. + // + if (std_module (cast (f.vars[c_module_name]))) + { + dir_path d (f.path ().directory ()); + + if (stdifc.empty ()) + { + // Go one directory up since /module:stdIfcDir will look in + // either Release or Debug subdirectories. Keeping the result + // absolute feels right. + // + stor.push_back ("/module:stdIfcDir"); + stor.push_back (d.directory ().string ()); + stdifc = move (d); + } + else if (d != stdifc) // Absolute and normalized. + fail << "multiple std.* modules in different directories"; + } + else + { + stor.push_back ("/module:reference"); + stor.push_back (relative (f.path ()).string ()); + } + } + break; + } + case compiler_id::icc: + assert (false); + } +#else + size_t n (pts.size ()); + + // Clang embeds module file references so we only need to specify + // our direct imports. + // + // If/when we get the ability to specify the mapping in a file, we + // should probably pass the whole list. + // + switch (cid) + { + case compiler_id::gcc: break; // All of them. + case compiler_id::clang_apple: + case compiler_id::clang: n = ms.copied != 0 ? ms.copied : n; break; + case compiler_id::msvc: break; // All of them. + case compiler_id::icc: assert (false); + } + + for (size_t i (ms.start); i != n; ++i) + { + const target* pt (pts[i]); + + if (pt == nullptr) + continue; + + // Here we use whatever bmi type has been added. And we know all of + // these are bmi's. + // + const file& f (pt->as ()); + string s (relative (f.path ()).string ()); + + switch (cid) + { + case compiler_id::gcc: + { + s.insert (0, 1, '='); + s.insert (0, cast (f.vars[c_module_name])); + s.insert (0, "-fmodule-file="); + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + // In Clang the module implementation's unit .pcm is special and + // must be "loaded". + // + if (md.type == translation_type::module_impl && i == ms.start) + s.insert (0, "-fmodule-file="); + else + { + s.insert (0, 1, '='); + s.insert (0, cast (f.vars[c_module_name])); + s.insert (0, "-fmodule-file="); + } + break; + } + case compiler_id::msvc: + { + // In VC std.* modules can only come from a single directory + // specified with the IFCPATH environment variable or the + // /module:stdIfcDir option. + // + if (std_module (cast (f.vars[c_module_name]))) + { + dir_path d (f.path ().directory ()); + + if (stdifc.empty ()) + { + // Go one directory up since /module:stdIfcDir will look in + // either Release or Debug subdirectories. Keeping the result + // absolute feels right. + // + s = d.directory ().string (); + stor.push_back ("/module:stdIfcDir"); + stdifc = move (d); + } + else + { + if (d != stdifc) // Absolute and normalized. + fail << "multiple std.* modules in different directories"; + + continue; // Skip. + } + } + else + stor.push_back ("/module:reference"); + + break; + } + case compiler_id::icc: + assert (false); + } + + stor.push_back (move (s)); + } +#endif + + // Shallow-copy storage to args. Why not do it as we go along pushing + // into storage? Because of potential reallocations. + // + for (const string& a: stor) + args.push_back (a.c_str ()); + + // VC's IFCPATH takes precedence over /module:stdIfcDir so unset it + // if we are using our own std modules. + // + if (!stdifc.empty ()) + env.push_back ("IFCPATH"); + } + + target_state compile_rule:: + perform_update (action a, const target& xt) const + { + const file& t (xt.as ()); + const path& tp (t.path ()); + + match_data md (move (t.data ())); + bool mod (md.type == translation_type::module_iface); + + // While all our prerequisites are already up-to-date, we still have to + // execute them to keep the dependency counts straight. Actually, no, we + // may also have to update the modules. + // + auto pr ( + execute_prerequisites ( + (mod ? *x_mod : x_src), + a, t, + md.mt, + [s = md.mods.start] (const target&, size_t i) + { + return s != 0 && i >= s; // Only compare timestamps for modules. + }, + md.mods.copied)); // See search_modules() for details. + + const file& s (pr.second); + const path* sp (&s.path ()); + + if (pr.first) + { + if (md.touch) + { + touch (tp, false, 2); + skip_count.fetch_add (1, memory_order_relaxed); + } + + t.mtime (md.mt); + return *pr.first; + } + + // Make sure depdb is no older than any of our prerequisites. + // + touch (md.dd, false, verb_never); + + const scope& bs (t.base_scope ()); + const scope& rs (*bs.root_scope ()); + + otype ot (compile_type (t, mod)); + linfo li (link_info (bs, ot)); + + environment env; + cstrings args {cpath.recall_string ()}; + + // If we are building a module, then the target is bmi*{} and its ad hoc + // member is obj*{}. + // + path relo, relm; + if (mod) + { + relm = relative (tp); + relo = relative (t.member->is_a ()->path ()); + } + else + relo = relative (tp); + + // Build the command line. + // + if (md.pp != preprocessed::all) + { + append_options (args, t, c_poptions); + append_options (args, t, x_poptions); + + // Add *.export.poptions from prerequisite libraries. + // + append_lib_options (bs, args, a, t, li); + + // Extra system header dirs (last). + // + assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); + append_option_values ( + args, "-I", + sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), + [] (const dir_path& d) {return d.string ().c_str ();}); + + if (md.symexport) + append_symexport_options (args, t); + } + + append_options (args, t, c_coptions); + append_options (args, t, x_coptions); + append_options (args, tstd); + + string out, out1; // Output options storage. + strings mods; // Module options storage. + size_t out_i (0); // Index of the -o option. + + if (cclass == compiler_class::msvc) + { + // The /F*: option variants with separate names only became available + // in VS2013/12.0. Why do we bother? Because the command line suddenly + // becomes readable. + // + uint64_t ver (cast (rs[x_version_major])); + + args.push_back ("/nologo"); + + // While we want to keep the low-level build as "pure" as possible, + // the two misguided defaults, exceptions and runtime, just have to be + // fixed. Otherwise the default build is pretty much unusable. But we + // also make sure that the user can easily disable our defaults: if we + // see any relevant options explicitly specified, we take our hands + // off. + // + // For C looks like no /EH* (exceptions supported but no C++ objects + // destroyed) is a reasonable default. + // + if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) + args.push_back ("/EHsc"); + + // The runtime is a bit more interesting. At first it may seem like a + // good idea to be a bit clever and use the static runtime if we are + // building obja{}. And for obje{} we could decide which runtime to + // use based on the library link order: if it is static-only, then we + // could assume the static runtime. But it is indeed too clever: when + // building liba{} we have no idea who is going to use it. It could be + // an exe{} that links both static and shared libraries (and is + // therefore built with the shared runtime). And to safely use the + // static runtime, everything must be built with /MT and there should + // be no DLLs in the picture. So we are going to play it safe and + // always default to the shared runtime. + // + // In a similar vein, it would seem reasonable to use the debug runtime + // if we are compiling with debug. But, again, there will be fireworks + // if we have some projects built with debug and some without and then + // we try to link them together (which is not an unreasonable thing to + // do). So by default we will always use the release runtime. + // + if (!find_option_prefixes ({"/MD", "/MT"}, args)) + args.push_back ("/MD"); + + if (md.mods.start != 0) + append_modules (env, args, mods, a, t, md); + + // The presence of /Zi or /ZI causes the compiler to write debug info + // to the .pdb file. By default it is a shared file called vcNN.pdb + // (where NN is the VC version) created (wait for it) in the current + // working directory (and not the directory of the .obj file). Also, + // because it is shared, there is a special Windows service that + // serializes access. We, of course, want none of that so we will + // create a .pdb per object file. + // + // Note that this also changes the name of the .idb file (used for + // minimal rebuild and incremental compilation): cl.exe take the /Fd + // value and replaces the .pdb extension with .idb. + // + // Note also that what we are doing here appears to be incompatible + // with PCH (/Y* options) and /Gm (minimal rebuild). + // + if (find_options ({"/Zi", "/ZI"}, args)) + { + if (ver >= 18) + args.push_back ("/Fd:"); + else + out1 = "/Fd"; + + out1 += relo.string (); + out1 += ".pdb"; + + args.push_back (out1.c_str ()); + } + + if (ver >= 18) + { + args.push_back ("/Fo:"); + args.push_back (relo.string ().c_str ()); + } + else + { + out = "/Fo" + relo.string (); + args.push_back (out.c_str ()); + } + + if (mod) + { + args.push_back ("/module:interface"); + args.push_back ("/module:output"); + args.push_back (relm.string ().c_str ()); + } + + // Note: no way to indicate that the source if already preprocessed. + + args.push_back ("/c"); // Compile only. + args.push_back (langopt (md)); // Compile as. + args.push_back (sp->string ().c_str ()); // Note: relied on being last. + } + else + { + if (ot == otype::s) + { + // On Darwin, Win32 -fPIC is the default. + // + if (tclass == "linux" || tclass == "bsd") + args.push_back ("-fPIC"); + } + + if (md.mods.start != 0) + append_modules (env, args, mods, a, t, md); + + // Note: the order of the following options is relied upon below. + // + out_i = args.size (); // Index of the -o option. + + if (mod) + { + switch (cid) + { + case compiler_id::gcc: + { + args.push_back ("-o"); + args.push_back (relo.string ().c_str ()); + + out = "-fmodule-output="; + out += relm.string (); + args.push_back (out.c_str ()); + + args.push_back ("-c"); + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + args.push_back ("-o"); + args.push_back (relm.string ().c_str ()); + args.push_back ("--precompile"); + + // Without this option Clang's .pcm will reference source files. + // In our case this file may be transient (.ii). Plus, it won't + // play nice with distributed compilation. + // + args.push_back ("-Xclang"); + args.push_back ("-fmodules-embed-all-files"); + + break; + } + case compiler_id::msvc: + case compiler_id::icc: + assert (false); + } + } + else + { + args.push_back ("-o"); + args.push_back (relo.string ().c_str ()); + args.push_back ("-c"); + } + + args.push_back ("-x"); + args.push_back (langopt (md)); + + if (md.pp == preprocessed::all) + { + // Note that the mode we select must still handle comments and line + // continuations. So some more compiler-specific voodoo. + // + switch (cid) + { + case compiler_id::gcc: + { + // -fdirectives-only is available since GCC 4.3.0. + // + if (cmaj > 4 || (cmaj == 4 && cmin >= 3)) + { + args.push_back ("-fpreprocessed"); + args.push_back ("-fdirectives-only"); + } + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + // Clang handles comments and line continuations in the + // preprocessed source (it does not have -fpreprocessed). + // + break; + } + case compiler_id::icc: + break; // Compile as normal source for now. + case compiler_id::msvc: + assert (false); + } + } + + args.push_back (sp->string ().c_str ()); + } + + args.push_back (nullptr); + + if (!env.empty ()) + env.push_back (nullptr); + + // With verbosity level 2 print the command line as if we are compiling + // the source file, not its preprocessed version (so that it's easy to + // copy and re-run, etc). Only at level 3 and above print the real deal. + // + if (verb == 1) + text << x_name << ' ' << s; + else if (verb == 2) + print_process (args); + + // If we have the (partially) preprocessed output, switch to that. + // + bool psrc (!md.psrc.path.empty ()); + bool pact (md.psrc.active); + if (psrc) + { + args.pop_back (); // nullptr + args.pop_back (); // sp + + sp = &md.psrc.path; + + // This should match with how we setup preprocessing. + // + switch (cid) + { + case compiler_id::gcc: + { + // The -fpreprocessed is implied by .i/.ii. + // + args.pop_back (); // lang() + args.pop_back (); // -x + args.push_back ("-fdirectives-only"); + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + // Note that without -x Clang will treat .i/.ii as fully + // preprocessed. + // + break; + } + case compiler_id::msvc: + { + // Nothing to do (/TP or /TC already there). + // + break; + } + case compiler_id::icc: + assert (false); + } + + args.push_back (sp->string ().c_str ()); + args.push_back (nullptr); + + // Let's keep the preprocessed file in case of an error but only at + // verbosity level 3 and up (when one actually sees it mentioned on + // the command line). We also have to re-arm on success (see below). + // + if (pact && verb >= 3) + md.psrc.active = false; + } + + if (verb >= 3) + print_process (args); + + try + { + // VC cl.exe sends diagnostics to stdout. It also prints the file name + // being compiled as the first line. So for cl.exe we redirect stdout + // to a pipe, filter that noise out, and send the rest to stderr. + // + // For other compilers redirect stdout to stderr, in case any of them + // tries to pull off something similar. For sane compilers this should + // be harmless. + // + bool filter (cid == compiler_id::msvc); + + process pr (cpath, + args.data (), + 0, (filter ? -1 : 2), 2, + nullptr, // CWD + env.empty () ? nullptr : env.data ()); + + if (filter) + { + try + { + ifdstream is ( + move (pr.in_ofd), fdstream_mode::text, ifdstream::badbit); + + msvc_filter_cl (is, *sp); + + // If anything remains in the stream, send it all to stderr. Note + // that the eof check is important: if the stream is at eof, this + // and all subsequent writes to the diagnostics stream will fail + // (and you won't see a thing). + // + if (is.peek () != ifdstream::traits_type::eof ()) + diag_stream_lock () << is.rdbuf (); + + is.close (); + } + catch (const io_error&) {} // Assume exits with error. + } + + run_finish (args, pr); + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + + if (pact && verb >= 3) + md.psrc.active = true; + + // Clang's module compilation requires two separate compiler + // invocations. + // + if (mod && (cid == compiler_id::clang || + cid == compiler_id::clang_apple)) + { + // Remove the target file if this fails. If we don't do that, we will + // end up with a broken build that is up-to-date. + // + auto_rmfile rm (relm); + + // Adjust the command line. First discard everything after -o then + // build the new "tail". + // + args.resize (out_i + 1); + args.push_back (relo.string ().c_str ()); // Produce .o. + args.push_back ("-c"); // By compiling .pcm. + args.push_back ("-Wno-unused-command-line-argument"); + args.push_back (relm.string ().c_str ()); + args.push_back (nullptr); + + if (verb >= 2) + print_process (args); + + try + { + process pr (cpath, + args.data (), + 0, 2, 2, + nullptr, // CWD + env.empty () ? nullptr : env.data ()); + + run_finish (args, pr); + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + + rm.cancel (); + } + + // Should we go to the filesystem and get the new mtime? We know the + // file has been modified, so instead just use the current clock time. + // It has the advantage of having the subseconds precision. + // + t.mtime (system_clock::now ()); + return target_state::changed; + } + + target_state compile_rule:: + perform_clean (action a, const target& xt) const + { + const file& t (xt.as ()); + + using id = compiler_id; + + switch (cid) + { + case id::gcc: return clean_extra (a, t, {".d", x_pext, ".t"}); + case id::clang_apple: + case id::clang: return clean_extra (a, t, {".d", x_pext}); + case id::msvc: return clean_extra (a, t, {".d", x_pext, ".idb", ".pdb"}); + case id::icc: return clean_extra (a, t, {".d"}); + } + + assert (false); + return target_state::unchanged; + } + } +} diff --git a/build2/cc/compile-rule.hxx b/build2/cc/compile-rule.hxx new file mode 100644 index 0000000..6bf63bf --- /dev/null +++ b/build2/cc/compile-rule.hxx @@ -0,0 +1,156 @@ +// file : build2/cc/compile-rule.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BUILD2_CC_COMPILE_RULE_HXX +#define BUILD2_CC_COMPILE_RULE_HXX + +#include + +#include +#include + +#include +#include // auto_rmfile + +#include +#include + +namespace build2 +{ + class depdb; + + namespace cc + { + // The order is arranged so that their integral values indicate whether + // one is a "stronger" than another. + // + enum class preprocessed: uint8_t {none, includes, modules, all}; + + // Positions of the re-exported bmi{}s. See search_modules() for + // details. + // + struct module_positions + { + size_t start; // First imported bmi*{}, 0 if none. + size_t exported; // First re-exported bmi*{}, 0 if none. + size_t copied; // First copied-over bmi*{}, 0 if none. + }; + + class compile_rule: public rule, virtual common + { + public: + compile_rule (data&&); + + virtual bool + match (action, target&, const string&) const override; + + virtual recipe + apply (action, target&) const override; + + target_state + perform_update (action, const target&) const; + + target_state + perform_clean (action, const target&) const; + + private: + struct match_data; + using environment = small_vector; + + void + append_lib_options (const scope&, + cstrings&, + action, + const target&, + linfo) const; + + void + hash_lib_options (const scope&, + sha256&, + action, + const target&, + linfo) const; + + // Mapping of include prefixes (e.g., foo in ) for auto- + // generated headers to directories where they will be generated. + // + // We are using a prefix map of directories (dir_path_map) instead of + // just a map in order to also cover sub-paths (e.g., if + // we continue with the example). Specifically, we need to make sure we + // don't treat foobar as a sub-directory of foo. + // + // The priority is used to decide who should override whom. Lesser + // values are considered higher priority. See append_prefixes() for + // details. + // + // @@ The keys should be normalized. + // + struct prefix_value + { + dir_path directory; + size_t priority; + }; + using prefix_map = butl::dir_path_map; + + void + append_prefixes (prefix_map&, const target&, const variable&) const; + + void + append_lib_prefixes (const scope&, + prefix_map&, + action, + target&, + linfo) const; + + prefix_map + build_prefix_map (const scope&, action, target&, linfo) const; + + // Reverse-lookup target type from extension. + // + const target_type* + map_extension (const scope&, const string&, const string&) const; + + pair + extract_headers (action, const scope&, file&, linfo, + const file&, const match_data&, + depdb&, bool&, timestamp) const; + + pair + parse_unit (action, file&, linfo, + const file&, auto_rmfile&, const match_data&) const; + + void + extract_modules (action, const scope&, file&, linfo, + const compile_target_types&, + const file&, match_data&, + module_info&&, depdb&, bool&) const; + + module_positions + search_modules (action, const scope&, file&, linfo, + const target_type&, + const file&, module_imports&, sha256&) const; + + const target& + make_module_sidebuild (action, const scope&, const target&, + const target&, const string&) const; + + void + append_modules (environment&, cstrings&, strings&, + action, const file&, const match_data&) const; + + // Language selection option (for VC) or the value for the -x option. + // + const char* + langopt (const match_data&) const; + + void + append_symexport_options (cstrings&, const target&) const; + + private: + const string rule_id; + }; + } +} + +#endif // BUILD2_CC_COMPILE_RULE_HXX diff --git a/build2/cc/compile.cxx b/build2/cc/compile.cxx deleted file mode 100644 index 94b3478..0000000 --- a/build2/cc/compile.cxx +++ /dev/null @@ -1,4627 +0,0 @@ -// file : build2/cc/compile.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#include - -#include // exit() -#include // strlen() - -#include -#include -#include -#include -#include -#include -#include - -#include -#include // create_project() - -#include -#include // h -#include -#include - -using namespace std; -using namespace butl; - -namespace build2 -{ - namespace cc - { - using namespace bin; - - // module_info string serialization. - // - // The string representation is a space-separated list of module names - // with the following rules: - // - // 1. If this is a module interface unit, then the first name is the - // module name intself following by either '!' for an interface unit or - // by '+' for an implementation unit. - // - // 2. If an imported module is re-exported, then the module name is - // followed by '*'. - // - // For example: - // - // foo! foo.core* foo.base* foo.impl - // foo.base+ foo.impl - // foo.base foo.impl - // - static string - to_string (const module_info& m) - { - string s; - - if (!m.name.empty ()) - { - s += m.name; - s += m.iface ? '!' : '+'; - } - - for (const module_import& i: m.imports) - { - if (!s.empty ()) - s += ' '; - - s += i.name; - - if (i.exported) - s += '*'; - } - - return s; - } - - static module_info - to_module_info (const string& s) - { - module_info m; - - for (size_t b (0), e (0), n; (n = next_word (s, b, e, ' ')) != 0; ) - { - char c (s[e - 1]); - switch (c) - { - case '!': - case '+': - case '*': break; - default: c = '\0'; - } - - string w (s, b, n - (c == '\0' ? 0 : 1)); - - if (c == '!' || c == '+') - { - m.name = move (w); - m.iface = (c == '!'); - } - else - m.imports.push_back (module_import {move (w), c == '*', 0}); - } - - return m; - } - - // preprocessed - // - template - inline bool - operator< (preprocessed l, T r) // Template because of VC14 bug. - { - return static_cast (l) < static_cast (r); - } - - preprocessed - to_preprocessed (const string& s) - { - if (s == "none") return preprocessed::none; - if (s == "includes") return preprocessed::includes; - if (s == "modules") return preprocessed::modules; - if (s == "all") return preprocessed::all; - throw invalid_argument ("invalid preprocessed value '" + s + "'"); - } - - struct compile::match_data - { - explicit - match_data (translation_type t, const prerequisite_member& s) - : type (t), src (s) {} - - translation_type type; - preprocessed pp = preprocessed::none; - bool symexport = false; // Target uses __symexport. - bool touch = false; // Target needs to be touched. - timestamp mt = timestamp_unknown; // Target timestamp. - prerequisite_member src; - auto_rmfile psrc; // Preprocessed source, if any. - path dd; // Dependency database path. - module_positions mods = {0, 0, 0}; - }; - - compile:: - compile (data&& d) - : common (move (d)), - rule_id (string (x) += ".compile 4") - { - static_assert (sizeof (compile::match_data) <= target::data_size, - "insufficient space"); - } - - const char* compile:: - langopt (const match_data& md) const - { - bool m (md.type == translation_type::module_iface); - //preprocessed p (md.pp); - - switch (cid) - { - case compiler_id::gcc: - { - // Ignore the preprocessed value since for GCC it is handled via - // -fpreprocessed -fdirectives-only. - // - switch (x_lang) - { - case lang::c: return "c"; - case lang::cxx: return "c++"; - } - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - // Clang has *-cpp-output (but not c++-module-cpp-output) and they - // handle comments and line continuations. However, currently this - // is only by accident since these modes are essentially equivalent - // to their cpp-output-less versions. - // - switch (x_lang) - { - case lang::c: return "c"; - case lang::cxx: return m ? "c++-module" : "c++"; - } - } - case compiler_id::msvc: - { - switch (x_lang) - { - case lang::c: return "/TC"; - case lang::cxx: return "/TP"; - } - } - case compiler_id::icc: - { - switch (x_lang) - { - case lang::c: return "c"; - case lang::cxx: return "c++"; - } - } - } - - return nullptr; - } - - inline void compile:: - append_symexport_options (cstrings& args, const target& t) const - { - // With VC if a BMI is compiled with dllexport, then when such BMI is - // imported, it is auto-magically treated as dllimport. Let's hope - // other compilers follow suit. - // - args.push_back (t.is_a () && tclass == "windows" - ? "-D__symexport=__declspec(dllexport)" - : "-D__symexport="); - } - - match_result compile:: - match (action act, target& t, const string&) const - { - tracer trace (x, "compile::match"); - - bool mod (t.is_a () || t.is_a () || t.is_a ()); - - // Link-up to our group (this is the obj/bmi{} target group protocol - // which means this can be done whether we match or not). - // - if (t.group == nullptr) - t.group = &search (t, - mod ? bmi::static_type : obj::static_type, - t.dir, t.out, t.name); - - // See if we have a source file. Iterate in reverse so that a source - // file specified for a member overrides the one specified for the - // group. Also "see through" groups. - // - for (prerequisite_member p: reverse_group_prerequisite_members (act, t)) - { - if (p.is_a (mod ? *x_mod : x_src)) - { - // Save in the target's auxiliary storage. Translation type will - // be refined in apply(). - // - t.data (match_data (mod - ? translation_type::module_iface - : translation_type::plain, - p)); - return true; - } - } - - l4 ([&]{trace << "no " << x_lang << " source file for target " << t;}); - return false; - } - - // Append or hash library options from a pair of *.export.* variables - // (first one is cc.export.*) recursively, prerequisite libraries first. - // - void compile:: - append_lib_options (const scope& bs, - cstrings& args, - const target& t, - action act, - linfo li) const - { - // See through utility libraries. - // - auto imp = [] (const file& l, bool la) {return la && l.is_a ();}; - - auto opt = [&args, this] ( - const file& l, const string& t, bool com, bool exp) - { - // Note that in our model *.export.poptions are always "interface", - // even if set on liba{}/libs{}, unlike loptions. - // - if (!exp) // Ignore libux. - return; - - const variable& var ( - com - ? c_export_poptions - : (t == x ? x_export_poptions : var_pool[t + ".export.poptions"])); - - append_options (args, l, var); - }; - - // In case we don't have the "small function object" optimization. - // - const function impf (imp); - const function optf (opt); - - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - // Should be already searched and matched for libraries. - // - if (const target* pt = p.load ()) - { - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, act, li); - - bool a; - if (!((a = pt->is_a ()) || - (a = pt->is_a ()) || - pt->is_a ())) - continue; - - process_libraries (act, bs, li, sys_lib_dirs, - pt->as (), a, 0, // Hack: lflags unused. - impf, nullptr, optf); - } - } - } - - void compile:: - hash_lib_options (const scope& bs, - sha256& cs, - const target& t, - action act, - linfo li) const - { - auto imp = [] (const file& l, bool la) {return la && l.is_a ();}; - - auto opt = [&cs, this] ( - const file& l, const string& t, bool com, bool exp) - { - if (!exp) - return; - - const variable& var ( - com - ? c_export_poptions - : (t == x ? x_export_poptions : var_pool[t + ".export.poptions"])); - - hash_options (cs, l, var); - }; - - // The same logic as in append_lib_options(). - // - const function impf (imp); - const function optf (opt); - - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - if (const target* pt = p.load ()) - { - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, act, li); - - bool a; - if (!((a = pt->is_a ()) || - (a = pt->is_a ()) || - pt->is_a ())) - continue; - - process_libraries (act, bs, li, sys_lib_dirs, - pt->as (), a, 0, // Hack: lflags unused. - impf, nullptr, optf); - } - } - } - - // Append library prefixes based on the *.export.poptions variables - // recursively, prerequisite libraries first. - // - void compile:: - append_lib_prefixes (const scope& bs, - prefix_map& m, - target& t, - action act, - linfo li) const - { - auto imp = [] (const file& l, bool la) {return la && l.is_a ();}; - - auto opt = [&m, this] ( - const file& l, const string& t, bool com, bool exp) - { - if (!exp) - return; - - const variable& var ( - com - ? c_export_poptions - : (t == x ? x_export_poptions : var_pool[t + ".export.poptions"])); - - append_prefixes (m, l, var); - }; - - // The same logic as in append_lib_options(). - // - const function impf (imp); - const function optf (opt); - - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - if (const target* pt = p.load ()) - { - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, act, li); - - bool a; - if (!((a = pt->is_a ()) || - (a = pt->is_a ()) || - pt->is_a ())) - continue; - - process_libraries (act, bs, li, sys_lib_dirs, - pt->as (), a, 0, // Hack: lflags unused. - impf, nullptr, optf); - } - } - } - - // Update the target during the match phase. Return true if it has changed - // or if the passed timestamp is not timestamp_unknown and is older than - // the target. - // - // This function is used to make sure header dependencies are up to date. - // - // There would normally be a lot of headers for every source file (think - // all the system headers) and just calling execute_direct() on all of - // them can get expensive. At the same time, most of these headers are - // existing files that we will never be updating (again, system headers, - // for example) and the rule that will match them is the fallback - // file_rule. That rule has an optimization: it returns noop_recipe (which - // causes the target state to be automatically set to unchanged) if the - // file is known to be up to date. So we do the update "smartly". - // - static bool - update (tracer& trace, action act, const target& t, timestamp ts) - { - const path_target* pt (t.is_a ()); - - if (pt == nullptr) - ts = timestamp_unknown; - - target_state os (t.matched_state (act)); - - if (os == target_state::unchanged) - { - if (ts == timestamp_unknown) - return false; - else - { - // We expect the timestamp to be known (i.e., existing file). - // - timestamp mt (pt->mtime ()); // @@ MT perf: know target state. - assert (mt != timestamp_unknown); - return mt > ts; - } - } - else - { - // We only want to return true if our call to execute() actually - // caused an update. In particular, the target could already have been - // in target_state::changed because of a dependency extraction run for - // some other source file. - // - // @@ MT perf: so we are going to switch the phase and execute for - // any generated header. - // - phase_switch ps (run_phase::execute); - target_state ns (execute_direct (act, t)); - - if (ns != os && ns != target_state::unchanged) - { - l6 ([&]{trace << "updated " << t - << "; old state " << os - << "; new state " << ns;}); - return true; - } - else - return ts != timestamp_unknown ? pt->newer (ts) : false; - } - } - - recipe compile:: - apply (action act, target& xt) const - { - tracer trace (x, "compile::apply"); - - file& t (xt.as ()); // Either obj*{} or bmi*{}. - - match_data& md (t.data ()); - bool mod (md.type == translation_type::module_iface); - - const scope& bs (t.base_scope ()); - const scope& rs (*bs.root_scope ()); - - otype ot (compile_type (t, mod)); - linfo li (link_info (bs, ot)); // Link info for selecting libraries. - compile_target_types tt (compile_types (ot)); - - // Derive file name from target name. - // - string e; // Primary target extension (module or object). - { - const char* o ("o"); // Object extension (.o or .obj). - - if (tsys == "win32-msvc") - { - switch (ot) - { - case otype::e: e = "exe."; break; - case otype::a: e = "lib."; break; - case otype::s: e = "dll."; break; - } - o = "obj"; - } - else if (tsys == "mingw32") - { - switch (ot) - { - case otype::e: e = "exe."; break; - case otype::a: e = "a."; break; - case otype::s: e = "dll."; break; - } - } - else if (tsys == "darwin") - { - switch (ot) - { - case otype::e: e = ""; break; - case otype::a: e = "a."; break; - case otype::s: e = "dylib."; break; - } - } - else - { - switch (ot) - { - case otype::e: e = ""; break; - case otype::a: e = "a."; break; - case otype::s: e = "so."; break; - } - } - - switch (cid) - { - case compiler_id::gcc: - { - e += mod ? "nms" : o; - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - e += mod ? "pcm" : o; - break; - } - case compiler_id::msvc: - { - e += mod ? "ifc" : o; - break; - } - case compiler_id::icc: - { - assert (!mod); - e += o; - } - } - - // If we are compiling a module, then the obj*{} is an ad hoc member - // of bmi*{}. - // - if (mod) - { - // The module interface unit can be the same as an implementation - // (e.g., foo.mxx and foo.cxx) which means obj*{} targets could - // collide. So we add the module extension to the target name. - // - target_lock obj (add_adhoc_member (act, t, tt.obj, e.c_str ())); - obj.target->as ().derive_path (o); - match_recipe (obj, group_recipe); // Set recipe and unlock. - } - } - - const path& tp (t.derive_path (e.c_str ())); - - // Inject dependency on the output directory. - // - const fsdir* dir (inject_fsdir (act, t)); - - // Match all the existing prerequisites. The injection code takes care - // of the ones it is adding. - // - // When cleaning, ignore prerequisites that are not in the same or a - // subdirectory of our project root. - // - auto& pts (t.prerequisite_targets); - optional usr_lib_dirs; // Extract lazily. - - // Start asynchronous matching of prerequisites. Wait with unlocked - // phase to allow phase switching. - // - wait_guard wg (target::count_busy (), t.task_count, true); - - size_t start (pts.size ()); // Index of the first to be added. - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - const target* pt (nullptr); - - // A dependency on a library is there so that we can get its - // *.export.poptions, modules, etc. This is the "library - // meta-information protocol". See also append_lib_options(). - // - if (p.is_a () || - p.is_a () || - p.is_a () || - p.is_a ()) - { - if (act.operation () == update_id) - { - // Handle (phase two) imported libraries. We know that for such - // libraries we don't need to do match() in order to get options - // (if any, they would be set by search_library()). - // - if (p.proj ()) - { - if (search_library (act, - sys_lib_dirs, - usr_lib_dirs, - p.prerequisite) != nullptr) - continue; - } - - pt = &p.search (t); - - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, act, li); - } - else - continue; - } - // - // For modules we pick only what we import which is done below so - // skip it here. One corner case is clean: we assume that someone - // else (normally library/executable) also depends on it and will - // clean it up. - // - else if (p.is_a () || p.is_a (tt.bmi)) - continue; - else - { - pt = &p.search (t); - - if (act.operation () == clean_id && !pt->dir.sub (rs.out_path ())) - continue; - } - - match_async (act, *pt, target::count_busy (), t.task_count); - pts.push_back (pt); - } - - wg.wait (); - - // Finish matching all the targets that we have started. - // - for (size_t i (start), n (pts.size ()); i != n; ++i) - { - const target*& pt (pts[i]); - - // Making sure a library is updated before us will only restrict - // parallelism. But we do need to match it in order to get its imports - // resolved and prerequisite_targets populated. So we match it but - // then unmatch if it is safe. And thanks to the two-pass prerequisite - // match in link::apply() it will be safe unless someone is building - // an obj?{} target directory. - // - if (build2::match ( - act, - *pt, - pt->is_a () || pt->is_a () || pt->is_a () - ? unmatch::safe - : unmatch::none)) - pt = nullptr; // Ignore in execute. - } - - // Inject additional prerequisites. We only do it when performing update - // since chances are we will have to update some of our prerequisites in - // the process (auto-generated source code). - // - if (act == perform_update_id) - { - // The cached prerequisite target should be the same as what is in - // t.prerequisite_targets since we used standard search() and match() - // above. - // - const file& src (*md.src.search (t).is_a ()); - - // Figure out if __symexport is used. While normally it is specified - // on the project root (which we cached), it can be overridden with - // a target-specific value for installed modules (which we sidebuild - // as part of our project). - // - if (modules && src.is_a (*x_mod)) - { - lookup l (src.vars[x_symexport]); - md.symexport = l ? cast (l) : symexport; - } - - // Make sure the output directory exists. - // - // Is this the right thing to do? It does smell a bit, but then we do - // worse things in inject_prerequisites() below. There is also no way - // to postpone this until update since we need to extract and inject - // header dependencies now (we don't want to be calling search() and - // match() in update), which means we need to cache them now as well. - // So the only alternative, it seems, is to cache the updates to the - // database until later which will sure complicate (and slow down) - // things. - // - if (dir != nullptr) - { - // We can do it properly by using execute_direct(). But this means - // we will be switching to the execute phase with all the associated - // overheads. At the same time, in case of update, creation of a - // directory is not going to change the external state in any way - // that would affect any parallel efforts in building the internal - // state. So we are just going to create the directory directly. - // Note, however, that we cannot modify the fsdir{} target since - // this can very well be happening in parallel. But that's not a - // problem since fsdir{}'s update is idempotent. - // - fsdir_rule::perform_update_direct (act, t); - } - - // Note: the leading '@' is reserved for the module map prefix (see - // extract_modules()) and no other line must start with it. - // - md.dd = tp + ".d"; - depdb dd (md.dd); - - // First should come the rule name/version. - // - if (dd.expect (rule_id) != nullptr) - l4 ([&]{trace << "rule mismatch forcing update of " << t;}); - - // Then the compiler checksum. Note that here we assume it - // incorporates the (default) target so that if the compiler changes - // but only in what it targets, then the checksum will still change. - // - if (dd.expect (cast (rs[x_checksum])) != nullptr) - l4 ([&]{trace << "compiler mismatch forcing update of " << t;}); - - // Then the options checksum. - // - // The idea is to keep them exactly as they are passed to the compiler - // since the order may be significant. - // - { - sha256 cs; - - // These flags affect how we compile the source and/or the format of - // depdb so factor them in. - // - cs.append (&md.pp, sizeof (md.pp)); - cs.append (&md.symexport, sizeof (md.symexport)); - - if (md.pp != preprocessed::all) - { - hash_options (cs, t, c_poptions); - hash_options (cs, t, x_poptions); - - // Hash *.export.poptions from prerequisite libraries. - // - hash_lib_options (bs, cs, t, act, li); - - // Extra system header dirs (last). - // - assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); - hash_option_values ( - cs, "-I", - sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), - [] (const dir_path& d) {return d.string ();}); - } - - hash_options (cs, t, c_coptions); - hash_options (cs, t, x_coptions); - hash_options (cs, tstd); - - if (ot == otype::s) - { - // On Darwin, Win32 -fPIC is the default. - // - if (tclass == "linux" || tclass == "bsd") - cs.append ("-fPIC"); - } - - if (dd.expect (cs.string ()) != nullptr) - l4 ([&]{trace << "options mismatch forcing update of " << t;}); - } - - // Finally the source file. - // - if (dd.expect (src.path ()) != nullptr) - l4 ([&]{trace << "source file mismatch forcing update of " << t;}); - - // If any of the above checks resulted in a mismatch (different - // compiler, options, or source file) or if the depdb is newer than - // the target (interrupted update), then do unconditional update. - // - timestamp mt; - bool u (dd.writing () || dd.mtime () > (mt = file_mtime (tp))); - if (u) - mt = timestamp_nonexistent; // Treat as if it doesn't exist. - - // Update prerequisite targets (normally just the source file). - // - // This is an unusual place and time to do it. But we have to do it - // before extracting dependencies. The reasoning for source file is - // pretty clear. What other prerequisites could we have? While - // normally they will be some other sources (as in, static content - // from src_root), it's possible they are some auto-generated stuff. - // And it's possible they affect the preprocessor result. Say some ad - // hoc/out-of-band compiler input file that is passed via the command - // line. So, to be safe, we make sure everything is up to date. - // - for (const target* pt: pts) - { - if (pt == nullptr || pt == dir) - continue; - - u = update (trace, act, *pt, u ? timestamp_unknown : mt) || u; - } - - // Check if the source is already preprocessed to a certain degree. - // This determines which of the following steps we perform and on - // what source (original or preprocessed). - // - // Note: must be set of the src target. - // - if (const string* v = cast_null (src[x_preprocessed])) - try - { - md.pp = to_preprocessed (*v); - } - catch (const invalid_argument& e) - { - fail << "invalid " << x_preprocessed.name << " variable value " - << "for target " << src << ": " << e; - } - - // If we have no #include directives, then skip header dependency - // extraction. - // - pair psrc (auto_rmfile (), false); - if (md.pp < preprocessed::includes) - psrc = extract_headers (act, bs, t, li, src, md, dd, u, mt); - - // Next we "obtain" the translation unit information. What exactly - // "obtain" entails is tricky: If things changed, then we re-parse the - // translation unit. Otherwise, we re-create this information from - // depdb. We, however, have to do it here and now in case the database - // is invalid and we still have to fallback to re-parse. - // - // Store a translation unit's checksum to detect ignorable changes - // (whitespaces, comments, etc). - // - { - string cs; - if (string* l = dd.read ()) - cs = move (*l); - else - u = true; // Database is invalid, force re-parse. - - translation_unit tu; - for (bool f (true);; f = false) - { - if (u) - { - auto p (parse_unit (act, t, li, src, psrc.first, md)); - - if (cs != p.second) - { - assert (f); // Unchanged TU has a different checksum? - dd.write (p.second); - } - else if (f) // Don't clear if it was forced. - { - // Clear the update flag and set the touch flag. Unless there - // is no object file, of course. See also the md.mt logic - // below. - // - if (mt != timestamp_nonexistent) - { - u = false; - md.touch = true; - } - } - - tu = move (p.first); - } - - if (modules) - { - if (u || !f) - { - string s (to_string (tu.mod)); - - if (f) - dd.expect (s); - else - dd.write (s); - } - else - { - if (string* l = dd.read ()) - tu.mod = to_module_info (*l); - else - { - u = true; // Database is invalid, force re-parse. - continue; - } - } - } - - break; - } - - // Make sure the translation unit type matches the resulting target - // type. - // - switch (tu.type ()) - { - case translation_type::plain: - case translation_type::module_impl: - { - if (mod) - fail << "translation unit " << src << " is not a module interface" << - info << "consider using " << x_src.name << "{} instead"; - break; - } - case translation_type::module_iface: - { - if (!mod) - fail << "translation unit " << src << " is a module interface" << - info << "consider using " << x_mod->name << "{} instead"; - break; - } - } - - md.type = tu.type (); - - // Extract the module dependency information in addition to header - // dependencies. - // - // NOTE: assumes that no further targets will be added into - // t.prerequisite_targets! - // - extract_modules (act, bs, t, li, tt, src, md, move (tu.mod), dd, u); - } - - // If anything got updated, then we didn't rely on the cache. However, - // the cached data could actually have been valid and the compiler run - // in extract_headers() as well as the code above merely validated it. - // - // We do need to update the database timestamp, however. Failed that, - // we will keep re-validating the cached data over and over again. - // - if (u && dd.reading ()) - dd.touch (); - - dd.close (); - - // If the preprocessed output is suitable for compilation and is not - // disabled, then pass it along. - // - if (psrc.second && !cast_false (t[c_reprocess])) - { - md.psrc = move (psrc.first); - - // Without modules keeping the (partially) preprocessed output - // around doesn't buy us much: if the source/headers haven't changed - // then neither will the object file. Modules make things more - // interesting: now we may have to recompile an otherwise unchanged - // translation unit because a BMI it depends on has changed. In this - // case re-processing the translation unit would be a waste and - // compiling the original source would break distributed - // compilation. - // - // Note also that the long term trend will (hopefully) be for - // modularized projects to get rid of #include's which means the - // need for producing this partially preprocessed output will - // (hopefully) gradually disappear. - // - if (modules) - md.psrc.active = false; // Keep. - } - - // Above we may have ignored changes to the translation unit. The - // problem is, unless we also update the target's timestamp, we will - // keep re-checking this on subsequent runs and it is not cheap. - // Updating the target's timestamp is not without problems either: it - // will cause a re-link on a subsequent run. So, essentially, we - // somehow need to remember two timestamps: one for checking - // "preprocessor prerequisites" above and one for checking other - // prerequisites (like modules) below. So what we are going to do is - // store the first in the target file (so we do touch it) and the - // second in depdb (which is never newer that the target). - // - md.mt = u ? timestamp_nonexistent : dd.mtime (); - } - - switch (act) - { - case perform_update_id: return [this] (action a, const target& t) - { - return perform_update (a, t); - }; - case perform_clean_id: return [this] (action a, const target& t) - { - return perform_clean (a, t); - }; - default: return noop_recipe; // Configure update. - } - } - - // Reverse-lookup target type from extension. - // - const target_type* compile:: - map_extension (const scope& s, const string& n, const string& e) const - { - // We will just have to try all of the possible ones, in the "most - // likely to match" order. - // - auto test = [&s, &n, &e] (const target_type& tt) -> bool - { - // Call the extension derivation function. Here we know that it will - // only use the target type and name from the target key so we can - // pass bogus values for the rest. - // - target_key tk {&tt, nullptr, nullptr, &n, nullopt}; - - // This is like prerequisite search. - // - if (optional de = tt.default_extension (tk, s, true)) - if (*de == e) - return true; - - return false; - }; - - for (const target_type* const* p (x_inc); *p != nullptr; ++p) - if (test (**p)) return *p; - - return nullptr; - } - - void compile:: - append_prefixes (prefix_map& m, const target& t, const variable& var) const - { - tracer trace (x, "compile::append_prefixes"); - - // If this target does not belong to any project (e.g, an "imported as - // installed" library), then it can't possibly generate any headers for - // us. - // - const scope& bs (t.base_scope ()); - const scope* rs (bs.root_scope ()); - if (rs == nullptr) - return; - - const dir_path& out_base (t.dir); - const dir_path& out_root (rs->out_path ()); - - if (auto l = t[var]) - { - const auto& v (cast (l)); - - for (auto i (v.begin ()), e (v.end ()); i != e; ++i) - { - // -I can either be in the "-Ifoo" or "-I foo" form. For VC it can - // also be /I. - // - const string& o (*i); - - if (o.size () < 2 || (o[0] != '-' && o[0] != '/') || o[1] != 'I') - continue; - - dir_path d; - if (o.size () == 2) - { - if (++i == e) - break; // Let the compiler complain. - - d = dir_path (*i); - } - else - d = dir_path (*i, 2, string::npos); - - l6 ([&]{trace << "-I " << d;}); - - if (d.relative ()) - fail << "relative -I directory " << d - << " in variable " << var.name - << " for target " << t; - - // If we are not inside our project root, then ignore. - // - if (!d.sub (out_root)) - continue; - - // If the target directory is a sub-directory of the include - // directory, then the prefix is the difference between the - // two. Otherwise, leave it empty. - // - // The idea here is to make this "canonical" setup work auto- - // magically: - // - // 1. We include all files with a prefix, e.g., . - // 2. The library target is in the foo/ sub-directory, e.g., - // /tmp/foo/. - // 3. The poptions variable contains -I/tmp. - // - dir_path p (out_base.sub (d) ? out_base.leaf (d) : dir_path ()); - - // We use the target's directory as out_base but that doesn't work - // well for targets that are stashed in subdirectories. So as a - // heuristics we are going to also enter the outer directories of - // the original prefix. It is, however, possible, that another -I - // option after this one will produce one of these outer prefixes as - // its original prefix in which case we should override it. - // - // So we are going to assign the original prefix priority value 0 - // (highest) and then increment it for each outer prefix. - // - auto enter = [&trace, &m] (dir_path p, dir_path d, size_t prio) - { - auto j (m.find (p)); - - if (j != m.end ()) - { - prefix_value& v (j->second); - - // We used to reject duplicates but it seems this can be - // reasonably expected to work according to the order of the - // -I options. - // - // Seeing that we normally have more "specific" -I paths first, - // (so that we don't pick up installed headers, etc), we ignore - // it. - // - if (v.directory == d) - { - if (v.priority > prio) - v.priority = prio; - } - else if (v.priority <= prio) - { - if (verb >= 4) - trace << "ignoring dependency prefix " << p << '\n' - << " existing mapping to " << v.directory - << " priority " << v.priority << '\n' - << " another mapping to " << d - << " priority " << prio; - } - else - { - if (verb >= 4) - trace << "overriding dependency prefix " << p << '\n' - << " existing mapping to " << v.directory - << " priority " << v.priority << '\n' - << " new mapping to " << d - << " priority " << prio; - - v.directory = move (d); - v.priority = prio; - } - } - else - { - l6 ([&]{trace << p << " -> " << d << " priority " << prio;}); - m.emplace (move (p), prefix_value {move (d), prio}); - } - }; - - size_t prio (0); - for (bool e (false); !e; ++prio) - { - dir_path n (p.directory ()); - e = n.empty (); - enter ((e ? move (p) : p), (e ? move (d) : d), prio); - p = move (n); - } - } - } - } - - auto compile:: - build_prefix_map (const scope& bs, - target& t, - action act, - linfo li) const -> prefix_map - { - prefix_map m; - - // First process our own. - // - append_prefixes (m, t, c_poptions); - append_prefixes (m, t, x_poptions); - - // Then process the include directories from prerequisite libraries. - // - append_lib_prefixes (bs, m, t, act, li); - - return m; - } - - // Return the next make prerequisite starting from the specified - // position and update position to point to the start of the - // following prerequisite or l.size() if there are none left. - // - static string - next_make (const string& l, size_t& p) - { - size_t n (l.size ()); - - // Skip leading spaces. - // - for (; p != n && l[p] == ' '; p++) ; - - // Lines containing multiple prerequisites are 80 characters max. - // - string r; - r.reserve (n); - - // Scan the next prerequisite while watching out for escape sequences. - // - for (; p != n && l[p] != ' '; p++) - { - char c (l[p]); - - if (p + 1 != n) - { - if (c == '$') - { - // Got to be another (escaped) '$'. - // - if (l[p + 1] == '$') - ++p; - } - else if (c == '\\') - { - // This may or may not be an escape sequence depending on whether - // what follows is "escapable". - // - switch (c = l[++p]) - { - case '\\': break; - case ' ': break; - default: c = '\\'; --p; // Restore. - } - } - } - - r += c; - } - - // Skip trailing spaces. - // - for (; p != n && l[p] == ' '; p++) ; - - // Skip final '\'. - // - if (p == n - 1 && l[p] == '\\') - p++; - - return r; - } - - // VC /showIncludes output. The first line is the file being compiled - // (handled by our caller). Then we have the list of headers, one per - // line, in this form (text can presumably be translated): - // - // Note: including file: C:\Program Files (x86)\[...]\iostream - // - // Finally, if we hit a non-existent header, then we end with an error - // line in this form: - // - // x.cpp(3): fatal error C1083: Cannot open include file: 'd/h.hpp': - // No such file or directory - // - // Distinguishing between the include note and the include error is - // easy: we can just check for C1083. Distinguising between the note and - // other errors/warnings is harder: an error could very well end with - // what looks like a path so we cannot look for the note but rather have - // to look for an error. Here we assume that a line containing ' CNNNN:' - // is an error. Should be robust enough in the face of language - // translation, etc. - // - // It turns out C1083 is also used when we are unable to open the main - // source file and the error line looks like this: - // - // c1xx: fatal error C1083: Cannot open source file: 's.cpp': No such - // file or directory - - // Sense whether this is an include note (return npos) or a diagnostics - // line (return postion of the NNNN code in CNNNN). - // - static inline size_t - next_show_sense (const string& l) - { - size_t p (l.find (':')); - - for (size_t n (l.size ()); - p != string::npos; - p = ++p != n ? l.find (':', p) : string::npos) - { - auto isnum = [](char c) {return c >= '0' && c <= '9';}; - - if (p > 5 && - l[p - 6] == ' ' && - l[p - 5] == 'C' && - isnum (l[p - 4]) && - isnum (l[p - 3]) && - isnum (l[p - 2]) && - isnum (l[p - 1])) - { - p -= 4; // Start of the error code. - break; - } - } - - return p; - } - - // Extract the include path from the VC /showIncludes output line. Return - // empty string if the line is not an include note or include error. Set - // the good_error flag if it is an include error (which means the process - // will terminate with the error status that needs to be ignored). - // - static string - next_show (const string& l, bool& good_error) - { - // The include error should be the last line that we handle. - // - assert (!good_error); - - size_t p (next_show_sense (l)); - if (p == string::npos) - { - // Include note. We assume the path is always at the end but need to - // handle both absolute Windows and POSIX ones. - // - // Note that VC appears to always write the absolute path to the - // included file even if it is ""-included and the source path is - // relative. Aren't we lucky today? - // - p = l.rfind (':'); - - if (p != string::npos) - { - // See if this one is part of the Windows drive letter. - // - if (p > 1 && p + 1 < l.size () && // 2 chars before, 1 after. - l[p - 2] == ' ' && - alpha (l[p - 1]) && - path::traits::is_separator (l[p + 1])) - p = l.rfind (':', p - 2); - } - - if (p != string::npos) - { - // VC uses indentation to indicate the include nesting so there - // could be any number of spaces after ':'. Skip them. - // - p = l.find_first_not_of (' ', p + 1); - } - - if (p == string::npos) - fail << "unable to parse /showIncludes include note line"; - - return string (l, p); - } - else if (l.compare (p, 4, "1083") == 0 && - l.compare (0, 5, "c1xx:") != 0 /* Not the main source file. */ ) - { - // Include error. The path is conveniently quoted with ''. - // - size_t p2 (l.rfind ('\'')); - - if (p2 != string::npos && p2 != 0) - { - size_t p1 (l.rfind ('\'', p2 - 1)); - - if (p1 != string::npos) - { - good_error = true; - return string (l, p1 + 1 , p2 - p1 - 1); - } - } - - fail << "unable to parse /showIncludes include error line" << endf; - } - else - { - // Some other error. - // - return string (); - } - } - - // Extract and inject header dependencies. Return the preprocessed source - // file as well as an indication if it is usable for compilation (see - // below for details). - // - pair compile:: - extract_headers (action act, - const scope& bs, - file& t, - linfo li, - const file& src, - const match_data& md, - depdb& dd, - bool& updating, - timestamp mt) const - { - tracer trace (x, "compile::extract_headers"); - - l5 ([&]{trace << "target: " << t;}); - - auto_rmfile psrc; - bool puse (true); - - // If things go wrong (and they often do in this area), give the user a - // bit extra context. - // - auto df = make_diag_frame ( - [&src](const diag_record& dr) - { - if (verb != 0) - dr << info << "while extracting header dependencies from " << src; - }); - - const scope& rs (*bs.root_scope ()); - - // Preprocess mode that preserves as much information as possible while - // still performing inclusions. Also serves as a flag indicating whether - // this compiler uses the separate preprocess and compile setup. - // - const char* pp (nullptr); - - switch (cid) - { - case compiler_id::gcc: - { - // -fdirectives-only is available since GCC 4.3.0. - // - if (cmaj > 4 || (cmaj == 4 && cmin >= 3)) - pp = "-fdirectives-only"; - - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - // -frewrite-includes is available since vanilla Clang 3.2.0. - // - // Apple Clang 5.0 is based on LLVM 3.3svn so it should have this - // option (4.2 is based on 3.2svc so it may or may not have it and, - // no, we are not going to try to find out). - // - if (cid == compiler_id::clang_apple - ? (cmaj >= 5) - : (cmaj > 3 || (cmaj == 3 && cmin >= 2))) - pp = "-frewrite-includes"; - - break; - } - case compiler_id::msvc: - { - pp = "/C"; - break; - } - case compiler_id::icc: - break; - } - - // Initialize lazily, only if required. - // - environment env; - cstrings args; - string out; // Storage. - - // Some compilers in certain modes (e.g., when also producing the - // preprocessed output) are incapable of writing the dependecy - // information to stdout. In this case we use a temporary file. - // - auto_rmfile drm; - - // Here is the problem: neither GCC nor Clang allow -MG (treat missing - // header as generated) when we produce any kind of other output (-MD). - // And that's probably for the best since otherwise the semantics gets - // pretty hairy (e.g., what is the exit code and state of the output)? - // - // One thing to note about generated headers: if we detect one, then, - // after generating it, we re-run the compiler since we need to get - // this header's dependencies. - // - // So this is how we are going to work around this problem: we first run - // with -E but without -MG. If there are any errors (maybe because of - // generated headers maybe not), we restart with -MG and without -E. If - // this fixes the error (so it was a generated header after all), then - // we have to restart at which point we go back to -E and no -MG. And we - // keep yo-yoing like this. Missing generated headers will probably be - // fairly rare occurrence so this shouldn't be too expensive. - // - // Actually, there is another error case we would like to handle: an - // outdated generated header that is now causing an error (e.g., because - // of a check that is now triggering #error or some such). So there are - // actually three error cases: outdated generated header, missing - // generated header, and some other error. To handle the outdated case - // we need the compiler to produce the dependency information even in - // case of an error. Clang does it, for VC we parse diagnostics - // ourselves, but GCC does not (but a patch has been submitted). - // - // So the final plan is then as follows: - // - // 1. Start wothout -MG and with suppressed diagnostics. - // 2. If error but we've updated a header, then repeat step 1. - // 3. Otherwise, restart with -MG and diagnostics. - // - // Note that below we don't even check if the compiler supports the - // dependency info on error. We just try to use it and if it's not - // there we ignore the io error since the compiler has failed. - // - bool args_gen; // Current state of args. - size_t args_i; // Start of the -M/-MD "tail". - - // Ok, all good then? Not so fast, the rabbit hole is deeper than it - // seems: When we run with -E we have to discard diagnostics. This is - // not a problem for errors since they will be shown on the re-run but - // it is for (preprocessor) warnings. - // - // Clang's -frewrite-includes is nice in that it preserves the warnings - // so they will be shown during the compilation of the preprocessed - // source. They are also shown during -E but that we discard. And unlike - // GCC, in Clang -M does not imply -w (disable warnings) so it would - // have been shown in -M -MG re-runs but we suppress that with explicit - // -w. All is good in the Clang land then (even -Werror works nicely). - // - // GCC's -fdirective-only, on the other hand, processes all the - // directives so they are gone from the preprocessed source. Here is - // what we are going to do to work around this: we will detect if any - // diagnostics has been written to stderr on the -E run. If that's the - // case (but the compiler indicated success) then we assume they are - // warnings and disable the use of the preprocessed output for - // compilation. This in turn will result in compilation from source - // which will display the warnings. Note that we may still use the - // preprocessed output for other things (e.g., C++ module dependency - // discovery). BTW, another option would be to collect all the - // diagnostics and then dump it if the run is successful, similar to - // the VC semantics (and drawbacks) described below. - // - // Finally, for VC, things are completely different: there is no -MG - // equivalent and we handle generated headers by analyzing the - // diagnostics. This means that unlike in the above two cases, the - // preprocessor warnings are shown during dependency extraction, not - // compilation. Not ideal but that's the best we can do. Or is it -- we - // could implement ad hoc diagnostics sensing... It appears warnings are - // in the C4000-C4999 code range though there can also be note lines - // which don't have any C-code. - // - // BTW, triggering a warning in the VC preprocessor is not easy; there - // is no #warning and pragmas are passed through to the compiler. One - // way to do it is to redefine a macro, for example: - // - // hello.cxx(4): warning C4005: 'FOO': macro redefinition - // hello.cxx(3): note: see previous definition of 'FOO' - // - // So seeing that it is hard to trigger a legitimate VC preprocessor - // warning, for now, we will just treat them as errors by adding /WX. - // - // Note: diagnostics sensing is currently only supported if dependency - // info is written to a file (see above). - // - bool sense_diag (false); - - // And here is another problem: if we have an already generated header - // in src and the one in out does not yet exist, then the compiler will - // pick the one in src and we won't even notice. Note that this is not - // only an issue with mixing in- and out-of-tree builds (which does feel - // wrong but is oh so convenient): this is also a problem with - // pre-generated headers, a technique we use to make installing the - // generator by end-users optional by shipping pre-generated headers. - // - // This is a nasty problem that doesn't seem to have a perfect solution - // (except, perhaps, C++ modules). So what we are going to do is try to - // rectify the situation by detecting and automatically remapping such - // mis-inclusions. It works as follows. - // - // First we will build a map of src/out pairs that were specified with - // -I. Here, for performance and simplicity, we will assume that they - // always come in pairs with out first and src second. We build this - // map lazily only if we are running the preprocessor and reuse it - // between restarts. - // - // With the map in hand we can then check each included header for - // potentially having a doppelganger in the out tree. If this is the - // case, then we calculate a corresponding header in the out tree and, - // (this is the most important part), check if there is a target for - // this header in the out tree. This should be fairly accurate and not - // require anything explicit from the user except perhaps for a case - // where the header is generated out of nothing (so there is no need to - // explicitly mention its target in the buildfile). But this probably - // won't be very common. - // - // One tricky area in this setup are target groups: if the generated - // sources are mentioned in the buildfile as a group, then there might - // be no header target (yet). The way we solve this is by requiring code - // generator rules to cooperate and create at least the header target as - // part of the group creation. While not all members of the group may be - // generated depending on the options (e.g., inline files might be - // suppressed), headers are usually non-optional. - // - // Note that we use path_map instead of dir_path_map to allow searching - // using path (file path). - // - using srcout_map = path_map; - srcout_map so_map; - - // The gen argument to init_args() is in/out. The caller signals whether - // to force the generated header support and on return it signals - // whether this support is enabled. The first call to init_args is - // expected to have gen false. - // - // Return NULL if the dependency information goes to stdout and a - // pointer to the temporary file path otherwise. - // - auto init_args = [&t, act, li, - &src, &md, &psrc, &sense_diag, - &rs, &bs, - pp, &env, &args, &args_gen, &args_i, &out, &drm, - &so_map, this] - (bool& gen) -> const path* - { - const path* r (nullptr); - - if (args.empty ()) // First call. - { - assert (!gen); - - // We use absolute/relative paths in the dependency output to - // distinguish existing headers from (missing) generated. Which - // means we have to (a) use absolute paths in -I and (b) pass - // absolute source path (for ""-includes). That (b) is a problem: - // if we use an absolute path, then all the #line directives will be - // absolute and all the diagnostics will have long, noisy paths - // (actually, we will still have long paths for diagnostics in - // headers). - // - // To work around this we used to pass a relative path to the source - // file and then check every relative path in the dependency output - // for existence in the source file's directory. This is not without - // issues: it is theoretically possible for a generated header that - // is <>-included and found via -I to exist in the source file's - // directory. Note, however, that this is a lot more likely to - // happen with prefix-less inclusion (e.g., ) and in this case - // we assume the file is in the project anyway. And if there is a - // conflict with a prefixed include (e.g., ), then, well, - // we will just have to get rid of quoted includes (which are - // generally a bad idea, anyway). - // - // But then this approach (relative path) fell apart further when we - // tried to implement precise changed detection: the preprocessed - // output would change depending from where it was compiled because - // of #line (which we could work around) and __FILE__/assert() - // (which we can't really do anything about). So it looks like using - // the absolute path is the lesser of all the evils (and there are - // many). - // - // Note that we detect and diagnose relative -I directories lazily - // when building the include prefix map. - // - args.push_back (cpath.recall_string ()); - - // Add *.export.poptions from prerequisite libraries. - // - append_lib_options (bs, args, t, act, li); - - append_options (args, t, c_poptions); - append_options (args, t, x_poptions); - - // Populate the src-out with the -I$out_base -I$src_base pairs. - // - { - // Try to be fast and efficient by reusing buffers as much as - // possible. - // - string ds; - - // Previous -I innermost scope if out_base plus the difference - // between the scope path and the -I path (normally empty). - // - const scope* s (nullptr); - dir_path p; - - for (auto i (args.begin ()), e (args.end ()); i != e; ++i) - { - // -I can either be in the "-Ifoo" or "-I foo" form. For VC it - // can also be /I. - // - const char* o (*i); - size_t n (strlen (o)); - - if (n < 2 || (o[0] != '-' && o[0] != '/') || o[1] != 'I') - { - s = nullptr; - continue; - } - - if (n == 2) - { - if (++i == e) - break; // Let the compiler complain. - - ds = *i; - } - else - ds.assign (o + 2, n - 2); - - if (!ds.empty ()) - { - // Note that we don't normalize the paths since it would be - // quite expensive and normally the pairs we are inerested in - // are already normalized (since they are usually specified as - // -I$src/out_*). We just need to add a trailing directory - // separator if it's not already there. - // - if (!dir_path::traits::is_separator (ds.back ())) - ds += dir_path::traits::directory_separator; - - dir_path d (move (ds), dir_path::exact); // Move the buffer in. - - // Ignore invalid paths (buffer is not moved). - // - if (!d.empty ()) - { - // Ignore any paths containing '.', '..' components. Allow - // any directory separators thought (think -I$src_root/foo - // on Windows). - // - if (d.absolute () && d.normalized (false)) - { - // If we have a candidate out_base, see if this is its - // src_base. - // - if (s != nullptr) - { - const dir_path& bp (s->src_path ()); - - if (d.sub (bp)) - { - if (p.empty () || d.leaf (bp) == p) - { - // We've got a pair. - // - so_map.emplace (move (d), s->out_path () / p); - s = nullptr; // Taken. - continue; - } - } - - // Not a pair. Fall through to consider as out_base. - // - s = nullptr; - } - - // See if this path is inside a project with an out-of- - // tree build and is in the out directory tree. - // - const scope& bs (scopes.find (d)); - if (bs.root_scope () != nullptr) - { - const dir_path& bp (bs.out_path ()); - if (bp != bs.src_path ()) - { - bool e; - if ((e = (d == bp)) || d.sub (bp)) - { - s = &bs; - if (e) - p.clear (); - else - p = d.leaf (bp); - } - } - } - } - else - s = nullptr; - - ds = move (d).string (); // Move the buffer out. - } - else - s = nullptr; - } - else - s = nullptr; - } - } - - // Extra system header dirs (last). - // - assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); - append_option_values ( - args, "-I", - sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), - [] (const dir_path& d) {return d.string ().c_str ();}); - - if (md.symexport) - append_symexport_options (args, t); - - // Some compile options (e.g., -std, -m) affect the preprocessor. - // - // Currently Clang supports importing "header modules" even when in - // the TS mode. And "header modules" support macros which means - // imports have to be resolved during preprocessing. Which poses a - // bit of a chicken and egg problem for us. For now, the workaround - // is to remove the -fmodules-ts option when preprocessing. Hopefully - // there will be a "pure modules" mode at some point. - // - - // Don't treat warnings as errors. - // - const char* werror (nullptr); - switch (cclass) - { - case compiler_class::gcc: werror = "-Werror"; break; - case compiler_class::msvc: werror = "/WX"; break; - } - - bool clang (cid == compiler_id::clang || - cid == compiler_id::clang_apple); - - append_options (args, t, c_coptions, werror); - append_options (args, t, x_coptions, werror); - append_options (args, tstd, - tstd.size () - (modules && clang ? 1 : 0)); - - switch (cclass) - { - case compiler_class::msvc: - { - assert (pp != nullptr); - - args.push_back ("/nologo"); - - // See perform_update() for details on overriding the default - // exceptions and runtime. - // - if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) - args.push_back ("/EHsc"); - - if (!find_option_prefixes ({"/MD", "/MT"}, args)) - args.push_back ("/MD"); - - args.push_back ("/P"); // Preprocess to file. - args.push_back ("/showIncludes"); // Goes to stdout (with diag). - args.push_back (pp); // /C (preserve comments). - args.push_back ("/WX"); // Warning as error (see above). - - psrc = auto_rmfile (t.path () + x_pext); - - if (cast (rs[x_version_major]) >= 18) - { - args.push_back ("/Fi:"); - args.push_back (psrc.path.string ().c_str ()); - } - else - { - out = "/Fi" + psrc.path.string (); - args.push_back (out.c_str ()); - } - - args.push_back (langopt (md)); // Compile as. - gen = args_gen = true; - break; - } - case compiler_class::gcc: - { - if (t.is_a ()) - { - // On Darwin, Win32 -fPIC is the default. - // - if (tclass == "linux" || tclass == "bsd") - args.push_back ("-fPIC"); - } - - // Depending on the compiler, decide whether (and how) we can - // produce preprocessed output as a side effect of dependency - // extraction. - // - // Note: -MM -MG skips missing <>-included. - - // Clang's -M does not imply -w (disable warnings). We also - // don't need them in the -MD case (see above) so disable for - // both. - // - if (clang) - args.push_back ("-w"); - - // Previously we used '*' as a target name but it gets expanded - // to the current directory file names by GCC (4.9) that comes - // with MSYS2 (2.4). Yes, this is the (bizarre) behavior of GCC - // being executed in the shell with -MQ '*' option and not just - // -MQ *. - // - args.push_back ("-MQ"); // Quoted target name. - args.push_back ("^"); // Old versions can't do empty target. - - args.push_back ("-x"); - args.push_back (langopt (md)); - - if (pp != nullptr) - { - // Note that the options are carefully laid out to be easy to - // override (see below). - // - args_i = args.size (); - - args.push_back ("-MD"); - args.push_back ("-E"); - args.push_back (pp); - - // Dependency output. - // - args.push_back ("-MF"); - - // GCC is not capable of writing the dependency info to - // stdout. We also need to sense the diagnostics on the -E - // runs. - // - if (cid == compiler_id::gcc) - { - // Use the .t extension (for "temporary"; .d is taken). - // - r = &(drm = auto_rmfile (t.path () + ".t")).path; - args.push_back (r->string ().c_str ()); - - sense_diag = true; - } - else - args.push_back ("-"); - - // Preprocessor output. - // - psrc = auto_rmfile (t.path () + x_pext); - args.push_back ("-o"); - args.push_back (psrc.path.string ().c_str ()); - } - else - { - args.push_back ("-M"); - args.push_back ("-MG"); // Treat missing headers as generated. - } - - gen = args_gen = (pp == nullptr); - break; - } - } - - args.push_back (src.path ().string ().c_str ()); - args.push_back (nullptr); - - // Note: only doing it here. - // - if (!env.empty ()) - env.push_back (nullptr); - } - else - { - assert (gen != args_gen); - - size_t i (args_i); - - if (gen) - { - // Overwrite. - // - args[i++] = "-M"; - args[i++] = "-MG"; - args[i++] = src.path ().string ().c_str (); - args[i] = nullptr; - - if (cid == compiler_id::gcc) - { - sense_diag = false; - } - } - else - { - // Restore. - // - args[i++] = "-MD"; - args[i++] = "-E"; - args[i++] = pp; - args[i] = "-MF"; - - if (cid == compiler_id::gcc) - { - r = &drm.path; - sense_diag = true; - } - } - - args_gen = gen; - } - - return r; - }; - - // Build the prefix map lazily only if we have non-existent files. - // Also reuse it over restarts since it doesn't change. - // - optional pfx_map; - - // If any prerequisites that we have extracted changed, then we have to - // redo the whole thing. The reason for this is auto-generated headers: - // the updated header may now include a yet-non-existent header. Unless - // we discover this and generate it (which, BTW, will trigger another - // restart since that header, in turn, can also include auto-generated - // headers), we will end up with an error during compilation proper. - // - // One complication with this restart logic is that we will see a - // "prefix" of prerequisites that we have already processed (i.e., they - // are already in our prerequisite_targets list) and we don't want to - // keep redoing this over and over again. One thing to note, however, is - // that the prefix that we have seen on the previous run must appear - // exactly the same in the subsequent run. The reason for this is that - // none of the files that it can possibly be based on have changed and - // thus it should be exactly the same. To put it another way, the - // presence or absence of a file in the dependency output can only - // depend on the previous files (assuming the compiler outputs them as - // it encounters them and it is hard to think of a reason why would - // someone do otherwise). And we have already made sure that all those - // files are up to date. And here is the way we are going to exploit - // this: we are going to keep track of how many prerequisites we have - // processed so far and on restart skip right to the next one. - // - // And one more thing: most of the time this list of headers would stay - // unchanged and extracting them by running the compiler every time is a - // bit wasteful. So we are going to cache them in the depdb. If the db - // hasn't been invalidated yet (e.g., because the compiler options have - // changed), then we start by reading from it. If anything is out of - // date then we use the same restart and skip logic to switch to the - // compiler run. - // - size_t skip_count (0); - - // Update and add a header file to the list of prerequisite targets. - // Depending on the cache flag, the file is assumed to either have come - // from the depdb cache or from the compiler run. Return whether the - // extraction process should be restarted. - // - auto add = [&trace, &pfx_map, &so_map, - act, &t, li, - &dd, &updating, &skip_count, - &bs, this] - (path f, bool cache, timestamp mt) -> bool - { - // Find or maybe insert the target. The directory is only moved - // from if insert is true. - // - auto find = [&trace, &t, this] - (dir_path&& d, path&& f, bool insert) -> const path_target* - { - // Split the file into its name part and extension. Here we can - // assume the name part is a valid filesystem name. - // - // Note that if the file has no extension, we record an empty - // extension rather than NULL (which would signify that the default - // extension should be added). - // - string e (f.extension ()); - string n (move (f).string ()); - - if (!e.empty ()) - n.resize (n.size () - e.size () - 1); // One for the dot. - - // Determine the target type. - // - const target_type* tt (nullptr); - - // See if this directory is part of any project out_root hierarchy. - // Note that this will miss all the headers that come from src_root - // (so they will be treated as generic C headers below). Generally, - // we don't have the ability to determine that some file belongs to - // src_root of some project. But that's not a problem for our - // purposes: it is only important for us to accurately determine - // target types for headers that could be auto-generated. - // - // While at it also try to determine if this target is from the src - // or out tree of said project. - // - dir_path out; - - const scope& bs (scopes.find (d)); - if (const scope* rs = bs.root_scope ()) - { - tt = map_extension (bs, n, e); - - if (bs.out_path () != bs.src_path () && d.sub (bs.src_path ())) - out = out_src (d, *rs); - } - - // If it is outside any project, or the project doesn't have such an - // extension, assume it is a plain old C header. - // - if (tt == nullptr) - { - // If the project doesn't "know" this extension then we won't - // possibly find an explicit target of this type. - // - if (!insert) - return nullptr; - - tt = &h::static_type; - } - - // Find or insert target. - // - // @@ OPT: move d, out, n - // - const target* r; - if (insert) - r = &search (t, *tt, d, out, n, &e, nullptr); - else - { - // Note that we skip any target type-specific searches (like for - // an existing file) and go straight for the target object since - // we need to find the target explicitly spelled out. - // - r = targets.find (*tt, d, out, n, e, trace); - } - - return static_cast (r); - }; - - // If it's not absolute then it either does not (yet) exist or is - // a relative ""-include (see init_args() for details). Reduce the - // second case to absolute. - // - // Note: we now always use absolute path to the translation unit so - // this no longer applies. - // -#if 0 - if (f.relative () && rels.relative ()) - { - // If the relative source path has a directory component, make sure - // it matches since ""-include will always start with that (none of - // the compilers we support try to normalize this path). Failed that - // we may end up searching for a generated header in a random - // (working) directory. - // - const string& fs (f.string ()); - const string& ss (rels.string ()); - - size_t p (path::traits::rfind_separator (ss)); - - if (p == string::npos || // No directory. - (fs.size () > p + 1 && - path::traits::compare (fs.c_str (), p, ss.c_str (), p) == 0)) - { - path t (work / f); // The rels path is relative to work. - - if (exists (t)) - f = move (t); - } - } -#endif - - const path_target* pt (nullptr); - - // If still relative then it does not exist. - // - if (f.relative ()) - { - f.normalize (); - - // This is probably as often an error as an auto-generated file, so - // trace at level 4. - // - l4 ([&]{trace << "non-existent header '" << f << "'";}); - - if (!pfx_map) - pfx_map = build_prefix_map (bs, t, act, li); - - // First try the whole file. Then just the directory. - // - // @@ Has to be a separate map since the prefix can be the same as - // the file name. - // - // auto i (pfx_map->find (f)); - - // Find the most qualified prefix of which we are a sub-path. - // - if (!pfx_map->empty ()) - { - dir_path d (f.directory ()); - auto i (pfx_map->find_sup (d)); - - if (i != pfx_map->end ()) - { - const dir_path& pd (i->second.directory); - - // If this is a prefixless mapping, then only use it if we can - // resolve it to an existing target (i.e., it is explicitly - // spelled out in a buildfile). - // - // Note that at some point we will probably have a list of - // directories. - // - pt = find (pd / d, f.leaf (), !i->first.empty ()); - if (pt != nullptr) - { - f = pd / f; - l4 ([&]{trace << "mapped as auto-generated " << f;}); - } - } - } - - if (pt == nullptr) - { - diag_record dr (fail); - dr << "header '" << f << "' not found and cannot be generated"; - //for (const auto& p: pm) - // dr << info << p.first.string () << " -> " << p.second.string (); - } - } - else - { - // We used to just normalize the path but that could result in an - // invalid path (e.g., on CentOS 7 with Clang 3.4) because of the - // symlinks. So now we realize (i.e., realpath(3)) it instead. - // Unless it comes from the depdb, in which case we've already done - // that. This is also where we handle src-out remap (again, not - // needed if cached) - // - if (!cache) - { - // While we can reasonably expect this path to exit, things do - // go south from time to time (like compiling under wine with - // file wlantypes.h included as WlanTypes.h). - // - try - { - f.realize (); - } - catch (const invalid_path&) - { - fail << "invalid header path '" << f << "'"; - } - catch (const system_error& e) - { - fail << "invalid header path '" << f << "': " << e; - } - - if (!so_map.empty ()) - { - // Find the most qualified prefix of which we are a sub-path. - // - auto i (so_map.find_sup (f)); - if (i != so_map.end ()) - { - // Ok, there is an out tree for this headers. Remap to a path - // from the out tree and see if there is a target for it. - // - dir_path d (i->second); - d /= f.leaf (i->first).directory (); - pt = find (move (d), f.leaf (), false); // d is not moved from. - - if (pt != nullptr) - { - path p (d / f.leaf ()); - l4 ([&]{trace << "remapping " << f << " to " << p;}); - f = move (p); - } - } - } - } - - if (pt == nullptr) - { - l6 ([&]{trace << "injecting " << f;}); - pt = find (f.directory (), f.leaf (), true); - } - } - - // Cache the path. - // - const path& pp (pt->path (move (f))); - - // Match to a rule. - // - // If we are reading the cache, then it is possible the file has since - // been removed (think of a header in /usr/local/include that has been - // uninstalled and now we need to use one from /usr/include). This - // will lead to the match failure which we translate to a restart. - // - if (!cache) - build2::match (act, *pt); - else if (!build2::try_match (act, *pt).first) - { - dd.write (); // Invalidate this line. - updating = true; - return true; - } - - // Update. - // - bool restart (update (trace, act, *pt, mt)); - - // Verify/add it to the dependency database. We do it after update in - // order not to add bogus files (non-existent and without a way to - // update). - // - if (!cache) - dd.expect (pp); - - // Add to our prerequisite target list. - // - t.prerequisite_targets.push_back (pt); - skip_count++; - - updating = updating || restart; - return restart; - }; - - // If nothing so far has invalidated the dependency database, then try - // the cached data before running the compiler. - // - bool cache (!updating); - - // See init_args() above for details on generated header support. - // - bool gen (false); - optional force_gen; - optional force_gen_skip; // Skip count at last force_gen run. - - const path* drmp (nullptr); // Points to drm.path () if active. - - for (bool restart (true); restart; cache = false) - { - restart = false; - - if (cache) - { - // If any, this is always the first run. - // - assert (skip_count == 0); - - // We should always end with a blank line. - // - for (;;) - { - string* l (dd.read ()); - - // If the line is invalid, run the compiler. - // - if (l == nullptr) - { - restart = true; - break; - } - - if (l->empty ()) // Done, nothing changed. - { - // If modules are enabled, then we keep the preprocessed output - // around (see apply() for details). - // - return modules - ? make_pair (auto_rmfile (t.path () + x_pext, false), true) - : make_pair (auto_rmfile (), false); - } - - // If this header came from the depdb, make sure it is no older - // than the target (if it has changed since the target was - // updated, then the cached data is stale). - // - restart = add (path (move (*l)), true, mt); - - if (restart) - { - l6 ([&]{trace << "restarting (cache)";}); - break; - } - } - } - else - { - try - { - if (force_gen) - gen = *force_gen; - - if (args.empty () || gen != args_gen) - drmp = init_args (gen); - - if (verb >= 3) - print_process (args.data ()); // Disable pipe mode. - - process pr; - - try - { - // Assume the preprocessed output (if produced) is usable - // until proven otherwise. - // - puse = true; - - // Save the timestamp just before we start preprocessing. If - // we depend on any header that has been updated since, then - // we should assume we've "seen" the old copy and re-process. - // - timestamp pmt (system_clock::now ()); - - // If we have no generated header support, then suppress all - // diagnostics (if things go badly we will restart with this - // support). - // - if (drmp == nullptr) - { - // Dependency info goes to stdout. - // - assert (!sense_diag); - - // For VC with /P the dependency info and diagnostics all go - // to stderr so redirect it to stdout. - // - pr = process ( - cpath, - args.data (), - 0, - -1, - cclass == compiler_class::msvc ? 1 : gen ? 2 : -2, - nullptr, // CWD - env.empty () ? nullptr : env.data ()); - } - else - { - // Dependency info goes to a temporary file. - // - pr = process (cpath, - args.data (), - 0, - 2, // Send stdout to stderr. - gen ? 2 : sense_diag ? -1 : -2, - nullptr, // CWD - env.empty () ? nullptr : env.data ()); - - // If requested, monitor for diagnostics and if detected, mark - // the preprocessed output as unusable for compilation. - // - if (sense_diag) - { - ifdstream is (move (pr.in_efd), fdstream_mode::skip); - puse = puse && (is.peek () == ifdstream::traits_type::eof ()); - is.close (); - } - - // The idea is to reduce it to the stdout case. - // - pr.wait (); - pr.in_ofd = fdopen (*drmp, fdopen_mode::in); - } - - // We may not read all the output (e.g., due to a restart). - // Before we used to just close the file descriptor to signal to - // the other end that we are not interested in the rest. This - // works fine with GCC but Clang (3.7.0) finds this impolite and - // complains, loudly (broken pipe). So now we are going to skip - // until the end. - // - ifdstream is (move (pr.in_ofd), - fdstream_mode::text | fdstream_mode::skip, - ifdstream::badbit); - - // In some cases we may need to ignore the error return status. - // The good_error flag keeps track of that. Similarly we - // sometimes expect the error return status based on the output - // we see. The bad_error flag is for that. - // - bool good_error (false), bad_error (false); - - size_t skip (skip_count); - string l; // Reuse. - for (bool first (true), second (false); !restart; ) - { - if (eof (getline (is, l))) - break; - - l6 ([&]{trace << "header dependency line '" << l << "'";}); - - // Parse different dependency output formats. - // - switch (cclass) - { - case compiler_class::msvc: - { - if (first) - { - // The first line should be the file we are compiling. - // If it is not, then something went wrong even before - // we could compile anything (e.g., file does not - // exist). In this case the first line (and everything - // after it) is presumably diagnostics. - // - if (l != src.path ().leaf ().string ()) - { - text << l; - bad_error = true; - break; - } - - first = false; - continue; - } - - string f (next_show (l, good_error)); - - if (f.empty ()) // Some other diagnostics. - { - text << l; - bad_error = true; - break; - } - - // Skip until where we left off. - // - if (skip != 0) - { - // We can't be skipping over a non-existent header. - // - assert (!good_error); - skip--; - } - else - { - restart = add (path (move (f)), false, pmt); - - // If the header does not exist (good_error), then - // restart must be true. Except that it is possible that - // someone running in parallel has already updated it. - // In this case we must force a restart since we haven't - // yet seen what's after this at-that-time-non-existent - // header. - // - // We also need to force the target update (normally - // done by add()). - // - if (good_error) - restart = updating = true; - // - // And if we have updated the header (restart is true), - // then we may end up in this situation: an old header - // got included which caused the preprocessor to fail - // down the line. So if we are restarting, set the good - // error flag in case the process fails because of - // something like this (and if it is for a valid reason, - // then we will pick it up on the next round). - // - else if (restart) - good_error = true; - - if (restart) - l6 ([&]{trace << "restarting";}); - } - - break; - } - case compiler_class::gcc: - { - // Make dependency declaration. - // - size_t pos (0); - - if (first) - { - // Empty/invalid output should mean the wait() call - // below will return false. - // - if (l.empty () || - l[0] != '^' || l[1] != ':' || l[2] != ' ') - { - if (!l.empty ()) - text << l; - - bad_error = true; - break; - } - - first = false; - second = true; - - // While normally we would have the source file on the - // first line, if too long, it will be moved to the next - // line and all we will have on this line is "^: \". - // - if (l.size () == 4 && l[3] == '\\') - continue; - else - pos = 3; // Skip "^: ". - - // Fall through to the 'second' block. - } - - if (second) - { - second = false; - next_make (l, pos); // Skip the source file. - } - - while (pos != l.size ()) - { - string f (next_make (l, pos)); - - // Skip until where we left off. - // - if (skip != 0) - { - skip--; - continue; - } - - restart = add (path (move (f)), false, pmt); - - if (restart) - { - // The same "preprocessor may fail down the line" - // logic as above. - // - good_error = true; - - l6 ([&]{trace << "restarting";}); - break; - } - } - - break; - } - } - - if (bad_error) - break; - } - - // In case of VC, we are parsing stderr and if things go south, - // we need to copy the diagnostics for the user to see. - // - if (bad_error && cclass == compiler_class::msvc) - { - // We used to just dump the whole rdbuf but it turns out VC - // may continue writing include notes interleaved with the - // diagnostics. So we have to filter them out. - // - for (; !eof (getline (is, l)); ) - { - size_t p (next_show_sense (l)); - if (p != string::npos && l.compare (p, 4, "1083") != 0) - diag_stream_lock () << l << endl; - } - } - - is.close (); - - // This is tricky: it is possible that in parallel someone has - // generated all our missing headers and we wouldn't restart - // normally. - // - // In this case we also need to force the target update - // (normally done by add()). - // - if (force_gen && *force_gen) - { - restart = updating = true; - force_gen = false; - } - - if (pr.wait ()) - { - if (!bad_error) - continue; - - fail << "expected error exist status from " << x_lang - << " compiler"; - } - else if (pr.exit->normal ()) - { - if (good_error) // Ignore expected errors (restart). - continue; - } - - // Fall through. - } - catch (const io_error&) - { - if (pr.wait ()) - fail << "unable to read " << x_lang << " compiler header " - << "dependency output"; - - // Fall through. - } - - assert (pr.exit && !*pr.exit); - const process_exit& e (*pr.exit); - - // For normal exit we assume the child process issued some - // diagnostics. - // - if (e.normal ()) - { - // If this run was with the generated header support then we - // have issued diagnostics and it's time to give up. - // - if (gen) - throw failed (); - - // Just to recap, being here means something is wrong with the - // source: it can be a missing generated header, it can be an - // outdated generated header (e.g., some check triggered #error - // which will go away if only we updated the generated header), - // or it can be a real error that is not going away. - // - // So this is what we are going to do here: if anything got - // updated on this run (i.e., the compiler has produced valid - // dependency information even though there were errors and we - // managed to find and update a header based on this - // informaion), then we restart in the same mode hoping that - // this fixes things. Otherwise, we force the generated header - // support which will either uncover a missing generated header - // or will issue diagnostics. - // - if (restart) - l6 ([&]{trace << "trying again without generated headers";}); - else - { - // In some pathological situations (e.g., we are out of disk - // space) we may end up switching back and forth indefinitely - // without making any headway. So we use skip_count to track - // our progress. - // - if (force_gen_skip && *force_gen_skip == skip_count) - { - diag_record dr (fail); - - dr << "inconsistent " << x_lang << " compiler behavior"; - - // Show the yo-yo'ing command lines. - // - dr << info; - print_process (dr, args.data ()); // No pipes. - - init_args ((gen = true)); - dr << info << ""; - print_process (dr, args.data ()); // No pipes. - - dr << info << "perhaps you are running out of disk space?"; - } - - restart = true; - force_gen = true; - force_gen_skip = skip_count; - l6 ([&]{trace << "restarting with forced generated headers";}); - } - continue; - } - else - run_finish (args, pr); // Throws. - } - catch (const process_error& e) - { - error << "unable to execute " << args[0] << ": " << e; - - // In a multi-threaded program that fork()'ed but did not exec(), - // it is unwise to try to do any kind of cleanup (like unwinding - // the stack and running destructors). - // - if (e.child) - { - drm.cancel (); - exit (1); - } - - throw failed (); - } - } - } - - // Add the terminating blank line (we are updated depdb). - // - dd.expect (""); - - puse = puse && !psrc.path.empty (); - return make_pair (move (psrc), puse); - } - - pair compile:: - parse_unit (action act, - file& t, - linfo lo, - const file& src, - auto_rmfile& psrc, - const match_data& md) const - { - tracer trace (x, "compile::parse_unit"); - - // If things go wrong give the user a bit extra context. - // - auto df = make_diag_frame ( - [&src](const diag_record& dr) - { - if (verb != 0) - dr << info << "while parsing " << src; - }); - - // For some compilers (GCC, Clang) the preporcessed output is only - // partially preprocessed. For others (VC), it is already fully - // preprocessed (well, almost: it still has comments but we can handle - // that). Plus, the source file might already be (sufficiently) - // preprocessed. - // - // So the plan is to start the compiler process that writes the fully - // preprocessed output to stdout and reduce the already preprocessed - // case to it. - // - environment env; - cstrings args; - const path* sp; // Source path. - - bool ps; // True if extracting from psrc. - if (md.pp < preprocessed::modules) - { - ps = !psrc.path.empty (); - sp = &(ps ? psrc.path : src.path ()); - - // VC's preprocessed output, if present, is fully preprocessed. - // - if (cclass != compiler_class::msvc || !ps) - { - // This should match with how we setup preprocessing and is pretty - // similar to init_args() from extract_headers(). - // - args.push_back (cpath.recall_string ()); - - append_lib_options (t.base_scope (), args, t, act, lo); - - append_options (args, t, c_poptions); - append_options (args, t, x_poptions); - - assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); - append_option_values ( - args, "-I", - sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), - [] (const dir_path& d) {return d.string ().c_str ();}); - - if (md.symexport) - append_symexport_options (args, t); - - // Make sure we don't fail because of warnings. - // - // @@ Can be both -WX and /WX. - // - const char* werror (nullptr); - switch (cclass) - { - case compiler_class::gcc: werror = "-Werror"; break; - case compiler_class::msvc: werror = "/WX"; break; - } - - bool clang (cid == compiler_id::clang || - cid == compiler_id::clang_apple); - - append_options (args, t, c_coptions, werror); - append_options (args, t, x_coptions, werror); - append_options (args, tstd, - tstd.size () - (modules && clang ? 1 : 0)); - - switch (cclass) - { - case compiler_class::msvc: - { - args.push_back ("/nologo"); - - if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) - args.push_back ("/EHsc"); - - if (!find_option_prefixes ({"/MD", "/MT"}, args)) - args.push_back ("/MD"); - - args.push_back ("/E"); - args.push_back ("/C"); - args.push_back (langopt (md)); // Compile as. - - break; - } - case compiler_class::gcc: - { - if (t.is_a ()) - { - if (tclass == "linux" || tclass == "bsd") - args.push_back ("-fPIC"); - } - - // Options that trigger preprocessing of partially preprocessed - // output are a bit of a compiler-specific voodoo. - // - args.push_back ("-E"); - - if (ps) - { - args.push_back ("-x"); - args.push_back (langopt (md)); - - if (cid == compiler_id::gcc) - { - args.push_back ("-fpreprocessed"); - args.push_back ("-fdirectives-only"); - } - } - - break; - } - } - - args.push_back (sp->string ().c_str ()); - args.push_back (nullptr); - } - - if (!env.empty ()) - env.push_back (nullptr); - } - else - { - // Extracting directly from source. - // - ps = false; - sp = &src.path (); - } - - // Preprocess and parse. - // - for (;;) // Breakout loop. - try - { - // Disarm the removal of the preprocessed file in case of an error. - // We re-arm it below. - // - if (ps) - psrc.active = false; - - process pr; - - try - { - if (args.empty ()) - { - pr = process (process_exit (0)); // Successfully exited. - pr.in_ofd = fdopen (*sp, fdopen_mode::in); - } - else - { - if (verb >= 3) - print_process (args); - - // We don't want to see warnings multiple times so ignore all - // diagnostics. - // - pr = process (cpath, - args.data (), - 0, -1, -2, - nullptr, // CWD - env.empty () ? nullptr : env.data ()); - } - - // Use binary mode to obtain consistent positions. - // - ifdstream is (move (pr.in_ofd), - fdstream_mode::binary | fdstream_mode::skip); - - parser p; - translation_unit tu (p.parse (is, *sp)); - - is.close (); - - if (pr.wait ()) - { - if (ps) - psrc.active = true; // Re-arm. - - // Prior to 15u5 VC was not using the 'export module' syntax so we - // use the preprequisite type to distinguish between interface and - // implementation units. - // - if (cid == compiler_id::msvc && - cmaj == 19 && cmin <= 11 && - x_mod != nullptr && src.is_a (*x_mod)) - { - // It's quite painful to guard the export with an #if/#endif so - // if it is present, "fixup" the (temporary) preprocessed output - // by getting rid of the keyword. - // - // Note: when removing this also remember to remove the test. - // - if (tu.mod.iface) - { - // We can only fixup a temporary file. - // - if (!ps) - fail (relative (src)) << "fixup requires preprocessor"; - - // Stomp out the export keyword with spaces. We are using - // std::fstream since our fdstream does not yet support - // seeking. - // - fstream os (psrc.path.string (), fstream::out | fstream::in); - auto pos (static_cast (p.export_pos)); - - if (!os.is_open () || - !os.seekp (pos) || - !os.write (" ", 6)) - fail << "unable to overwrite preprocessor output"; - } - else - tu.mod.iface = true; - } - - return pair (move (tu), p.checksum); - } - - // Fall through. - } - catch (const io_error&) - { - if (pr.wait ()) - fail << "unable to read " << x_lang << " preprocessor output"; - - // Fall through. - } - - assert (pr.exit && !*pr.exit); - const process_exit& e (*pr.exit); - - // What should we do with a normal error exit? Remember we suppressed - // the compiler's diagnostics. We used to issue a warning and continue - // with the assumption that the compilation step will fail with - // diagnostics. The problem with this approach is that we may fail - // before that because the information we return (e.g., module name) - // is bogus. So looks like failing is the only option. - // - if (e.normal ()) - { - fail << "unable to preprocess " << src << - info << "re-run with -s -V to display failing command" << - info << "then run failing command to display compiler diagnostics"; - } - else - run_finish (args, pr); // Throws. - } - catch (const process_error& e) - { - error << "unable to execute " << args[0] << ": " << e; - - if (e.child) - exit (1); - } - - throw failed (); - } - - // Extract and inject module dependencies. - // - void compile:: - extract_modules (action act, - const scope& bs, - file& t, - linfo li, - const compile_target_types& tt, - const file& src, - match_data& md, - module_info&& mi, - depdb& dd, - bool& updating) const - { - tracer trace (x, "compile::extract_modules"); - l5 ([&]{trace << "target: " << t;}); - - // If things go wrong, give the user a bit extra context. - // - auto df = make_diag_frame ( - [&src](const diag_record& dr) - { - if (verb != 0) - dr << info << "while extracting module dependencies from " << src; - }); - - if (!modules) - { - if (!mi.name.empty () || !mi.imports.empty ()) - fail (relative (src)) << "modules support not enabled/available"; - - return; - } - - // Sanity checks. - // - // If we are compiling a module interface unit, make sure it has the - // necessary declarations. - // - if (src.is_a (*x_mod) && (mi.name.empty () || !mi.iface)) - fail << src << " is not a module interface unit"; - - // Search and match all the modules we depend on. If this is a module - // implementation unit, then treat the module itself as if it was - // imported (we insert it first since for some compilers we have to - // differentiate between this special module and real imports). Note: - // move. - // - if (!mi.iface && !mi.name.empty ()) - mi.imports.insert (mi.imports.begin (), - module_import {move (mi.name), false, 0}); - - // The change to the set of imports would have required a change to - // source code (or options). Changes to the bmi{}s themselves will be - // detected via the normal prerequisite machinery. However, the same set - // of imports could be resolved to a different set of bmi{}s (in a sense - // similar to changing the source file). To detect this we calculate and - // store a hash of all (not just direct) bmi{}'s paths. - // - sha256 cs; - - if (!mi.imports.empty ()) - md.mods = search_modules (act, bs, t, li, tt.bmi, src, mi.imports, cs); - - if (dd.expect (cs.string ()) != nullptr) - updating = true; - -#if 0 - // Save the module map for compilers that use it. - // - if (md.mods.start != 0) - { - switch (cid) - { - case compiler_id::gcc: - case compiler_id::clang: - case compiler_id::clang_apple: - { - // We don't need to redo this if the above hash hasn't changed and - // the database is valid. - // - if (dd.writing () || !dd.skip ()) - { - const auto& pts (t.prerequisite_targets); - - for (size_t i (md.mods.start); i != pts.size (); ++i) - { - if (const target* m = pts[i]) - { - // Save a variable lookup by getting the module name from - // the import list (see search_modules()). - // - dd.write ('@', false); - dd.write (mi.imports[i - md.mods.start].name, false); - dd.write ('=', false); - dd.write (m->as ().path ()); - } - } - } - break; - } - default: - break; - } - } -#endif - - // Set the cc.module_name variable if this is an interface unit. Note - // that it may seem like a good idea to set it on the bmi{} group to - // avoid duplication. We, however, cannot do it MT-safely since we don't - // match the group. - // - if (mi.iface) - { - if (value& v = t.vars.assign (c_module_name)) - assert (cast (v) == mi.name); - else - v = move (mi.name); // Note: move. - } - } - - inline bool - std_module (const string& m) - { - size_t n (m.size ()); - return (n >= 3 && - m[0] == 's' && m[1] == 't' && m[2] == 'd' && - (n == 3 || m[3] == '.')); - }; - - // Resolve imported modules to bmi*{} targets. - // - module_positions compile:: - search_modules (action act, - const scope& bs, - file& t, - linfo li, - const target_type& mtt, - const file& src, - module_imports& imports, - sha256& cs) const - { - tracer trace (x, "compile::search_modules"); - - // So we have a list of imports and a list of "potential" module - // prerequisites. They are potential in the sense that they may or may - // not be required by this translation unit. In other words, they are - // the pool where we can resolve actual imports. - // - // Because we may not need all of these prerequisites, we cannot just go - // ahead and match all of them (and they can even have cycles; see rule - // synthesis). This poses a bit of a problem: the only way to discover - // the module's actual name (see cc.module_name) is by matching it. - // - // One way to solve this would be to make the user specify the module - // name for each mxx{} explicitly. This will be a major pain, however. - // Another would be to require encoding of the module name in the - // interface unit file name. For example, hello.core -> hello-core.mxx. - // This is better but still too restrictive: some will want to call it - // hello_core.mxx or HelloCore.mxx (because that's their file naming - // convention) or place it in a subdirectory, say, hello/core.mxx. - // - // In the above examples one common theme about all the file names is - // that they contain, in one form or another, the "tail" of the module - // name ('core'). So what we are going to do is require that the - // interface file names contain enough of the module name tail to - // unambiguously resolve all the module imports. On our side we are - // going to implement a "fuzzy" module name to file name match. This - // should be reliable enough since we will always verify our guesses - // once we match the target and extract the actual module name. Plus, - // the user will always have the option of resolving any impasses by - // specifying the module name explicitly. - // - // So, the fuzzy match: the idea is that each match gets a score, the - // number of characters in the module name that got matched. A match - // with the highest score is used. And we use the (length + 1) for a - // match against an actual module name. - // - // For std.* modules we only accept non-fuzzy matches (think std.core vs - // some core.mxx). And if such a module is unresolved, then we assume it - // is pre-built and will be found by some other means (e.g., VC's - // IFCPATH). - // - auto match = [] (const string& f, const string& m) -> size_t - { - size_t fi (f.size ()); - size_t mi (m.size ()); - - // Scan backwards for as long as we match. Keep track of the previous - // character for case change detection. - // - for (char fc, mc, fp ('\0'), mp ('\0'); - fi != 0 && mi != 0; - fp = fc, mp = mc, --fi, --mi) - { - fc = f[fi - 1]; - mc = m[mi - 1]; - - if (casecmp (fc, mc) == 0) - continue; - - // We consider all separators equal and character case change being - // a separators. Some examples of the latter: - // - // foo.bar - // fooBAR - // FOObar - // - bool fs (fc == '_' || fc == '-' || fc == '.' || - path::traits::is_separator (fc)); - bool ms (mc == '_' || mc == '.'); - - if (fs && ms) - continue; - - // Only if one is a real separator do we consider case change. - // - if (fs || ms) - { - auto cc = [] (char c1, char c2) -> bool - { - return (alpha (c1) && - alpha (c2) && - (ucase (c1) == c1) != (ucase (c2) == c2)); - }; - - bool fa (false), ma (false); - if ((fs || (fa = cc (fp, fc))) && (ms || (ma = cc (mp, mc)))) - { - // Stay on this character if imaginary punctuation (note: cannot - // be both true). - // - if (fa) ++fi; - if (ma) ++mi; - continue; - } - } - - break; // No match. - } - - // Return the number of characters matched in the module name and not - // in the file (this may not be the same because of the imaginary - // separators). - // - return m.size () - mi; - }; - - auto& pts (t.prerequisite_targets); - size_t start (pts.size ()); // Index of the first to be added. - - // We have two parallel vectors: module names/scores in imports and - // targets in prerequisite_targets (offset with start). Pre-allocate - // NULL entries in the latter. - // - size_t n (imports.size ()); - pts.resize (start + n, nullptr); - - // Oh, yes, there is one "minor" complication. It's the last one, I - // promise. It has to do with module re-exporting (export import M;). - // In this case (currently) all implementations simply treat it as a - // shallow (from the BMI's point of view) reference to the module (or an - // implicit import, if you will). Do you see where it's going? Nowever - // good, that's right. This shallow reference means that the compiler - // should be able to find BMIs for all the re-exported modules, - // recursive. The good news is we are actually in a pretty good shape to - // handle this: after match all our prerequisite BMIs will have their - // prerequisite BMIs known, recursively. The only bit that is missing is - // the re-export flag of some sorts. As well as deciding where to handle - // it: here or in append_modules(). After some meditation it became - // clear handling it here will be simpler: We need to weed out - // duplicates for which we can re-use the imports vector. And we may - // also need to save this "flattened" list of modules in depdb. - // - // Ok, so, here is the plan: - // - // 1. There is no good place in prerequisite_targets to store the - // exported flag (no, using the marking facility across match/execute - // is a bad idea). So what we are going to do is put re-exported - // bmi{}s at the back and store (in the target's data pad) the start - // position. One bad aspect about this part is that we assume those - // bmi{}s have been matched by the same rule. But let's not kid - // ourselves, there will be no other rule that matches bmi{}s. - // - // 2. Once we have matched all the bmi{}s we are importing directly - // (with all the re-exported by us at the back), we will go over them - // and copy all of their re-exported bmi{}s (using the position we - // saved on step #1). The end result will be a recursively-explored - // list of imported bmi{}s that append_modules() can simply convert - // to the list of options. - // - // One issue with this approach is that these copied targets will be - // executed which means we need to adjust their dependent counts - // (which is normally done by match). While this seems conceptually - // correct (especially if you view re-exports as implicit imports), - // it's just extra overhead (we know they will be updated). So what - // we are going to do is save another position, that of the start of - // these copied-over targets, and will only execute up to this point. - // - // And after implementing this came the reality check: all the current - // implementations require access to all the imported BMIs, not only - // re-exported. Some (like Clang) store references to imported BMI files - // so we actually don't need to pass any extra options (unless things - // get moved) but they still need access to the BMIs (and things will - // most likely have to be done differenly for distributed compilation). - // - // So the revised plan: on the off chance that some implementation will - // do it differently we will continue maintaing the imported/re-exported - // split and how much to copy-over can be made compiler specific. - // - // As a first sub-step of step #1, move all the re-exported imports to - // the end of the vector. This will make sure they end up at the end - // of prerequisite_targets. Note: the special first import, if any, - // should be unaffected. - // - sort (imports.begin (), imports.end (), - [] (const module_import& x, const module_import& y) - { - return !x.exported && y.exported; - }); - - // Go over the prerequisites once. - // - // For (direct) library prerequisites, check their prerequisite bmi{}s - // (which should be searched and matched with module names discovered; - // see the library meta-information protocol for details). - // - // For our own bmi{} prerequisites, checking if each (better) matches - // any of the imports. - - // For fuzzy check if a file name (better) resolves any of our imports - // and if so make it the new selection. For exact the name is the actual - // module name and it can only resolve one import (there are no - // duplicates). - // - // Set done to true if all the imports have now been resolved to actual - // module names (which means we can stop searching). This will happens - // if all the modules come from libraries. Which will be fairly common - // (think of all the tests) so it's worth optimizing for. - // - bool done (false); - - auto check_fuzzy = [&trace, &imports, &pts, &match, start, n] - (const target* pt, const string& name) - { - for (size_t i (0); i != n; ++i) - { - module_import& m (imports[i]); - - if (std_module (m.name)) // No fuzzy std.* matches. - continue; - - size_t n (m.name.size ()); - - if (m.score > n) // Resolved to module name. - continue; - - size_t s (match (name, m.name)); - - l5 ([&]{trace << name << " ~ " << m.name << ": " << s;}); - - if (s > m.score) - { - pts[start + i] = pt; - m.score = s; - } - } - }; - - // If resolved, return the "slot" in pts (we don't want to create a - // side build until we know we match; see below for details). - // - auto check_exact = [&trace, &imports, &pts, start, n, &done] - (const string& name) -> const target** - { - const target** r (nullptr); - done = true; - - for (size_t i (0); i != n; ++i) - { - module_import& m (imports[i]); - - size_t n (m.name.size ()); - - if (m.score > n) // Resolved to module name (no effect on done). - continue; - - if (r == nullptr) - { - size_t s (name == m.name ? n + 1 : 0); - - l5 ([&]{trace << name << " ~ " << m.name << ": " << s;}); - - if (s > m.score) - { - r = &pts[start + i].target; - m.score = s; - continue; // Scan the rest to detect if all done. - } - } - - done = false; - } - - return r; - }; - - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - const target* pt (p.load ()); // Should be cached for libraries. - - if (pt != nullptr) - { - const target* lt (nullptr); - - if (const libx* l = pt->is_a ()) - lt = &link_member (*l, act, li); - else if (pt->is_a () || pt->is_a () || pt->is_a ()) - lt = pt; - - // If this is a library, check its bmi{}s and mxx{}s. - // - if (lt != nullptr) - { - for (const target* bt: lt->prerequisite_targets) - { - if (bt == nullptr) - continue; - - // Note that here we (try) to use whatever flavor of bmi*{} is - // available. - // - // @@ MOD: BMI compatibility check. - // @@ UTL: we need to (recursively) see through libux{} (and - // also in pkgconfig_save()). - // - if (bt->is_a () || - bt->is_a () || - bt->is_a ()) - { - const string& n (cast (bt->vars[c_module_name])); - - if (const target** p = check_exact (n)) - *p = bt; - } - else if (bt->is_a (*x_mod)) - { - // This is an installed library with a list of module sources - // (the source are specified as prerequisites but the fallback - // file rule puts them into prerequisite_targets for us). - // - // The module names should be specified but if not assume - // something else is going on and ignore. - // - const string* n (cast_null (bt->vars[c_module_name])); - if (n == nullptr) - continue; - - if (const target** p = check_exact (*n)) - *p = &make_module_sidebuild (act, bs, *lt, *bt, *n); - } - else - continue; - - if (done) - break; - } - - if (done) - break; - - continue; - } - - // Fall through. - } - - // While it would have been even better not to search for a target, we - // need to get hold of the corresponding mxx{} (unlikely but possible - // for bmi{} to have a different name). - // - if (p.is_a ()) - pt = &search (t, mtt, p.key ()); // Same logic as in picking obj*{}. - else if (p.is_a (mtt)) - { - if (pt == nullptr) - pt = &p.search (t); - } - else - continue; - - // Find the mxx{} prerequisite and extract its "file name" for the - // fuzzy match unless the user specified the module name explicitly. - // - for (prerequisite_member p: group_prerequisite_members (act, *pt)) - { - if (p.is_a (*x_mod)) - { - // Check for an explicit module name. Only look for an existing - // target (which means the name can only be specified on the - // target itself, no target type/pattern-spec). - // - const target* t (p.search_existing ()); - const string* n (t != nullptr - ? cast_null (t->vars[c_module_name]) - : nullptr); - if (n != nullptr) - { - if (const target** p = check_exact (*n)) - *p = pt; - } - else - { - // Fuzzy match. - // - string f; - - // Add the directory part if it is relative. The idea is to - // include it into the module match, say hello.core vs - // hello/mxx{core}. - // - // @@ MOD: Why not for absolute? Good question. What if it - // contains special components, say, ../mxx{core}? - // - const dir_path& d (p.dir ()); - - if (!d.empty () && d.relative ()) - f = d.representation (); // Includes trailing slash. - - f += p.name (); - check_fuzzy (pt, f); - } - break; - } - } - - if (done) - break; - } - - // Diagnose unresolved modules. - // - if (!done) - { - for (size_t i (0); i != n; ++i) - { - if (pts[start + i] == nullptr && !std_module (imports[i].name)) - { - // It would have been nice to print the location of the import - // declaration. And we could save it during parsing at the expense - // of a few paths (that can be pooled). The question is what to do - // when we re-create this information from depdb? We could have - // saved the location information there but the relative paths - // (e.g., from the #line directives) could end up being wrong if - // the we re-run from a different working directory. - // - // It seems the only workable approach is to extract full location - // info during parse, not save it in depdb, when re-creating, - // fallback to just src path without any line/column information. - // This will probably cover the majority of case (most of the time - // it will be a misspelled module name, not a removal of module - // from buildfile). - // - // But at this stage this doesn't seem worth the trouble. - // - fail (relative (src)) << "unable to resolve module " - << imports[i].name; - } - } - } - - // Match in parallel and wait for completion. - // - match_members (act, t, pts, start); - - // Post-process the list of our (direct) imports. While at it, calculate - // the checksum of all (direct and indirect) bmi{} paths. - // - size_t exported (n); - size_t copied (pts.size ()); - - for (size_t i (0); i != n; ++i) - { - const module_import& m (imports[i]); - - // Determine the position of the first re-exported bmi{}. - // - if (m.exported && exported == n) - exported = i; - - const target* bt (pts[start + i]); - - if (bt == nullptr) - continue; // Unresolved (std.*). - - // Verify our guesses against extracted module names but don't waste - // time if it was a match against the actual module name. - // - const string& in (m.name); - - if (m.score <= in.size ()) - { - const string& mn (cast (bt->vars[c_module_name])); - - if (in != mn) - { - for (prerequisite_member p: group_prerequisite_members (act, *bt)) - { - if (p.is_a (*x_mod)) // Got to be there. - { - fail (relative (src)) - << "failed to correctly guess module name from " << p << - info << "guessed: " << in << - info << "actual: " << mn << - info << "consider adjusting module interface file names or" << - info << "consider specifying module name with " << x - << ".module_name"; - } - } - } - } - - // Hash (we know it's a file). - // - cs.append (static_cast (*bt).path ().string ()); - - // Copy over bmi{}s from our prerequisites weeding out duplicates. - // - if (size_t j = bt->data ().mods.start) - { - // Hard to say whether we should reserve or not. We will probably - // get quite a bit of duplications. - // - for (size_t m (bt->prerequisite_targets.size ()); j != m; ++j) - { - const target* et (bt->prerequisite_targets[j]); - - if (et == nullptr) - continue; // Unresolved (std.*). - - const string& mn (cast (et->vars[c_module_name])); - - if (find_if (imports.begin (), imports.end (), - [&mn] (const module_import& i) - { - return i.name == mn; - }) == imports.end ()) - { - pts.push_back (et); - cs.append (static_cast (*et).path ().string ()); - - // Add to the list of imports for further duplicate suppression. - // We could have probably stored reference to the name (e.g., in - // score) but it's probably not worth it if we have a small - // string optimization. - // - imports.push_back (module_import {mn, true, 0}); - } - } - } - } - - if (copied == pts.size ()) // No copied tail. - copied = 0; - - if (exported == n) // No (own) re-exported imports. - exported = copied; - else - exported += start; // Rebase. - - return module_positions {start, exported, copied}; - } - - // Synthesize a dependency for building a module binary interface on - // the side. - // - const target& compile:: - make_module_sidebuild (action act, - const scope& bs, - const target& lt, - const target& mt, - const string& mn) const - { - tracer trace (x, "compile::make_module_sidebuild"); - - // First figure out where we are going to build. We want to avoid - // multiple sidebuilds so the outermost scope that has loaded the - // cc.config module and that is within our amalgmantion seems like a - // good place. - // - const scope& rs (*bs.root_scope ()); - const scope* as (&rs); - { - const scope* ws (as->weak_scope ()); - if (as != ws) - { - const scope* s (as); - do - { - s = s->parent_scope ()->root_scope (); - - // Use cc.core.vars as a proxy for {c,cxx}.config (a bit smelly). - // - // This is also the module that registers the scope operation - // callback that cleans up the subproject. - // - if (cast_false ((*s)["cc.core.vars.loaded"])) - as = s; - - } while (s != ws); - } - } - - // We build modules in a subproject (since there might be no full - // language support module loaded in the amalgamation, only *.config). - // So the first step is to check if the project has already been created - // and/or loaded and if not, then to go ahead and do so. - // - dir_path pd (as->out_path () / modules_sidebuild_dir /= x); - { - const scope* ps (&scopes.find (pd)); - - if (ps->out_path () != pd) - { - // Switch the phase to load then create and load the subproject. - // - phase_switch phs (run_phase::load); - - // Re-test again now that we are in exclusive phase (another thread - // could have already created and loaded the subproject). - // - ps = &scopes.find (pd); - - if (ps->out_path () != pd) - { - // The project might already be created in which case we just need - // to load it. - // - if (!is_src_root (pd)) - { - // Copy our standard and force modules. - // - string extra; - - if (const string* std = cast_null (rs[x_std])) - extra += string (x) + ".std = " + *std + '\n'; - - extra += string (x) + ".features.modules = true"; - - config::create_project ( - pd, - as->out_path ().relative (pd), /* amalgamation */ - {}, /* boot_modules */ - extra, /* root_pre */ - {string (x) + '.'}, /* root_modules */ - "", /* root_post */ - false, /* config */ - false, /* buildfile */ - "the cc module", - 2); /* verbosity */ - } - - ps = &load_project (as->rw () /* lock */, pd, pd); - } - } - - // Some sanity checks. - // -#ifndef NDEBUG - assert (ps->root ()); - const module* m (ps->modules.lookup (x)); - assert (m != nullptr && m->modules); -#endif - } - - // Next we need to come up with a file/target name that will be unique - // enough not to conflict with other modules. If we assume that within - // an amalgamation there is only one "version" of each module, then the - // module name itself seems like a good fit. We just replace '.' with - // '-'. - // - string mf; - transform (mn.begin (), mn.end (), - back_inserter (mf), - [] (char c) {return c == '.' ? '-' : c;}); - - // It seems natural to build a BMI type that corresponds to the library - // type. After all, this is where the object file part of the BMI is - // going to come from (though things will probably be different for - // module-only libraries). - // - const target_type* tt (nullptr); - switch (link_type (lt).type) - { - case otype::a: tt = &bmia::static_type; break; - case otype::s: tt = &bmis::static_type; break; - case otype::e: assert (false); - } - - // Store the BMI target in the subproject root. If the target already - // exists then we assume all this is already done (otherwise why would - // someone have created such a target). - // - if (const target* bt = targets.find ( - *tt, - pd, - dir_path (), // Always in the out tree. - mf, - nullopt, // Use default extension. - trace)) - return *bt; - - prerequisites ps; - ps.push_back (prerequisite (mt)); - - // We've added the mxx{} but it may import other modules from this - // library. Or from (direct) dependencies of this library. We add them - // all as prerequisites so that the standard module search logic can - // sort things out. This is pretty similar to what we do in link when - // synthesizing dependencies for bmi{}'s. - // - ps.push_back (prerequisite (lt)); - for (prerequisite_member p: group_prerequisite_members (act, lt)) - { - // @@ TODO: will probably need revision if using sidebuild for - // non-installed libraries (e.g., direct BMI dependencies - // will probably have to be translated to mxx{} or some such). - // - if (p.is_a () || - p.is_a () || p.is_a () || p.is_a ()) - { - ps.push_back (p.as_prerequisite ()); - } - } - - auto p (targets.insert_locked (*tt, - move (pd), - dir_path (), // Always in the out tree. - move (mf), - nullopt, // Use default extension. - true, // Implied. - trace)); - const target& bt (p.first); - - // Note that this is racy and someone might have created this target - // while we were preparing the prerequisite list. - // - if (p.second.owns_lock ()) - bt.prerequisites (move (ps)); - - return bt; - } - - // Filter cl.exe noise (msvc.cxx). - // - void - msvc_filter_cl (ifdstream&, const path& src); - - void compile:: - append_modules (environment& env, - cstrings& args, - strings& stor, - const file& t, - const match_data& md) const - { - const module_positions& ms (md.mods); - assert (ms.start != 0); - - dir_path stdifc; // See the VC case below. - -#if 0 - switch (cid) - { - case compiler_id::gcc: - { - // Use the module map stored in depdb. - // - string s (relative (md.dd).string ()); - s.insert (0, "-fmodule-file-map=@="); - stor.push_back (move (s)); - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - // In Clang the module implementation's unit .pcm is special and - // must be "loaded". - // - if (md.type == translation_type::module_impl) - { - const file& f (t.prerequisite_targets[ms.start]->as ()); - string s (relative (f.path ()).string ()); - s.insert (0, "-fmodule-file="); - stor.push_back (move (s)); - } - - // Use the module map stored in depdb for others. - // - string s (relative (md.dd).string ()); - s.insert (0, "-fmodule-file-map=@="); - stor.push_back (move (s)); - break; - } - case compiler_id::msvc: - { - for (size_t i (ms.start), n (t.prerequisite_targets.size ()); - i != n; - ++i) - { - const target* pt (t.prerequisite_targets[i]); - - if (pt == nullptr) - continue; - - // Here we use whatever bmi type has been added. And we know all - // of these are bmi's. - // - const file& f (pt->as ()); - - // In VC std.* modules can only come from a single directory - // specified with the IFCPATH environment variable or the - // /module:stdIfcDir option. - // - if (std_module (cast (f.vars[c_module_name]))) - { - dir_path d (f.path ().directory ()); - - if (stdifc.empty ()) - { - // Go one directory up since /module:stdIfcDir will look in - // either Release or Debug subdirectories. Keeping the result - // absolute feels right. - // - stor.push_back ("/module:stdIfcDir"); - stor.push_back (d.directory ().string ()); - stdifc = move (d); - } - else if (d != stdifc) // Absolute and normalized. - fail << "multiple std.* modules in different directories"; - } - else - { - stor.push_back ("/module:reference"); - stor.push_back (relative (f.path ()).string ()); - } - } - break; - } - case compiler_id::icc: - assert (false); - } -#else - size_t n (t.prerequisite_targets.size ()); - - // Clang embeds module file references so we only need to specify - // our direct imports. - // - // If/when we get the ability to specify the mapping in a file, we - // should probably pass the whole list. - // - switch (cid) - { - case compiler_id::gcc: break; // All of them. - case compiler_id::clang_apple: - case compiler_id::clang: n = ms.copied != 0 ? ms.copied : n; break; - case compiler_id::msvc: break; // All of them. - case compiler_id::icc: assert (false); - } - - for (size_t i (ms.start); i != n; ++i) - { - const target* pt (t.prerequisite_targets[i]); - - if (pt == nullptr) - continue; - - // Here we use whatever bmi type has been added. And we know all of - // these are bmi's. - // - const file& f (pt->as ()); - string s (relative (f.path ()).string ()); - - switch (cid) - { - case compiler_id::gcc: - { - s.insert (0, 1, '='); - s.insert (0, cast (f.vars[c_module_name])); - s.insert (0, "-fmodule-file="); - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - // In Clang the module implementation's unit .pcm is special and - // must be "loaded". - // - if (md.type == translation_type::module_impl && i == ms.start) - s.insert (0, "-fmodule-file="); - else - { - s.insert (0, 1, '='); - s.insert (0, cast (f.vars[c_module_name])); - s.insert (0, "-fmodule-file="); - } - break; - } - case compiler_id::msvc: - { - // In VC std.* modules can only come from a single directory - // specified with the IFCPATH environment variable or the - // /module:stdIfcDir option. - // - if (std_module (cast (f.vars[c_module_name]))) - { - dir_path d (f.path ().directory ()); - - if (stdifc.empty ()) - { - // Go one directory up since /module:stdIfcDir will look in - // either Release or Debug subdirectories. Keeping the result - // absolute feels right. - // - s = d.directory ().string (); - stor.push_back ("/module:stdIfcDir"); - stdifc = move (d); - } - else - { - if (d != stdifc) // Absolute and normalized. - fail << "multiple std.* modules in different directories"; - - continue; // Skip. - } - } - else - stor.push_back ("/module:reference"); - - break; - } - case compiler_id::icc: - assert (false); - } - - stor.push_back (move (s)); - } -#endif - - // Shallow-copy storage to args. Why not do it as we go along pushing - // into storage? Because of potential reallocations. - // - for (const string& a: stor) - args.push_back (a.c_str ()); - - // VC's IFCPATH takes precedence over /module:stdIfcDir so unset it - // if we are using our own std modules. - // - if (!stdifc.empty ()) - env.push_back ("IFCPATH"); - } - - target_state compile:: - perform_update (action act, const target& xt) const - { - const file& t (xt.as ()); - const path& tp (t.path ()); - - match_data md (move (t.data ())); - bool mod (md.type == translation_type::module_iface); - - // While all our prerequisites are already up-to-date, we still have to - // execute them to keep the dependency counts straight. Actually, no, we - // may also have to update the modules. - // - auto pr ( - execute_prerequisites ( - (mod ? *x_mod : x_src), - act, t, - md.mt, - [s = md.mods.start] (const target&, size_t i) - { - return s != 0 && i >= s; // Only compare timestamps for modules. - }, - md.mods.copied)); // See search_modules() for details. - - const file& s (pr.second); - const path* sp (&s.path ()); - - if (pr.first) - { - if (md.touch) - { - touch (tp, false, 2); - skip_count.fetch_add (1, memory_order_relaxed); - } - - t.mtime (md.mt); - return *pr.first; - } - - // Make sure depdb is no older than any of our prerequisites. - // - touch (md.dd, false, verb_never); - - const scope& bs (t.base_scope ()); - const scope& rs (*bs.root_scope ()); - - otype ot (compile_type (t, mod)); - linfo li (link_info (bs, ot)); - - environment env; - cstrings args {cpath.recall_string ()}; - - // If we are building a module, then the target is bmi*{} and its ad hoc - // member is obj*{}. - // - path relo, relm; - if (mod) - { - relm = relative (tp); - relo = relative (t.member->is_a ()->path ()); - } - else - relo = relative (tp); - - // Build the command line. - // - if (md.pp != preprocessed::all) - { - append_options (args, t, c_poptions); - append_options (args, t, x_poptions); - - // Add *.export.poptions from prerequisite libraries. - // - append_lib_options (bs, args, t, act, li); - - // Extra system header dirs (last). - // - assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); - append_option_values ( - args, "-I", - sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), - [] (const dir_path& d) {return d.string ().c_str ();}); - - if (md.symexport) - append_symexport_options (args, t); - } - - append_options (args, t, c_coptions); - append_options (args, t, x_coptions); - append_options (args, tstd); - - string out, out1; // Output options storage. - strings mods; // Module options storage. - size_t out_i (0); // Index of the -o option. - - if (cclass == compiler_class::msvc) - { - // The /F*: option variants with separate names only became available - // in VS2013/12.0. Why do we bother? Because the command line suddenly - // becomes readable. - // - uint64_t ver (cast (rs[x_version_major])); - - args.push_back ("/nologo"); - - // While we want to keep the low-level build as "pure" as possible, - // the two misguided defaults, exceptions and runtime, just have to be - // fixed. Otherwise the default build is pretty much unusable. But we - // also make sure that the user can easily disable our defaults: if we - // see any relevant options explicitly specified, we take our hands - // off. - // - // For C looks like no /EH* (exceptions supported but no C++ objects - // destroyed) is a reasonable default. - // - if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) - args.push_back ("/EHsc"); - - // The runtime is a bit more interesting. At first it may seem like a - // good idea to be a bit clever and use the static runtime if we are - // building obja{}. And for obje{} we could decide which runtime to - // use based on the library link order: if it is static-only, then we - // could assume the static runtime. But it is indeed too clever: when - // building liba{} we have no idea who is going to use it. It could be - // an exe{} that links both static and shared libraries (and is - // therefore built with the shared runtime). And to safely use the - // static runtime, everything must be built with /MT and there should - // be no DLLs in the picture. So we are going to play it safe and - // always default to the shared runtime. - // - // In a similar vein, it would seem reasonable to use the debug runtime - // if we are compiling with debug. But, again, there will be fireworks - // if we have some projects built with debug and some without and then - // we try to link them together (which is not an unreasonable thing to - // do). So by default we will always use the release runtime. - // - if (!find_option_prefixes ({"/MD", "/MT"}, args)) - args.push_back ("/MD"); - - if (md.mods.start != 0) - append_modules (env, args, mods, t, md); - - // The presence of /Zi or /ZI causes the compiler to write debug info - // to the .pdb file. By default it is a shared file called vcNN.pdb - // (where NN is the VC version) created (wait for it) in the current - // working directory (and not the directory of the .obj file). Also, - // because it is shared, there is a special Windows service that - // serializes access. We, of course, want none of that so we will - // create a .pdb per object file. - // - // Note that this also changes the name of the .idb file (used for - // minimal rebuild and incremental compilation): cl.exe take the /Fd - // value and replaces the .pdb extension with .idb. - // - // Note also that what we are doing here appears to be incompatible - // with PCH (/Y* options) and /Gm (minimal rebuild). - // - if (find_options ({"/Zi", "/ZI"}, args)) - { - if (ver >= 18) - args.push_back ("/Fd:"); - else - out1 = "/Fd"; - - out1 += relo.string (); - out1 += ".pdb"; - - args.push_back (out1.c_str ()); - } - - if (ver >= 18) - { - args.push_back ("/Fo:"); - args.push_back (relo.string ().c_str ()); - } - else - { - out = "/Fo" + relo.string (); - args.push_back (out.c_str ()); - } - - if (mod) - { - args.push_back ("/module:interface"); - args.push_back ("/module:output"); - args.push_back (relm.string ().c_str ()); - } - - // Note: no way to indicate that the source if already preprocessed. - - args.push_back ("/c"); // Compile only. - args.push_back (langopt (md)); // Compile as. - args.push_back (sp->string ().c_str ()); // Note: relied on being last. - } - else - { - if (ot == otype::s) - { - // On Darwin, Win32 -fPIC is the default. - // - if (tclass == "linux" || tclass == "bsd") - args.push_back ("-fPIC"); - } - - if (md.mods.start != 0) - append_modules (env, args, mods, t, md); - - // Note: the order of the following options is relied upon below. - // - out_i = args.size (); // Index of the -o option. - - if (mod) - { - switch (cid) - { - case compiler_id::gcc: - { - args.push_back ("-o"); - args.push_back (relo.string ().c_str ()); - - out = "-fmodule-output="; - out += relm.string (); - args.push_back (out.c_str ()); - - args.push_back ("-c"); - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - args.push_back ("-o"); - args.push_back (relm.string ().c_str ()); - args.push_back ("--precompile"); - - // Without this option Clang's .pcm will reference source files. - // In our case this file may be transient (.ii). Plus, it won't - // play nice with distributed compilation. - // - args.push_back ("-Xclang"); - args.push_back ("-fmodules-embed-all-files"); - - break; - } - case compiler_id::msvc: - case compiler_id::icc: - assert (false); - } - } - else - { - args.push_back ("-o"); - args.push_back (relo.string ().c_str ()); - args.push_back ("-c"); - } - - args.push_back ("-x"); - args.push_back (langopt (md)); - - if (md.pp == preprocessed::all) - { - // Note that the mode we select must still handle comments and line - // continuations. So some more compiler-specific voodoo. - // - switch (cid) - { - case compiler_id::gcc: - { - // -fdirectives-only is available since GCC 4.3.0. - // - if (cmaj > 4 || (cmaj == 4 && cmin >= 3)) - { - args.push_back ("-fpreprocessed"); - args.push_back ("-fdirectives-only"); - } - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - // Clang handles comments and line continuations in the - // preprocessed source (it does not have -fpreprocessed). - // - break; - } - case compiler_id::icc: - break; // Compile as normal source for now. - case compiler_id::msvc: - assert (false); - } - } - - args.push_back (sp->string ().c_str ()); - } - - args.push_back (nullptr); - - if (!env.empty ()) - env.push_back (nullptr); - - // With verbosity level 2 print the command line as if we are compiling - // the source file, not its preprocessed version (so that it's easy to - // copy and re-run, etc). Only at level 3 and above print the real deal. - // - if (verb == 1) - text << x_name << ' ' << s; - else if (verb == 2) - print_process (args); - - // If we have the (partially) preprocessed output, switch to that. - // - bool psrc (!md.psrc.path.empty ()); - bool pact (md.psrc.active); - if (psrc) - { - args.pop_back (); // nullptr - args.pop_back (); // sp - - sp = &md.psrc.path; - - // This should match with how we setup preprocessing. - // - switch (cid) - { - case compiler_id::gcc: - { - // The -fpreprocessed is implied by .i/.ii. - // - args.pop_back (); // lang() - args.pop_back (); // -x - args.push_back ("-fdirectives-only"); - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - // Note that without -x Clang will treat .i/.ii as fully - // preprocessed. - // - break; - } - case compiler_id::msvc: - { - // Nothing to do (/TP or /TC already there). - // - break; - } - case compiler_id::icc: - assert (false); - } - - args.push_back (sp->string ().c_str ()); - args.push_back (nullptr); - - // Let's keep the preprocessed file in case of an error but only at - // verbosity level 3 and up (when one actually sees it mentioned on - // the command line). We also have to re-arm on success (see below). - // - if (pact && verb >= 3) - md.psrc.active = false; - } - - if (verb >= 3) - print_process (args); - - try - { - // VC cl.exe sends diagnostics to stdout. It also prints the file name - // being compiled as the first line. So for cl.exe we redirect stdout - // to a pipe, filter that noise out, and send the rest to stderr. - // - // For other compilers redirect stdout to stderr, in case any of them - // tries to pull off something similar. For sane compilers this should - // be harmless. - // - bool filter (cid == compiler_id::msvc); - - process pr (cpath, - args.data (), - 0, (filter ? -1 : 2), 2, - nullptr, // CWD - env.empty () ? nullptr : env.data ()); - - if (filter) - { - try - { - ifdstream is ( - move (pr.in_ofd), fdstream_mode::text, ifdstream::badbit); - - msvc_filter_cl (is, *sp); - - // If anything remains in the stream, send it all to stderr. Note - // that the eof check is important: if the stream is at eof, this - // and all subsequent writes to the diagnostics stream will fail - // (and you won't see a thing). - // - if (is.peek () != ifdstream::traits_type::eof ()) - diag_stream_lock () << is.rdbuf (); - - is.close (); - } - catch (const io_error&) {} // Assume exits with error. - } - - run_finish (args, pr); - } - catch (const process_error& e) - { - error << "unable to execute " << args[0] << ": " << e; - - if (e.child) - exit (1); - - throw failed (); - } - - if (pact && verb >= 3) - md.psrc.active = true; - - // Clang's module compilation requires two separate compiler - // invocations. - // - if (mod && (cid == compiler_id::clang || - cid == compiler_id::clang_apple)) - { - // Remove the target file if this fails. If we don't do that, we will - // end up with a broken build that is up-to-date. - // - auto_rmfile rm (relm); - - // Adjust the command line. First discard everything after -o then - // build the new "tail". - // - args.resize (out_i + 1); - args.push_back (relo.string ().c_str ()); // Produce .o. - args.push_back ("-c"); // By compiling .pcm. - args.push_back ("-Wno-unused-command-line-argument"); - args.push_back (relm.string ().c_str ()); - args.push_back (nullptr); - - if (verb >= 2) - print_process (args); - - try - { - process pr (cpath, - args.data (), - 0, 2, 2, - nullptr, // CWD - env.empty () ? nullptr : env.data ()); - - run_finish (args, pr); - } - catch (const process_error& e) - { - error << "unable to execute " << args[0] << ": " << e; - - if (e.child) - exit (1); - - throw failed (); - } - - rm.cancel (); - } - - // Should we go to the filesystem and get the new mtime? We know the - // file has been modified, so instead just use the current clock time. - // It has the advantage of having the subseconds precision. - // - t.mtime (system_clock::now ()); - return target_state::changed; - } - - target_state compile:: - perform_clean (action a, const target& xt) const - { - const file& t (xt.as ()); - - using id = compiler_id; - - switch (cid) - { - case id::gcc: return clean_extra (a, t, {".d", x_pext, ".t"}); - case id::clang_apple: - case id::clang: return clean_extra (a, t, {".d", x_pext}); - case id::msvc: return clean_extra (a, t, {".d", x_pext, ".idb", ".pdb"}); - case id::icc: return clean_extra (a, t, {".d"}); - } - - assert (false); - return target_state::unchanged; - } - } -} diff --git a/build2/cc/compile.hxx b/build2/cc/compile.hxx deleted file mode 100644 index 2878e3d..0000000 --- a/build2/cc/compile.hxx +++ /dev/null @@ -1,153 +0,0 @@ -// file : build2/cc/compile.hxx -*- C++ -*- -// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#ifndef BUILD2_CC_COMPILE_HXX -#define BUILD2_CC_COMPILE_HXX - -#include - -#include -#include - -#include -#include // auto_rmfile - -#include -#include - -namespace build2 -{ - class depdb; - - namespace cc - { - // The order is arranged so that their integral values indicate whether - // one is a "stronger" than another. - // - enum class preprocessed: uint8_t {none, includes, modules, all}; - - // Positions of the re-exported bmi{}s. See search_modules() for - // details. - // - struct module_positions - { - size_t start; // First imported bmi*{}, 0 if none. - size_t exported; // First re-exported bmi*{}, 0 if none. - size_t copied; // First copied-over bmi*{}, 0 if none. - }; - - class compile: public rule, virtual common - { - public: - compile (data&&); - - virtual match_result - match (action, target&, const string&) const override; - - virtual recipe - apply (action, target&) const override; - - target_state - perform_update (action, const target&) const; - - target_state - perform_clean (action, const target&) const; - - private: - struct match_data; - using environment = small_vector; - - void - append_lib_options (const scope&, - cstrings&, - const target&, - action, linfo) const; - - void - hash_lib_options (const scope&, - sha256&, - const target&, - action, linfo) const; - - // Mapping of include prefixes (e.g., foo in ) for auto- - // generated headers to directories where they will be generated. - // - // We are using a prefix map of directories (dir_path_map) instead of - // just a map in order to also cover sub-paths (e.g., if - // we continue with the example). Specifically, we need to make sure we - // don't treat foobar as a sub-directory of foo. - // - // The priority is used to decide who should override whom. Lesser - // values are considered higher priority. See append_prefixes() for - // details. - // - // @@ The keys should be normalized. - // - struct prefix_value - { - dir_path directory; - size_t priority; - }; - using prefix_map = butl::dir_path_map; - - void - append_prefixes (prefix_map&, const target&, const variable&) const; - - void - append_lib_prefixes (const scope&, - prefix_map&, - target&, - action, linfo) const; - - prefix_map - build_prefix_map (const scope&, target&, action, linfo) const; - - // Reverse-lookup target type from extension. - // - const target_type* - map_extension (const scope&, const string&, const string&) const; - - pair - extract_headers (action, const scope&, file&, linfo, - const file&, const match_data&, - depdb&, bool&, timestamp) const; - - pair - parse_unit (action, file&, linfo, - const file&, auto_rmfile&, const match_data&) const; - - void - extract_modules (action, const scope&, file&, linfo, - const compile_target_types&, - const file&, match_data&, - module_info&&, depdb&, bool&) const; - - module_positions - search_modules (action, const scope&, file&, linfo, - const target_type&, - const file&, module_imports&, sha256&) const; - - const target& - make_module_sidebuild (action, const scope&, const target&, - const target&, const string&) const; - - void - append_modules (environment&, cstrings&, strings&, - const file&, const match_data&) const; - - // Language selection option (for VC) or the value for the -x option. - // - const char* - langopt (const match_data&) const; - - void - append_symexport_options (cstrings&, const target&) const; - - private: - const string rule_id; - }; - } -} - -#endif // BUILD2_CC_COMPILE_HXX diff --git a/build2/cc/install-rule.cxx b/build2/cc/install-rule.cxx new file mode 100644 index 0000000..4e232ff --- /dev/null +++ b/build2/cc/install-rule.cxx @@ -0,0 +1,269 @@ +// file : build2/cc/install-rule.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include + +#include + +#include +#include // match() + +using namespace std; + +namespace build2 +{ + namespace cc + { + using namespace bin; + + // install_rule + // + install_rule:: + install_rule (data&& d, const link_rule& l) + : common (move (d)), link_ (l) {} + + const target* install_rule:: + filter (action a, const target& t, prerequisite_member p) const + { + // NOTE: see libux_install_rule::filter() if changing anything here. + + otype ot (link_type (t).type); + + // Don't install executable's prerequisite headers. + // + if (t.is_a () && x_header (p)) + return nullptr; + + // Here is a problem: if the user spells the obj*/bmi*{} targets + // explicitly, then the source files, including headers/modules may be + // specified as preprequisites of those targets and not of this target. + // While this can be worked around for headers by also listing them as + // prerequisites of this target, this won't work for modules (since they + // are compiled). So what we are going to do here is detect bmi*{} and + // translate them to their mxx{} (this doesn't quite work for headers + // since there would normally be several of them). + // + if (p.is_a () || p.is_a (compile_types (ot).bmi)) + { + const target& mt (p.search (t)); + + for (prerequisite_member mp: group_prerequisite_members (a, mt)) + { + if (mp.is_a (*x_mod)) + return t.is_a () ? nullptr : file_rule::filter (a, mt, mp); + } + } + + // If this is a shared library prerequisite, install it as long as it + // is in the same amalgamation as we are. + // + // Less obvious: we also want to install a static library prerequisite + // of a library (since it could be referenced from its .pc file, etc). + // + bool st (t.is_a () || t.is_a ()); // Target needs shared. + bool at (t.is_a () || t.is_a ()); // Target needs static. + + if ((st && (p.is_a () || p.is_a ())) || + (at && (p.is_a () || p.is_a ()))) + { + const target* pt (&p.search (t)); + + // If this is the lib{}/libu{} group, pick a member which we would + // link. For libu{} we want the "see through" logic. + // + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, link_info (t.base_scope (), ot)); + + if ((st && pt->is_a ()) || (at && pt->is_a ())) + return pt->in (t.weak_scope ()) ? pt : nullptr; + + // See through libux{}. Note that we are always in the same project + // (and thus amalgamation). + // + if (pt->is_a ()) + return pt; + } + + return file_rule::filter (a, t, p); + } + + bool install_rule:: + match (action a, target& t, const string& hint) const + { + // @@ How do we split the hint between the two? + // + + // We only want to handle installation if we are also the ones building + // this target. So first run link's match(). + // + return link_.match (a, t, hint) && file_rule::match (a, t, ""); + } + + recipe install_rule:: + apply (action a, target& t) const + { + recipe r (file_rule::apply (a, t)); + + if (a.operation () == update_id) + { + // Signal to the link rule that this is update for install. And if the + // update has already been executed, verify it was done for install. + // + auto& md (t.data ()); + + if (md.for_install) + { + if (!*md.for_install) + fail << "target " << t << " already updated but not for install"; + } + else + md.for_install = true; + } + else // install or uninstall + { + // Derive shared library paths and cache them in the target's aux + // storage if we are un/installing (used in *_extra() functions + // below). + // + static_assert (sizeof (link_rule::libs_paths) <= target::data_size, + "insufficient space"); + + file* f; + if ((f = t.is_a ()) != nullptr && tclass != "windows") + { + const string* p (cast_null (t["bin.lib.prefix"])); + const string* s (cast_null (t["bin.lib.suffix"])); + t.data ( + link_.derive_libs_paths (*f, + p != nullptr ? p->c_str (): nullptr, + s != nullptr ? s->c_str (): nullptr)); + } + } + + return r; + } + + bool install_rule:: + install_extra (const file& t, const install_dir& id) const + { + bool r (false); + + if (t.is_a () && tclass != "windows") + { + // Here we may have a bunch of symlinks that we need to install. + // + const scope& rs (t.root_scope ()); + auto& lp (t.data ()); + + auto ln = [&rs, &id] (const path& f, const path& l) + { + install_l (rs, id, f.leaf (), l.leaf (), false); + return true; + }; + + const path& lk (lp.link); + const path& so (lp.soname); + const path& in (lp.interm); + + const path* f (lp.real); + + if (!in.empty ()) {r = ln (*f, in) || r; f = ∈} + if (!so.empty ()) {r = ln (*f, so) || r; f = &so;} + if (!lk.empty ()) {r = ln (*f, lk) || r; } + } + + return r; + } + + bool install_rule:: + uninstall_extra (const file& t, const install_dir& id) const + { + bool r (false); + + if (t.is_a () && tclass != "windows") + { + // Here we may have a bunch of symlinks that we need to uninstall. + // + const scope& rs (t.root_scope ()); + auto& lp (t.data ()); + + auto rm = [&rs, &id] (const path& l) + { + return uninstall_f (rs, id, nullptr, l.leaf (), false); + }; + + const path& lk (lp.link); + const path& so (lp.soname); + const path& in (lp.interm); + + if (!lk.empty ()) r = rm (lk) || r; + if (!so.empty ()) r = rm (so) || r; + if (!in.empty ()) r = rm (in) || r; + } + + return r; + } + + // libux_install_rule + // + libux_install_rule:: + libux_install_rule (data&& d, const link_rule& l) + : common (move (d)), link_ (l) {} + + const target* libux_install_rule:: + filter (action a, const target& t, prerequisite_member p) const + { + // The "see through" semantics that should be parallel to install_rule + // above. In particular, here we use libue/libua/libus{} as proxies for + // exe/liba/libs{} there. + + otype ot (link_type (t).type); + + if (t.is_a () && x_header (p)) + return nullptr; + + if (p.is_a () || p.is_a (compile_types (ot).bmi)) + { + const target& mt (p.search (t)); + + for (prerequisite_member mp: group_prerequisite_members (a, mt)) + { + if (mp.is_a (*x_mod)) + return t.is_a () ? nullptr : alias_rule::filter (a, mt, mp); + } + } + + bool st (t.is_a () || t.is_a ()); // Target needs shared. + bool at (t.is_a () || t.is_a ()); // Target needs static. + + if ((st && (p.is_a () || p.is_a ())) || + (at && (p.is_a () || p.is_a ()))) + { + const target* pt (&p.search (t)); + + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, link_info (t.base_scope (), ot)); + + if ((st && pt->is_a ()) || (at && pt->is_a ())) + return pt->in (t.weak_scope ()) ? pt : nullptr; + + if (pt->is_a ()) + return pt; + } + + return alias_rule::filter (a, t, p); + } + + bool libux_install_rule:: + match (action a, target& t, const string& hint) const + { + // We only want to handle installation if we are also the ones building + // this target. So first run link's match(). + // + return link_.match (a, t, hint) && alias_rule::match (a, t, ""); + } + } +} diff --git a/build2/cc/install-rule.hxx b/build2/cc/install-rule.hxx new file mode 100644 index 0000000..ac2f93a --- /dev/null +++ b/build2/cc/install-rule.hxx @@ -0,0 +1,77 @@ +// file : build2/cc/install-rule.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BUILD2_CC_INSTALL_RULE_HXX +#define BUILD2_CC_INSTALL_RULE_HXX + +#include +#include + +#include + +#include +#include + +namespace build2 +{ + namespace cc + { + class link_rule; + + // Installation rule for exe{} and lib*{}. Here we do: + // + // 1. Signal to the link rule that this is update for install. + // + // 2. Additional filtering of prerequisites (e.g., headers of an exe{}). + // + // 3. Extra un/installation (e.g., libs{} symlinks). + // + class install_rule: public install::file_rule, virtual common + { + public: + install_rule (data&&, const link_rule&); + + virtual const target* + filter (action, const target&, prerequisite_member) const override; + + virtual bool + match (action, target&, const string&) const override; + + virtual recipe + apply (action, target&) const override; + + virtual bool + install_extra (const file&, const install_dir&) const override; + + virtual bool + uninstall_extra (const file&, const install_dir&) const override; + + private: + const link_rule& link_; + }; + + // Installation rule for libu*{}. + // + // While libu*{} themselves are not installable, we need to see through + // them in case they depend on stuff that we need to install (e.g., + // headers). Note that we use the alias_rule as a base. + // + class libux_install_rule: public install::alias_rule, virtual common + { + public: + libux_install_rule (data&&, const link_rule&); + + virtual const target* + filter (action, const target&, prerequisite_member) const override; + + virtual bool + match (action, target&, const string&) const override; + + private: + const link_rule& link_; + }; + } +} + +#endif // BUILD2_CC_INSTALL_RULE_HXX diff --git a/build2/cc/install.cxx b/build2/cc/install.cxx deleted file mode 100644 index fcaf626..0000000 --- a/build2/cc/install.cxx +++ /dev/null @@ -1,246 +0,0 @@ -// file : build2/cc/install.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#include - -#include - -#include - -#include // match() -#include - -using namespace std; - -namespace build2 -{ - namespace cc - { - using namespace bin; - - // file_install - // - file_install:: - file_install (data&& d, const link& l): common (move (d)), link_ (l) {} - - const target* file_install:: - filter (action a, const target& t, prerequisite_member p) const - { - // NOTE: see also alias_install::filter() below if changing anything - // here. - - otype ot (link_type (t).type); - - // Don't install executable's prerequisite headers. - // - if (t.is_a () && x_header (p)) - return nullptr; - - // Here is a problem: if the user spells the obj*/bmi*{} targets - // explicitly, then the source files, including headers/modules may be - // specified as preprequisites of those targets and not of this target. - // While this can be worked around for headers by also listing them as - // prerequisites of this target, this won't work for modules (since they - // are compiled). So what we are going to do here is detect bmi*{} and - // translate them to their mxx{} (this doesn't quite work for headers - // since there would normally be several of them). - // - if (p.is_a () || p.is_a (compile_types (ot).bmi)) - { - const target& mt (p.search (t)); - - for (prerequisite_member mp: group_prerequisite_members (a, mt)) - { - if (mp.is_a (*x_mod)) - return t.is_a () ? nullptr : file_rule::filter (a, mt, mp); - } - } - - // If this is a shared library prerequisite, install it as long as it - // is in the same amalgamation as we are. - // - // Less obvious: we also want to install a static library prerequisite - // of a library (since it could be referenced from its .pc file, etc). - // - bool st (t.is_a () || t.is_a ()); // Target needs shared. - bool at (t.is_a () || t.is_a ()); // Target needs static. - - if ((st && (p.is_a () || p.is_a ())) || - (at && (p.is_a () || p.is_a ()))) - { - const target* pt (&p.search (t)); - - // If this is the lib{}/libu{} group, pick a member which we would - // link. For libu{} we want to the "see through" logic. - // - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, a, link_info (t.base_scope (), ot)); - - if ((st && pt->is_a ()) || (at && pt->is_a ())) - return pt->in (t.weak_scope ()) ? pt : nullptr; - - // See through libux{}. Note that we are always in the same project - // (and thus amalgamation). - // - if (pt->is_a ()) - return pt; - } - - return file_rule::filter (a, t, p); - } - - match_result file_install:: - match (action a, target& t, const string& hint) const - { - // @@ How do we split the hint between the two? - // - - // We only want to handle installation if we are also the ones building - // this target. So first run link's match(). - // - match_result r (link_.match (a, t, hint)); - return r ? file_rule::match (a, t, "") : r; - } - - recipe file_install:: - apply (action a, target& t) const - { - recipe r (file_rule::apply (a, t)); - - // Derive shared library paths and cache them in the target's aux - // storage if we are (un)installing (used in *_extra() functions below). - // - if (a.operation () == install_id || a.operation () == uninstall_id) - { - file* f; - if ((f = t.is_a ()) != nullptr && tclass != "windows") - { - const string* p (cast_null (t["bin.lib.prefix"])); - const string* s (cast_null (t["bin.lib.suffix"])); - t.data ( - link_.derive_libs_paths (*f, - p != nullptr ? p->c_str (): nullptr, - s != nullptr ? s->c_str (): nullptr)); - } - } - - return r; - } - - void file_install:: - install_extra (const file& t, const install_dir& id) const - { - if (t.is_a () && tclass != "windows") - { - // Here we may have a bunch of symlinks that we need to install. - // - const scope& rs (t.root_scope ()); - auto& lp (t.data ()); - - auto ln = [&rs, &id] (const path& f, const path& l) - { - install_l (rs, id, f.leaf (), l.leaf (), false); - }; - - const path& lk (lp.link); - const path& so (lp.soname); - const path& in (lp.interm); - - const path* f (&lp.real); - - if (!in.empty ()) {ln (*f, in); f = ∈} - if (!so.empty ()) {ln (*f, so); f = &so;} - if (!lk.empty ()) {ln (*f, lk);} - } - } - - bool file_install:: - uninstall_extra (const file& t, const install_dir& id) const - { - bool r (false); - - if (t.is_a () && tclass != "windows") - { - // Here we may have a bunch of symlinks that we need to uninstall. - // - const scope& rs (t.root_scope ()); - auto& lp (t.data ()); - - auto rm = [&rs, &id] (const path& l) - { - return uninstall_f (rs, id, nullptr, l.leaf (), false); - }; - - const path& lk (lp.link); - const path& so (lp.soname); - const path& in (lp.interm); - - if (!lk.empty ()) r = rm (lk) || r; - if (!so.empty ()) r = rm (so) || r; - if (!in.empty ()) r = rm (in) || r; - } - - return r; - } - - // alias_install - // - alias_install:: - alias_install (data&& d, const link& l): common (move (d)), link_ (l) {} - - const target* alias_install:: - filter (action a, const target& t, prerequisite_member p) const - { - // The "see through" semantics that should be parallel to file_install - // above. In particular, here we use libue/libua/libus{} as proxies for - // exe/liba/libs{} there. - - otype ot (link_type (t).type); - - if (t.is_a () && x_header (p)) - return nullptr; - - if (p.is_a () || p.is_a (compile_types (ot).bmi)) - { - const target& mt (p.search (t)); - - for (prerequisite_member mp: group_prerequisite_members (a, mt)) - { - if (mp.is_a (*x_mod)) - return t.is_a () ? nullptr : alias_rule::filter (a, mt, mp); - } - } - - bool st (t.is_a () || t.is_a ()); // Target needs shared. - bool at (t.is_a () || t.is_a ()); // Target needs static. - - if ((st && (p.is_a () || p.is_a ())) || - (at && (p.is_a () || p.is_a ()))) - { - const target* pt (&p.search (t)); - - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, a, link_info (t.base_scope (), ot)); - - if ((st && pt->is_a ()) || (at && pt->is_a ())) - return pt->in (t.weak_scope ()) ? pt : nullptr; - - if (pt->is_a ()) - return pt; - } - - return alias_rule::filter (a, t, p); - } - - match_result alias_install:: - match (action a, target& t, const string& hint) const - { - // We only want to handle installation if we are also the ones building - // this target. So first run link's match(). - // - match_result r (link_.match (a, t, hint)); - return r ? alias_rule::match (a, t, "") : r; - } - } -} diff --git a/build2/cc/install.hxx b/build2/cc/install.hxx deleted file mode 100644 index 28a0a94..0000000 --- a/build2/cc/install.hxx +++ /dev/null @@ -1,67 +0,0 @@ -// file : build2/cc/install.hxx -*- C++ -*- -// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#ifndef BUILD2_CC_INSTALL_HXX -#define BUILD2_CC_INSTALL_HXX - -#include -#include - -#include - -#include -#include - -namespace build2 -{ - namespace cc - { - class link; - - // Installation rule for exe{}, lib*{}, etc. - // - class file_install: public install::file_rule, virtual common - { - public: - file_install (data&&, const link&); - - virtual const target* - filter (action, const target&, prerequisite_member) const override; - - virtual match_result - match (action, target&, const string&) const override; - - virtual recipe - apply (action, target&) const override; - - virtual void - install_extra (const file&, const install_dir&) const override; - - virtual bool - uninstall_extra (const file&, const install_dir&) const override; - - private: - const link& link_; - }; - - // Installation rule for libux{}. - // - class alias_install: public install::alias_rule, virtual common - { - public: - alias_install (data&&, const link&); - - virtual const target* - filter (action, const target&, prerequisite_member) const override; - - virtual match_result - match (action, target&, const string&) const override; - - private: - const link& link_; - }; - } -} - -#endif // BUILD2_CC_INSTALL_HXX diff --git a/build2/cc/link-rule.cxx b/build2/cc/link-rule.cxx new file mode 100644 index 0000000..d06a835 --- /dev/null +++ b/build2/cc/link-rule.cxx @@ -0,0 +1,2104 @@ +// file : build2/cc/link-rule.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include // exit() + +#include +#include // file_exists() + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include // c, pc* +#include + +using namespace std; +using namespace butl; + +namespace build2 +{ + namespace cc + { + using namespace bin; + + link_rule:: + link_rule (data&& d) + : common (move (d)), + rule_id (string (x) += ".link 1") + { + static_assert (sizeof (match_data) <= target::data_size, + "insufficient space"); + } + + bool link_rule:: + match (action a, target& t, const string& hint) const + { + tracer trace (x, "link_rule::match"); + + // NOTE: may be called multiple times (see install rules). + + ltype lt (link_type (t)); + otype ot (lt.type); + + // If this is a library, link-up to our group (this is the target group + // protocol which means this can be done whether we match or not). + // + if (lt.library ()) + { + if (t.group == nullptr) + t.group = &search (t, + lt.utility ? libu::static_type : lib::static_type, + t.dir, t.out, t.name); + } + + // Scan prerequisites and see if we can work with what we've got. Note + // that X could be C. We handle this by always checking for X first. + // + // Note also that we treat bmi{} as obj{}. + // + bool seen_x (false), seen_c (false), seen_obj (false), seen_lib (false); + + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + if (p.is_a (x_src) || (x_mod != nullptr && p.is_a (*x_mod))) + { + seen_x = seen_x || true; + } + else if (p.is_a ()) + { + seen_c = seen_c || true; + } + else if (p.is_a () || p.is_a ()) + { + seen_obj = seen_obj || true; + } + else if (p.is_a () || p.is_a ()) + { + if (ot != otype::e) + fail << p.type ().name << "{} as prerequisite of " << t; + + seen_obj = seen_obj || true; + } + else if (p.is_a () || p.is_a ()) + { + if (ot != otype::a) + fail << p.type ().name << "{} as prerequisite of " << t; + + seen_obj = seen_obj || true; + } + else if (p.is_a () || p.is_a ()) + { + if (ot != otype::s) + fail << p.type ().name << "{} as prerequisite of " << t; + + seen_obj = seen_obj || true; + } + else if (p.is_a () || + p.is_a () || + p.is_a () || + p.is_a ()) + { + seen_lib = seen_lib || true; + } + // If this is some other c-common source (say C++ in a C rule), then + // it will most definitely need to be compiled but we can't do that. + // + else if (p.is_a ()) + return false; + } + + if (!(seen_x || seen_c || seen_obj || seen_lib)) + return false; + + // We will only chain a C source if there is also an X source or we were + // explicitly told to. + // + if (seen_c && !seen_x && hint < x) + { + l4 ([&]{trace << "C prerequisite without " << x_lang << " or hint";}); + return false; + } + + return true; + } + + auto link_rule:: + derive_libs_paths (file& ls, const char* pfx, const char* sfx) const + -> libs_paths + { + const char* ext (nullptr); + + bool win (tclass == "windows"); + + if (win) + { + if (tsys == "mingw32") + { + if (pfx == nullptr) + pfx = "lib"; + } + + ext = "dll"; + } + else + { + if (pfx == nullptr) + pfx = "lib"; + + if (tclass == "macos") + ext = "dylib"; + else + ext = "so"; + } + + // First sort out which extension we are using. + // + const string& e (ls.derive_extension (ext)); + + auto append_ext = [&e] (path& p) + { + if (!e.empty ()) + { + p += '.'; + p += e; + } + }; + + // Figure out the version. + // + string v; + using verion_map = map; + if (const verion_map* m = cast_null (ls["bin.lib.version"])) + { + // First look for the target system. + // + auto i (m->find (tsys)); + + // Then look for the target class. + // + if (i == m->end ()) + i = m->find (tclass); + + // Then look for the wildcard. Since it is higly unlikely one can have + // a version that will work across platforms, this is only useful to + // say "all others -- no version". + // + if (i == m->end ()) + i = m->find ("*"); + + // At this stage the only platform-specific version we support is the + // "no version" override. + // + if (i != m->end () && !i->second.empty ()) + fail << i->first << "-specific bin.lib.version not yet supported"; + + // Finally look for the platform-independent version. + // + if (i == m->end ()) + i = m->find (""); + + // If we didn't find anything, fail. If the bin.lib.version was + // specified, then it should explicitly handle all the targets. + // + if (i == m->end ()) + fail << "no version for " << ctgt << " in bin.lib.version" << + info << "considere adding " << tsys << "@ or " << tclass + << "@"; + + v = i->second; + } + + // Now determine the paths. + // + path lk, so, in; + + // We start with the basic path. + // + path b (ls.dir); + path cp; // Clean pattern. + { + if (pfx == nullptr || pfx[0] == '\0') + { + b /= ls.name; + } + else + { + b /= pfx; + b += ls.name; + } + + cp = b; + cp += "?*"; // Don't match empty (like the libfoo.so symlink). + + if (sfx != nullptr) + { + b += sfx; + cp += sfx; + } + } + + append_ext (cp); + + // On Windows the real path is to libs{} and the link path is to the + // import library. + // + if (win) + { + // Usually on Windows the import library is called the same as the DLL + // but with the .lib extension. Which means it clashes with the static + // library. Instead of decorating the static library name with ugly + // suffixes (as is customary), let's use the MinGW approach (one must + // admit it's quite elegant) and call it .dll.lib. + // + lk = b; + append_ext (lk); + + libi& li (ls.member->as ()); // Note: libi is locked. + lk = li.derive_path (move (lk), tsys == "mingw32" ? "a" : "lib"); + } + else if (!v.empty ()) + { + lk = b; + append_ext (lk); + } + + if (!v.empty ()) + b += v; + + const path& re (ls.derive_path (move (b))); + + return libs_paths {move (lk), move (so), move (in), &re, move (cp)}; + } + + recipe link_rule:: + apply (action a, target& xt) const + { + tracer trace (x, "link_rule::apply"); + + file& t (xt.as ()); + + // Note that for_install is signalled by install_rule and therefore + // can only be relied upon during execute. + // + match_data& md (t.data (match_data ())); + + const scope& bs (t.base_scope ()); + const scope& rs (*bs.root_scope ()); + + ltype lt (link_type (t)); + otype ot (lt.type); + linfo li (link_info (bs, ot)); + + // Set the library type (C, C++, etc). + // + if (lt.library ()) + t.vars.assign (c_type) = string (x); + + // Derive file name(s) and add ad hoc group members. + // + { + target_lock libi; // Have to hold until after PDB member addition. + + const char* e (nullptr); // Extension. + const char* p (nullptr); // Prefix. + const char* s (nullptr); // Suffix. + + if (lt.utility) + { + // These are all static libraries with names indicating the kind of + // object files they contain (similar to how we name object files + // themselves). We add the 'u' extension to avoid clashes with + // real libraries/import stubs. + // + // libue libhello.u.a hello.exe.u.lib + // libua libhello.a.u.a hello.lib.u.lib + // libus libhello.so.u.a hello.dll.u.lib hello.dylib.u.lib + // + // Note that we currently don't add bin.lib.{prefix,suffix} since + // these are not installed. + // + if (tsys == "win32-msvc") + { + switch (ot) + { + case otype::e: e = "exe.u.lib"; break; + case otype::a: e = "lib.u.lib"; break; + case otype::s: e = "dll.u.lib"; break; + } + } + else + { + p = "lib"; + + if (tsys == "mingw32") + { + switch (ot) + { + case otype::e: e = "exe.u.a"; break; + case otype::a: e = "a.u.a"; break; + case otype::s: e = "dll.u.a"; break; + } + + } + else if (tsys == "darwin") + { + switch (ot) + { + case otype::e: e = "u.a"; break; + case otype::a: e = "a.u.a"; break; + case otype::s: e = "dylib.u.a"; break; + } + } + else + { + switch (ot) + { + case otype::e: e = "u.a"; break; + case otype::a: e = "a.u.a"; break; + case otype::s: e = "so.u.a"; break; + } + } + } + + t.derive_path (e, p, s); + } + else + { + if (auto l = t[ot == otype::e ? "bin.exe.prefix" : "bin.lib.prefix"]) + p = cast (l).c_str (); + if (auto l = t[ot == otype::e ? "bin.exe.suffix" : "bin.lib.suffix"]) + s = cast (l).c_str (); + + switch (ot) + { + case otype::e: + { + if (tclass == "windows") + e = "exe"; + else + e = ""; + + t.derive_path (e, p, s); + break; + } + case otype::a: + { + if (tsys == "win32-msvc") + e = "lib"; + else + { + if (p == nullptr) p = "lib"; + e = "a"; + } + + t.derive_path (e, p, s); + break; + } + case otype::s: + { + // On Windows libs{} is an ad hoc group. The libs{} itself is + // the DLL and we add libi{} import library as its member. + // + if (tclass == "windows") + libi = add_adhoc_member (a, t); + + md.libs_data = derive_libs_paths (t, p, s); + + if (libi) + match_recipe (libi, group_recipe); // Set recipe and unlock. + + break; + } + } + + // Add VC's .pdb. Note that we are looking for the link.exe /DEBUG + // option. + // + if (ot != otype::a && + tsys == "win32-msvc" && + (find_option ("/DEBUG", t, c_loptions, true) || + find_option ("/DEBUG", t, x_loptions, true))) + { + // Note: add after the import library if any. + // + target_lock pdb ( + add_adhoc_member (a, t, *bs.find_target_type ("pdb"))); + + // We call it foo.{exe,dll}.pdb rather than just foo.pdb because + // we can have both foo.exe and foo.dll in the same directory. + // + pdb.target->as ().derive_path (t.path (), "pdb"); + + match_recipe (pdb, group_recipe); // Set recipe and unlock. + } + + // Add pkg-config's .pc file. + // + // Note that we do it here regardless of whether we are installing + // or not for two reasons. Firstly, it is not easy to detect this + // situation in apply() since the for_install hasn't yet been + // communicated by install_rule. Secondly, always having the member + // takes care of cleanup automagically. The actual generation + // happens in perform_update() below. + // + if (ot != otype::e) + { + target_lock pc ( + add_adhoc_member ( + a, t, + ot == otype::a ? pca::static_type : pcs::static_type)); + + // Note that here we always use the lib name prefix, even on + // Windows with VC. The reason is the user needs a consistent name + // across platforms by which they can refere to the library. This + // is also the reason why we use the static/shared suffixes rather + // that a./.lib/.so/.dylib/.dll. + // + pc.target->as ().derive_path (nullptr, + (p == nullptr ? "lib" : p), + s); + + match_recipe (pc, group_recipe); // Set recipe and unlock. + } + } + } + + // Inject dependency on the output directory. + // + inject_fsdir (a, t); + + // Process prerequisites, pass 1: search and match prerequisite + // libraries, search obj/bmi{} targets, and search targets we do rule + // chaining for. + // + // We do libraries first in order to indicate that we will execute these + // targets before matching any of the obj/bmi{}. This makes it safe for + // compile::apply() to unmatch them and therefore not to hinder + // parallelism. + // + // We also create obj/bmi{} chain targets because we need to add + // (similar to lib{}) all the bmi{} as prerequisites to all the other + // obj/bmi{} that we are creating. Note that this doesn't mean that the + // compile rule will actually treat them all as prerequisite targets. + // Rather, they are used to resolve actual module imports. We don't + // really have to search obj{} targets here but it's the same code so we + // do it here to avoid duplication. + // + // Also, when cleaning, we ignore prerequisites that are not in the same + // or a subdirectory of our project root. + // + optional usr_lib_dirs; // Extract lazily. + compile_target_types tt (compile_types (ot)); + + auto skip = [&a, &rs] (const target*& pt) + { + if (a.operation () == clean_id && !pt->dir.sub (rs.out_path ())) + pt = nullptr; + + return pt == nullptr; + }; + + auto& pts (t.prerequisite_targets[a]); + size_t start (pts.size ()); + + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + // We pre-allocate a NULL slot for each (potential; see clean) + // prerequisite target. + // + pts.push_back (nullptr); + const target*& pt (pts.back ()); + + uint8_t m (0); // Mark: lib (0), src (1), mod (2), obj/bmi (3). + + bool mod (x_mod != nullptr && p.is_a (*x_mod)); + + if (mod || p.is_a (x_src) || p.is_a ()) + { + // Rule chaining, part 1. + // + + // Which scope shall we use to resolve the root? Unlikely, but + // possible, the prerequisite is from a different project + // altogether. So we are going to use the target's project. + // + + // If the source came from the lib{} group, then create the obj{} + // group and add the source as a prerequisite of the obj{} group, + // not the obj*{} member. This way we only need one prerequisite + // for, say, both liba{} and libs{}. The same goes for bmi{}. + // + bool group (!p.prerequisite.belongs (t)); // Group's prerequisite. + + const target_type& rtt (mod + ? (group ? bmi::static_type : tt.bmi) + : (group ? obj::static_type : tt.obj)); + + const prerequisite_key& cp (p.key ()); // Source key. + + // Come up with the obj*/bmi*{} target. The source prerequisite + // directory can be relative (to the scope) or absolute. If it is + // relative, then use it as is. If absolute, then translate it to + // the corresponding directory under out_root. While the source + // directory is most likely under src_root, it is also possible it + // is under out_root (e.g., generated source). + // + dir_path d; + { + const dir_path& cpd (*cp.tk.dir); + + if (cpd.relative () || cpd.sub (rs.out_path ())) + d = cpd; + else + { + if (!cpd.sub (rs.src_path ())) + fail << "out of project prerequisite " << cp << + info << "specify corresponding " << rtt.name << "{} " + << "target explicitly"; + + d = rs.out_path () / cpd.leaf (rs.src_path ()); + } + } + + // obj/bmi{} is always in the out tree. Note that currently it could + // be the group -- we will pick a member in part 2 below. + // + pt = &search (t, rtt, d, dir_path (), *cp.tk.name, nullptr, cp.scope); + + // If we shouldn't clean obj{}, then it is fair to assume we + // shouldn't clean the source either (generated source will be in + // the same directory as obj{} and if not, well, go find yourself + // another build system ;-)). + // + if (skip (pt)) + continue; + + m = mod ? 2 : 1; + } + else if (p.is_a () || + p.is_a () || + p.is_a () || + p.is_a ()) + { + // Handle imported libraries. + // + // Note that since the search is rule-specific, we don't cache the + // target in the prerequisite. + // + if (p.proj ()) + pt = search_library ( + a, sys_lib_dirs, usr_lib_dirs, p.prerequisite); + + // The rest is the same basic logic as in search_and_match(). + // + if (pt == nullptr) + pt = &p.search (t); + + if (skip (pt)) + continue; + + // If this is the lib{}/libu{} group, then pick the appropriate + // member. + // + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, li); + } + else + { + // If this is the obj{} or bmi{} target group, then pick the + // appropriate member. + // + if (p.is_a ()) pt = &search (t, tt.obj, p.key ()); + else if (p.is_a ()) pt = &search (t, tt.bmi, p.key ()); + else pt = &p.search (t); + + if (skip (pt)) + continue; + + m = 3; + } + + mark (pt, m); + } + + // Match lib{} (the only unmarked) in parallel and wait for completion. + // + match_members (a, t, pts, start); + + // Process prerequisites, pass 2: finish rule chaining but don't start + // matching anything yet since that may trigger recursive matching of + // bmi{} targets we haven't completed yet. Hairy, I know. + // + + // Parallel prerequisite_targets loop. + // + size_t i (start), n (pts.size ()); + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + const target*& pt (pts[i].target); + uintptr_t& pd (pts[i++].data); + + if (pt == nullptr) + continue; + + uint8_t m (unmark (pt)); // New mark: completion (1), verfication (2). + + if (m == 3) // obj/bmi{} + m = 1; // Just completion. + else if (m == 1 || m == 2) // Source/module chain. + { + bool mod (m == 2); + + m = 1; + + const target& rt (*pt); + bool group (!p.prerequisite.belongs (t)); // Group's prerequisite. + + // If we have created a obj/bmi{} target group, pick one of its + // members; the rest would be primarily concerned with it. + // + pt = + group + ? &search (t, (mod ? tt.bmi : tt.obj), rt.dir, rt.out, rt.name) + : &rt; + + // If this obj*{} already has prerequisites, then verify they are + // "compatible" with what we are doing here. Otherwise, synthesize + // the dependency. Note that we may also end up synthesizing with + // someone beating up to it. In this case also verify. + // + bool verify (true); + + if (!pt->has_prerequisites ()) + { + prerequisites ps; + ps.push_back (p.as_prerequisite ()); // Source. + + // Add our lib*{} (see the export.* machinery for details) and + // bmi*{} (both original and chained; see module search logic) + // prerequisites. + // + // Note that we don't resolve lib{} to liba{}/libs{} here + // instead leaving it to whomever (e.g., the compile rule) will + // be needing *.export.*. One reason for doing it there is that + // the object target might be specified explicitly by the user + // in which case they will have to specify the set of lib{} + // prerequisites and it's much cleaner to do as lib{} rather + // than liba{}/libs{}. + // + // Initially, we were only adding imported libraries, but there + // is a problem with this approach: the non-imported library + // might depend on the imported one(s) which we will never "see" + // unless we start with this library. + // + // Note: have similar logic in make_module_sidebuild(). + // + size_t j (start); + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + const target* pt (pts[j++]); + + if (p.is_a () || + p.is_a () || p.is_a () || p.is_a () || + p.is_a () || p.is_a (tt.bmi)) + { + ps.push_back (p.as_prerequisite ()); + } + else if (x_mod != nullptr && p.is_a (*x_mod)) // Chained module. + { + // Searched during pass 1 but can be NULL or marked. + // + if (pt != nullptr && i != j) // Don't add self (note: both +1). + { + // This is sticky: pt might have come before us and if it + // was a group, then we would have picked up a member. So + // here we may have to "unpick" it. + // + bool group (j < i && !p.prerequisite.belongs (t)); + + unmark (pt); + ps.push_back (prerequisite (group ? *pt->group : *pt)); + } + } + } + + // Note: add to the group, not the member. + // + verify = !rt.prerequisites (move (ps)); + } + + if (verify) + { + // This gets a bit tricky. We need to make sure the source files + // are the same which we can only do by comparing the targets to + // which they resolve. But we cannot search ot's prerequisites -- + // only the rule that matches can. Note, however, that if all this + // works out, then our next step is to match the obj*{} target. If + // things don't work out, then we fail, in which case searching + // and matching speculatively doesn't really hurt. So we start the + // async match here and finish this verification in the "harvest" + // loop below. + // + const target_type& rtt (mod + ? (group ? bmi::static_type : tt.bmi) + : (group ? obj::static_type : tt.obj)); + + bool src (false); + for (prerequisite_member p1: group_prerequisite_members (a, *pt)) + { + // Most of the time we will have just a single source so fast- + // path that case. + // + if (p1.is_a (mod ? *x_mod : x_src) || p1.is_a ()) + { + src = true; + continue; // Check the rest of the prerequisites. + } + + // Ignore some known target types (fsdir, headers, libraries, + // modules). + // + if (p1.is_a () || + p1.is_a () || + p1.is_a () || p1.is_a () || p1.is_a () || + p1.is_a () || + p1.is_a () || p1.is_a () || p1.is_a () || + (p.is_a (mod ? *x_mod : x_src) && x_header (p1)) || + (p.is_a () && p1.is_a ())) + continue; + + fail << "synthesized dependency for prerequisite " << p + << " would be incompatible with existing target " << *pt << + info << "unexpected existing prerequisite type " << p1 << + info << "specify corresponding " << rtt.name << "{} " + << "dependency explicitly"; + } + + if (!src) + fail << "synthesized dependency for prerequisite " << p + << " would be incompatible with existing target " << *pt << + info << "no existing c/" << x_name << " source prerequisite" << + info << "specify corresponding " << rtt.name << "{} " + << "dependency explicitly"; + + m = 2; // Needs verification. + } + } + else // lib*{} + { + // If this is a static library, see if we need to link it whole. + // Note that we have to do it after match since we rely on the + // group link-up. + // + bool u; + if ((u = pt->is_a ()) || pt->is_a ()) + { + const variable& var (var_pool["bin.whole"]); // @@ Cache. + + // See the bin module for the lookup semantics discussion. Note + // that the variable is not overridable so we omit find_override() + // calls. + // + lookup l (p.prerequisite.vars[var]); + + if (!l.defined ()) + l = pt->find_original (var, true).first; + + if (!l.defined ()) + { + bool g (pt->group != nullptr); + l = bs.find_original (var, + &pt->type (), + &pt->name, + (g ? &pt->group->type () : nullptr), + (g ? &pt->group->name : nullptr)).first; + } + + if (l ? cast (*l) : u) + pd |= lflag_whole; + } + } + + mark (pt, m); + } + + // Process prerequisites, pass 3: match everything and verify chains. + // + + // Wait with unlocked phase to allow phase switching. + // + wait_guard wg (target::count_busy (), t[a].task_count, true); + + for (i = start; i != n; ++i) + { + const target*& pt (pts[i]); + + if (pt == nullptr) + continue; + + if (uint8_t m = unmark (pt)) + { + match_async (a, *pt, target::count_busy (), t[a].task_count); + mark (pt, m); + } + } + + wg.wait (); + + // The "harvest" loop: finish matching the targets we have started. Note + // that we may have bailed out early (thus the parallel i/n for-loop). + // + i = start; + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + const target*& pt (pts[i++]); + + // Skipped or not marked for completion. + // + uint8_t m; + if (pt == nullptr || (m = unmark (pt)) == 0) + continue; + + build2::match (a, *pt); + + // Nothing else to do if not marked for verification. + // + if (m == 1) + continue; + + // Finish verifying the existing dependency (which is now matched) + // compared to what we would have synthesized. + // + bool mod (x_mod != nullptr && p.is_a (*x_mod)); + + for (prerequisite_member p1: group_prerequisite_members (a, *pt)) + { + if (p1.is_a (mod ? *x_mod : x_src) || p1.is_a ()) + { + // Searching our own prerequisite is ok, p1 must already be + // resolved. + // + if (&p.search (t) != &p1.search (*pt)) + { + bool group (!p.prerequisite.belongs (t)); + + const target_type& rtt (mod + ? (group ? bmi::static_type : tt.bmi) + : (group ? obj::static_type : tt.obj)); + + fail << "synthesized dependency for prerequisite " << p << " " + << "would be incompatible with existing target " << *pt << + info << "existing prerequisite " << p1 << " does not match " + << p << + info << "specify corresponding " << rtt.name << "{} " + << "dependency explicitly"; + } + + break; + } + } + } + + switch (a) + { + case perform_update_id: return [this] (action a, const target& t) + { + return perform_update (a, t); + }; + case perform_clean_id: return [this] (action a, const target& t) + { + return perform_clean (a, t); + }; + default: return noop_recipe; // Configure update. + } + } + + void link_rule:: + append_libraries (strings& args, + const file& l, bool la, lflags lf, + const scope& bs, action a, linfo li) const + { + // Note: lack of the "small function object" optimization will really + // kill us here since we are called in a loop. + // + auto imp = [] (const file&, bool la) {return la;}; + + auto lib = [&args, this] (const file* l, const string& p, lflags f, bool) + { + if (l != nullptr) + { + // On Windows a shared library is a DLL with the import library as a + // first ad hoc group member. MinGW though can link directly to DLLs + // (see search_library() for details). + // + if (l->member != nullptr && l->is_a () && tclass == "windows") + l = &l->member->as (); + + string p (relative (l->path ()).string ()); + + if (f & lflag_whole) + { + if (tsys == "win32-msvc") + { + p.insert (0, "/WHOLEARCHIVE:"); // Only available from VC14U2. + } + else if (tsys == "darwin") + { + p.insert (0, "-Wl,-force_load,"); + } + else + { + args.push_back ("-Wl,--whole-archive"); + args.push_back (move (p)); + args.push_back ("-Wl,--no-whole-archive"); + return; + } + } + + args.push_back (move (p)); + } + else + args.push_back (p); + }; + + auto opt = [&args, this] ( + const file& l, const string& t, bool com, bool exp) + { + // If we need an interface value, then use the group (lib{}). + // + if (const target* g = exp && l.is_a () ? l.group : &l) + { + const variable& var ( + com + ? (exp ? c_export_loptions : c_loptions) + : (t == x + ? (exp ? x_export_loptions : x_loptions) + : var_pool[t + (exp ? ".export.loptions" : ".loptions")])); + + append_options (args, *g, var); + } + }; + + process_libraries ( + a, bs, li, sys_lib_dirs, l, la, lf, imp, lib, opt, true); + } + + void link_rule:: + hash_libraries (sha256& cs, + bool& update, timestamp mt, + const file& l, bool la, lflags lf, + const scope& bs, action a, linfo li) const + { + auto imp = [] (const file&, bool la) {return la;}; + + struct data + { + sha256& cs; + bool& update; + timestamp mt; + } d {cs, update, mt}; + + auto lib = [&d, this] (const file* l, const string& p, lflags f, bool) + { + if (l != nullptr) + { + // Check if this library renders us out of date. + // + d.update = d.update || l->newer (d.mt); + + // On Windows a shared library is a DLL with the import library as a + // first ad hoc group member. MinGW though can link directly to DLLs + // (see search_library() for details). + // + if (l->member != nullptr && l->is_a () && tclass == "windows") + l = &l->member->as (); + + d.cs.append (f); + d.cs.append (l->path ().string ()); + } + else + d.cs.append (p); + }; + + auto opt = [&cs, this] ( + const file& l, const string& t, bool com, bool exp) + { + if (const target* g = exp && l.is_a () ? l.group : &l) + { + const variable& var ( + com + ? (exp ? c_export_loptions : c_loptions) + : (t == x + ? (exp ? x_export_loptions : x_loptions) + : var_pool[t + (exp ? ".export.loptions" : ".loptions")])); + + hash_options (cs, *g, var); + } + }; + + process_libraries ( + a, bs, li, sys_lib_dirs, l, la, lf, imp, lib, opt, true); + } + + void link_rule:: + rpath_libraries (strings& args, + const target& t, + const scope& bs, + action a, + linfo li, + bool for_install) const + { + // Use -rpath-link on targets that support it (Linux, *BSD). Note + // that we don't really need it for top-level libraries. + // + if (for_install) + { + if (tclass != "linux" && tclass != "bsd") + return; + } + + auto imp = [for_install] (const file& l, bool la) + { + // If we are not installing, then we only need to rpath interface + // libraries (they will include rpath's for their implementations) + // Otherwise, we have to do this recursively. In both cases we also + // want to see through utility libraries. + // + // The rpath-link part is tricky: ideally we would like to get only + // implementations and only of shared libraries. We are not interested + // in interfaces because we are linking their libraries explicitly. + // However, in our model there is no such thing as "implementation + // only"; it is either interface or interface and implementation. So + // we are going to rpath-link all of them which should be harmless + // except for some noise on the command line. + // + // + return (for_install ? !la : false) || l.is_a (); + }; + + // Package the data to keep within the 2-pointer small std::function + // optimization limit. + // + struct + { + strings& args; + bool for_install; + } d {args, for_install}; + + auto lib = [&d, this] (const file* l, const string& f, lflags, bool sys) + { + // We don't rpath system libraries. Why, you may ask? There are many + // good reasons and I have them written on a napkin somewhere... + // + if (sys) + return; + + if (l != nullptr) + { + if (!l->is_a ()) + return; + } + else + { + // This is an absolute path and we need to decide whether it is + // a shared or static library. Doesn't seem there is anything + // better than checking for a platform-specific extension (maybe + // we should cache it somewhere). + // + size_t p (path::traits::find_extension (f)); + + if (p == string::npos) + return; + + ++p; // Skip dot. + + bool c (true); + const char* e; + + if (tclass == "windows") {e = "dll"; c = false;} + else if (tsys == "darwin") e = "dylib"; + else e = "so"; + + if ((c + ? f.compare (p, string::npos, e) + : casecmp (f.c_str () + p, e)) != 0) + return; + } + + // Ok, if we are here then it means we have a non-system, shared + // library and its absolute path is in f. + // + string o (d.for_install ? "-Wl,-rpath-link," : "-Wl,-rpath,"); + + size_t p (path::traits::rfind_separator (f)); + assert (p != string::npos); + + o.append (f, 0, (p != 0 ? p : 1)); // Don't include trailing slash. + d.args.push_back (move (o)); + }; + + // In case we don't have the "small function object" optimization. + // + const function impf (imp); + const function libf (lib); + + for (const prerequisite_target& pt: t.prerequisite_targets[a]) + { + bool la; + const file* f; + + if ((la = (f = pt->is_a ())) || + (la = (f = pt->is_a ())) || + ( f = pt->is_a ())) + { + if (!for_install && !la) + { + // Top-level sharen library dependency. It is either matched or + // imported so should be a cc library. + // + if (!cast_false (f->vars[c_system])) + args.push_back ( + "-Wl,-rpath," + f->path ().directory ().string ()); + } + + process_libraries (a, bs, li, sys_lib_dirs, + *f, la, pt.data, + impf, libf, nullptr); + } + } + } + + // Filter link.exe noise (msvc.cxx). + // + void + msvc_filter_link (ifdstream&, const file&, otype); + + // Translate target CPU to the link.exe/lib.exe /MACHINE option. + // + const char* + msvc_machine (const string& cpu); // msvc.cxx + + target_state link_rule:: + perform_update (action a, const target& xt) const + { + tracer trace (x, "link_rule::perform_update"); + + const file& t (xt.as ()); + const path& tp (t.path ()); + + match_data& md (t.data ()); + + // Unless the outer install rule signalled that this is update for + // install, signal back that we've performed plain update. + // + if (!md.for_install) + md.for_install = false; + + bool for_install (*md.for_install); + + const scope& bs (t.base_scope ()); + const scope& rs (*bs.root_scope ()); + + ltype lt (link_type (t)); + otype ot (lt.type); + linfo li (link_info (bs, ot)); + + // Update prerequisites. We determine if any relevant ones render us + // out-of-date manually below. + // + bool update (false); + timestamp mt (t.load_mtime ()); + target_state ts (straight_execute_prerequisites (a, t)); + + // If targeting Windows, take care of the manifest. + // + path manifest; // Manifest itself (msvc) or compiled object file. + timestamp rpath_timestamp (timestamp_nonexistent); // DLLs timestamp. + + if (lt.executable () && tclass == "windows") + { + // First determine if we need to add our rpath emulating assembly. The + // assembly itself is generated later, after updating the target. Omit + // it if we are updating for install. + // + if (!for_install) + rpath_timestamp = windows_rpath_timestamp (t, bs, a, li); + + pair p ( + windows_manifest (t, + rpath_timestamp != timestamp_nonexistent)); + + path& mf (p.first); + bool mf_cf (p.second); // Changed flag (timestamp resolution). + + timestamp mf_mt (file_mtime (mf)); + + if (tsys == "mingw32") + { + // Compile the manifest into the object file with windres. While we + // are going to synthesize an .rc file to pipe to windres' stdin, we + // will still use .manifest to check if everything is up-to-date. + // + manifest = mf + ".o"; + + if (mf_mt > file_mtime (manifest) || mf_cf) + { + path of (relative (manifest)); + + const process_path& rc (cast (rs["bin.rc.path"])); + + // @@ Would be good to add this to depdb (e.g,, rc changes). + // + const char* args[] = { + rc.recall_string (), + "--input-format=rc", + "--output-format=coff", + "-o", of.string ().c_str (), + nullptr}; + + if (verb >= 3) + print_process (args); + + try + { + process pr (rc, args, -1); + + try + { + ofdstream os (move (pr.out_fd)); + + // 1 is resource ID, 24 is RT_MANIFEST. We also need to escape + // Windows path backslashes. + // + os << "1 24 \""; + + const string& s (mf.string ()); + for (size_t i (0), j;; i = j + 1) + { + j = s.find ('\\', i); + os.write (s.c_str () + i, + (j == string::npos ? s.size () : j) - i); + + if (j == string::npos) + break; + + os.write ("\\\\", 2); + } + + os << "\"" << endl; + + os.close (); + } + catch (const io_error& e) + { + if (pr.wait ()) // Ignore if child failed. + fail << "unable to pipe resource file to " << args[0] + << ": " << e; + } + + run_finish (args, pr); + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + + update = true; // Manifest changed, force update. + } + } + else + { + manifest = move (mf); // Save for link.exe's /MANIFESTINPUT. + + if (mf_mt > mt || mf_cf) + update = true; // Manifest changed, force update. + } + } + + // Check/update the dependency database. + // + depdb dd (tp + ".d"); + + // First should come the rule name/version. + // + if (dd.expect (rule_id) != nullptr) + l4 ([&]{trace << "rule mismatch forcing update of " << t;}); + + lookup ranlib; + + // Then the linker checksum (ar/ranlib or the compiler). + // + if (lt.static_library ()) + { + ranlib = rs["bin.ranlib.path"]; + + const char* rl ( + ranlib + ? cast (rs["bin.ranlib.checksum"]).c_str () + : "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + + if (dd.expect (cast (rs["bin.ar.checksum"])) != nullptr) + l4 ([&]{trace << "ar mismatch forcing update of " << t;}); + + if (dd.expect (rl) != nullptr) + l4 ([&]{trace << "ranlib mismatch forcing update of " << t;}); + } + else + { + // For VC we use link.exe directly. + // + const string& cs ( + cast ( + rs[tsys == "win32-msvc" + ? var_pool["bin.ld.checksum"] + : x_checksum])); + + if (dd.expect (cs) != nullptr) + l4 ([&]{trace << "linker mismatch forcing update of " << t;}); + } + + // Next check the target. While it might be incorporated into the linker + // checksum, it also might not (e.g., VC link.exe). + // + if (dd.expect (ctgt.string ()) != nullptr) + l4 ([&]{trace << "target mismatch forcing update of " << t;}); + + // Start building the command line. While we don't yet know whether we + // will really need it, we need to hash it to find out. So the options + // are to either replicate the exact process twice, first for hashing + // then for building or to go ahead and start building and hash the + // result. The first approach is probably more efficient while the + // second is simpler. Let's got with the simpler for now (actually it's + // kind of a hybrid). + // + cstrings args {nullptr}; // Reserve one for config.bin.ar/config.x. + + // Storage. + // + string soname1, soname2; + strings sargs; + + if (lt.static_library ()) + { + if (tsys == "win32-msvc") ; + else + { + // If the user asked for ranlib, don't try to do its function with + // -s. Some ar implementations (e.g., the LLVM one) don't support + // leading '-'. + // + args.push_back (ranlib ? "rc" : "rcs"); + } + } + else + { + if (tsys == "win32-msvc") + { + // We are using link.exe directly so don't pass the compiler + // options. + } + else + { + append_options (args, t, c_coptions); + append_options (args, t, x_coptions); + append_options (args, tstd); + } + + append_options (args, t, c_loptions); + append_options (args, t, x_loptions); + + // Extra system library dirs (last). + // + // @@ /LIBPATH:, not /LIBPATH + // + assert (sys_lib_dirs_extra <= sys_lib_dirs.size ()); + append_option_values ( + args, + cclass == compiler_class::msvc ? "/LIBPATH:" : "-L", + sys_lib_dirs.begin () + sys_lib_dirs_extra, sys_lib_dirs.end (), + [] (const dir_path& d) {return d.string ().c_str ();}); + + // Handle soname/rpath. + // + if (tclass == "windows") + { + // Limited emulation for Windows with no support for user-defined + // rpaths. + // + auto l (t["bin.rpath"]); + + if (l && !l->empty ()) + fail << ctgt << " does not support rpath"; + } + else + { + // Set soname. + // + if (lt.shared_library ()) + { + const libs_paths& paths (md.libs_data); + const string& leaf (paths.effect_soname ().leaf ().string ()); + + if (tclass == "macos") + { + // With Mac OS 10.5 (Leopard) Apple finally caved in and gave us + // a way to emulate vanilla -rpath. + // + // It may seem natural to do something different on update for + // install. However, if we don't make it @rpath, then the user + // won't be able to use config.bin.rpath for installed libraries. + // + soname1 = "-install_name"; + soname2 = "@rpath/" + leaf; + } + else + soname1 = "-Wl,-soname," + leaf; + + if (!soname1.empty ()) + args.push_back (soname1.c_str ()); + + if (!soname2.empty ()) + args.push_back (soname2.c_str ()); + } + + // Add rpaths. We used to first add the ones specified by the user + // so that they take precedence. But that caused problems if we have + // old versions of the libraries sitting in the rpath location + // (e.g., installed libraries). And if you think about this, it's + // probably correct to prefer libraries that we explicitly imported + // to the ones found via rpath. + // + // Note also that if this is update for install, then we don't add + // rpath of the imported libraries (i.e., we assume they are also + // installed). But we add -rpath-link for some platforms. + // + rpath_libraries (sargs, t, bs, a, li, for_install); + + if (auto l = t["bin.rpath"]) + for (const dir_path& p: cast (l)) + sargs.push_back ("-Wl,-rpath," + p.string ()); + } + } + + // All the options should now be in. Hash them and compare with the db. + // + { + sha256 cs; + + for (size_t i (1); i != args.size (); ++i) + cs.append (args[i]); + + for (size_t i (0); i != sargs.size (); ++i) + cs.append (sargs[i]); + + if (dd.expect (cs.string ()) != nullptr) + l4 ([&]{trace << "options mismatch forcing update of " << t;}); + } + + // Finally, hash and compare the list of input files. + // + // Should we capture actual files or their checksum? The only good + // reason for capturing actual files is diagnostics: we will be able + // to pinpoint exactly what is causing the update. On the other hand, + // the checksum is faster and simpler. And we like simple. + // + { + sha256 cs; + + for (const prerequisite_target& p: t.prerequisite_targets[a]) + { + const target* pt (p.target); + + // If this is bmi*{}, then obj*{} is its ad hoc member. + // + if (modules) + { + if (pt->is_a () || pt->is_a () || pt->is_a ()) + pt = pt->member; + } + + const file* f; + bool la (false), ls (false); + + if ((f = pt->is_a ()) || + (f = pt->is_a ()) || + (f = pt->is_a ()) || + (!lt.static_library () && // @@ UTL: TODO libua to liba link. + ((la = (f = pt->is_a ())) || + (la = (f = pt->is_a ())) || + (ls = (f = pt->is_a ()))))) + { + // Link all the dependent interface libraries (shared) or interface + // and implementation (static), recursively. + // + // Also check if any of them render us out of date. The tricky + // case is, say, a utility library (static) that depends on a + // shared library. When the shared library is updated, there is no + // reason to re-archive the utility but those who link the utility + // have to "see through" the changes in the shared library. + // + if (la || ls) + { + hash_libraries (cs, update, mt, *f, la, p.data, bs, a, li); + f = nullptr; // Timestamp checked by hash_libraries(). + } + else + cs.append (f->path ().string ()); + } + else + f = pt->is_a (); // Consider executable mtime (e.g., linker). + + // Check if this input renders us out of date. + // + if (f != nullptr) + update = update || f->newer (mt); + } + + // Treat it as input for both MinGW and VC (mtime checked above). + // + if (!manifest.empty ()) + cs.append (manifest.string ()); + + // Treat .libs as inputs, not options. + // + if (!lt.static_library ()) + { + hash_options (cs, t, c_libs); + hash_options (cs, t, x_libs); + } + + if (dd.expect (cs.string ()) != nullptr) + l4 ([&]{trace << "file set mismatch forcing update of " << t;}); + } + + // If any of the above checks resulted in a mismatch (different linker, + // options or input file set), or if the database is newer than the + // target (interrupted update) then force the target update. Also note + // this situation in the "from scratch" flag. + // + bool scratch (false); + if (dd.writing () || dd.mtime () > mt) + scratch = update = true; + + dd.close (); + + // (Re)generate pkg-config's .pc file. While the target itself might be + // up-to-date from a previous run, there is no guarantee that .pc exists + // or also up-to-date. So to keep things simple we just regenerate it + // unconditionally. + // + // Also, if you are wondering why don't we just always produce this .pc, + // install or no install, the reason is unless and until we are updating + // for install, we have no idea where-to things will be installed. + // + if (for_install) + { + bool la; + const file* f; + + if ((la = (f = t.is_a ())) || + ( f = t.is_a ())) + pkgconfig_save (a, *f, la); + } + + // If nothing changed, then we are done. + // + if (!update) + return ts; + + // Ok, so we are updating. Finish building the command line. + // + string out, out1, out2, out3; // Storage. + + // Translate paths to relative (to working directory) ones. This results + // in easier to read diagnostics. + // + path relt (relative (tp)); + + const process_path* ld (nullptr); + if (lt.static_library ()) + { + ld = &cast (rs["bin.ar.path"]); + + if (tsys == "win32-msvc") + { + // lib.exe has /LIBPATH but it's not clear/documented what it's used + // for. Perhaps for link-time code generation (/LTCG)? If that's the + // case, then we may need to pass *.loptions. + // + args.push_back ("/NOLOGO"); + + // Add /MACHINE. + // + args.push_back (msvc_machine (cast (rs[x_target_cpu]))); + + out = "/OUT:" + relt.string (); + args.push_back (out.c_str ()); + } + else + args.push_back (relt.string ().c_str ()); + } + else + { + // The options are usually similar enough to handle executables + // and shared libraries together. + // + if (tsys == "win32-msvc") + { + // Using link.exe directly. + // + ld = &cast (rs["bin.ld.path"]); + args.push_back ("/NOLOGO"); + + if (ot == otype::s) + args.push_back ("/DLL"); + + // Add /MACHINE. + // + args.push_back (msvc_machine (cast (rs[x_target_cpu]))); + + // Unless explicitly enabled with /INCREMENTAL, disable incremental + // linking (it is implicitly enabled if /DEBUG is specified). The + // reason is the .ilk file: its name cannot be changed and if we + // have, say, foo.exe and foo.dll, then they will end up stomping on + // each other's .ilk's. + // + // So the idea is to disable it by default but let the user request + // it explicitly if they are sure their project doesn't suffer from + // the above issue. We can also have something like 'incremental' + // config initializer keyword for this. + // + // It might also be a good idea to ask Microsoft to add an option. + // + if (!find_option ("/INCREMENTAL", args, true)) + args.push_back ("/INCREMENTAL:NO"); + + if (cid == compiler_id::clang) + { + // According to Clang's MSVC.cpp, we shall link libcmt.lib (static + // multi-threaded runtime) unless -nostdlib or -nostartfiles is + // specified. + // + if (!find_options ({"-nostdlib", "-nostartfiles"}, t, c_coptions) && + !find_options ({"-nostdlib", "-nostartfiles"}, t, x_coptions)) + args.push_back ("/DEFAULTLIB:libcmt.lib"); + } + + // If you look at the list of libraries Visual Studio links by + // default, it includes everything and a couple of kitchen sinks + // (winspool32.lib, ole32.lib, odbc32.lib, etc) while we want to + // keep our low-level build as pure as possible. However, there seem + // to be fairly essential libraries that are not linked by link.exe + // by default (use /VERBOSE:LIB to see the list). For example, MinGW + // by default links advapi32, shell32, user32, and kernel32. And so + // we follow suit and make sure those are linked. advapi32 and + // kernel32 are already on the default list and we only need to add + // the other two. + // + // The way we are going to do it is via the /DEFAULTLIB option + // rather than specifying the libraries as normal inputs (as VS + // does). This way the user can override our actions with the + // /NODEFAULTLIB option. + // + args.push_back ("/DEFAULTLIB:shell32.lib"); + args.push_back ("/DEFAULTLIB:user32.lib"); + + // Take care of the manifest (will be empty for the DLL). + // + if (!manifest.empty ()) + { + out3 = "/MANIFESTINPUT:"; + out3 += relative (manifest).string (); + args.push_back ("/MANIFEST:EMBED"); + args.push_back (out3.c_str ()); + } + + if (ot == otype::s) + { + // On Windows libs{} is the DLL and its first ad hoc group member + // is the import library. + // + // This will also create the .exp export file. Its name will be + // derived from the import library by changing the extension. + // Lucky for us -- there is no option to name it. + // + auto& imp (t.member->as ()); + out2 = "/IMPLIB:" + relative (imp.path ()).string (); + args.push_back (out2.c_str ()); + } + + // If we have /DEBUG then name the .pdb file. It is either the first + // (exe) or the second (dll) ad hoc group member. + // + if (find_option ("/DEBUG", args, true)) + { + auto& pdb ( + (ot == otype::e ? t.member : t.member->member)->as ()); + out1 = "/PDB:" + relative (pdb.path ()).string (); + args.push_back (out1.c_str ()); + } + + // @@ An executable can have an import library and VS seems to + // always name it. I wonder what would trigger its generation? + // Could it be the presence of export symbols? Yes, link.exe will + // generate the import library iff there are exported symbols. + // Which means there could be a DLL without an import library + // (which we currently don't handle very well). + // + out = "/OUT:" + relt.string (); + args.push_back (out.c_str ()); + } + else + { + switch (cclass) + { + case compiler_class::gcc: + { + ld = &cpath; + + // Add the option that triggers building a shared library and + // take care of any extras (e.g., import library). + // + if (ot == otype::s) + { + if (tclass == "macos") + args.push_back ("-dynamiclib"); + else + args.push_back ("-shared"); + + if (tsys == "mingw32") + { + // On Windows libs{} is the DLL and its first ad hoc group + // member is the import library. + // + auto& imp (t.member->as ()); + out = "-Wl,--out-implib=" + relative (imp.path ()).string (); + args.push_back (out.c_str ()); + } + } + + args.push_back ("-o"); + args.push_back (relt.string ().c_str ()); + break; + } + case compiler_class::msvc: assert (false); + } + } + } + + args[0] = ld->recall_string (); + + // The same logic as during hashing above. + // + for (const prerequisite_target& p: t.prerequisite_targets[a]) + { + const target* pt (p.target); + + if (modules) + { + if (pt->is_a () || pt->is_a () || pt->is_a ()) + pt = pt->member; + } + + const file* f; + bool la (false), ls (false); + + if ((f = pt->is_a ()) || + (f = pt->is_a ()) || + (f = pt->is_a ()) || + (!lt.static_library () && // @@ UTL: TODO libua to liba link. + ((la = (f = pt->is_a ())) || + (la = (f = pt->is_a ())) || + (ls = (f = pt->is_a ()))))) + { + // Link all the dependent interface libraries (shared) or interface + // and implementation (static), recursively. + // + if (la || ls) + append_libraries (sargs, *f, la, p.data, bs, a, li); + else + sargs.push_back (relative (f->path ()).string ()); // string()&& + } + } + + // For MinGW manifest is an object file. + // + if (!manifest.empty () && tsys == "mingw32") + sargs.push_back (relative (manifest).string ()); + + // Shallow-copy sargs to args. Why not do it as we go along pushing into + // sargs? Because of potential reallocations. + // + for (const string& a: sargs) + args.push_back (a.c_str ()); + + if (!lt.static_library ()) + { + append_options (args, t, c_libs); + append_options (args, t, x_libs); + } + + args.push_back (nullptr); + + // Cleanup old (versioned) libraries. + // + if (lt.shared_library ()) + { + const libs_paths& paths (md.libs_data); + const path& p (paths.clean); + + if (!p.empty ()) + try + { + if (verb >= 4) // Seeing this with -V doesn't really add any value. + text << "rm " << p; + + auto rm = [&paths, this] (path&& m, const string&, bool interm) + { + if (!interm) + { + // Filter out paths that have one of the current paths as a + // prefix. + // + auto test = [&m] (const path& p) + { + const string& s (p.string ()); + return s.empty () || m.string ().compare (0, s.size (), s) != 0; + }; + + if (test (*paths.real) && + test (paths.interm) && + test (paths.soname) && + test (paths.link)) + { + try_rmfile (m); + try_rmfile (m + ".d"); + + if (tsys == "win32-msvc") + { + try_rmfile (m.base () += ".ilk"); + try_rmfile (m += ".pdb"); + } + } + } + return true; + }; + + path_search (p, rm, dir_path (), false); // Don't follow symlinks. + } + catch (const system_error&) {} // Ignore errors. + } + else if (lt.static_library ()) + { + // We use relative paths to the object files which means we may end + // up with different ones depending on CWD and some implementation + // treat them as different archive members. So remote the file to + // be sure. Note that we ignore errors leaving it to the achiever + // to complain. + // + if (mt != timestamp_nonexistent) + try_rmfile (relt, true); + } + + if (verb >= 2) + print_process (args); + else if (verb) + text << "ld " << t; + + try + { + // VC tools (both lib.exe and link.exe) send diagnostics to stdout. + // Also, link.exe likes to print various gratuitous messages. So for + // link.exe we redirect stdout to a pipe, filter that noise out, and + // send the rest to stderr. + // + // For lib.exe (and any other insane compiler that may try to pull off + // something like this) we are going to redirect stdout to stderr. For + // sane compilers this should be harmless. + // + bool filter (tsys == "win32-msvc" && !lt.static_library ()); + + process pr (*ld, args.data (), 0, (filter ? -1 : 2)); + + if (filter) + { + try + { + ifdstream is ( + move (pr.in_ofd), fdstream_mode::text, ifdstream::badbit); + + msvc_filter_link (is, t, ot); + + // If anything remains in the stream, send it all to stderr. Note + // that the eof check is important: if the stream is at eof, this + // and all subsequent writes to the diagnostics stream will fail + // (and you won't see a thing). + // + if (is.peek () != ifdstream::traits_type::eof ()) + diag_stream_lock () << is.rdbuf (); + + is.close (); + } + catch (const io_error&) {} // Assume exits with error. + } + + run_finish (args, pr); + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + // In a multi-threaded program that fork()'ed but did not exec(), + // it is unwise to try to do any kind of cleanup (like unwinding + // the stack and running destructors). + // + if (e.child) + exit (1); + + throw failed (); + } + + // Remove the target file if any of the subsequent actions fail. If we + // don't do that, we will end up with a broken build that is up-to-date. + // + auto_rmfile rm (relt); + + if (ranlib) + { + const process_path& rl (cast (ranlib)); + + const char* args[] = { + rl.recall_string (), + relt.string ().c_str (), + nullptr}; + + if (verb >= 2) + print_process (args); + + run (rl, args); + } + + if (tclass == "windows") + { + // For Windows generate rpath-emulating assembly (unless updating for + // install). + // + if (lt.executable () && !for_install) + windows_rpath_assembly (t, bs, a, li, + cast (rs[x_target_cpu]), + rpath_timestamp, + scratch); + } + else if (lt.shared_library ()) + { + // For shared libraries we may need to create a bunch of symlinks. + // + auto ln = [] (const path& f, const path& l) + { + if (verb >= 3) + text << "ln -sf " << f << ' ' << l; + + try + { + if (file_exists (l, false)) // The -f part. + try_rmfile (l); + + mksymlink (f, l); + } + catch (const system_error& e) + { + fail << "unable to create symlink " << l << ": " << e; + } + }; + + const libs_paths& paths (md.libs_data); + + const path& lk (paths.link); + const path& so (paths.soname); + const path& in (paths.interm); + + const path* f (paths.real); + + if (!in.empty ()) {ln (f->leaf (), in); f = ∈} + if (!so.empty ()) {ln (f->leaf (), so); f = &so;} + if (!lk.empty ()) {ln (f->leaf (), lk);} + } + + rm.cancel (); + + // Should we go to the filesystem and get the new mtime? We know the + // file has been modified, so instead just use the current clock time. + // It has the advantage of having the subseconds precision. + // + t.mtime (system_clock::now ()); + return target_state::changed; + } + + target_state link_rule:: + perform_clean (action a, const target& xt) const + { + const file& t (xt.as ()); + ltype lt (link_type (t)); + + if (lt.executable ()) + { + if (tclass == "windows") + { + if (tsys == "mingw32") + return clean_extra ( + a, t, {".d", ".dlls/", ".manifest.o", ".manifest"}); + else + // Assuming it's VC or alike. Clean up .ilk in case the user + // enabled incremental linking (note that .ilk replaces .exe). + // + return clean_extra ( + a, t, {".d", ".dlls/", ".manifest", "-.ilk"}); + } + } + else if (lt.shared_library ()) + { + if (tclass == "windows") + { + // Assuming it's VC or alike. Clean up .exp and .ilk. + // + // Note that .exp is based on the .lib, not .dll name. And with + // versioning their bases may not be the same. + // + if (tsys != "mingw32") + return clean_extra (a, t, {{".d", "-.ilk"}, {"-.exp"}}); + } + else + { + // Here we can have a bunch of symlinks that we need to remove. If + // the paths are empty, then they will be ignored. + // + const libs_paths& paths (t.data ().libs_data); + + return clean_extra (a, t, {".d", + paths.link.string ().c_str (), + paths.soname.string ().c_str (), + paths.interm.string ().c_str ()}); + } + } + // For static library it's just the defaults. + + return clean_extra (a, t, {".d"}); + } + } +} diff --git a/build2/cc/link-rule.hxx b/build2/cc/link-rule.hxx new file mode 100644 index 0000000..ba40410 --- /dev/null +++ b/build2/cc/link-rule.hxx @@ -0,0 +1,166 @@ +// file : build2/cc/link-rule.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BUILD2_CC_LINK_RULE_HXX +#define BUILD2_CC_LINK_RULE_HXX + +#include + +#include +#include + +#include + +#include +#include + +namespace build2 +{ + namespace cc + { + class link_rule: public rule, virtual common + { + public: + link_rule (data&&); + + virtual bool + match (action, target&, const string&) const override; + + virtual recipe + apply (action, target&) const override; + + target_state + perform_update (action, const target&) const; + + target_state + perform_clean (action, const target&) const; + + private: + friend class install_rule; + friend class libux_install_rule; + + // Shared library paths. + // + struct libs_paths + { + // If any (except real) is empty, then it is the same as the next + // one. Except for intermediate, for which empty indicates that it is + // not used. + // + // The libs{} path is always the real path. On Windows the link path + // is the import library. + // + path link; // What we link: libfoo.so + path soname; // SONAME: libfoo-1.so, libfoo.so.1 + path interm; // Intermediate: libfoo.so.1.2 + const path* real; // Real: libfoo.so.1.2.3 + + inline const path& + effect_link () const {return link.empty () ? effect_soname () : link;} + + inline const path& + effect_soname () const {return soname.empty () ? *real : soname;} + + // Cleanup pattern used to remove previous versions. If empty, no + // cleanup is performed. The above (current) names are automatically + // filtered out. + // + path clean; + }; + + libs_paths + derive_libs_paths (file&, const char*, const char*) const; + + struct match_data + { + // The "for install" condition is signalled to us by install_rule when + // it is matched for the update operation. It also verifies that if we + // have already been executed, then it was for install. + // + // This has an interesting implication: it means that this rule cannot + // be used to update targets during match. Specifically, we cannot be + // executed for group resolution purposes (not a problem) nor as part + // of the generated source update. The latter case can be a problem: + // imagine a code generator that itself may need to be updated before + // it can be used to re-generate some out-of-date source code. As an + // aside, note that even if we were somehow able to communicate the + // "for install" in this case, the result of such an update may not + // actually be "usable" (e.g., not runnable because of the missing + // rpaths). There is another prominent case where the result may not + // be usable: cross-compilation. + // + // So the current (admittedly fuzzy) thinking is that a project shall + // not try to use its own build for update since it may not be usable + // (because of cross-compilations, being "for install", etc). Instead, + // it should rely on another, "usable" build of itself (this, BTW, is + // related to bpkg's build-time vs run-time dependencies). + // + optional for_install; + + libs_paths libs_data; + }; + + // Library handling. + // + void + append_libraries (strings&, + const file&, bool, lflags, + const scope&, action, linfo) const; + + void + hash_libraries (sha256&, + bool&, timestamp, + const file&, bool, lflags, + const scope&, action, linfo) const; + + void + rpath_libraries (strings&, + const target&, + const scope&, action, linfo, + bool) const; + + // Windows rpath emulation (windows-rpath.cxx). + // + struct windows_dll + { + const string& dll; + const string* pdb; // NULL if none. + string pdb_storage; + + bool operator< (const windows_dll& y) const {return dll < y.dll;} + }; + + using windows_dlls = std::set; + + timestamp + windows_rpath_timestamp (const file&, + const scope&, + action, linfo) const; + + windows_dlls + windows_rpath_dlls (const file&, const scope&, action, linfo) const; + + void + windows_rpath_assembly (const file&, const scope&, action, linfo, + const string&, + timestamp, + bool) const; + + // Windows-specific (windows-manifest.cxx). + // + pair + windows_manifest (const file&, bool rpath_assembly) const; + + // pkg-config's .pc file generation (pkgconfig.cxx). + // + void + pkgconfig_save (action, const file&, bool) const; + + private: + const string rule_id; + }; + } +} + +#endif // BUILD2_CC_LINK_RULE_HXX diff --git a/build2/cc/link.cxx b/build2/cc/link.cxx deleted file mode 100644 index f69d549..0000000 --- a/build2/cc/link.cxx +++ /dev/null @@ -1,2108 +0,0 @@ -// file : build2/cc/link.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#include - -#include -#include // exit() - -#include -#include // file_exists() - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include // c, pc* -#include - -using namespace std; -using namespace butl; - -namespace build2 -{ - namespace cc - { - using namespace bin; - - link:: - link (data&& d) - : common (move (d)), - rule_id (string (x) += ".link 1") - { - } - - match_result link:: - match (action act, target& t, const string& hint) const - { - tracer trace (x, "link::match"); - - // @@ TODO: - // - // - if path already assigned, verify extension? - // - // @@ Q: - // - // - if there is no .o, are we going to check if the one derived - // from target exist or can be built? A: No. - // What if there is a library. Probably ok if static, not if shared, - // (i.e., a utility library). - // - - ltype lt (link_type (t)); - otype ot (lt.type); - - // If this is a library, link-up to our group (this is the target group - // protocol which means this can be done whether we match or not). - // - if (lt.library ()) - { - if (t.group == nullptr) - t.group = &search (t, - lt.utility ? libu::static_type : lib::static_type, - t.dir, t.out, t.name); - } - - // Scan prerequisites and see if we can work with what we've got. Note - // that X could be C. We handle this by always checking for X first. - // - // Note also that we treat bmi{} as obj{}. - // - bool seen_x (false), seen_c (false), seen_obj (false), seen_lib (false); - - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - if (p.is_a (x_src) || (x_mod != nullptr && p.is_a (*x_mod))) - { - seen_x = seen_x || true; - } - else if (p.is_a ()) - { - seen_c = seen_c || true; - } - else if (p.is_a () || p.is_a ()) - { - seen_obj = seen_obj || true; - } - else if (p.is_a () || p.is_a ()) - { - if (ot != otype::e) - fail << p.type ().name << "{} as prerequisite of " << t; - - seen_obj = seen_obj || true; - } - else if (p.is_a () || p.is_a ()) - { - if (ot != otype::a) - fail << p.type ().name << "{} as prerequisite of " << t; - - seen_obj = seen_obj || true; - } - else if (p.is_a () || p.is_a ()) - { - if (ot != otype::s) - fail << p.type ().name << "{} as prerequisite of " << t; - - seen_obj = seen_obj || true; - } - else if (p.is_a () || - p.is_a () || - p.is_a () || - p.is_a ()) - { - seen_lib = seen_lib || true; - } - // If this is some other c-common source (say C++ in a C rule), then - // it will most definitely need to be compiled but we can't do that. - // - else if (p.is_a ()) - return false; - } - - if (!(seen_x || seen_c || seen_obj || seen_lib)) - return false; - - // We will only chain a C source if there is also an X source or we were - // explicitly told to. - // - if (seen_c && !seen_x && hint < x) - { - l4 ([&]{trace << "C prerequisite without " << x_lang << " or hint";}); - return false; - } - - return true; - } - - auto link:: - derive_libs_paths (file& ls, const char* pfx, const char* sfx) const - -> libs_paths - { - const char* ext (nullptr); - - bool win (tclass == "windows"); - - if (win) - { - if (tsys == "mingw32") - { - if (pfx == nullptr) - pfx = "lib"; - } - - ext = "dll"; - } - else - { - if (pfx == nullptr) - pfx = "lib"; - - if (tclass == "macos") - ext = "dylib"; - else - ext = "so"; - } - - // First sort out which extension we are using. - // - const string& e (ls.derive_extension (ext)); - - auto append_ext = [&e] (path& p) - { - if (!e.empty ()) - { - p += '.'; - p += e; - } - }; - - // Figure out the version. - // - string v; - using verion_map = map; - if (const verion_map* m = cast_null (ls["bin.lib.version"])) - { - // First look for the target system. - // - auto i (m->find (tsys)); - - // Then look for the target class. - // - if (i == m->end ()) - i = m->find (tclass); - - // Then look for the wildcard. Since it is higly unlikely one can have - // a version that will work across platforms, this is only useful to - // say "all others -- no version". - // - if (i == m->end ()) - i = m->find ("*"); - - // At this stage the only platform-specific version we support is the - // "no version" override. - // - if (i != m->end () && !i->second.empty ()) - fail << i->first << "-specific bin.lib.version not yet supported"; - - // Finally look for the platform-independent version. - // - if (i == m->end ()) - i = m->find (""); - - // If we didn't find anything, fail. If the bin.lib.version was - // specified, then it should explicitly handle all the targets. - // - if (i == m->end ()) - fail << "no version for " << ctgt << " in bin.lib.version" << - info << "considere adding " << tsys << "@ or " << tclass - << "@"; - - v = i->second; - } - - // Now determine the paths. - // - path lk, so, in; - - // We start with the basic path. - // - path b (ls.dir); - path cp; // Clean pattern. - { - if (pfx == nullptr || pfx[0] == '\0') - { - b /= ls.name; - } - else - { - b /= pfx; - b += ls.name; - } - - cp = b; - cp += "?*"; // Don't match empty (like the libfoo.so symlink). - - if (sfx != nullptr) - { - b += sfx; - cp += sfx; - } - } - - append_ext (cp); - - // On Windows the real path is to libs{} and the link path is to the - // import library. - // - if (win) - { - // Usually on Windows the import library is called the same as the DLL - // but with the .lib extension. Which means it clashes with the static - // library. Instead of decorating the static library name with ugly - // suffixes (as is customary), let's use the MinGW approach (one must - // admit it's quite elegant) and call it .dll.lib. - // - lk = b; - append_ext (lk); - - libi& li (ls.member->as ()); // Note: libi is locked. - lk = li.derive_path (move (lk), tsys == "mingw32" ? "a" : "lib"); - } - else if (!v.empty ()) - { - lk = b; - append_ext (lk); - } - - if (!v.empty ()) - b += v; - - const path& re (ls.derive_path (move (b))); - - return libs_paths {move (lk), move (so), move (in), re, move (cp)}; - } - - recipe link:: - apply (action act, target& xt) const - { - static_assert (sizeof (link::libs_paths) <= target::data_size, - "insufficient space"); - - tracer trace (x, "link::apply"); - - file& t (xt.as ()); - - const scope& bs (t.base_scope ()); - const scope& rs (*bs.root_scope ()); - - ltype lt (link_type (t)); - otype ot (lt.type); - linfo li (link_info (bs, ot)); - - // Set the library type (C, C++, etc). - // - if (lt.library ()) - t.vars.assign (c_type) = string (x); - - // Derive file name(s) and add ad hoc group members. - // - { - target_lock libi; // Have to hold until after PDB member addition. - - const char* e (nullptr); // Extension. - const char* p (nullptr); // Prefix. - const char* s (nullptr); // Suffix. - - if (lt.utility) - { - // These are all static libraries with names indicating the kind of - // object files they contain (similar to how we name object files - // themselves). We add the 'u' extension to avoid clashes with - // real libraries/import stubs. - // - // libue libhello.u.a hello.exe.u.lib - // libua libhello.a.u.a hello.lib.u.lib - // libus libhello.so.u.a hello.dll.u.lib hello.dylib.u.lib - // - // Note that we currently don't add bin.lib.{prefix,suffix} since - // these are not installed. - // - if (tsys == "win32-msvc") - { - switch (ot) - { - case otype::e: e = "exe.u.lib"; break; - case otype::a: e = "lib.u.lib"; break; - case otype::s: e = "dll.u.lib"; break; - } - } - else - { - p = "lib"; - - if (tsys == "mingw32") - { - switch (ot) - { - case otype::e: e = "exe.u.a"; break; - case otype::a: e = "a.u.a"; break; - case otype::s: e = "dll.u.a"; break; - } - - } - else if (tsys == "darwin") - { - switch (ot) - { - case otype::e: e = "u.a"; break; - case otype::a: e = "a.u.a"; break; - case otype::s: e = "dylib.u.a"; break; - } - } - else - { - switch (ot) - { - case otype::e: e = "u.a"; break; - case otype::a: e = "a.u.a"; break; - case otype::s: e = "so.u.a"; break; - } - } - } - - t.derive_path (e, p, s); - } - else - { - if (auto l = t[ot == otype::e ? "bin.exe.prefix" : "bin.lib.prefix"]) - p = cast (l).c_str (); - if (auto l = t[ot == otype::e ? "bin.exe.suffix" : "bin.lib.suffix"]) - s = cast (l).c_str (); - - switch (ot) - { - case otype::e: - { - if (tclass == "windows") - e = "exe"; - else - e = ""; - - t.derive_path (e, p, s); - break; - } - case otype::a: - { - if (tsys == "win32-msvc") - e = "lib"; - else - { - if (p == nullptr) p = "lib"; - e = "a"; - } - - t.derive_path (e, p, s); - break; - } - case otype::s: - { - // On Windows libs{} is an ad hoc group. The libs{} itself is - // the DLL and we add libi{} import library as its member. - // - if (tclass == "windows") - libi = add_adhoc_member (act, t); - - t.data (derive_libs_paths (t, p, s)); // Cache in target. - - if (libi) - match_recipe (libi, group_recipe); // Set recipe and unlock. - - break; - } - } - - // Add VC's .pdb. Note that we are looking for the link.exe /DEBUG - // option. - // - if (ot != otype::a && - tsys == "win32-msvc" && - (find_option ("/DEBUG", t, c_loptions, true) || - find_option ("/DEBUG", t, x_loptions, true))) - { - // Note: add after the import library if any. - // - target_lock pdb ( - add_adhoc_member (act, t, *bs.find_target_type ("pdb"))); - - // We call it foo.{exe,dll}.pdb rather than just foo.pdb because - // we can have both foo.exe and foo.dll in the same directory. - // - pdb.target->as ().derive_path (t.path (), "pdb"); - - match_recipe (pdb, group_recipe); // Set recipe and unlock. - } - - // Add pkg-config's .pc file. - // - // Note that we do it here regardless of whether we are installing - // or not for two reasons. Firstly, it is not easy to detect this - // situation in apply() since the action may (and is) overridden to - // unconditional install. Secondly, always having the member takes - // care of cleanup automagically. The actual generation happens in - // the install rule. - // - if (ot != otype::e) - { - target_lock pc ( - add_adhoc_member ( - act, t, - ot == otype::a ? pca::static_type : pcs::static_type)); - - // Note that here we always use the lib name prefix, even on - // Windows with VC. The reason is the user needs a consistent name - // across platforms by which they can refere to the library. This - // is also the reason why we use the static/shared suffixes rather - // that a./.lib/.so/.dylib/.dll. - // - pc.target->as ().derive_path (nullptr, - (p == nullptr ? "lib" : p), - s); - - match_recipe (pc, group_recipe); // Set recipe and unlock. - } - } - } - - // Inject dependency on the output directory. - // - inject_fsdir (act, t); - - // Process prerequisites, pass 1: search and match prerequisite - // libraries, search obj/bmi{} targets, and search targets we do rule - // chaining for. - // - // We do libraries first in order to indicate that we will execute these - // targets before matching any of the obj/bmi{}. This makes it safe for - // compile::apply() to unmatch them and therefore not to hinder - // parallelism. - // - // We also create obj/bmi{} chain targets because we need to add - // (similar to lib{}) all the bmi{} as prerequisites to all the other - // obj/bmi{} that we are creating. Note that this doesn't mean that the - // compile rule will actually treat them all as prerequisite targets. - // Rather, they are used to resolve actual module imports. We don't - // really have to search obj{} targets here but it's the same code so we - // do it here to avoid duplication. - // - // Also, when cleaning, we ignore prerequisites that are not in the same - // or a subdirectory of our project root. - // - optional usr_lib_dirs; // Extract lazily. - compile_target_types tt (compile_types (ot)); - - auto skip = [&act, &rs] (const target*& pt) - { - if (act.operation () == clean_id && !pt->dir.sub (rs.out_path ())) - pt = nullptr; - - return pt == nullptr; - }; - - size_t start (t.prerequisite_targets.size ()); - - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - // We pre-allocate a NULL slot for each (potential; see clean) - // prerequisite target. - // - t.prerequisite_targets.push_back (nullptr); - const target*& pt (t.prerequisite_targets.back ()); - - uint8_t m (0); // Mark: lib (0), src (1), mod (2), obj/bmi (3). - - bool mod (x_mod != nullptr && p.is_a (*x_mod)); - - if (mod || p.is_a (x_src) || p.is_a ()) - { - // Rule chaining, part 1. - // - - // Which scope shall we use to resolve the root? Unlikely, but - // possible, the prerequisite is from a different project - // altogether. So we are going to use the target's project. - // - - // If the source came from the lib{} group, then create the obj{} - // group and add the source as a prerequisite of the obj{} group, - // not the obj*{} member. This way we only need one prerequisite - // for, say, both liba{} and libs{}. The same goes for bmi{}. - // - bool group (!p.prerequisite.belongs (t)); // Group's prerequisite. - - const target_type& rtt (mod - ? (group ? bmi::static_type : tt.bmi) - : (group ? obj::static_type : tt.obj)); - - const prerequisite_key& cp (p.key ()); // Source key. - - // Come up with the obj*/bmi*{} target. The source prerequisite - // directory can be relative (to the scope) or absolute. If it is - // relative, then use it as is. If absolute, then translate it to - // the corresponding directory under out_root. While the source - // directory is most likely under src_root, it is also possible it - // is under out_root (e.g., generated source). - // - dir_path d; - { - const dir_path& cpd (*cp.tk.dir); - - if (cpd.relative () || cpd.sub (rs.out_path ())) - d = cpd; - else - { - if (!cpd.sub (rs.src_path ())) - fail << "out of project prerequisite " << cp << - info << "specify corresponding " << rtt.name << "{} " - << "target explicitly"; - - d = rs.out_path () / cpd.leaf (rs.src_path ()); - } - } - - // obj/bmi{} is always in the out tree. Note that currently it could - // be the group -- we will pick a member in part 2 below. - // - pt = &search (t, rtt, d, dir_path (), *cp.tk.name, nullptr, cp.scope); - - // If we shouldn't clean obj{}, then it is fair to assume we - // shouldn't clean the source either (generated source will be in - // the same directory as obj{} and if not, well, go find yourself - // another build system ;-)). - // - if (skip (pt)) - continue; - - m = mod ? 2 : 1; - } - else if (p.is_a () || - p.is_a () || - p.is_a () || - p.is_a ()) - { - // Handle imported libraries. - // - // Note that since the search is rule-specific, we don't cache the - // target in the prerequisite. - // - if (p.proj ()) - pt = search_library ( - act, sys_lib_dirs, usr_lib_dirs, p.prerequisite); - - // The rest is the same basic logic as in search_and_match(). - // - if (pt == nullptr) - pt = &p.search (t); - - if (skip (pt)) - continue; - - // If this is the lib{}/libu{} group, then pick the appropriate - // member. - // - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, act, li); - } - else - { - // If this is the obj{} or bmi{} target group, then pick the - // appropriate member. - // - if (p.is_a ()) pt = &search (t, tt.obj, p.key ()); - else if (p.is_a ()) pt = &search (t, tt.bmi, p.key ()); - else pt = &p.search (t); - - if (skip (pt)) - continue; - - m = 3; - } - - mark (pt, m); - } - - // Match lib{} (the only unmarked) in parallel and wait for completion. - // - match_members (act, t, t.prerequisite_targets, start); - - // Process prerequisites, pass 2: finish rule chaining but don't start - // matching anything yet since that may trigger recursive matching of - // bmi{} targets we haven't completed yet. Hairy, I know. - // - - // Parallel prerequisite_targets loop. - // - size_t i (start), n (t.prerequisite_targets.size ()); - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - const target*& pt (t.prerequisite_targets[i].target); - uintptr_t& pd (t.prerequisite_targets[i++].data); - - if (pt == nullptr) - continue; - - uint8_t m (unmark (pt)); // New mark: completion (1), verfication (2). - - if (m == 3) // obj/bmi{} - m = 1; // Just completion. - else if (m == 1 || m == 2) // Source/module chain. - { - bool mod (m == 2); - - m = 1; - - const target& rt (*pt); - bool group (!p.prerequisite.belongs (t)); // Group's prerequisite. - - // If we have created a obj/bmi{} target group, pick one of its - // members; the rest would be primarily concerned with it. - // - pt = - group - ? &search (t, (mod ? tt.bmi : tt.obj), rt.dir, rt.out, rt.name) - : &rt; - - // If this obj*{} already has prerequisites, then verify they are - // "compatible" with what we are doing here. Otherwise, synthesize - // the dependency. Note that we may also end up synthesizing with - // someone beating up to it. In this case also verify. - // - bool verify (true); - - if (!pt->has_prerequisites ()) - { - prerequisites ps; - ps.push_back (p.as_prerequisite ()); // Source. - - // Add our lib*{} (see the export.* machinery for details) and - // bmi*{} (both original and chained; see module search logic) - // prerequisites. - // - // Note that we don't resolve lib{} to liba{}/libs{} here - // instead leaving it to whomever (e.g., the compile rule) will - // be needing *.export.*. One reason for doing it there is that - // the object target might be specified explicitly by the user - // in which case they will have to specify the set of lib{} - // prerequisites and it's much cleaner to do as lib{} rather - // than liba{}/libs{}. - // - // Initially, we were only adding imported libraries, but there - // is a problem with this approach: the non-imported library - // might depend on the imported one(s) which we will never "see" - // unless we start with this library. - // - // Note: have similar logic in make_module_sidebuild(). - // - size_t j (start); - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - const target* pt (t.prerequisite_targets[j++]); - - if (p.is_a () || - p.is_a () || p.is_a () || p.is_a () || - p.is_a () || p.is_a (tt.bmi)) - { - ps.push_back (p.as_prerequisite ()); - } - else if (x_mod != nullptr && p.is_a (*x_mod)) // Chained module. - { - // Searched during pass 1 but can be NULL or marked. - // - if (pt != nullptr && i != j) // Don't add self (note: both +1). - { - // This is sticky: pt might have come before us and if it - // was a group, then we would have picked up a member. So - // here we may have to "unpick" it. - // - bool group (j < i && !p.prerequisite.belongs (t)); - - unmark (pt); - ps.push_back (prerequisite (group ? *pt->group : *pt)); - } - } - } - - // Note: add to the group, not the member. - // - verify = !rt.prerequisites (move (ps)); - } - - if (verify) - { - // This gets a bit tricky. We need to make sure the source files - // are the same which we can only do by comparing the targets to - // which they resolve. But we cannot search ot's prerequisites -- - // only the rule that matches can. Note, however, that if all this - // works out, then our next step is to match the obj*{} target. If - // things don't work out, then we fail, in which case searching - // and matching speculatively doesn't really hurt. So we start the - // async match here and finish this verification in the "harvest" - // loop below. - // - const target_type& rtt (mod - ? (group ? bmi::static_type : tt.bmi) - : (group ? obj::static_type : tt.obj)); - - bool src (false); - for (prerequisite_member p1: group_prerequisite_members (act, *pt)) - { - // Most of the time we will have just a single source so fast- - // path that case. - // - if (p1.is_a (mod ? *x_mod : x_src) || p1.is_a ()) - { - src = true; - continue; // Check the rest of the prerequisites. - } - - // Ignore some known target types (fsdir, headers, libraries, - // modules). - // - if (p1.is_a () || - p1.is_a () || - p1.is_a () || p1.is_a () || p1.is_a () || - p1.is_a () || - p1.is_a () || p1.is_a () || p1.is_a () || - (p.is_a (mod ? *x_mod : x_src) && x_header (p1)) || - (p.is_a () && p1.is_a ())) - continue; - - fail << "synthesized dependency for prerequisite " << p - << " would be incompatible with existing target " << *pt << - info << "unexpected existing prerequisite type " << p1 << - info << "specify corresponding " << rtt.name << "{} " - << "dependency explicitly"; - } - - if (!src) - fail << "synthesized dependency for prerequisite " << p - << " would be incompatible with existing target " << *pt << - info << "no existing c/" << x_name << " source prerequisite" << - info << "specify corresponding " << rtt.name << "{} " - << "dependency explicitly"; - - m = 2; // Needs verification. - } - } - else // lib*{} - { - // If this is a static library, see if we need to link it whole. - // Note that we have to do it after match since we rely on the - // group link-up. - // - bool u; - if ((u = pt->is_a ()) || pt->is_a ()) - { - const variable& var (var_pool["bin.whole"]); // @@ Cache. - - // See the bin module for the lookup semantics discussion. Note - // that the variable is not overridable so we omit find_override() - // calls. - // - lookup l (p.prerequisite.vars[var]); - - if (!l.defined ()) - l = pt->find_original (var, true).first; - - if (!l.defined ()) - { - bool g (pt->group != nullptr); - l = bs.find_original (var, - &pt->type (), - &pt->name, - (g ? &pt->group->type () : nullptr), - (g ? &pt->group->name : nullptr)).first; - } - - if (l ? cast (*l) : u) - pd |= lflag_whole; - } - } - - mark (pt, m); - } - - // Process prerequisites, pass 3: match everything and verify chains. - // - - // Wait with unlocked phase to allow phase switching. - // - wait_guard wg (target::count_busy (), t.task_count, true); - - for (i = start; i != n; ++i) - { - const target*& pt (t.prerequisite_targets[i]); - - if (pt == nullptr) - continue; - - if (uint8_t m = unmark (pt)) - { - match_async (act, *pt, target::count_busy (), t.task_count); - mark (pt, m); - } - } - - wg.wait (); - - // The "harvest" loop: finish matching the targets we have started. Note - // that we may have bailed out early (thus the parallel i/n for-loop). - // - i = start; - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - const target*& pt (t.prerequisite_targets[i++]); - - // Skipped or not marked for completion. - // - uint8_t m; - if (pt == nullptr || (m = unmark (pt)) == 0) - continue; - - build2::match (act, *pt); - - // Nothing else to do if not marked for verification. - // - if (m == 1) - continue; - - // Finish verifying the existing dependency (which is now matched) - // compared to what we would have synthesized. - // - bool mod (x_mod != nullptr && p.is_a (*x_mod)); - - for (prerequisite_member p1: group_prerequisite_members (act, *pt)) - { - if (p1.is_a (mod ? *x_mod : x_src) || p1.is_a ()) - { - // Searching our own prerequisite is ok, p1 must already be - // resolved. - // - if (&p.search (t) != &p1.search (*pt)) - { - bool group (!p.prerequisite.belongs (t)); - - const target_type& rtt (mod - ? (group ? bmi::static_type : tt.bmi) - : (group ? obj::static_type : tt.obj)); - - fail << "synthesized dependency for prerequisite " << p << " " - << "would be incompatible with existing target " << *pt << - info << "existing prerequisite " << p1 << " does not match " - << p << - info << "specify corresponding " << rtt.name << "{} " - << "dependency explicitly"; - } - - break; - } - } - } - - switch (act) - { - case perform_update_id: return [this] (action a, const target& t) - { - return perform_update (a, t); - }; - case perform_clean_id: return [this] (action a, const target& t) - { - return perform_clean (a, t); - }; - default: return noop_recipe; // Configure update. - } - } - - void link:: - append_libraries (strings& args, - const file& l, bool la, lflags lf, - const scope& bs, action act, linfo li) const - { - // Note: lack of the "small function object" optimization will really - // kill us here since we are called in a loop. - // - auto imp = [] (const file&, bool la) {return la;}; - - auto lib = [&args, this] (const file* l, const string& p, lflags f, bool) - { - if (l != nullptr) - { - // On Windows a shared library is a DLL with the import library as a - // first ad hoc group member. MinGW though can link directly to DLLs - // (see search_library() for details). - // - if (l->member != nullptr && l->is_a () && tclass == "windows") - l = &l->member->as (); - - string p (relative (l->path ()).string ()); - - if (f & lflag_whole) - { - if (tsys == "win32-msvc") - { - p.insert (0, "/WHOLEARCHIVE:"); // Only available from VC14U2. - } - else if (tsys == "darwin") - { - p.insert (0, "-Wl,-force_load,"); - } - else - { - args.push_back ("-Wl,--whole-archive"); - args.push_back (move (p)); - args.push_back ("-Wl,--no-whole-archive"); - return; - } - } - - args.push_back (move (p)); - } - else - args.push_back (p); - }; - - auto opt = [&args, this] ( - const file& l, const string& t, bool com, bool exp) - { - // If we need an interface value, then use the group (lib{}). - // - if (const target* g = exp && l.is_a () ? l.group : &l) - { - const variable& var ( - com - ? (exp ? c_export_loptions : c_loptions) - : (t == x - ? (exp ? x_export_loptions : x_loptions) - : var_pool[t + (exp ? ".export.loptions" : ".loptions")])); - - append_options (args, *g, var); - } - }; - - process_libraries ( - act, bs, li, sys_lib_dirs, l, la, lf, imp, lib, opt, true); - } - - void link:: - hash_libraries (sha256& cs, - bool& update, timestamp mt, - const file& l, bool la, lflags lf, - const scope& bs, action act, linfo li) const - { - auto imp = [] (const file&, bool la) {return la;}; - - struct data - { - sha256& cs; - bool& update; - timestamp mt; - } d {cs, update, mt}; - - auto lib = [&d, this] (const file* l, const string& p, lflags f, bool) - { - if (l != nullptr) - { - // Check if this library renders us out of date. - // - d.update = d.update || l->newer (d.mt); - - // On Windows a shared library is a DLL with the import library as a - // first ad hoc group member. MinGW though can link directly to DLLs - // (see search_library() for details). - // - if (l->member != nullptr && l->is_a () && tclass == "windows") - l = &l->member->as (); - - d.cs.append (f); - d.cs.append (l->path ().string ()); - } - else - d.cs.append (p); - }; - - auto opt = [&cs, this] ( - const file& l, const string& t, bool com, bool exp) - { - if (const target* g = exp && l.is_a () ? l.group : &l) - { - const variable& var ( - com - ? (exp ? c_export_loptions : c_loptions) - : (t == x - ? (exp ? x_export_loptions : x_loptions) - : var_pool[t + (exp ? ".export.loptions" : ".loptions")])); - - hash_options (cs, *g, var); - } - }; - - process_libraries ( - act, bs, li, sys_lib_dirs, l, la, lf, imp, lib, opt, true); - } - - void link:: - rpath_libraries (strings& args, - const target& t, - const scope& bs, - action act, - linfo li, - bool for_install) const - { - // Use -rpath-link on targets that support it (Linux, *BSD). Note - // that we don't really need it for top-level libraries. - // - if (for_install) - { - if (tclass != "linux" && tclass != "bsd") - return; - } - - auto imp = [for_install] (const file& l, bool la) - { - // If we are not installing, then we only need to rpath interface - // libraries (they will include rpath's for their implementations) - // Otherwise, we have to do this recursively. In both cases we also - // want to see through utility libraries. - // - // The rpath-link part is tricky: ideally we would like to get only - // implementations and only of shared libraries. We are not interested - // in interfaces because we are linking their libraries explicitly. - // However, in our model there is no such thing as "implementation - // only"; it is either interface or interface and implementation. So - // we are going to rpath-link all of them which should be harmless - // except for some noise on the command line. - // - // - return (for_install ? !la : false) || l.is_a (); - }; - - // Package the data to keep within the 2-pointer small std::function - // optimization limit. - // - struct - { - strings& args; - bool for_install; - } d {args, for_install}; - - auto lib = [&d, this] (const file* l, const string& f, lflags, bool sys) - { - // We don't rpath system libraries. Why, you may ask? There are many - // good reasons and I have them written on a napkin somewhere... - // - if (sys) - return; - - if (l != nullptr) - { - if (!l->is_a ()) - return; - } - else - { - // This is an absolute path and we need to decide whether it is - // a shared or static library. Doesn't seem there is anything - // better than checking for a platform-specific extension (maybe - // we should cache it somewhere). - // - size_t p (path::traits::find_extension (f)); - - if (p == string::npos) - return; - - ++p; // Skip dot. - - bool c (true); - const char* e; - - if (tclass == "windows") {e = "dll"; c = false;} - else if (tsys == "darwin") e = "dylib"; - else e = "so"; - - if ((c - ? f.compare (p, string::npos, e) - : casecmp (f.c_str () + p, e)) != 0) - return; - } - - // Ok, if we are here then it means we have a non-system, shared - // library and its absolute path is in f. - // - string o (d.for_install ? "-Wl,-rpath-link," : "-Wl,-rpath,"); - - size_t p (path::traits::rfind_separator (f)); - assert (p != string::npos); - - o.append (f, 0, (p != 0 ? p : 1)); // Don't include trailing slash. - d.args.push_back (move (o)); - }; - - // In case we don't have the "small function object" optimization. - // - const function impf (imp); - const function libf (lib); - - for (auto pt: t.prerequisite_targets) - { - bool a; - const file* f; - - if ((a = (f = pt->is_a ())) || - (a = (f = pt->is_a ())) || - ( f = pt->is_a ())) - { - if (!for_install && !a) - { - // Top-level sharen library dependency. It is either matched or - // imported so should be a cc library. - // - if (!cast_false (f->vars[c_system])) - args.push_back ( - "-Wl,-rpath," + f->path ().directory ().string ()); - } - - process_libraries (act, bs, li, sys_lib_dirs, - *f, a, pt.data, - impf, libf, nullptr); - } - } - } - - // Filter link.exe noise (msvc.cxx). - // - void - msvc_filter_link (ifdstream&, const file&, otype); - - // Translate target CPU to the link.exe/lib.exe /MACHINE option. - // - const char* - msvc_machine (const string& cpu); // msvc.cxx - - target_state link:: - perform_update (action act, const target& xt) const - { - tracer trace (x, "link::perform_update"); - - auto oop (act.outer_operation ()); - bool for_install (oop == install_id || oop == uninstall_id); - - const file& t (xt.as ()); - const path& tp (t.path ()); - - const scope& bs (t.base_scope ()); - const scope& rs (*bs.root_scope ()); - - ltype lt (link_type (t)); - otype ot (lt.type); - linfo li (link_info (bs, ot)); - - // Update prerequisites. We determine if any relevant ones render us - // out-of-date manually below. - // - bool update (false); - timestamp mt (t.load_mtime ()); - target_state ts (straight_execute_prerequisites (act, t)); - - // If targeting Windows, take care of the manifest. - // - path manifest; // Manifest itself (msvc) or compiled object file. - timestamp rpath_timestamp (timestamp_nonexistent); // DLLs timestamp. - - if (lt.executable () && tclass == "windows") - { - // First determine if we need to add our rpath emulating assembly. The - // assembly itself is generated later, after updating the target. Omit - // it if we are updating for install. - // - if (!for_install) - rpath_timestamp = windows_rpath_timestamp (t, bs, act, li); - - pair p ( - windows_manifest (t, - rpath_timestamp != timestamp_nonexistent)); - - path& mf (p.first); - bool mf_cf (p.second); // Changed flag (timestamp resolution). - - timestamp mf_mt (file_mtime (mf)); - - if (tsys == "mingw32") - { - // Compile the manifest into the object file with windres. While we - // are going to synthesize an .rc file to pipe to windres' stdin, we - // will still use .manifest to check if everything is up-to-date. - // - manifest = mf + ".o"; - - if (mf_mt > file_mtime (manifest) || mf_cf) - { - path of (relative (manifest)); - - const process_path& rc (cast (rs["bin.rc.path"])); - - // @@ Would be good to add this to depdb (e.g,, rc changes). - // - const char* args[] = { - rc.recall_string (), - "--input-format=rc", - "--output-format=coff", - "-o", of.string ().c_str (), - nullptr}; - - if (verb >= 3) - print_process (args); - - try - { - process pr (rc, args, -1); - - try - { - ofdstream os (move (pr.out_fd)); - - // 1 is resource ID, 24 is RT_MANIFEST. We also need to escape - // Windows path backslashes. - // - os << "1 24 \""; - - const string& s (mf.string ()); - for (size_t i (0), j;; i = j + 1) - { - j = s.find ('\\', i); - os.write (s.c_str () + i, - (j == string::npos ? s.size () : j) - i); - - if (j == string::npos) - break; - - os.write ("\\\\", 2); - } - - os << "\"" << endl; - - os.close (); - } - catch (const io_error& e) - { - if (pr.wait ()) // Ignore if child failed. - fail << "unable to pipe resource file to " << args[0] - << ": " << e; - } - - run_finish (args, pr); - } - catch (const process_error& e) - { - error << "unable to execute " << args[0] << ": " << e; - - if (e.child) - exit (1); - - throw failed (); - } - - update = true; // Manifest changed, force update. - } - } - else - { - manifest = move (mf); // Save for link.exe's /MANIFESTINPUT. - - if (mf_mt > mt || mf_cf) - update = true; // Manifest changed, force update. - } - } - - // Check/update the dependency database. - // - depdb dd (tp + ".d"); - - // First should come the rule name/version. - // - if (dd.expect (rule_id) != nullptr) - l4 ([&]{trace << "rule mismatch forcing update of " << t;}); - - lookup ranlib; - - // Then the linker checksum (ar/ranlib or the compiler). - // - if (lt.static_library ()) - { - ranlib = rs["bin.ranlib.path"]; - - const char* rl ( - ranlib - ? cast (rs["bin.ranlib.checksum"]).c_str () - : "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); - - if (dd.expect (cast (rs["bin.ar.checksum"])) != nullptr) - l4 ([&]{trace << "ar mismatch forcing update of " << t;}); - - if (dd.expect (rl) != nullptr) - l4 ([&]{trace << "ranlib mismatch forcing update of " << t;}); - } - else - { - // For VC we use link.exe directly. - // - const string& cs ( - cast ( - rs[tsys == "win32-msvc" - ? var_pool["bin.ld.checksum"] - : x_checksum])); - - if (dd.expect (cs) != nullptr) - l4 ([&]{trace << "linker mismatch forcing update of " << t;}); - } - - // Next check the target. While it might be incorporated into the linker - // checksum, it also might not (e.g., VC link.exe). - // - if (dd.expect (ctgt.string ()) != nullptr) - l4 ([&]{trace << "target mismatch forcing update of " << t;}); - - // Start building the command line. While we don't yet know whether we - // will really need it, we need to hash it to find out. So the options - // are to either replicate the exact process twice, first for hashing - // then for building or to go ahead and start building and hash the - // result. The first approach is probably more efficient while the - // second is simpler. Let's got with the simpler for now (actually it's - // kind of a hybrid). - // - cstrings args {nullptr}; // Reserve one for config.bin.ar/config.x. - - // Storage. - // - string soname1, soname2; - strings sargs; - - if (lt.static_library ()) - { - if (tsys == "win32-msvc") ; - else - { - // If the user asked for ranlib, don't try to do its function with - // -s. Some ar implementations (e.g., the LLVM one) don't support - // leading '-'. - // - args.push_back (ranlib ? "rc" : "rcs"); - } - } - else - { - if (tsys == "win32-msvc") - { - // We are using link.exe directly so don't pass the compiler - // options. - } - else - { - append_options (args, t, c_coptions); - append_options (args, t, x_coptions); - append_options (args, tstd); - } - - append_options (args, t, c_loptions); - append_options (args, t, x_loptions); - - // Extra system library dirs (last). - // - // @@ /LIBPATH:, not /LIBPATH - // - assert (sys_lib_dirs_extra <= sys_lib_dirs.size ()); - append_option_values ( - args, - cclass == compiler_class::msvc ? "/LIBPATH:" : "-L", - sys_lib_dirs.begin () + sys_lib_dirs_extra, sys_lib_dirs.end (), - [] (const dir_path& d) {return d.string ().c_str ();}); - - // Handle soname/rpath. - // - if (tclass == "windows") - { - // Limited emulation for Windows with no support for user-defined - // rpaths. - // - auto l (t["bin.rpath"]); - - if (l && !l->empty ()) - fail << ctgt << " does not support rpath"; - } - else - { - // Set soname. - // - if (lt.shared_library ()) - { - const libs_paths& paths (t.data ()); - const string& leaf (paths.effect_soname ().leaf ().string ()); - - if (tclass == "macos") - { - // With Mac OS 10.5 (Leopard) Apple finally caved in and gave us - // a way to emulate vanilla -rpath. - // - // It may seem natural to do something different on update for - // install. However, if we don't make it @rpath, then the user - // won't be able to use config.bin.rpath for installed libraries. - // - soname1 = "-install_name"; - soname2 = "@rpath/" + leaf; - } - else - soname1 = "-Wl,-soname," + leaf; - - if (!soname1.empty ()) - args.push_back (soname1.c_str ()); - - if (!soname2.empty ()) - args.push_back (soname2.c_str ()); - } - - // Add rpaths. We used to first add the ones specified by the user - // so that they take precedence. But that caused problems if we have - // old versions of the libraries sitting in the rpath location - // (e.g., installed libraries). And if you think about this, it's - // probably correct to prefer libraries that we explicitly imported - // to the ones found via rpath. - // - // Note also that if this is update for install, then we don't add - // rpath of the imported libraries (i.e., we assume they are also - // installed). But we add -rpath-link for some platforms. - // - rpath_libraries (sargs, t, bs, act, li, for_install); - - if (auto l = t["bin.rpath"]) - for (const dir_path& p: cast (l)) - sargs.push_back ("-Wl,-rpath," + p.string ()); - } - } - - // All the options should now be in. Hash them and compare with the db. - // - { - sha256 cs; - - for (size_t i (1); i != args.size (); ++i) - cs.append (args[i]); - - for (size_t i (0); i != sargs.size (); ++i) - cs.append (sargs[i]); - - if (dd.expect (cs.string ()) != nullptr) - l4 ([&]{trace << "options mismatch forcing update of " << t;}); - } - - // Finally, hash and compare the list of input files. - // - // Should we capture actual files or their checksum? The only good - // reason for capturing actual files is diagnostics: we will be able - // to pinpoint exactly what is causing the update. On the other hand, - // the checksum is faster and simpler. And we like simple. - // - { - sha256 cs; - - for (auto p: t.prerequisite_targets) - { - const target* pt (p.target); - - // If this is bmi*{}, then obj*{} is its ad hoc member. - // - if (modules) - { - if (pt->is_a () || pt->is_a () || pt->is_a ()) - pt = pt->member; - } - - const file* f; - bool a (false), s (false); - - if ((f = pt->is_a ()) || - (f = pt->is_a ()) || - (f = pt->is_a ()) || - (!lt.static_library () && // @@ UTL: TODO libua to liba link. - ((a = (f = pt->is_a ())) || - (a = (f = pt->is_a ())) || - (s = (f = pt->is_a ()))))) - { - // Link all the dependent interface libraries (shared) or interface - // and implementation (static), recursively. - // - // Also check if any of them render us out of date. The tricky - // case is, say, a utility library (static) that depends on a - // shared library. When the shared library is updated, there is no - // reason to re-archive the utility but those who link the utility - // have to "see through" the changes in the shared library. - // - if (a || s) - { - hash_libraries (cs, update, mt, *f, a, p.data, bs, act, li); - f = nullptr; // Timestamp checked by hash_libraries(). - } - else - cs.append (f->path ().string ()); - } - else - f = pt->is_a (); // Consider executable mtime (e.g., linker). - - // Check if this input renders us out of date. - // - if (f != nullptr) - update = update || f->newer (mt); - } - - // Treat it as input for both MinGW and VC (mtime checked above). - // - if (!manifest.empty ()) - cs.append (manifest.string ()); - - // Treat .libs as inputs, not options. - // - if (!lt.static_library ()) - { - hash_options (cs, t, c_libs); - hash_options (cs, t, x_libs); - } - - if (dd.expect (cs.string ()) != nullptr) - l4 ([&]{trace << "file set mismatch forcing update of " << t;}); - } - - // If any of the above checks resulted in a mismatch (different linker, - // options or input file set), or if the database is newer than the - // target (interrupted update) then force the target update. Also note - // this situation in the "from scratch" flag. - // - bool scratch (false); - if (dd.writing () || dd.mtime () > mt) - scratch = update = true; - - dd.close (); - - // (Re)generate pkg-config's .pc file. While the target itself might be - // up-to-date from a previous run, there is no guarantee that .pc exists - // or also up-to-date. So to keep things simple we just regenerate it - // unconditionally. - // - // Also, if you are wondering why don't we just always produce this .pc, - // install or no install, the reason is unless and until we are updating - // for install, we have no idea where to things will be installed. - // - if (for_install) - { - bool a; - const file* f; - - if ((a = (f = t.is_a ())) || - ( f = t.is_a ())) - { - // @@ Hack: this should really be in install:update_extra() where we - // (should) what we are installing and what not. - // - if (rs["install.root"]) - pkgconfig_save (act, *f, a); - } - } - - // If nothing changed, then we are done. - // - if (!update) - return ts; - - // Ok, so we are updating. Finish building the command line. - // - string out, out1, out2, out3; // Storage. - - // Translate paths to relative (to working directory) ones. This results - // in easier to read diagnostics. - // - path relt (relative (tp)); - - const process_path* ld (nullptr); - if (lt.static_library ()) - { - ld = &cast (rs["bin.ar.path"]); - - if (tsys == "win32-msvc") - { - // lib.exe has /LIBPATH but it's not clear/documented what it's used - // for. Perhaps for link-time code generation (/LTCG)? If that's the - // case, then we may need to pass *.loptions. - // - args.push_back ("/NOLOGO"); - - // Add /MACHINE. - // - args.push_back (msvc_machine (cast (rs[x_target_cpu]))); - - out = "/OUT:" + relt.string (); - args.push_back (out.c_str ()); - } - else - args.push_back (relt.string ().c_str ()); - } - else - { - // The options are usually similar enough to handle executables - // and shared libraries together. - // - if (tsys == "win32-msvc") - { - // Using link.exe directly. - // - ld = &cast (rs["bin.ld.path"]); - args.push_back ("/NOLOGO"); - - if (ot == otype::s) - args.push_back ("/DLL"); - - // Add /MACHINE. - // - args.push_back (msvc_machine (cast (rs[x_target_cpu]))); - - // Unless explicitly enabled with /INCREMENTAL, disable incremental - // linking (it is implicitly enabled if /DEBUG is specified). The - // reason is the .ilk file: its name cannot be changed and if we - // have, say, foo.exe and foo.dll, then they will end up stomping on - // each other's .ilk's. - // - // So the idea is to disable it by default but let the user request - // it explicitly if they are sure their project doesn't suffer from - // the above issue. We can also have something like 'incremental' - // config initializer keyword for this. - // - // It might also be a good idea to ask Microsoft to add an option. - // - if (!find_option ("/INCREMENTAL", args, true)) - args.push_back ("/INCREMENTAL:NO"); - - if (cid == compiler_id::clang) - { - // According to Clang's MSVC.cpp, we shall link libcmt.lib (static - // multi-threaded runtime) unless -nostdlib or -nostartfiles is - // specified. - // - if (!find_options ({"-nostdlib", "-nostartfiles"}, t, c_coptions) && - !find_options ({"-nostdlib", "-nostartfiles"}, t, x_coptions)) - args.push_back ("/DEFAULTLIB:libcmt.lib"); - } - - // If you look at the list of libraries Visual Studio links by - // default, it includes everything and a couple of kitchen sinks - // (winspool32.lib, ole32.lib, odbc32.lib, etc) while we want to - // keep our low-level build as pure as possible. However, there seem - // to be fairly essential libraries that are not linked by link.exe - // by default (use /VERBOSE:LIB to see the list). For example, MinGW - // by default links advapi32, shell32, user32, and kernel32. And so - // we follow suit and make sure those are linked. advapi32 and - // kernel32 are already on the default list and we only need to add - // the other two. - // - // The way we are going to do it is via the /DEFAULTLIB option - // rather than specifying the libraries as normal inputs (as VS - // does). This way the user can override our actions with the - // /NODEFAULTLIB option. - // - args.push_back ("/DEFAULTLIB:shell32.lib"); - args.push_back ("/DEFAULTLIB:user32.lib"); - - // Take care of the manifest (will be empty for the DLL). - // - if (!manifest.empty ()) - { - out3 = "/MANIFESTINPUT:"; - out3 += relative (manifest).string (); - args.push_back ("/MANIFEST:EMBED"); - args.push_back (out3.c_str ()); - } - - if (ot == otype::s) - { - // On Windows libs{} is the DLL and its first ad hoc group member - // is the import library. - // - // This will also create the .exp export file. Its name will be - // derived from the import library by changing the extension. - // Lucky for us -- there is no option to name it. - // - auto& imp (t.member->as ()); - out2 = "/IMPLIB:" + relative (imp.path ()).string (); - args.push_back (out2.c_str ()); - } - - // If we have /DEBUG then name the .pdb file. It is either the first - // (exe) or the second (dll) ad hoc group member. - // - if (find_option ("/DEBUG", args, true)) - { - auto& pdb ( - (ot == otype::e ? t.member : t.member->member)->as ()); - out1 = "/PDB:" + relative (pdb.path ()).string (); - args.push_back (out1.c_str ()); - } - - // @@ An executable can have an import library and VS seems to - // always name it. I wonder what would trigger its generation? - // Could it be the presence of export symbols? Yes, link.exe will - // generate the import library iff there are exported symbols. - // Which means there could be a DLL without an import library - // (which we currently don't handle very well). - // - out = "/OUT:" + relt.string (); - args.push_back (out.c_str ()); - } - else - { - switch (cclass) - { - case compiler_class::gcc: - { - ld = &cpath; - - // Add the option that triggers building a shared library and - // take care of any extras (e.g., import library). - // - if (ot == otype::s) - { - if (tclass == "macos") - args.push_back ("-dynamiclib"); - else - args.push_back ("-shared"); - - if (tsys == "mingw32") - { - // On Windows libs{} is the DLL and its first ad hoc group - // member is the import library. - // - auto& imp (t.member->as ()); - out = "-Wl,--out-implib=" + relative (imp.path ()).string (); - args.push_back (out.c_str ()); - } - } - - args.push_back ("-o"); - args.push_back (relt.string ().c_str ()); - break; - } - case compiler_class::msvc: assert (false); - } - } - } - - args[0] = ld->recall_string (); - - // The same logic as during hashing above. - // - for (auto p: t.prerequisite_targets) - { - const target* pt (p.target); - - if (modules) - { - if (pt->is_a () || pt->is_a () || pt->is_a ()) - pt = pt->member; - } - - const file* f; - bool a (false), s (false); - - if ((f = pt->is_a ()) || - (f = pt->is_a ()) || - (f = pt->is_a ()) || - (!lt.static_library () && // @@ UTL: TODO libua to liba link. - ((a = (f = pt->is_a ())) || - (a = (f = pt->is_a ())) || - (s = (f = pt->is_a ()))))) - { - // Link all the dependent interface libraries (shared) or interface - // and implementation (static), recursively. - // - if (a || s) - append_libraries (sargs, *f, a, p.data, bs, act, li); - else - sargs.push_back (relative (f->path ()).string ()); // string()&& - } - } - - // For MinGW manifest is an object file. - // - if (!manifest.empty () && tsys == "mingw32") - sargs.push_back (relative (manifest).string ()); - - // Shallow-copy sargs to args. Why not do it as we go along pushing into - // sargs? Because of potential reallocations. - // - for (const string& a: sargs) - args.push_back (a.c_str ()); - - if (!lt.static_library ()) - { - append_options (args, t, c_libs); - append_options (args, t, x_libs); - } - - args.push_back (nullptr); - - // Cleanup old (versioned) libraries. - // - if (lt.shared_library ()) - { - const libs_paths& paths (t.data ()); - const path& p (paths.clean); - - if (!p.empty ()) - try - { - if (verb >= 4) // Seeing this with -V doesn't really add any value. - text << "rm " << p; - - auto rm = [&paths, this] (path&& m, const string&, bool interm) - { - if (!interm) - { - // Filter out paths that have one of the current paths as a - // prefix. - // - auto test = [&m] (const path& p) - { - const string& s (p.string ()); - return s.empty () || m.string ().compare (0, s.size (), s) != 0; - }; - - if (test (paths.real) && - test (paths.interm) && - test (paths.soname) && - test (paths.link)) - { - try_rmfile (m); - try_rmfile (m + ".d"); - - if (tsys == "win32-msvc") - { - try_rmfile (m.base () += ".ilk"); - try_rmfile (m += ".pdb"); - } - } - } - return true; - }; - - path_search (p, rm, dir_path (), false); // Don't follow symlinks. - } - catch (const system_error&) {} // Ignore errors. - } - else if (lt.static_library ()) - { - // We use relative paths to the object files which means we may end - // up with different ones depending on CWD and some implementation - // treat them as different archive members. So remote the file to - // be sure. Note that we ignore errors leaving it to the achiever - // to complain. - // - if (mt != timestamp_nonexistent) - try_rmfile (relt, true); - } - - if (verb >= 2) - print_process (args); - else if (verb) - text << "ld " << t; - - try - { - // VC tools (both lib.exe and link.exe) send diagnostics to stdout. - // Also, link.exe likes to print various gratuitous messages. So for - // link.exe we redirect stdout to a pipe, filter that noise out, and - // send the rest to stderr. - // - // For lib.exe (and any other insane compiler that may try to pull off - // something like this) we are going to redirect stdout to stderr. For - // sane compilers this should be harmless. - // - bool filter (tsys == "win32-msvc" && !lt.static_library ()); - - process pr (*ld, args.data (), 0, (filter ? -1 : 2)); - - if (filter) - { - try - { - ifdstream is ( - move (pr.in_ofd), fdstream_mode::text, ifdstream::badbit); - - msvc_filter_link (is, t, ot); - - // If anything remains in the stream, send it all to stderr. Note - // that the eof check is important: if the stream is at eof, this - // and all subsequent writes to the diagnostics stream will fail - // (and you won't see a thing). - // - if (is.peek () != ifdstream::traits_type::eof ()) - diag_stream_lock () << is.rdbuf (); - - is.close (); - } - catch (const io_error&) {} // Assume exits with error. - } - - run_finish (args, pr); - } - catch (const process_error& e) - { - error << "unable to execute " << args[0] << ": " << e; - - // In a multi-threaded program that fork()'ed but did not exec(), - // it is unwise to try to do any kind of cleanup (like unwinding - // the stack and running destructors). - // - if (e.child) - exit (1); - - throw failed (); - } - - // Remove the target file if any of the subsequent actions fail. If we - // don't do that, we will end up with a broken build that is up-to-date. - // - auto_rmfile rm (relt); - - if (ranlib) - { - const process_path& rl (cast (ranlib)); - - const char* args[] = { - rl.recall_string (), - relt.string ().c_str (), - nullptr}; - - if (verb >= 2) - print_process (args); - - run (rl, args); - } - - if (tclass == "windows") - { - // For Windows generate rpath-emulating assembly (unless updating for - // install). - // - if (lt.executable () && !for_install) - windows_rpath_assembly (t, bs, act, li, - cast (rs[x_target_cpu]), - rpath_timestamp, - scratch); - } - else if (lt.shared_library ()) - { - // For shared libraries we may need to create a bunch of symlinks. - // - auto ln = [] (const path& f, const path& l) - { - if (verb >= 3) - text << "ln -sf " << f << ' ' << l; - - try - { - if (file_exists (l, false)) // The -f part. - try_rmfile (l); - - mksymlink (f, l); - } - catch (const system_error& e) - { - fail << "unable to create symlink " << l << ": " << e; - } - }; - - const libs_paths& paths (t.data ()); - - const path& lk (paths.link); - const path& so (paths.soname); - const path& in (paths.interm); - - const path* f (&paths.real); - - if (!in.empty ()) {ln (f->leaf (), in); f = ∈} - if (!so.empty ()) {ln (f->leaf (), so); f = &so;} - if (!lk.empty ()) {ln (f->leaf (), lk);} - } - - rm.cancel (); - - // Should we go to the filesystem and get the new mtime? We know the - // file has been modified, so instead just use the current clock time. - // It has the advantage of having the subseconds precision. - // - t.mtime (system_clock::now ()); - return target_state::changed; - } - - target_state link:: - perform_clean (action act, const target& xt) const - { - const file& t (xt.as ()); - ltype lt (link_type (t)); - - if (lt.executable ()) - { - if (tclass == "windows") - { - if (tsys == "mingw32") - return clean_extra ( - act, t, {".d", ".dlls/", ".manifest.o", ".manifest"}); - else - // Assuming it's VC or alike. Clean up .ilk in case the user - // enabled incremental linking (note that .ilk replaces .exe). - // - return clean_extra ( - act, t, {".d", ".dlls/", ".manifest", "-.ilk"}); - } - } - else if (lt.shared_library ()) - { - if (tclass == "windows") - { - // Assuming it's VC or alike. Clean up .exp and .ilk. - // - // Note that .exp is based on the .lib, not .dll name. And with - // versioning their bases may not be the same. - // - if (tsys != "mingw32") - return clean_extra (act, t, {{".d", "-.ilk"}, {"-.exp"}}); - } - else - { - // Here we can have a bunch of symlinks that we need to remove. If - // the paths are empty, then they will be ignored. - // - const libs_paths& paths (t.data ()); - - return clean_extra (act, t, {".d", - paths.link.string ().c_str (), - paths.soname.string ().c_str (), - paths.interm.string ().c_str ()}); - } - } - // For static library it's just the defaults. - - return clean_extra (act, t, {".d"}); - } - } -} diff --git a/build2/cc/link.hxx b/build2/cc/link.hxx deleted file mode 100644 index c26102d..0000000 --- a/build2/cc/link.hxx +++ /dev/null @@ -1,137 +0,0 @@ -// file : build2/cc/link.hxx -*- C++ -*- -// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#ifndef BUILD2_CC_LINK_HXX -#define BUILD2_CC_LINK_HXX - -#include - -#include -#include - -#include - -#include -#include - -namespace build2 -{ - namespace cc - { - class link: public rule, virtual common - { - public: - link (data&&); - - virtual match_result - match (action, target&, const string&) const override; - - virtual recipe - apply (action, target&) const override; - - target_state - perform_update (action, const target&) const; - - target_state - perform_clean (action, const target&) const; - - private: - friend class file_install; - friend class alias_install; - - // Shared library paths. - // - struct libs_paths - { - // If any (except real) is empty, then it is the same as the next - // one. Except for intermediate, for which empty indicates that it is - // not used. - // - // The libs{} path is always the real path. On Windows the link path - // is the import library. - // - const path link; // What we link: libfoo.so - const path soname; // SONAME: libfoo-1.so, libfoo.so.1 - const path interm; // Intermediate: libfoo.so.1.2 - const path& real; // Real: libfoo.so.1.2.3 - - inline const path& - effect_link () const {return link.empty () ? effect_soname () : link;} - - inline const path& - effect_soname () const {return soname.empty () ? real : soname;} - - // Cleanup pattern used to remove previous versions. If empty, no - // cleanup is performed. The above (current) names are automatically - // filtered out. - // - const path clean; - }; - - libs_paths - derive_libs_paths (file&, const char*, const char*) const; - - // Library handling. - // - void - append_libraries (strings&, - const file&, bool, lflags, - const scope&, action, linfo) const; - - void - hash_libraries (sha256&, - bool&, timestamp, - const file&, bool, lflags, - const scope&, action, linfo) const; - - void - rpath_libraries (strings&, - const target&, - const scope&, action, linfo, - bool) const; - - // Windows rpath emulation (windows-rpath.cxx). - // - struct windows_dll - { - const string& dll; - const string* pdb; // NULL if none. - string pdb_storage; - - bool operator< (const windows_dll& y) const {return dll < y.dll;} - }; - - using windows_dlls = std::set; - - timestamp - windows_rpath_timestamp (const file&, - const scope&, - action, linfo) const; - - windows_dlls - windows_rpath_dlls (const file&, const scope&, action, linfo) const; - - void - windows_rpath_assembly (const file&, const scope&, action, linfo, - const string&, - timestamp, - bool) const; - - // Windows-specific (windows-manifest.cxx). - // - pair - windows_manifest (const file&, bool rpath_assembly) const; - - // pkg-config's .pc file generation (pkgconfig.cxx). - // - void - pkgconfig_save (action, const file&, bool) const; - - private: - const string rule_id; - }; - } -} - -#endif // BUILD2_CC_LINK_HXX diff --git a/build2/cc/module.cxx b/build2/cc/module.cxx index c56bca9..ae64220 100644 --- a/build2/cc/module.cxx +++ b/build2/cc/module.cxx @@ -499,8 +499,8 @@ namespace build2 // We register for configure so that we detect unresolved imports // during configuration rather that later, e.g., during update. // - const compile& cr (*this); - const link& lr (*this); + const compile_rule& cr (*this); + const link_rule& lr (*this); r.insert (perform_update_id, x_compile, cr); r.insert (perform_clean_id, x_compile, cr); @@ -559,26 +559,27 @@ namespace build2 // if (install_loaded) { - const file_install& fr (*this); - const alias_install& ar (*this); + const install_rule& ir (*this); - r.insert (perform_install_id, x_install, fr); - r.insert (perform_uninstall_id, x_uninstall, fr); + r.insert (perform_install_id, x_install, ir); + r.insert (perform_uninstall_id, x_uninstall, ir); - r.insert (perform_install_id, x_install, fr); - r.insert (perform_uninstall_id, x_uninstall, fr); + r.insert (perform_install_id, x_install, ir); + r.insert (perform_uninstall_id, x_uninstall, ir); - r.insert (perform_install_id, x_install, fr); - r.insert (perform_uninstall_id, x_uninstall, fr); + r.insert (perform_install_id, x_install, ir); + r.insert (perform_uninstall_id, x_uninstall, ir); - r.insert (perform_install_id, x_install, ar); - r.insert (perform_uninstall_id, x_uninstall, ar); + const libux_install_rule& lr (*this); - r.insert (perform_install_id, x_install, ar); - r.insert (perform_uninstall_id, x_uninstall, ar); + r.insert (perform_install_id, x_install, lr); + r.insert (perform_uninstall_id, x_uninstall, lr); - r.insert (perform_install_id, x_install, ar); - r.insert (perform_uninstall_id, x_uninstall, ar); + r.insert (perform_install_id, x_install, lr); + r.insert (perform_uninstall_id, x_uninstall, lr); + + r.insert (perform_install_id, x_install, lr); + r.insert (perform_uninstall_id, x_uninstall, lr); } } } diff --git a/build2/cc/module.hxx b/build2/cc/module.hxx index de61611..58aa184 100644 --- a/build2/cc/module.hxx +++ b/build2/cc/module.hxx @@ -13,9 +13,9 @@ #include -#include -#include -#include +#include +#include +#include namespace build2 { @@ -76,19 +76,19 @@ namespace build2 }; class module: public module_base, public virtual common, - link, - compile, - file_install, - alias_install + link_rule, + compile_rule, + install_rule, + libux_install_rule { public: explicit module (data&& d) : common (move (d)), - link (move (d)), - compile (move (d)), - file_install (move (d), *this), - alias_install (move (d), *this) {} + link_rule (move (d)), + compile_rule (move (d)), + install_rule (move (d), *this), + libux_install_rule (move (d), *this) {} void init (scope&, const location&, const variable_map&); diff --git a/build2/cc/pkgconfig.cxx b/build2/cc/pkgconfig.cxx index 0ffd135..697a60e 100644 --- a/build2/cc/pkgconfig.cxx +++ b/build2/cc/pkgconfig.cxx @@ -26,8 +26,8 @@ #include #include -#include -#include +#include +#include using namespace std; using namespace butl; @@ -451,7 +451,7 @@ namespace build2 // #ifndef BUILD2_BOOTSTRAP bool common:: - pkgconfig_load (action act, + pkgconfig_load (action a, const scope& s, lib& lt, liba* at, @@ -592,12 +592,13 @@ namespace build2 // Extract --cflags and set them as lib?{}:export.poptions. Note that we // still pass --static in case this is pkgconf which has Cflags.private. // - auto parse_cflags = [&trace, this] (target& t, const pkgconf& pc, bool a) + auto parse_cflags = + [&trace, this] (target& t, const pkgconf& pc, bool la) { strings pops; bool arg (false); - for (auto& o: pc.cflags (a)) + for (auto& o: pc.cflags (la)) { if (arg) { @@ -646,8 +647,8 @@ namespace build2 // Parse --libs into loptions/libs (interface and implementation). If // ps is not NULL, add each resolves library target as a prerequisite. // - auto parse_libs = [act, &s, top_sysd, this] - (target& t, const pkgconf& pc, bool a, prerequisites* ps) + auto parse_libs = [a, &s, top_sysd, this] + (target& t, const pkgconf& pc, bool la, prerequisites* ps) { strings lops; vector libs; @@ -664,7 +665,7 @@ namespace build2 // library names (without -l) after seeing an unknown option. // bool arg (false), first (true), known (true), have_L; - for (auto& o: pc.libs (a)) + for (auto& o: pc.libs (la)) { if (arg) { @@ -726,10 +727,10 @@ namespace build2 // Space-separated list of escaped library flags. // - auto lflags = [&pc, a] () -> string + auto lflags = [&pc, la] () -> string { string r; - for (const auto& o: pc.libs (a)) + for (const auto& o: pc.libs (la)) { if (!r.empty ()) r += ' '; @@ -831,7 +832,7 @@ namespace build2 prerequisite_key pk { nullopt, {&lib::static_type, &out, &out, &name, nullopt}, &s}; - if (const target* lt = search_library (act, top_sysd, usrd, pk)) + if (const target* lt = search_library (a, top_sysd, usrd, pk)) { // We used to pick a member but that doesn't seem right since the // same target could be used with different link orders. @@ -1112,8 +1113,8 @@ namespace build2 #endif - void link:: - pkgconfig_save (action act, const file& l, bool la) const + void link_rule:: + pkgconfig_save (action a, const file& l, bool la) const { tracer trace (x, "pkgconfig_save"); @@ -1258,7 +1259,7 @@ namespace build2 os << " -L" << escape (ld.string ()); // Now process ourselves as if we were being linked to something (so - // pretty similar to link::append_libraries()). + // pretty similar to link_rule::append_libraries()). // bool priv (false); auto imp = [&priv] (const file&, bool la) {return priv && la;}; @@ -1307,7 +1308,7 @@ namespace build2 // linfo li {otype::e, la ? lorder::a_s : lorder::s_a}; - process_libraries (act, bs, li, sys_lib_dirs, + process_libraries (a, bs, li, sys_lib_dirs, l, la, 0, // Link flags. imp, lib, opt, true); os << endl; @@ -1317,7 +1318,7 @@ namespace build2 os << "Libs.private:"; priv = true; - process_libraries (act, bs, li, sys_lib_dirs, + process_libraries (a, bs, li, sys_lib_dirs, l, la, 0, // Link flags. imp, lib, opt, false); os << endl; @@ -1339,7 +1340,7 @@ namespace build2 }; vector modules; - for (const target* pt: l.prerequisite_targets) + for (const target* pt: l.prerequisite_targets[a]) { // @@ UTL: we need to (recursively) see through libux{} (and // also in search_modules()). @@ -1354,7 +1355,7 @@ namespace build2 // the first mxx{} target that we see. // const target* mt (nullptr); - for (const target* t: pt->prerequisite_targets) + for (const target* t: pt->prerequisite_targets[a]) { if ((mt = t->is_a (*x_mod))) break; diff --git a/build2/cc/windows-manifest.cxx b/build2/cc/windows-manifest.cxx index 4393fbf..ae33f66 100644 --- a/build2/cc/windows-manifest.cxx +++ b/build2/cc/windows-manifest.cxx @@ -9,7 +9,7 @@ #include #include -#include +#include using namespace std; using namespace butl; @@ -39,10 +39,10 @@ namespace build2 // file corresponding to the exe{} target. Return the manifest file path // as well as whether it was changed. // - pair link:: + pair link_rule:: windows_manifest (const file& t, bool rpath_assembly) const { - tracer trace (x, "link::windows_manifest"); + tracer trace (x, "link_rule::windows_manifest"); const scope& rs (t.root_scope ()); diff --git a/build2/cc/windows-rpath.cxx b/build2/cc/windows-rpath.cxx index b28ce42..8854542 100644 --- a/build2/cc/windows-rpath.cxx +++ b/build2/cc/windows-rpath.cxx @@ -13,7 +13,7 @@ #include -#include +#include using namespace std; using namespace butl; @@ -46,10 +46,10 @@ namespace build2 // Return the greatest (newest) timestamp of all the DLLs that we will be // adding to the assembly or timestamp_nonexistent if there aren't any. // - timestamp link:: + timestamp link_rule:: windows_rpath_timestamp (const file& t, const scope& bs, - action act, + action a, linfo li) const { timestamp r (timestamp_nonexistent); @@ -103,19 +103,19 @@ namespace build2 r = t; }; - for (auto pt: t.prerequisite_targets) + for (const prerequisite_target& pt: t.prerequisite_targets[a]) { if (pt == nullptr) continue; - bool a; + bool la; const file* f; - if ((a = (f = pt->is_a ())) || - (a = (f = pt->is_a ())) || // See through. + if ((la = (f = pt->is_a ())) || + (la = (f = pt->is_a ())) || // See through. ( f = pt->is_a ())) - process_libraries (act, bs, li, sys_lib_dirs, - *f, a, pt.data, + process_libraries (a, bs, li, sys_lib_dirs, + *f, la, pt.data, imp, lib, nullptr, true); } @@ -125,10 +125,10 @@ namespace build2 // Like *_timestamp() but actually collect the DLLs (and weed out the // duplicates). // - auto link:: + auto link_rule:: windows_rpath_dlls (const file& t, const scope& bs, - action act, + action a, linfo li) const -> windows_dlls { windows_dlls r; @@ -193,19 +193,19 @@ namespace build2 } }; - for (auto pt: t.prerequisite_targets) + for (const prerequisite_target& pt: t.prerequisite_targets[a]) { if (pt == nullptr) continue; - bool a; + bool la; const file* f; - if ((a = (f = pt->is_a ())) || - (a = (f = pt->is_a ())) || // See through. - ( f = pt->is_a ())) - process_libraries (act, bs, li, sys_lib_dirs, - *f, a, pt.data, + if ((la = (f = pt->is_a ())) || + (la = (f = pt->is_a ())) || // See through. + ( f = pt->is_a ())) + process_libraries (a, bs, li, sys_lib_dirs, + *f, la, pt.data, imp, lib, nullptr, true); } @@ -223,10 +223,10 @@ namespace build2 // unnecessary work by comparing the DLLs timestamp against the assembly // manifest file. // - void link:: + void link_rule:: windows_rpath_assembly (const file& t, const scope& bs, - action act, + action a, linfo li, const string& tcpu, timestamp ts, @@ -264,7 +264,7 @@ namespace build2 windows_dlls dlls; if (!empty) - dlls = windows_rpath_dlls (t, bs, act, li); + dlls = windows_rpath_dlls (t, bs, a, li); // Clean the assembly directory and make sure it exists. Maybe it would // have been faster to overwrite the existing manifest rather than diff --git a/build2/cli/init.cxx b/build2/cli/init.cxx index a4403a9..df123ba 100644 --- a/build2/cli/init.cxx +++ b/build2/cli/init.cxx @@ -23,7 +23,7 @@ namespace build2 { namespace cli { - static const compile compile_; + static const compile_rule compile_rule_; bool config_init (scope& rs, @@ -306,10 +306,10 @@ namespace build2 auto reg = [&r] (meta_operation_id mid, operation_id oid) { - r.insert (mid, oid, "cli.compile", compile_); - r.insert (mid, oid, "cli.compile", compile_); - r.insert (mid, oid, "cli.compile", compile_); - r.insert (mid, oid, "cli.compile", compile_); + r.insert (mid, oid, "cli.compile", compile_rule_); + r.insert (mid, oid, "cli.compile", compile_rule_); + r.insert (mid, oid, "cli.compile", compile_rule_); + r.insert (mid, oid, "cli.compile", compile_rule_); }; reg (perform_id, update_id); diff --git a/build2/cli/rule.cxx b/build2/cli/rule.cxx index d05b190..42f2176 100644 --- a/build2/cli/rule.cxx +++ b/build2/cli/rule.cxx @@ -43,10 +43,10 @@ namespace build2 return false; } - match_result compile:: + bool compile_rule:: match (action a, target& xt, const string&) const { - tracer trace ("cli::compile::match"); + tracer trace ("cli::compile_rule::match"); if (cli_cxx* pt = xt.is_a ()) { @@ -149,7 +149,7 @@ namespace build2 } } - recipe compile:: + recipe compile_rule:: apply (action a, target& xt) const { if (cli_cxx* pt = xt.is_a ()) @@ -208,7 +208,7 @@ namespace build2 } } - target_state compile:: + target_state compile_rule:: perform_update (action a, const target& xt) { const cli_cxx& t (xt.as ()); diff --git a/build2/cli/rule.hxx b/build2/cli/rule.hxx index 9af1da4..ba6337a 100644 --- a/build2/cli/rule.hxx +++ b/build2/cli/rule.hxx @@ -16,12 +16,12 @@ namespace build2 { // @@ Redo as two separate rules? // - class compile: public rule + class compile_rule: public rule { public: - compile () {} + compile_rule () {} - virtual match_result + virtual bool match (action, target&, const string&) const override; virtual recipe diff --git a/build2/cli/target.cxx b/build2/cli/target.cxx index c35ee18..be3098c 100644 --- a/build2/cli/target.cxx +++ b/build2/cli/target.cxx @@ -33,7 +33,7 @@ namespace build2 // cli.cxx // group_view cli_cxx:: - group_members (action_type) const + group_members (action) const { static_assert (sizeof (cli_cxx_members) == sizeof (const target*) * 3, "member layout incompatible with array"); diff --git a/build2/cli/target.hxx b/build2/cli/target.hxx index 1247172..d595856 100644 --- a/build2/cli/target.hxx +++ b/build2/cli/target.hxx @@ -41,7 +41,7 @@ namespace build2 using mtime_target::mtime_target; virtual group_view - group_members (action_type) const override; + group_members (action) const override; public: static const target_type static_type; diff --git a/build2/context.hxx b/build2/context.hxx index d2daad6..71035ad 100644 --- a/build2/context.hxx +++ b/build2/context.hxx @@ -33,7 +33,7 @@ namespace build2 // Match can be interrupted with "exclusive load" in order to load // additional buildfiles. Similarly, it can be interrupted with (parallel) // execute in order to build targetd required to complete the match (for - // example, generated source code or source code generators themselves. + // example, generated source code or source code generators themselves). // // Such interruptions are performed by phase change that is protected by // phase_mutex (which is also used to synchronize the state changes between diff --git a/build2/dist/rule.cxx b/build2/dist/rule.cxx index b288a66..a131000 100644 --- a/build2/dist/rule.cxx +++ b/build2/dist/rule.cxx @@ -15,7 +15,7 @@ namespace build2 { namespace dist { - match_result rule:: + bool rule:: match (action, target&, const string&) const { return true; // We always match. diff --git a/build2/dist/rule.hxx b/build2/dist/rule.hxx index 0524029..ffeed9e 100644 --- a/build2/dist/rule.hxx +++ b/build2/dist/rule.hxx @@ -19,7 +19,6 @@ namespace build2 // This is the default rule that simply matches all the prerequisites. // // A custom rule (usually the same as perform_update) may be necessary to - // enter ad hoc prerequisites (like generated test input/output) or // establishing group links (so that we see the dist variable set on a // group). // @@ -28,7 +27,7 @@ namespace build2 public: rule () {} - virtual match_result + virtual bool match (action, target&, const string&) const override; virtual recipe diff --git a/build2/dump.cxx b/build2/dump.cxx index f88dcfd..263f1b5 100644 --- a/build2/dump.cxx +++ b/build2/dump.cxx @@ -243,12 +243,14 @@ namespace build2 // Note: running serial and task_count is 0 before any operation has // started. // - if (size_t c = t.task_count.load (memory_order_relaxed)) + action inner; // @@ Only for the inner part of the action currently. + + if (size_t c = t[inner].task_count.load (memory_order_relaxed)) { if (c == target::count_applied () || c == target::count_executed ()) { bool f (false); - for (const target* pt: t.prerequisite_targets) + for (const target* pt: t.prerequisite_targets[inner]) { if (pt == nullptr) // Skipped. continue; diff --git a/build2/install/init.cxx b/build2/install/init.cxx index 0eb9521..a11f3e5 100644 --- a/build2/install/init.cxx +++ b/build2/install/init.cxx @@ -217,7 +217,7 @@ namespace build2 bs.rules.insert (perform_uninstall_id, "uninstall.alias", ar); bs.rules.insert (perform_install_id, "install.file", fr); - bs.rules.insert (perform_uninstall_id, "uinstall.file", fr); + bs.rules.insert (perform_uninstall_id, "uninstall.file", fr); } // Configuration. diff --git a/build2/install/rule.cxx b/build2/install/rule.cxx index 79287f8..4d4cb51 100644 --- a/build2/install/rule.cxx +++ b/build2/install/rule.cxx @@ -40,9 +40,15 @@ namespace build2 // const alias_rule alias_rule::instance; - match_result alias_rule:: + bool alias_rule:: match (action, target&, const string&) const { + // We always match. + // + // Note that we are called both as the outer part during the "update for + // un/install" pre-operation and as the inner part during the + // un/install operation itself. + // return true; } @@ -57,78 +63,134 @@ namespace build2 { tracer trace ("install::alias_rule::apply"); + // Pass-through to our installable prerequisites. + // + // @@ Shouldn't we do match in parallel (here and below)? + // + auto& pts (t.prerequisite_targets[a]); for (prerequisite_member p: group_prerequisite_members (a, t)) { + // Ignore unresolved targets that are imported from other projects. + // We are definitely not installing those. + // + if (p.proj ()) + continue; + // Let a customized rule have its say. // const target* pt (filter (a, t, p)); if (pt == nullptr) + { + l5 ([&]{trace << "ignoring " << p << " (filtered out)";}); continue; + } - // Check if this prerequisite is explicitly "not installable", - // that is, there is the 'install' variable and its value is - // false. + // Check if this prerequisite is explicitly "not installable", that + // is, there is the 'install' variable and its value is false. // - // At first, this might seem redundand since we could have let - // the file_rule below take care of it. The nuance is this: this - // prerequsite can be in a different subproject that hasn't loaded - // the install module (and therefore has no file_rule registered). - // The typical example would be the 'tests' subproject. + // At first, this might seem redundand since we could have let the + // file_rule below take care of it. The nuance is this: this + // prerequsite can be in a different subproject that hasn't loaded the + // install module (and therefore has no file_rule registered). The + // typical example would be the 'tests' subproject. // // Note: not the same as lookup() above. // auto l ((*pt)["install"]); if (l && cast (l).string () == "false") { - l5 ([&]{trace << "ignoring " << *pt;}); + l5 ([&]{trace << "ignoring " << *pt << " (not installable)";}); continue; } build2::match (a, *pt); - t.prerequisite_targets.push_back (pt); + pts.push_back (pt); } return default_recipe; } - // file_rule + // group_rule // - const file_rule file_rule::instance; + const group_rule group_rule::instance; - struct match_data + const target* group_rule:: + filter (action, const target&, const target& m) const { - bool install; - }; - - static_assert (sizeof (match_data) <= target::data_size, - "insufficient space"); + return &m; + } - match_result file_rule:: - match (action a, target& t, const string&) const + recipe group_rule:: + apply (action a, target& t) const { - // First determine if this target should be installed (called - // "installable" for short). + tracer trace ("install::group_rule::apply"); + + // Resolve group members. + // + // Remember that we are called twice: first during update for install + // (pre-operation) and then during install. During the former, we rely + // on the normall update rule to resolve the group members. During the + // latter, there will be no rule to do this but the group will already + // have been resolved by the pre-operation. // - match_data md {lookup_install (t, "install") != nullptr}; - match_result mr (true); + // If the rule could not resolve the group, then we ignore it. + // + group_view gv (a.outer () + ? resolve_group_members (a, t) + : t.group_members (a)); - if (a.operation () == update_id) + if (gv.members != nullptr) { - // If this is the update pre-operation and the target is installable, - // change the recipe action to (update, 0) (i.e., "unconditional - // update") so that we don't get matched for its prerequisites. - // - if (md.install) - mr.recipe_action = action (a.meta_operation (), update_id); - else - // Otherwise, signal that we don't match so that some other rule can - // take care of it. + auto& pts (t.prerequisite_targets[a]); + for (size_t i (0); i != gv.count; ++i) + { + const target* m (gv.members[i]); + + if (m == nullptr) + continue; + + // Let a customized rule have its say. // - return false; + const target* mt (filter (a, t, *m)); + if (mt == nullptr) + { + l5 ([&]{trace << "ignoring " << *m << " (filtered out)";}); + continue; + } + + // See if we were explicitly instructed not to touch this target. + // + // Note: not the same as lookup() above. + // + auto l ((*mt)["install"]); + if (l && cast (l).string () == "false") + { + l5 ([&]{trace << "ignoring " << *mt << " (not installable)";}); + continue; + } + + build2::match (a, *mt); + pts.push_back (mt); + } } - t.data (md); // Save the data in the target's auxilary storage. - return mr; + // Delegate to the base rule. + // + return alias_rule::apply (a, t); + } + + + // file_rule + // + const file_rule file_rule::instance; + + bool file_rule:: + match (action, target&, const string&) const + { + // We always match, even if this target is not installable (so that we + // can ignore it; see apply()). + // + return true; } const target* file_rule:: @@ -141,28 +203,27 @@ namespace build2 recipe file_rule:: apply (action a, target& t) const { - match_data md (move (t.data ())); - t.clear_data (); // In case delegated-to rule (or the rule that overrides - // us; see cc/install) also uses aux storage. + tracer trace ("install::file_rule::apply"); - if (!md.install) // Not installable. - return noop_recipe; - - // Ok, if we are here, then this means: + // Note that we are called both as the outer part during the "update for + // un/install" pre-operation and as the inner part during the + // un/install operation itself. // - // 1. This target is installable. - // 2. The action is either - // a. (perform, [un]install, 0) or - // b. (*, update, [un]install) + // In both cases we first determine if the target is installable and + // return noop if it's not. Otherwise, in the first case (update for + // un/install) we delegate to the normal update and in the second + // (un/install) -- perform the test. // + if (lookup_install (t, "install") == nullptr) + return noop_recipe; + // In both cases, the next step is to search, match, and collect all the // installable prerequisites. // - // @@ Perhaps if [noinstall] will be handled by the - // group_prerequisite_members machinery, then we can just - // run standard search_and_match()? Will need an indicator - // that it was forced (e.g., [install]) for filter() below. + // @@ Unconditional group? How does it work for cli? Change to maybe + // same like test? If so, also in alias_rule. // + auto& pts (t.prerequisite_targets[a]); for (prerequisite_member p: group_prerequisite_members (a, t)) { // Ignore unresolved targets that are imported from other projects. @@ -175,84 +236,70 @@ namespace build2 // const target* pt (filter (a, t, p)); if (pt == nullptr) + { + l5 ([&]{trace << "ignoring " << p << " (filtered out)";}); continue; + } // See if we were explicitly instructed not to touch this target. // + // Note: not the same as lookup() above. + // auto l ((*pt)["install"]); if (l && cast (l).string () == "false") + { + l5 ([&]{trace << "ignoring " << *pt << " (not installable)";}); continue; + } - // If the matched rule returned noop_recipe, then the target - // state will be set to unchanged as an optimization. Use this - // knowledge to optimize things on our side as well since this - // will help a lot in case of any static installable content - // (headers, documentation, etc). + // If the matched rule returned noop_recipe, then the target state is + // set to unchanged as an optimization. Use this knowledge to optimize + // things on our side as well since this will help a lot when updating + // static installable content (headers, documentation, etc). // if (!build2::match (a, *pt, unmatch::unchanged)) - t.prerequisite_targets.push_back (pt); + pts.push_back (pt); } - // This is where we diverge depending on the operation. In the - // update pre-operation, we need to make sure that this target - // as well as all its installable prerequisites are up to date. - // if (a.operation () == update_id) { - // Save the prerequisite targets that we found since the - // call to match_delegate() below will wipe them out. - // - prerequisite_targets p; - - if (!t.prerequisite_targets.empty ()) - p.swap (t.prerequisite_targets); - - // Find the "real" update rule, that is, the rule that would - // have been found if we signalled that we do not match from - // match() above. - // - recipe d (match_delegate (a, t, *this)); - - // If we have no installable prerequisites, then simply redirect - // to it. + // For the update pre-operation match the inner rule (actual update). // - if (p.empty ()) - return d; - - // Ok, the worst case scenario: we need to cause update of - // prerequisite targets and also delegate to the real update. - // - return [pt = move (p), dr = move (d)] ( - action a, const target& t) mutable -> target_state + if (match_inner (a, t, unmatch::unchanged)) { - // Do the target update first. - // - target_state r (execute_delegate (dr, a, t)); - - // Swap our prerequisite targets back in and execute. - // - t.prerequisite_targets.swap (pt); - r |= straight_execute_prerequisites (a, t); - pt.swap (t.prerequisite_targets); // In case we get re-executed. + return pts.empty () ? noop_recipe : default_recipe; + } - return r; - }; + return &perform_update; } - else if (a.operation () == install_id) - return [this] (action a, const target& t) - { - return perform_install (a, t); - }; else + { return [this] (action a, const target& t) { - return perform_uninstall (a, t); + return a.operation () == install_id + ? perform_install (a, t) + : perform_uninstall (a, t); }; + } } - void file_rule:: + target_state file_rule:: + perform_update (action a, const target& t) + { + // First execute the inner recipe then prerequisites. + // + target_state ts (execute_inner (a, t)); + + if (t.prerequisite_targets[a].size () != 0) + ts |= straight_execute_prerequisites (a, t); + + return ts; + } + + bool file_rule:: install_extra (const file&, const install_dir&) const { + return false; } bool file_rule:: diff --git a/build2/install/rule.hxx b/build2/install/rule.hxx index 642ab96..ffab206 100644 --- a/build2/install/rule.hxx +++ b/build2/install/rule.hxx @@ -19,21 +19,39 @@ namespace build2 class alias_rule: public rule { public: - static const alias_rule instance; - - alias_rule () {} - - virtual match_result + virtual bool match (action, target&, const string&) const override; - virtual recipe - apply (action, target&) const override; - // Return NULL if this prerequisite should be ignored and pointer to its // target otherwise. The default implementation accepts all prerequsites. // virtual const target* filter (action, const target&, prerequisite_member) const; + + virtual recipe + apply (action, target&) const override; + + alias_rule () {} + static const alias_rule instance; + }; + + // In addition to the alias rule's semantics, this rule sees through to + // the group's members. + // + class group_rule: public alias_rule + { + public: + // Return NULL if this group member should be ignored and pointer to its + // target otherwise. The default implementation accepts all members. + // + virtual const target* + filter (action, const target&, const target& group_member) const; + + virtual recipe + apply (action, target&) const override; + + group_rule () {} + static const group_rule instance; }; struct install_dir; @@ -41,41 +59,38 @@ namespace build2 class file_rule: public rule { public: - static const file_rule instance; - - file_rule () {} - - virtual match_result + virtual bool match (action, target&, const string&) const override; - virtual recipe - apply (action, target&) const override; - // Return NULL if this prerequisite should be ignored and pointer to its - // target otherwise. The default implementation ignores prerequsites that - // are outside of this target's project. + // target otherwise. The default implementation ignores prerequsites + // that are outside of this target's project. // virtual const target* filter (action, const target&, prerequisite_member) const; - // Extra installation hooks. + virtual recipe + apply (action, target&) const override; + + static target_state + perform_update (action, const target&); + + // Extra un/installation hooks. Return true if anything was + // un/installed. // - using install_dir = install::install_dir; + using install_dir = install::install_dir; // For derived rules. - virtual void + virtual bool install_extra (const file&, const install_dir&) const; - // Return true if anything was uninstalled. - // virtual bool uninstall_extra (const file&, const install_dir&) const; - // Installation "commands". + // Installation/uninstallation "commands". // // If verbose is false, then only print the command at verbosity level 2 // or higher. - // - public: + // Install a symlink: base/link -> target. // static void @@ -96,16 +111,18 @@ namespace build2 static bool uninstall_f (const scope& rs, const install_dir& base, - const file* t, + const file* target, const path& name, bool verbose); - protected: target_state perform_install (action, const target&) const; target_state perform_uninstall (action, const target&) const; + + static const file_rule instance; + file_rule () {} }; } } diff --git a/build2/operation.cxx b/build2/operation.cxx index 698edbb..a43e20a 100644 --- a/build2/operation.cxx +++ b/build2/operation.cxx @@ -191,20 +191,9 @@ namespace build2 action_target& at (ts[j]); const target& t (at.as_target ()); - // Finish matching targets that we have started. Note that we use the - // state for the "final" action that will be executed and not our - // action. Failed that, we may fail to find a match for a "stronger" - // action but will still get unchanged for the original one. - // - target_state s; - if (j < i) - { - match (a, t, false); - s = t.serial_state (false); - } - else - s = target_state::postponed; - + target_state s (j < i + ? match (a, t, false) + : target_state::postponed); switch (s) { case target_state::postponed: @@ -369,7 +358,7 @@ namespace build2 { const target& t (at.as_target ()); - switch ((at.state = t.executed_state (false))) + switch ((at.state = t.executed_state (a, false))) { case target_state::unknown: { diff --git a/build2/operation.hxx b/build2/operation.hxx index a65fc3d..8c9818e 100644 --- a/build2/operation.hxx +++ b/build2/operation.hxx @@ -52,9 +52,6 @@ namespace build2 { action (): inner_id (0), outer_id (0) {} // Invalid action. - bool - valid () const {return inner_id != 0;} - // If this is not a nested operation, then outer should be 0. // action (meta_operation_id m, operation_id inner, operation_id outer = 0) @@ -70,6 +67,9 @@ namespace build2 operation_id outer_operation () const {return outer_id & 0xF;} + bool inner () const {return outer_id == 0;} + bool outer () const {return outer_id != 0;} + // Implicit conversion operator to action_id for the switch() statement, // etc. Most places only care about the inner operation. // @@ -88,24 +88,25 @@ namespace build2 inline bool operator!= (action x, action y) {return !(x == y);} - // This is an "overrides" comparison, i.e., it returns true if the recipe - // for x overrides recipe for y. The idea is that for the same inner - // operation, action with an outer operation is "weaker" than the one - // without. - // - inline bool - operator> (action x, action y) - { - return x.inner_id != y.inner_id || - (x.outer_id != y.outer_id && y.outer_id != 0); - } - - inline bool - operator>= (action x, action y) {return x == y || x > y;} + bool operator> (action, action) = delete; + bool operator< (action, action) = delete; + bool operator>= (action, action) = delete; + bool operator<= (action, action) = delete; ostream& operator<< (ostream&, action); + // Inner/outer operation state container. + // + template + struct action_state + { + T states[2]; // [0] -- inner, [1] -- outer. + + T& operator[] (action a) {return states[a.inner () ? 0 : 1];} + const T& operator[] (action a) const {return states[a.inner () ? 0 : 1];} + }; + // Id constants for build-in and pre-defined meta/operations. // const meta_operation_id noop_id = 1; // nomop? diff --git a/build2/parser.cxx b/build2/parser.cxx index b21d51c..5687b0f 100644 --- a/build2/parser.cxx +++ b/build2/parser.cxx @@ -747,14 +747,15 @@ namespace build2 // small_vector, 1> tgs; - for (auto& tn: ns) + for (auto i (ns.begin ()), e (ns.end ()); i != e; ++i) { - if (tn.qualified ()) - fail (nloc) << "project name in target " << tn; + name& n (*i); - // @@ OUT TODO - // - enter_target tg (*this, move (tn), name (), false, nloc, trace); + if (n.qualified ()) + fail (nloc) << "project name in target " << n; + + name o (n.pair ? move (*++i) : name ()); + enter_target tg (*this, move (n), move (o), false, nloc, trace); if (default_target_ == nullptr) default_target_ = target_; diff --git a/build2/rule-map.hxx b/build2/rule-map.hxx index f63d4b9..e754283 100644 --- a/build2/rule-map.hxx +++ b/build2/rule-map.hxx @@ -79,6 +79,8 @@ namespace build2 insert (a >> 4, a & 0x0F, hint, r); } + // 0 oid is a wildcard. + // template void insert (meta_operation_id mid, diff --git a/build2/rule.cxx b/build2/rule.cxx index 2ed0bbd..ef1d3a4 100644 --- a/build2/rule.cxx +++ b/build2/rule.cxx @@ -25,7 +25,7 @@ namespace build2 // that normal implementations should follow. So you probably shouldn't // use it as a guide to implement your own, normal, rules. // - match_result file_rule:: + bool file_rule:: match (action a, target& t, const string&) const { tracer trace ("file_rule::match"); @@ -131,7 +131,7 @@ namespace build2 // alias_rule // - match_result alias_rule:: + bool alias_rule:: match (action, target&, const string&) const { return true; @@ -153,7 +153,7 @@ namespace build2 // fsdir_rule // - match_result fsdir_rule:: + bool fsdir_rule:: match (action, target&, const string&) const { return true; @@ -220,7 +220,7 @@ namespace build2 // First update prerequisites (e.g. create parent directories) then create // this directory. // - if (!t.prerequisite_targets.empty ()) + if (!t.prerequisite_targets[a].empty ()) ts = straight_execute_prerequisites (a, t); // The same code as in perform_update_direct() below. @@ -243,9 +243,9 @@ namespace build2 { // First create the parent directory. If present, it is always first. // - const target* p (t.prerequisite_targets.empty () + const target* p (t.prerequisite_targets[a].empty () ? nullptr - : t.prerequisite_targets[0]); + : t.prerequisite_targets[a][0]); if (p != nullptr && p->is_a ()) perform_update_direct (a, *p); @@ -272,7 +272,7 @@ namespace build2 ? target_state::changed : target_state::unchanged); - if (!t.prerequisite_targets.empty ()) + if (!t.prerequisite_targets[a].empty ()) ts |= reverse_execute_prerequisites (a, t); return ts; diff --git a/build2/rule.hxx b/build2/rule.hxx index f6e23f6..8e43ca6 100644 --- a/build2/rule.hxx +++ b/build2/rule.hxx @@ -13,48 +13,16 @@ namespace build2 { - class match_result - { - public: - bool result; - - // If set, then this is a recipe's action. It must override the original - // action. Normally it is "unconditional inner operation". Only - // noop_recipe can be overridden. - // - // It is passed to rule::apply() so that prerequisites are matched for - // this action. It is also passed to target::recipe() so that if someone - // is matching this target for this action, we won't end-up re-matching - // it. However, the recipe itself is executed with the original action - // so that it can adjust its logic, if necessary. - // - action recipe_action = action (); - - explicit - operator bool () const {return result;} - - // Note that the from-bool constructor is intentionally implicit so that - // we can return true/false from match(). - // - match_result (bool r): result (r) {} - match_result (bool r, action a): result (r), recipe_action (a) {} - }; - // Once a rule is registered (for a scope), it is treated as immutable. If // you need to modify some state (e.g., counters or some such), then make // sure it is MT-safe. // - // Note that match() may not be followed by apply() or be called several - // times before the following apply() (see resolve_group_members()) which - // means that it should be idempotent. The target_data object in the call - // to match() may not be the same as target. - // - // match() can also be called by another rules (see cc/install). + // Note: match() is only called once but may not be followed by apply(). // class rule { public: - virtual match_result + virtual bool match (action, target&, const string& hint) const = 0; virtual recipe @@ -68,7 +36,7 @@ namespace build2 public: file_rule () {} - virtual match_result + virtual bool match (action, target&, const string&) const override; virtual recipe @@ -82,7 +50,7 @@ namespace build2 public: alias_rule () {} - virtual match_result + virtual bool match (action, target&, const string&) const override; virtual recipe @@ -96,7 +64,7 @@ namespace build2 public: fsdir_rule () {} - virtual match_result + virtual bool match (action, target&, const string&) const override; virtual recipe @@ -124,7 +92,7 @@ namespace build2 public: fallback_rule () {} - virtual match_result + virtual bool match (action, target&, const string&) const override { return true; diff --git a/build2/target.cxx b/build2/target.cxx index a5316ce..201bd15 100644 --- a/build2/target.cxx +++ b/build2/target.cxx @@ -92,7 +92,7 @@ namespace build2 } group_view target:: - group_members (action_type) const + group_members (action) const { assert (false); // Not a group or doesn't expose its members. return group_view {nullptr, 0}; @@ -117,83 +117,6 @@ namespace build2 return *r; } - pair target:: - state (action_type a) const - { - assert (phase == run_phase::match); - - // The tricky aspect here is that we my be re-matching the target for - // another (overriding action). Since it is only valid to call this - // function after the target has been matched (for this action), we assume - // that if the target is busy, then it is being overriden (note that it - // cannot be being executed since we are in the match phase). - // - // But that's not the end of it: in case of a re-match the task count - // might have been reset to, say, applied. The only way to know for sure - // that there isn't another match "underneath" is to compare actions. But - // that can only be done safely if we lock the target. At least we will be - // quick (and we don't need to wait since if it's busy, we know it is a - // re-match). This logic is similar to lock_impl(). - // - size_t b (target::count_base ()); - size_t e (task_count.load (memory_order_acquire)); - - size_t exec (b + target::offset_executed); - size_t lock (b + target::offset_locked); - size_t busy (b + target::offset_busy); - - for (;;) - { - for (; e == lock; e = task_count.load (memory_order_acquire)) - this_thread::yield (); - - if (e >= busy) // Override in progress. - return make_pair (true, target_state::unchanged); - - // Unlike lock_impl(), we are only called after being matched for this - // action so if we see executed, then it means executed for this action - // (or noop). - // - if (e == exec) - return make_pair (true, group_state () ? group->state_ : state_); - - // Try to grab the spin-lock. - // - if (task_count.compare_exchange_strong ( - e, - lock, - memory_order_acq_rel, // Synchronize on success. - memory_order_acquire)) // Synchronize on failure. - { - break; - } - - // Task count changed, try again. - } - - // We have the spin-lock. Quickly get the matched action and unlock. - // - action_type ma (action); - bool mf (state_ == target_state::failed); - task_count.store (e, memory_order_release); - - if (ma > a) // Overriden. - return make_pair (true, // Override may have failed but we had the rule. - mf ? target_state::failed: target_state::unchanged); - - // Otherwise we should have a matched target. - // - assert (ma == a); - - if (e == b + target::offset_tried) - return make_pair (false, target_state::unknown); - else - { - assert (e == b + target::offset_applied || e == exec); - return make_pair (true, group_state () ? group->state_ : state_); - } - } - pair target:: find_original (const variable& var, bool target_only) const { @@ -519,26 +442,19 @@ namespace build2 case run_phase::load: break; case run_phase::match: { - // Similar logic to state(action) except here we don't distinguish - // between original/overridden actions (an overridable action is by - // definition a noop and should never need to query the mtime). - // - size_t c (task_count.load (memory_order_acquire)); // For group_state() - - // Wait out the spin lock to get the meaningful count. + // Similar logic to matched_state_impl(). // - for (size_t lock (target::count_locked ()); - c == lock; - c = task_count.load (memory_order_acquire)) - this_thread::yield (); + const opstate& s (state[action () /* inner */]); + size_t o (s.task_count.load (memory_order_relaxed) - // Synchronized. + target::count_base ()); - if (c != target::count_applied () && c != target::count_executed ()) + if (o != target::offset_applied && o != target::offset_executed) break; } // Fall through. case run_phase::execute: { - if (group_state ()) + if (group_state (action () /* inner */)) t = &group->as (); break; diff --git a/build2/target.hxx b/build2/target.hxx index e15b970..105cc4b 100644 --- a/build2/target.hxx +++ b/build2/target.hxx @@ -104,6 +104,10 @@ namespace build2 }; using prerequisite_targets = vector; + // A rule match is an element of hint_rule_map. + // + using rule_match = pair>; + // Target. // class target @@ -135,6 +139,15 @@ namespace build2 const dir_path& out_dir () const {return out.empty () ? dir : out;} + // A target that is not (yet) entered as part of a real dependency + // declaration (for example, that is entered as part of a target-specific + // variable assignment, dependency extraction, etc) is called implied. + // + // The implied flag should only be cleared during the load phase via the + // MT-safe target_set::insert(). + // + bool implied; + // Target group to which this target belongs, if any. Note that we assume // that the group and all its members are in the same scope (for example, // in variable lookup). We also don't support nested groups (with a small @@ -178,7 +191,6 @@ namespace build2 // const target* group = nullptr; - // What has been described above is a "normal" group. That is, there is // a dedicated target type that explicitly serves as a group and there // is an explicit mechanism for discovering the group's members. @@ -240,13 +252,11 @@ namespace build2 } public: - using action_type = build2::action; - // You should not call this function directly; rather use // resolve_group_members() from . // virtual group_view - group_members (action_type) const; + group_members (action) const; // Note that the returned key "tracks" the target (except for the // extension). @@ -395,17 +405,7 @@ namespace build2 value& append (const variable&); - // A target that is not (yet) entered as part of a real dependency - // declaration (for example, that is entered as part of a target-specific - // variable assignment, dependency extraction, etc) is called implied. - // - // The implied flag should only be cleared during the load phase via the - // MT-safe target_set::insert(). - // - public: - bool implied; - - // Target state. + // Target operation state. // public: // Atomic task count that is used during match and execution to track the @@ -419,10 +419,6 @@ namespace build2 // automatically resetting the target to "not yet touched" state for this // operation. // - // For match we have a further complication in that we may re-match the - // target and override with a "stronger" recipe thus re-setting the state - // from, say, applied back to touched. - // // The target is said to be synchronized (in this thread) if we have // either observed the task count to reach applied or executed or we have // successfully changed it (via compare_exchange) to locked or busy. If @@ -434,8 +430,7 @@ namespace build2 static const size_t offset_matched = 3; // Rule has been matched. static const size_t offset_applied = 4; // Rule has been applied. static const size_t offset_executed = 5; // Recipe has been executed. - static const size_t offset_locked = 6; // Fast (spin) lock. - static const size_t offset_busy = 7; // Slow (wait) lock. + static const size_t offset_busy = 6; // Match/execute in progress. static size_t count_base () {return 5 * (current_on - 1);} @@ -444,51 +439,70 @@ namespace build2 static size_t count_matched () {return offset_matched + count_base ();} static size_t count_applied () {return offset_applied + count_base ();} static size_t count_executed () {return offset_executed + count_base ();} - static size_t count_locked () {return offset_locked + count_base ();} static size_t count_busy () {return offset_busy + count_base ();} - mutable atomic_count task_count {0}; // Start offset_touched - 1. + // Inner/outer operation state. + // + struct opstate + { + mutable atomic_count task_count {0}; // Start offset_touched - 1. + + // Number of direct targets that depend on this target in the current + // operation. It is incremented during match and then decremented during + // execution, before running the recipe. As a result, the recipe can + // detect the last chance (i.e., last dependent) to execute the command + // (see also the first/last execution modes in ). + // + mutable atomic_count dependents {0}; + + // Matched rule (pointer to hint_rule_map element). Note that in case of + // a direct recipe assignment we may not have a rule (NULL). + // + const rule_match* rule; + + // Applied recipe. + // + build2::recipe recipe; + + // Target state for this operation. Note that it is undetermined until + // a rule is matched and recipe applied (see set_recipe()). + // + target_state state; + }; + + action_state state; + + opstate& operator[] (action a) {return state[a];} + const opstate& operator[] (action a) const {return state[a];} // This function should only be called during match if we have observed // (synchronization-wise) that this target has been matched (i.e., the // rule has been applied) for this action. // target_state - matched_state (action a, bool fail = true) const; + matched_state (action, bool fail = true) const; // See try_match(). // pair - try_matched_state (action a, bool fail = true) const; + try_matched_state (action, bool fail = true) const; - // This function should only be called during execution if we have - // observed (synchronization-wise) that this target has been executed. + // After the target has been matched and synchronized, check if the target + // is known to be unchanged. Used for optimizations during search & match. // - target_state - executed_state (bool fail = true) const; + bool + unchanged (action a) const + { + return matched_state_impl (a).second == target_state::unchanged; + } - // This function should only be called between match and execute while - // running serially. It returns the group state for the "final" action - // that has been matched (and that will be executed). + // This function should only be called during execution if we have + // observed (synchronization-wise) that this target has been executed. // target_state - serial_state (bool fail = true) const; - - // Number of direct targets that depend on this target in the current - // operation. It is incremented during match and then decremented during - // execution, before running the recipe. As a result, the recipe can - // detect the last chance (i.e., last dependent) to execute the command - // (see also the first/last execution modes in ). - // - mutable atomic_count dependents; + executed_state (action, bool fail = true) const; protected: - // Return fail-untranslated (but group-translated) state assuming the - // target is executed and synchronized. - // - target_state - state () const; - // Version that should be used during match after the target has been // matched for this action (see the recipe override). // @@ -496,51 +510,22 @@ namespace build2 // result (see try_match()). // pair - state (action a) const; + matched_state_impl (action) const; + + // Return fail-untranslated (but group-translated) state assuming the + // target is executed and synchronized. + // + target_state + executed_state_impl (action) const; // Return true if the state comes from the group. Target must be at least // matched. // bool - group_state () const; - - // Raw state, normally not accessed directly. - // - public: - target_state state_ = target_state::unknown; + group_state (action a) const; - // Recipe. - // public: - using recipe_type = build2::recipe; - using rule_type = build2::rule; - - action_type action; // Action the rule/recipe is for. - - // Matched rule (pointer to hint_rule_map element). Note that in case of a - // direct recipe assignment we may not have a rule. - // - const pair>* rule; - - // Applied recipe. - // - recipe_type recipe_; - - // Note that the target must be locked in order to set the recipe. - // - void - recipe (recipe_type); - - // After the target has been matched and synchronized, check if the target - // is known to be unchanged. Used for optimizations during search & match. - // - bool - unchanged (action_type a) const - { - return state (a).second == target_state::unchanged; - } - - // Targets to which prerequisites resolve for this recipe. Note that + // Targets to which prerequisites resolve for this action. Note that // unlike prerequisite::target, these can be resolved to group members. // NULL means the target should be skipped (or the rule may simply not add // such a target to the list). @@ -552,7 +537,7 @@ namespace build2 // // Note that the recipe may modify this list. // - mutable build2::prerequisite_targets prerequisite_targets; + mutable action_state prerequisite_targets; // Auxilary data storage. // @@ -573,7 +558,8 @@ namespace build2 // // Currenly the data is not destroyed until the next match. // - // Note that the recipe may modify the data. + // Note that the recipe may modify the data. Currently reserved for the + // inner part of the action. // static constexpr size_t data_size = sizeof (string) * 16; mutable std::aligned_storage::type data_pad; @@ -1330,9 +1316,13 @@ namespace build2 // which racing updates happen because we do not modify the external state // (which is the source of timestemps) while updating the internal. // + // The modification time is reserved for the inner operation thus there is + // no action argument. + // // The rule for groups that utilize target_state::group is as follows: if // it has any members that are mtime_targets, then the group should be - // mtime_target and the members get the mtime from it. + // mtime_target and the members get the mtime from it. During match and + // execute the target should be synchronized. // // Note that this function can be called before the target is matched in // which case the value always comes from the target itself. In other diff --git a/build2/target.ixx b/build2/target.ixx index 6225aef..16dbf61 100644 --- a/build2/target.ixx +++ b/build2/target.ixx @@ -62,26 +62,48 @@ namespace build2 } } + inline pair target:: + matched_state_impl (action a) const + { + assert (phase == run_phase::match); + + const opstate& s (state[a]); + size_t o (s.task_count.load (memory_order_relaxed) - // Synchronized. + target::count_base ()); + + if (o == target::offset_tried) + return make_pair (false, target_state::unknown); + else + { + // Normally applied but can also be already executed. + // + assert (o == target::offset_applied || o == target::offset_executed); + return make_pair (true, (group_state (a) ? group->state[a] : s).state); + } + } + inline target_state target:: - state () const + executed_state_impl (action a) const { assert (phase == run_phase::execute); - return group_state () ? group->state_ : state_; + return (group_state (a) ? group->state : state)[a].state; } inline bool target:: - group_state () const + group_state (action a) const { // We go an extra step and short-circuit to the target state even if the // raw state is not group provided the recipe is group_recipe and the // state is not failed. + // + const opstate& s (state[a]); - if (state_ == target_state::group) + if (s.state == target_state::group) return true; - if (state_ != target_state::failed && group != nullptr) + if (s.state != target_state::failed && group != nullptr) { - if (recipe_function* const* f = recipe_.target ()) + if (recipe_function* const* f = s.recipe.target ()) return *f == &group_action; } @@ -89,11 +111,11 @@ namespace build2 } inline target_state target:: - matched_state (action_type a, bool fail) const + matched_state (action a, bool fail) const { // Note that the target could be being asynchronously re-matched. // - pair r (state (a)); + pair r (matched_state_impl (a)); if (fail && (!r.first || r.second == target_state::failed)) throw failed (); @@ -102,9 +124,9 @@ namespace build2 } inline pair target:: - try_matched_state (action_type a, bool fail) const + try_matched_state (action a, bool fail) const { - pair r (state (a)); + pair r (matched_state_impl (a)); if (fail && r.first && r.second == target_state::failed) throw failed (); @@ -113,22 +135,9 @@ namespace build2 } inline target_state target:: - executed_state (bool fail) const - { - target_state r (state ()); - - if (fail && r == target_state::failed) - throw failed (); - - return r; - } - - inline target_state target:: - serial_state (bool fail) const + executed_state (action a, bool fail) const { - //assert (sched.serial ()); - - target_state r (group_state () ? group->state_ : state_); + target_state r (executed_state_impl (a)); if (fail && r == target_state::failed) throw failed (); @@ -136,44 +145,6 @@ namespace build2 return r; } - extern atomic_count target_count; // context.hxx - - inline void target:: - recipe (recipe_type r) - { - recipe_ = move (r); - - // Do not clear the failed target state in case of an override (and we - // should never see the failed state from the previous operation since we - // should abort the execution in this case). - // - if (state_ != target_state::failed) - { - state_ = target_state::unknown; - - // If this is a noop recipe, then mark the target unchanged to allow for - // some optimizations. - // - recipe_function** f (recipe_.target ()); - - if (f != nullptr && *f == &noop_action) - state_ = target_state::unchanged; - else - { - // This gets tricky when we start considering overrides (which can - // only happen for noop recipes), direct execution, etc. So here seems - // like the best place to do this. - // - // We also ignore the group recipe since it is used for ad hoc - // groups (which are not executed). Plus, group action means real - // recipe is in the group so this also feels right conceptually. - // - if (f == nullptr || *f != &group_action) - target_count.fetch_add (1, memory_order_relaxed); - } - } - } - // mark()/unmark() // @@ -323,7 +294,8 @@ namespace build2 inline timestamp mtime_target:: load_mtime (const path& p) const { - assert (phase == run_phase::execute && !group_state ()); + assert (phase == run_phase::execute && + !group_state (action () /* inner */)); duration::rep r (mtime_.load (memory_order_consume)); if (r == timestamp_unknown_rep) @@ -349,7 +321,9 @@ namespace build2 // much we can do here except detect the case where the target was // changed on this run. // - return mt < mp || (mt == mp && state () == target_state::changed); + return mt < mp || (mt == mp && + executed_state_impl (action () /* inner */) == + target_state::changed); } // path_target diff --git a/build2/test/common.hxx b/build2/test/common.hxx index 89b7581..81cccb7 100644 --- a/build2/test/common.hxx +++ b/build2/test/common.hxx @@ -17,7 +17,24 @@ namespace build2 enum class output_before {fail, warn, clean}; enum class output_after {clean, keep}; - struct common + struct common_data + { + const variable& config_test; + const variable& config_test_output; + + const variable& var_test; + const variable& test_options; + const variable& test_arguments; + + const variable& test_stdin; + const variable& test_stdout; + const variable& test_roundtrip; + const variable& test_input; + + const variable& test_target; + }; + + struct common: common_data { // The config.test.output values. // @@ -45,6 +62,9 @@ namespace build2 // bool test (const target& test_target, const path& id_path) const; + + explicit + common (common_data&& d): common_data (move (d)) {} }; } } diff --git a/build2/test/init.cxx b/build2/test/init.cxx index 6119ae0..556de00 100644 --- a/build2/test/init.cxx +++ b/build2/test/init.cxx @@ -23,7 +23,7 @@ namespace build2 namespace test { bool - boot (scope& rs, const location&, unique_ptr&) + boot (scope& rs, const location&, unique_ptr& mod) { tracer trace ("test::boot"); @@ -38,53 +38,78 @@ namespace build2 // auto& vp (var_pool.rw (rs)); - // Tests to execute. - // - // Specified as @ pairs with both sides being optional. - // The variable is untyped (we want a list of name-pairs), overridable, - // and inheritable. The target is relative (in essence a prerequisite) - // which is resolved from the (root) scope where the config.test value - // is defined. - // - vp.insert ("config.test", true); + common_data d { - // Test working directory before/after cleanup (see Testscript spec for - // semantics). - // - vp.insert ("config.test.output", true); + // Tests to execute. + // + // Specified as @ pairs with both sides being + // optional. The variable is untyped (we want a list of name-pairs), + // overridable, and inheritable. The target is relative (in essence a + // prerequisite) which is resolved from the (root) scope where the + // config.test value is defined. + // + vp.insert ("config.test", true), - // Note: none are overridable. - // - // The test variable is a name which can be a path (with the - // true/false special values) or a target name. - // - vp.insert ("test", variable_visibility::target); - vp.insert ("test.input", variable_visibility::project); - vp.insert ("test.output", variable_visibility::project); - vp.insert ("test.roundtrip", variable_visibility::project); - vp.insert ("test.options", variable_visibility::project); - vp.insert ("test.arguments", variable_visibility::project); + // Test working directory before/after cleanup (see Testscript spec + // for semantics). + // + vp.insert ("config.test.output", true), + + // The test variable is a name which can be a path (with the + // true/false special values) or a target name. + // + // Note: none are overridable. + // + vp.insert ("test", variable_visibility::target), + vp.insert ("test.options", variable_visibility::project), + vp.insert ("test.arguments", variable_visibility::project), + + // Prerequisite-specific. + // + // test.stdin and test.stdout can be used to mark a prerequisite as a + // file to redirect stdin from and to compare stdout to, respectively. + // test.roundtrip is a shortcut to mark a prerequisite as both stdin + // and stdout. + // + // Prerequisites marked with test.input are treated as additional test + // inputs: they are made sure to be up to date and their paths are + // passed as additional command line arguments (after test.options and + // test.arguments). Their primary use is to pass inputs that may have + // varying file names/paths, for example: + // + // exe{parent}: exe{child}: test.input = true + // + // Note that currently this mechanism is only available to simple + // tests though we could also support it for testscript (e.g., by + // appending the input paths to test.arguments or by passing them in a + // separate test.inputs variable). + // + vp.insert ("test.stdin", variable_visibility::target), + vp.insert ("test.stdout", variable_visibility::target), + vp.insert ("test.roundtrip", variable_visibility::target), + vp.insert ("test.input", variable_visibility::target), + + // Test target platform. + // + vp.insert ("test.target", variable_visibility::project) + }; // These are only used in testscript. // vp.insert ("test.redirects", variable_visibility::project); vp.insert ("test.cleanups", variable_visibility::project); - // Test target platform. - // // Unless already set, default test.target to build.host. Note that it // can still be overriden by the user, e.g., in root.build. // { - value& v ( - rs.assign ( - vp.insert ( - "test.target", variable_visibility::project))); + value& v (rs.assign (d.test_target)); if (!v || v.empty ()) v = cast ((*global_scope)["build.host"]); } + mod.reset (new module (move (d))); return false; } @@ -108,8 +133,7 @@ namespace build2 const dir_path& out_root (rs.out_path ()); l5 ([&]{trace << "for " << out_root;}); - assert (mod == nullptr); - mod.reset (new module ()); + assert (mod != nullptr); module& m (static_cast (*mod)); // Configure. @@ -123,7 +147,7 @@ namespace build2 // config.test // - if (lookup l = config::omitted (rs, "config.test").first) + if (lookup l = config::omitted (rs, m.config_test).first) { // Figure out which root scope it came from. // @@ -139,7 +163,7 @@ namespace build2 // config.test.output // - if (lookup l = config::omitted (rs, "config.test.output").first) + if (lookup l = config::omitted (rs, m.config_test_output).first) { const name_pair& p (cast (l)); @@ -180,22 +204,13 @@ namespace build2 t.insert (); } - // Register rules. + // Register our test running rule. // { - const rule& r (m); - const alias_rule& ar (m); + default_rule& dr (m); - // Register our test running rule. - // - rs.rules.insert (perform_test_id, "test", r); - rs.rules.insert (perform_test_id, "test", ar); - - // Register our rule for the dist meta-operation. We need to do this - // because we may have ad hoc prerequisites (test input/output files) - // that need to be entered into the target list. - // - rs.rules.insert (dist_id, test_id, "test", r); + rs.rules.insert (perform_test_id, "test", dr); + rs.rules.insert (perform_test_id, "test", dr); } return true; diff --git a/build2/test/module.hxx b/build2/test/module.hxx index 3c9539f..529f826 100644 --- a/build2/test/module.hxx +++ b/build2/test/module.hxx @@ -17,8 +17,19 @@ namespace build2 { namespace test { - struct module: module_base, virtual common, rule, alias_rule + struct module: module_base, virtual common, default_rule, group_rule { + const test::group_rule& + group_rule () const + { + return *this; + } + + explicit + module (common_data&& d) + : common (move (d)), + test::default_rule (move (d)), + test::group_rule (move (d)) {} }; } } diff --git a/build2/test/rule.cxx b/build2/test/rule.cxx index 96941e6..9a00d84 100644 --- a/build2/test/rule.cxx +++ b/build2/test/rule.cxx @@ -23,38 +23,66 @@ namespace build2 { namespace test { - struct match_data + bool rule:: + match (action, target&, const string&) const { - bool pass; // Pass-through to prerequsites (for alias only). - bool test; - - bool script; - }; - - static_assert (sizeof (match_data) <= target::data_size, - "insufficient space"); + // We always match, even if this target is not testable (so that we can + // ignore it; see apply()). + // + return true; + } - match_result rule_common:: - match (action a, target& t, const string&) const + recipe rule:: + apply (action a, target& t) const { - // The (admittedly twisted) logic of this rule tries to achieve the - // following: If the target is testable, then we want both first update - // it and then test it. Otherwise, we want to ignore it for both - // operations. To achieve this the rule will be called to match during - // both operations. For update, if the target is not testable, then the - // rule matches with a noop recipe. If the target is testable, then the - // rule also matches but delegates to the real update rule. In this case - // (and this is tricky) the rule also changes the operation to - // unconditional update to make sure it doesn't match any prerequisites - // (which, if not testable, it will noop). + // Note that we are called both as the outer part during the "update for + // test" pre-operation and as the inner part during the test operation + // itself. + // + // In both cases we first determine if the target is testable and return + // noop if it's not. Otherwise, in the first case (update for test) we + // delegate to the normal update and in the second (test) -- perform the + // test. // // And to add a bit more complexity, we want to handle aliases slightly // differently: we may not want to ignore their prerequisites if the // alias is not testable since their prerequisites could be. + // + // Here is the state matrix: + // + // test'able | pass'able | neither + // | | + // update for test delegate (& pass) | pass | noop + // ---------------------------------------+-------------+--------- + // test test (& pass) | pass | noop + // + + // If we are passing-through, then match our prerequisites. + // + // Note that we may already have stuff in prerequisite_targets (see + // group_rule). + // + if (t.is_a () && pass (t)) + { + // For the test operation we have to implement our own search and + // match because we need to ignore prerequisites that are outside of + // our project. They can be from projects that don't use the test + // module (and thus won't have a suitable rule). Or they can be from + // no project at all (e.g., installed). Also, generally, not testing + // stuff that's not ours seems right. + // + match_prerequisites (a, t, t.root_scope ()); + } - match_data md {t.is_a () && pass (t), false, false}; + auto& pts (t.prerequisite_targets[a]); + size_t pass_n (pts.size ()); // Number of pass-through prerequisites. - if (test (t)) + // See if it's testable and if so, what kind. + // + bool test (false); + bool script (false); + + if (this->test (t)) { // We have two very different cases: testscript and simple test (plus // it may not be a testable target at all). So as the first step @@ -75,292 +103,223 @@ namespace build2 { if (p.is_a ()) { - md.script = true; + if (!script) + { + script = true; - // We treat this target as testable unless the test variable is - // explicitly set to false. + // We treat this target as testable unless the test variable is + // explicitly set to false. + // + const name* n (cast_null (t[var_test])); + test = (n == nullptr || !n->simple () || n->value != "false"); + + if (!test) + break; + } + + // If this is the test operation, collect testscripts after the + // pass-through prerequisites. + // + // Note that we don't match nor execute them relying on update to + // assign their paths and make sure they are up to date. // - const name* n (cast_null (t["test"])); - md.test = n == nullptr || !n->simple () || n->value != "false"; - break; + if (a.operation () != test_id) + break; + + pts.push_back (&p.search (t)); } } // If this is not a script, then determine if it is a simple test. - // Ignore aliases and testscripts files themselves at the outset. + // Ignore aliases and testscript files themselves at the outset. // - if (!md.script && !t.is_a () && !t.is_a ()) + if (!script && !t.is_a () && !t.is_a ()) { // For the simple case whether this is a test is controlled by the // test variable. Also, it feels redundant to specify, say, "test = - // true" and "test.output = test.out" -- the latter already says this + // true" and "test.stdout = test.out" -- the latter already says this // is a test. // + const name* n (cast_null (t[var_test])); - // Use lookup depths to figure out who "overrides" whom. + // If the test variable is explicitly set to false then we treat + // it as not testable regardless of what other test.* variables + // or prerequisites we might have. // - auto p (t.find (var_pool["test"])); - const name* n (cast_null (p.first)); - - // Note that test can be set to an "override" target. + // Note that the test variable can be set to an "override" target + // (which means 'true' for our purposes). // - if (n != nullptr && (!n->simple () || n->value != "false")) - md.test = true; + if (n != nullptr && n->simple () && n->value == "false") + test = false; else { - auto test = [&t, &p] (const char* var) + // Look for test input/stdin/stdout prerequisites. The same + // group logic as in the testscript case above. + // + for (prerequisite_member p: + group_prerequisite_members (a, t, members_mode::maybe)) { - return t.find (var_pool[var]).second < p.second; - }; - - md.test = - test ("test.input") || - test ("test.output") || - test ("test.roundtrip") || - test ("test.options") || - test ("test.arguments"); + const auto& vars (p.prerequisite.vars); + + if (vars.empty ()) // Common case. + continue; + + bool rt ( cast_false (vars[test_roundtrip])); + bool si (rt || cast_false (vars[test_stdin])); + bool so (rt || cast_false (vars[test_stdout])); + bool in ( cast_false (vars[test_input])); + + if (si || so || in) + { + // Note that we don't match nor execute them relying on update + // to assign their paths and make sure they are up to date. + // + const target& pt (p.search (t)); + + // Verify it is file-based. + // + if (!pt.is_a ()) + { + fail << "test." << (si ? "stdin" : so ? "stdout" : "input") + << " prerequisite " << p << " of target " << t + << " is not a file"; + } + + if (!test) + { + test = true; + + if (a.operation () != test_id) + break; + + // First matching prerequisite. Establish the structure in + // pts: the first element (after pass_n) is stdin (can be + // NULL), the second is stdout (can be NULL), and everything + // after that (if any) is inputs. + // + pts.push_back (nullptr); // stdin + pts.push_back (nullptr); // stdout + } + + if (si) + { + if (pts[pass_n] != nullptr) + fail << "multiple test.stdin prerequisites for target " + << t; + + pts[pass_n] = &pt; + } + + if (so) + { + if (pts[pass_n + 1] != nullptr) + fail << "multiple test.stdout prerequisites for target " + << t; + + pts[pass_n + 1] = &pt; + } + + if (in) + pts.push_back (&pt); + } + } + + if (!test) + test = (n != nullptr); // We have the test variable. + + if (!test) + test = t[test_options] || t[test_arguments]; } } } - match_result mr (true); - - // Theoretically if this target is testable and this is the update - // pre-operation, then all we need to do is say we are not a match and - // the standard matching machinery will find the rule to update this - // target. The problem with this approach is that the matching will - // still happen for "update for test" which means this rule may still - // match prerequisites (e.g., non-existent files) which we don't want. - // - // Also, for the simple case there is one more complication: test - // input/output. While normally they will be existing (in src_base) - // files, they could also be auto-generated. In fact, they could only be - // needed for testing, which means the normall update won't even know - // about them (nor clean, for that matter; this is why we need - // cleantest). - // - // @@ Maybe we should just say if input/output are generated, then they - // must be explicitly listed as prerequisites? Then no need for - // cleantest but they will be updated even when not needed. - // - // To make generated input/output work we will have to cause their - // update ourselves. In other words, we may have to do some actual work - // for (update, test), and not simply "guide" (update, 0) as to which - // targets need updating. For how exactly we are going to do it, see - // apply() below. - - // Change the recipe action to (update, 0) (i.e., "unconditional - // update") for "leaf" tests to make sure we won't match any - // prerequisites. Note that this doesn't cover the case where an alias - // is both a test and a pass for a test prerequisite with generated - // input/output. - // - if (a.operation () == update_id && md.test) - mr.recipe_action = action (a.meta_operation (), update_id); - - // Note that we match even if this target is not testable so that we can - // ignore it (see apply()). - // - t.data (md); // Save the data in the target's auxilary storage. - return mr; - } - - recipe alias_rule:: - apply (action a, target& t) const - { - match_data md (move (t.data ())); - t.clear_data (); // In case delegated-to rule also uses aux storage. - - // We can only test an alias via a testscript, not a simple test. + // Neither testing nor passing-through. // - assert (!md.test || md.script); - - if (!md.pass && !md.test) + if (!test && pass_n == 0) return noop_recipe; - // If this is the update pre-operation then simply redirect to the - // standard alias rule. - // - if (a.operation () == update_id) - return match_delegate (a, t, *this); - - // For the test operation we have to implement our own search and match - // because we need to ignore prerequisites that are outside of our - // project. They can be from projects that don't use the test module - // (and thus won't have a suitable rule). Or they can be from no project - // at all (e.g., installed). Also, generally, not testing stuff that's - // not ours seems right. Note that we still want to make sure they are - // up to date (via the above delegate) since our tests might use them. + // If we are only passing-through, then use the default recipe (which + // will execute all the matched prerequisites). // - match_prerequisites (a, t, t.root_scope ()); + if (!test) + return default_recipe; - // If not a test then also redirect to the alias rule. + // Being here means we are definitely testing and maybe passing-through. // - return md.test - ? [this] (action a, const target& t) {return perform_test (a, t);} - : default_recipe; - } - - recipe rule:: - apply (action a, target& t) const - { - tracer trace ("test::rule::apply"); - - match_data md (move (t.data ())); - t.clear_data (); // In case delegated-to rule also uses aux storage. - - if (!md.test) - return noop_recipe; - - // If we are here, then the target is testable and the action is either - // a. (perform, test, 0) or - // b. (*, update, 0) - // - if (md.script) + if (a.operation () == update_id) { - if (a.operation () == update_id) - return match_delegate (a, t, *this); - - // Collect all the testscript targets in prerequisite_targets. + // For the update pre-operation match the inner rule (actual update). + // Note that here we assume it will update (if required) all the + // testscript and input/output prerequisites. // - for (prerequisite_member p: - group_prerequisite_members (a, t, members_mode::maybe)) - { - if (p.is_a ()) - t.prerequisite_targets.push_back (&p.search (t)); - } - - return [this] (action a, const target& t) - { - return perform_script (a, t); - }; + match_inner (a, t); + return &perform_update; } else { - // In both cases, the next step is to see if we have test.{input, - // output,roundtrip}. - // - - // We should have either arguments or input/roundtrip. Again, use - // lookup depth to figure out who takes precedence. - // - auto ip (t.find (var_pool["test.input"])); - auto op (t.find (var_pool["test.output"])); - auto rp (t.find (var_pool["test.roundtrip"])); - auto ap (t.find (var_pool["test.arguments"])); - - auto test = [&t] (pair& x, const char* xn, - pair& y, const char* yn) + if (script) { - if (x.first && y.first) + return [pass_n, this] (action a, const target& t) { - if (x.second == y.second) - fail << "both " << xn << " and " << yn << " specified for " - << "target " << t; - - (x.second < y.second ? y : x) = make_pair (lookup (), size_t (~0)); - } - }; - - test (ip, "test.input", ap, "test.arguments"); - test (rp, "test.roundtrip", ap, "test.arguments"); - test (ip, "test.input", rp, "test.roundtrip"); - test (op, "test.output", rp, "test.roundtrip"); - - const name* in; - const name* on; - - // Reduce the roundtrip case to input/output. - // - if (rp.first) - { - in = on = &cast (rp.first); + return perform_script (a, t, pass_n); + }; } else { - in = ip.first ? &cast (ip.first) : nullptr; - on = op.first ? &cast (op.first) : nullptr; + return [pass_n, this] (action a, const target& t) + { + return perform_test (a, t, pass_n); + }; } + } + } - // Resolve them to targets, which normally would be existing files - // but could also be targets that need updating. - // - const scope& bs (t.base_scope ()); - - // @@ OUT: what if this is a @-qualified pair or names? - // - const target* it (in != nullptr ? &search (t, *in, bs) : nullptr); - const target* ot (on != nullptr - ? in == on ? it : &search (t, *on, bs) - : nullptr); + recipe group_rule:: + apply (action a, target& t) const + { + // Resolve group members. + // + // Remember that we are called twice: first during update for test + // (pre-operation) and then during test. During the former, we rely on + // the normall update rule to resolve the group members. During the + // latter, there will be no rule to do this but the group will already + // have been resolved by the pre-operation. + // + // If the rule could not resolve the group, then we ignore it. + // + group_view gv (a.outer () + ? resolve_group_members (a, t) + : t.group_members (a)); - if (a.operation () == update_id) + if (gv.members != nullptr) + { + auto& pts (t.prerequisite_targets[a]); + for (size_t i (0); i != gv.count; ++i) { - // First see if input/output are existing, up-to-date files. This - // is a common case optimization. - // - if (it != nullptr) - { - if (build2::match (a, *it, unmatch::unchanged)) - it = nullptr; - } + if (const target* m = gv.members[i]) + pts.push_back (m); + } - if (ot != nullptr) - { - if (in != on) - { - if (build2::match (a, *ot, unmatch::unchanged)) - ot = nullptr; - } - else - ot = it; - } + match_members (a, t, pts); + } - // Find the "real" update rule, that is, the rule that would have - // been found if we signalled that we do not match from match() - // above. - // - recipe d (match_delegate (a, t, *this)); + // Delegate to the base rule. + // + return rule::apply (a, t); + } - // If we have no input/output that needs updating, then simply - // redirect to it. - // - if (it == nullptr && ot == nullptr) - return d; + target_state rule:: + perform_update (action a, const target& t) + { + // First execute the inner recipe, then, if passing-through, execute + // prerequisites. + // + target_state ts (execute_inner (a, t)); - // Ok, time to handle the worst case scenario: we need to cause - // update of input/output targets and also delegate to the real - // update. - // - return [it, ot, dr = move (d)] ( - action a, const target& t) -> target_state - { - // Do the general update first. - // - target_state r (execute_delegate (dr, a, t)); + if (t.prerequisite_targets[a].size () != 0) + ts |= straight_execute_prerequisites (a, t); - const target* ts[] = {it, ot}; - return r |= straight_execute_members (a, t, ts); - }; - } - else - { - // Cache the targets in our prerequsite targets lists where they can - // be found by perform_test(). If we have either or both, then the - // first entry is input and the second -- output (either can be - // NULL). - // - if (it != nullptr || ot != nullptr) - { - auto& pts (t.prerequisite_targets); - pts.resize (2, nullptr); - pts[0] = it; - pts[1] = ot; - } - - return &perform_test; - } - } + return ts; } static script::scope_state @@ -404,31 +363,33 @@ namespace build2 return r; } - target_state rule_common:: - perform_script (action, const target& t) const + target_state rule:: + perform_script (action a, const target& t, size_t pass_n) const { + // First pass through. + // + if (pass_n != 0) + straight_execute_prerequisites (a, t, pass_n); + // Figure out whether the testscript file is called 'testscript', in // which case it should be the only one. // + auto& pts (t.prerequisite_targets[a]); + size_t pts_n (pts.size ()); + bool one; - size_t count (0); { optional o; - for (const target* pt: t.prerequisite_targets) + for (size_t i (pass_n); i != pts_n; ++i) { - // In case we are using the alias rule's list (see above). - // - if (const testscript* ts = pt->is_a ()) - { - count++; + const testscript& ts (*pts[i]->is_a ()); - bool r (ts->name == "testscript"); + bool r (ts.name == "testscript"); - if ((r && o) || (!r && o && *o)) - fail << "both 'testscript' and other names specified for " << t; + if ((r && o) || (!r && o && *o)) + fail << "both 'testscript' and other names specified for " << t; - o = r; - } + o = r; } assert (o); // We should have a testscript or we wouldn't be here. @@ -487,57 +448,56 @@ namespace build2 // Start asynchronous execution of the testscripts. // - wait_guard wg (target::count_busy (), t.task_count); + wait_guard wg (target::count_busy (), t[a].task_count); // Result vector. // using script::scope_state; vector result; - result.reserve (count); // Make sure there are no reallocations. + result.reserve (pts_n - pass_n); // Make sure there are no reallocations. - for (const target* pt: t.prerequisite_targets) + for (size_t i (pass_n); i != pts_n; ++i) { - if (const testscript* ts = pt->is_a ()) + const testscript& ts (*pts[i]->is_a ()); + + // If this is just the testscript, then its id path is empty (and it + // can only be ignored by ignoring the test target, which makes sense + // since it's the only testscript file). + // + if (one || test (t, path (ts.name))) { - // If this is just the testscript, then its id path is empty (and it - // can only be ignored by ignoring the test target, which makes - // sense since it's the only testscript file). - // - if (one || test (t, path (ts->name))) + if (mk) { - if (mk) - { - mkdir (wd, 2); - mk = false; - } + mkdir (wd, 2); + mk = false; + } - result.push_back (scope_state::unknown); - scope_state& r (result.back ()); - - if (!sched.async (target::count_busy (), - t.task_count, - [this] (scope_state& r, - const target& t, - const testscript& ts, - const dir_path& wd, - const diag_frame* ds) - { - diag_frame df (ds); - r = perform_script_impl (t, ts, wd, *this); - }, - ref (r), - cref (t), - cref (*ts), - cref (wd), - diag_frame::stack)) - { - // Executed synchronously. If failed and we were not asked to - // keep going, bail out. - // - if (r == scope_state::failed && !keep_going) - throw failed (); - } + result.push_back (scope_state::unknown); + scope_state& r (result.back ()); + + if (!sched.async (target::count_busy (), + t[a].task_count, + [this] (scope_state& r, + const target& t, + const testscript& ts, + const dir_path& wd, + const diag_frame* ds) + { + diag_frame df (ds); + r = perform_script_impl (t, ts, wd, *this); + }, + ref (r), + cref (t), + cref (ts), + cref (wd), + diag_frame::stack)) + { + // Executed synchronously. If failed and we were not asked to keep + // going, bail out. + // + if (r == scope_state::failed && !keep_going) + throw failed (); } } } @@ -597,28 +557,15 @@ namespace build2 try { - if (prev == nullptr) - { - // First process. - // - process p (args, 0, out); - pr = *next == nullptr || run_test (t, dr, next, &p); - p.wait (); + process p (prev == nullptr + ? process (args, 0, out) // First process. + : process (args, *prev, out)); // Next process. - assert (p.exit); - pe = *p.exit; - } - else - { - // Next process. - // - process p (args, *prev, out); - pr = *next == nullptr || run_test (t, dr, next, &p); - p.wait (); + pr = *next == nullptr || run_test (t, dr, next, &p); + p.wait (); - assert (p.exit); - pe = *p.exit; - } + assert (p.exit); + pe = *p.exit; } catch (const process_error& e) { @@ -646,10 +593,12 @@ namespace build2 } target_state rule:: - perform_test (action, const target& tt) + perform_test (action a, const target& tt, size_t pass_n) const { - // @@ Would be nice to print what signal/core was dumped. + // First pass through. // + if (pass_n != 0) + straight_execute_prerequisites (a, tt, pass_n); // See if we have the test executable override. // @@ -657,7 +606,7 @@ namespace build2 { // Note that the test variable's visibility is target. // - lookup l (tt["test"]); + lookup l (tt[var_test]); // Note that we have similar code for scripted tests. // @@ -686,7 +635,7 @@ namespace build2 { // Must be a target name. // - // @@ OUT: what if this is a @-qualified pair or names? + // @@ OUT: what if this is a @-qualified pair of names? // t = search_existing (*n, tt.base_scope ()); @@ -722,41 +671,74 @@ namespace build2 } } - process_path pp (run_search (p, true)); - cstrings args {pp.recall_string ()}; - - // Do we have options? + // See apply() for the structure of prerequisite_targets in the presence + // of test.{input,stdin,stdout}. // - if (auto l = tt["test.options"]) - append_options (args, cast (l)); + auto& pts (tt.prerequisite_targets[a]); + size_t pts_n (pts.size ()); + + cstrings args; - // Do we have input? + // Do we have stdin? // - auto& pts (tt.prerequisite_targets); - if (pts.size () != 0 && pts[0] != nullptr) + // We simulate stdin redirect (as ()); + const file& it (pts[pass_n]->as ()); const path& ip (it.path ()); assert (!ip.empty ()); // Should have been assigned by update. + + try + { + cat.in_ofd = fdopen (ip, fdopen_mode::in); + } + catch (const io_error& e) + { + fail << "unable to open " << ip << ": " << e; + } + + // Purely for diagnostics. + // + args.push_back ("cat"); args.push_back (ip.string ().c_str ()); + args.push_back (nullptr); } - // Maybe arguments then? + + process_path pp (run_search (p, true)); + args.push_back (pp.recall_string ()); + + // Do we have options and/or arguments? // - else + if (auto l = tt[test_options]) + append_options (args, cast (l)); + + if (auto l = tt[test_arguments]) + append_options (args, cast (l)); + + // Do we have inputs? + // + for (size_t i (pass_n + 2); i < pts_n; ++i) { - if (auto l = tt["test.arguments"]) - append_options (args, cast (l)); + const file& it (pts[i]->as ()); + const path& ip (it.path ()); + assert (!ip.empty ()); // Should have been assigned by update. + args.push_back (ip.string ().c_str ()); } args.push_back (nullptr); - // Do we have output? + // Do we have stdout? // path dp ("diff"); process_path dpp; - if (pts.size () != 0 && pts[1] != nullptr) + if (pass_n != pts_n && pts[pass_n + 1] != nullptr) { - const file& ot (pts[1]->as ()); + const file& ot (pts[pass_n + 1]->as ()); const path& op (ot.path ()); assert (!op.empty ()); // Should have been assigned by update. @@ -775,7 +757,7 @@ namespace build2 // Ignore Windows newline fluff if that's what we are running on. // - if (cast (tt["test.target"]).class_ == "windows") + if (cast (tt[test_target]).class_ == "windows") args.push_back ("--strip-trailing-cr"); args.push_back (op.string ().c_str ()); @@ -791,7 +773,10 @@ namespace build2 text << "test " << tt; diag_record dr; - if (!run_test (tt, dr, args.data ())) + if (!run_test (tt, + dr, + args.data () + (stdin ? 3 : 0), // Skip cat. + stdin ? &cat : nullptr)) { dr << info << "test command line: "; print_process (dr, args); @@ -800,18 +785,5 @@ namespace build2 return target_state::changed; } - - target_state alias_rule:: - perform_test (action a, const target& t) const - { - // Run the alias recipe first then the test. - // - target_state r (straight_execute_prerequisites (a, t)); - - // Note that we reuse the prerequisite_targets prepared by the standard - // search and match. - // - return r |= perform_script (a, t); - } } } diff --git a/build2/test/rule.hxx b/build2/test/rule.hxx index 819121c..b331263 100644 --- a/build2/test/rule.hxx +++ b/build2/test/rule.hxx @@ -17,34 +17,46 @@ namespace build2 { namespace test { - class rule_common: public build2::rule, protected virtual common + class rule: public build2::rule, protected virtual common { public: - virtual match_result + explicit + rule (common_data&& d): common (move (d)) {} + + virtual bool match (action, target&, const string&) const override; + virtual recipe + apply (action, target&) const override; + + static target_state + perform_update (action, const target&); + + target_state + perform_test (action, const target&, size_t) const; + target_state - perform_script (action, const target&) const; + perform_script (action, const target&, size_t) const; }; - class rule: public rule_common + class default_rule: public rule // For disambiguation in module. { public: - virtual recipe - apply (action, target&) const override; - - static target_state - perform_test (action, const target&); + explicit + default_rule (common_data&& d): common (move (d)), rule (move (d)) {} }; - class alias_rule: public rule_common + // In addition to the above rule's semantics, this rule sees through to + // the group's members. + // + class group_rule: public rule { public: + explicit + group_rule (common_data&& d): common (move (d)), rule (move (d)) {} + virtual recipe apply (action, target&) const override; - - target_state - perform_test (action, const target&) const; }; } } diff --git a/build2/test/script/runner.cxx b/build2/test/script/runner.cxx index cddd3a7..9588ac2 100644 --- a/build2/test/script/runner.cxx +++ b/build2/test/script/runner.cxx @@ -181,6 +181,8 @@ namespace build2 // For targets other than Windows leave the string intact. // + // @@ Would be nice to use cached value from test::common_data. + // if (cast (scr.test_target["test.target"]).class_ != "windows") return s; @@ -294,6 +296,8 @@ namespace build2 // Ignore Windows newline fluff if that's what we are running on. // + // @@ Would be nice to use cached value from test::common_data. + // if (cast ( sp.root->test_target["test.target"]).class_ == "windows") args.push_back ("--strip-trailing-cr"); diff --git a/build2/test/script/script.cxx b/build2/test/script/script.cxx index 51c08cb..0516b0f 100644 --- a/build2/test/script/script.cxx +++ b/build2/test/script/script.cxx @@ -503,6 +503,8 @@ namespace build2 // buildfiles except for test: while in buildfiles it can be a // target name, in testscripts it should be resolved to a path. // + // Note: entering in a custom variable pool. + // test_var (var_pool.insert ("test")), options_var (var_pool.insert ("test.options")), arguments_var (var_pool.insert ("test.arguments")), @@ -527,7 +529,9 @@ namespace build2 // script // script:: - script (const target& tt, const testscript& st, const dir_path& rwd) + script (const target& tt, + const testscript& st, + const dir_path& rwd) : group (st.name == "testscript" ? string () : st.name, this), test_target (tt), script_target (st) @@ -574,7 +578,7 @@ namespace build2 { // Must be a target name. // - // @@ OUT: what if this is a @-qualified pair or names? + // @@ OUT: what if this is a @-qualified pair of names? // t = search_existing (*n, tt.base_scope ()); diff --git a/build2/variable.cxx b/build2/variable.cxx index d1f95c5..eb74aad 100644 --- a/build2/variable.cxx +++ b/build2/variable.cxx @@ -302,7 +302,7 @@ namespace build2 } void - typify (value& v, const value_type& t, const variable* var) + typify (value& v, const value_type& t, const variable* var, memory_order mo) { if (v.type == nullptr) { @@ -312,11 +312,16 @@ namespace build2 // names ns (move (v).as ()); v = nullptr; - v.type = &t; - v.assign (move (ns), var); + + // Use value_type::assign directly to delay v.type change. + // + t.assign (v, move (ns), var); + v.null = false; } else v.type = &t; + + v.type.store (&t, mo); } else if (v.type != &t) { @@ -342,8 +347,10 @@ namespace build2 variable_cache_mutex_shard[ hash () (&v) % variable_cache_mutex_shard_size]); + // Note: v.type is rechecked by typify() under lock. + // ulock l (m); - typify (v, t, var); // v.type is rechecked by typify(), stored under lock. + typify (v, t, var, memory_order_release); } void diff --git a/build2/variable.ixx b/build2/variable.ixx index fd6b7b2..dcc1304 100644 --- a/build2/variable.ixx +++ b/build2/variable.ixx @@ -237,6 +237,16 @@ namespace build2 typify (v, t, var); } + void + typify (value&, const value_type&, const variable*, memory_order); + + inline void + typify (value& v, const value_type& t, const variable* var) + { + typify (v, t, var, memory_order_relaxed); + } + + inline vector_view reverse (const value& v, names& storage) { diff --git a/build2/version/init.cxx b/build2/version/init.cxx index 63e32ab..7b8bd01 100644 --- a/build2/version/init.cxx +++ b/build2/version/init.cxx @@ -27,8 +27,8 @@ namespace build2 { static const path manifest ("manifest"); - static const version_doc version_doc_; - static const version_in version_in_; + static const doc_rule doc_rule_; + static const in_rule in_rule_; bool boot (scope& rs, const location& l, unique_ptr& mod) @@ -311,13 +311,13 @@ namespace build2 { auto& r (rs.rules); - r.insert (perform_update_id, "version.doc", version_doc_); - r.insert (perform_clean_id, "version.doc", version_doc_); - r.insert (configure_update_id, "version.doc", version_doc_); + r.insert (perform_update_id, "version.doc", doc_rule_); + r.insert (perform_clean_id, "version.doc", doc_rule_); + r.insert (configure_update_id, "version.doc", doc_rule_); - r.insert (perform_update_id, "version.in", version_in_); - r.insert (perform_clean_id, "version.in", version_in_); - r.insert (configure_update_id, "version.in", version_in_); + r.insert (perform_update_id, "version.in", in_rule_); + r.insert (perform_clean_id, "version.in", in_rule_); + r.insert (configure_update_id, "version.in", in_rule_); } return true; diff --git a/build2/version/rule.cxx b/build2/version/rule.cxx index 9e127ca..bbfe1f6 100644 --- a/build2/version/rule.cxx +++ b/build2/version/rule.cxx @@ -45,12 +45,12 @@ namespace build2 return d == rs.src_path (); } - // version_doc + // doc_rule // - match_result version_doc:: + bool doc_rule:: match (action a, target& xt, const string&) const { - tracer trace ("version::version_doc::match"); + tracer trace ("version::doc_rule::match"); doc& t (static_cast (xt)); @@ -76,7 +76,7 @@ namespace build2 return false; } - recipe version_doc:: + recipe doc_rule:: apply (action a, target& xt) const { doc& t (static_cast (xt)); @@ -101,7 +101,7 @@ namespace build2 } } - target_state version_doc:: + target_state doc_rule:: perform_update (action a, const target& xt) { const doc& t (xt.as ()); @@ -168,12 +168,12 @@ namespace build2 return target_state::changed; } - // version_in + // in_rule // - match_result version_in:: + bool in_rule:: match (action a, target& xt, const string&) const { - tracer trace ("version::version_in::match"); + tracer trace ("version::in_rule::match"); file& t (static_cast (xt)); const scope& rs (t.root_scope ()); @@ -195,7 +195,7 @@ namespace build2 return fm && fi; } - recipe version_in:: + recipe in_rule:: apply (action a, target& xt) const { file& t (static_cast (xt)); @@ -220,10 +220,10 @@ namespace build2 } } - target_state version_in:: + target_state in_rule:: perform_update (action a, const target& xt) { - tracer trace ("version::version_in::perform_update"); + tracer trace ("version::in_rule::perform_update"); const file& t (xt.as ()); const path& tp (t.path ()); diff --git a/build2/version/rule.hxx b/build2/version/rule.hxx index e686694..9172ba3 100644 --- a/build2/version/rule.hxx +++ b/build2/version/rule.hxx @@ -16,12 +16,12 @@ namespace build2 { // Generate a version file. // - class version_doc: public rule + class doc_rule: public rule { public: - version_doc () {} + doc_rule () {} - virtual match_result + virtual bool match (action, target&, const string&) const override; virtual recipe @@ -33,12 +33,12 @@ namespace build2 // Preprocess an .in file. // - class version_in: public rule + class in_rule: public rule { public: - version_in () {} + in_rule () {} - virtual match_result + virtual bool match (action, target&, const string&) const override; virtual recipe diff --git a/doc/testscript.cli b/doc/testscript.cli index 7b3d472..666d553 100644 --- a/doc/testscript.cli +++ b/doc/testscript.cli @@ -39,10 +39,10 @@ target as a test along with passing options and arguments, providing the result. For example: \ +exe{hello}: file{names.txt}: test.stdin = true +exe{hello}: file{greetings.txt}: test.stdout = true exe{hello}: test.options = --greeting 'Hi' exe{hello}: test.arguments = - # Read names from stdin. -exe{hello}: test.input = names.txt -exe{hello}: test.output = greetings.txt \ This works well for simple, single-run tests. If, however, our testing diff --git a/tests/cc/preprocessed/testscript b/tests/cc/preprocessed/testscript index fb69e65..644e121 100644 --- a/tests/cc/preprocessed/testscript +++ b/tests/cc/preprocessed/testscript @@ -12,7 +12,7 @@ test.arguments = config.cxx="$recall($cxx.path)" update # trace: cxx::compile::extract_(headers|modules): target: .../obje{(test).o...} # filter = sed -n -e \ - \''s/^trace: cxx::compile::extract_([^:]+): target:[^{]+\{([^.]+).*/\1 \2/p'\' + \''s/^trace: cxx::compile_rule::extract_([^:]+): target:[^{]+\{([^.]+).*/\1 \2/p'\' +cat <=build/root.build cxx.std = latest diff --git a/tests/test/config-test/testscript b/tests/test/config-test/testscript index 0d08eb0..7423322 100644 --- a/tests/test/config-test/testscript +++ b/tests/test/config-test/testscript @@ -82,8 +82,7 @@ EOI +touch proj/units/simple/driver +cat <=proj/units/simple/buildfile driver = $src_root/../../exe{driver} -#@@ TMP file{driver}@./: $driver -./: file{driver} $driver +file{driver}@./: $driver file{driver}@./: test = $driver file{driver}@./: test.arguments = units/simple EOI -- cgit v1.1