aboutsummaryrefslogtreecommitdiff
path: root/build2/context.hxx
blob: 378c6c05ebafb403199528b1d0f741d83c40b3c2 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
// file      : build2/context.hxx -*- C++ -*-
// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
// license   : MIT; see accompanying LICENSE file

#ifndef BUILD2_CONTEXT_HXX
#define BUILD2_CONTEXT_HXX

#include <build2/types.hxx>
#include <build2/utility.hxx>

#include <build2/scope.hxx>
#include <build2/variable.hxx>
#include <build2/operation.hxx>
#include <build2/scheduler.hxx>

namespace build2
{
  // Main (and only) scheduler. Started up and shut down in main().
  //
  extern scheduler sched;

  // In order to perform each operation the build system goes through the
  // following phases:
  //
  // load     - load the buildfiles
  // match    - search prerequisites and match rules
  // execute  - execute the matched rule
  //
  // The build system starts with a "serial load" phase and then continues
  // with parallel search and execute. Match, however, can be interrupted
  // both with load and execute.
  //
  // Match can be interrupted with "exclusive load" in order to load
  // additional buildfiles. Similarly, it can be interrupted with (parallel)
  // execute in order to build targetd required to complete the match (for
  // example, generated source code or source code generators themselves).
  //
  // Such interruptions are performed by phase change that is protected by
  // phase_mutex (which is also used to synchronize the state changes between
  // phases).
  //
  // Serial load can perform arbitrary changes to the model. Exclusive load,
  // however, can only perform "island appends". That is, it can create new
  // "nodes" (variables, scopes, etc) but not (semantically) change already
  // existing nodes or invalidate any references to such (the idea here is
  // that one should be able to load additional buildfiles as long as they
  // don't interfere with the existing build state). The "islands" are
  // identified by the load_generation number (0 for the initial/serial
  // load). It is incremented in case of a phase switch and can be stored in
  // various "nodes" to verify modifications are only done "within the
  // islands".
  //
  extern run_phase phase;
  extern size_t load_generation;

  // A "tri-mutex" that keeps all the threads in one of the three phases. When
  // a thread wants to switch a phase, it has to wait for all the other
  // threads to do the same (or release their phase locks). The load phase is
  // exclusive.
  //
  // The interleaving match and execute is interesting: during match we read
  // the "external state" (e.g., filesystem entries, modifications times, etc)
  // and capture it in the "internal state" (our dependency graph). During
  // execute we are modifying the external state with controlled modifications
  // of the internal state to reflect the changes (e.g., update mtimes). If
  // you think about it, it's pretty clear that we cannot safely perform both
  // of these actions simultaneously. A good example would be running a code
  // generator and header dependency extraction simultaneously: the extraction
  // process may pick up headers as they are being generated. As a result, we
  // either have everyone treat the external state as read-only or write-only.
  //
  // There is also one more complication: if we are returning from a load
  // phase that has failed, then the build state could be seriously messed up
  // (things like scopes not being setup completely, etc). And once we release
  // the lock, other threads that are waiting will start relying on this
  // messed up state. So a load phase can mark the phase_mutex as failed in
  // which case all currently blocked and future lock()/relock() calls return
  // false. Note that in this case we still switch to the desired phase. See
  // the phase_{lock,switch,unlock} implementations for details.
  //
  class phase_mutex
  {
  public:
    // Acquire a phase lock potentially blocking (unless already in the
    // desired phase) until switching to the desired phase is possible.
    //
    bool
    lock (run_phase);

    // Release the phase lock potentially allowing (unless there are other
    // locks on this phase) switching to a different phase.
    //
    void
    unlock (run_phase);

    // Switch from one phase to another. Semantically, just unlock() followed
    // by lock() but more efficient.
    //
    bool
    relock (run_phase unlock, run_phase lock);

  private:
    friend struct phase_lock;
    friend struct phase_unlock;
    friend struct phase_switch;

    phase_mutex ()
        : fail_ (false), lc_ (0), mc_ (0), ec_ (0)
    {
      phase = run_phase::load;
    }

    static phase_mutex instance;

  private:
    // We have a counter for each phase which represents the number of threads
    // in or waiting for this phase.
    //
    // We use condition variables to wait for a phase switch. The load phase
    // is exclusive so we have a separate mutex to serialize it (think of it
    // as a second level locking).
    //
    // When the mutex is unlocked (all three counters become zero, the phase
    // is always changed to load (this is also the initial state).
    //
    mutex m_;

    bool fail_;

    size_t lc_;
    size_t mc_;
    size_t ec_;

    condition_variable lv_;
    condition_variable mv_;
    condition_variable ev_;

    mutex lm_;
  };

  // Grab a new phase lock releasing it on destruction. The lock can be
  // "owning" or "referencing" (recursive).
  //
  // On the referencing semantics: If there is already an instance of
  // phase_lock in this thread, then the new instance simply references it.
  //
  // The reason for this semantics is to support the following scheduling
  // pattern (in actual code we use wait_guard to RAII it):
  //
  // atomic_count task_count (0);
  //
  // {
  //   phase_lock l (run_phase::match);                    // (1)
  //
  //   for (...)
  //   {
  //     sched.async (task_count,
  //                  [] (...)
  //                  {
  //                    phase_lock pl (run_phase::match);  // (2)
  //                    ...
  //                  },
  //                  ...);
  //   }
  // }
  //
  // sched.wait (task_count);                              // (3)
  //
  // Here is what's going on here:
  //
  // 1. We first get a phase lock "for ourselves" since after the first
  //    iteration of the loop, things may become asynchronous (including
  //    attempts to switch the phase and modify the structure we are iteration
  //    upon).
  //
  // 2. The task can be queued or it can be executed synchronously inside
  //    async() (refer to the scheduler class for details on this semantics).
  //
  //    If this is an async()-synchronous execution, then the task will create
  //    a referencing phase_lock. If, however, this is a queued execution
  //    (including wait()-synchronous), then the task will create a top-level
  //    phase_lock.
  //
  //    Note that we only acquire the lock once the task starts executing
  //    (there is no reason to hold the lock while the task is sitting in the
  //    queue). This optimization assumes that whatever else we pass to the
  //    task (for example, a reference to a target) is stable (in other words,
  //    such a reference cannot become invalid).
  //
  // 3. Before calling wait(), we release our phase lock to allow switching
  //    the phase.
  //
  struct phase_lock
  {
    explicit phase_lock (run_phase);
    ~phase_lock ();

    phase_lock (phase_lock&&) = delete;
    phase_lock (const phase_lock&) = delete;

    phase_lock& operator= (phase_lock&&) = delete;
    phase_lock& operator= (const phase_lock&) = delete;

    run_phase p;

    static
#ifdef __cpp_thread_local
    thread_local
#else
    __thread
#endif
    phase_lock* instance;
  };

  // Assuming we have a lock on the current phase, temporarily release it
  // and reacquire on destruction.
  //
  struct phase_unlock
  {
    phase_unlock (bool unlock = true);
    ~phase_unlock () noexcept (false);

    phase_lock* l;
  };

  // Assuming we have a lock on the current phase, temporarily switch to a
  // new phase and switch back on destruction.
  //
  struct phase_switch
  {
    explicit phase_switch (run_phase);
    ~phase_switch () noexcept (false);

    run_phase o, n;
  };

  // Wait for a task count optionally and temporarily unlocking the phase.
  //
  struct wait_guard
  {
    ~wait_guard () noexcept (false);

    explicit
    wait_guard (atomic_count& task_count,
                bool phase = false);

    wait_guard (size_t start_count,
                atomic_count& task_count,
                bool phase = false);

    void
    wait ();

    size_t start_count;
    atomic_count* task_count;
    bool phase;
  };

  // Cached variables.
  //
  // Note: consider printing in info meta-operation if adding anything here.
  //
  extern const variable* var_src_root;
  extern const variable* var_out_root;
  extern const variable* var_src_base;
  extern const variable* var_out_base;
  extern const variable* var_forwarded;

  extern const variable* var_project;
  extern const variable* var_amalgamation;
  extern const variable* var_subprojects;
  extern const variable* var_version;

  extern const variable* var_project_url;     // project.url
  extern const variable* var_project_summary; // project.summary

  extern const variable* var_import_target;   // import.target

  extern const variable* var_clean;           // [bool] target visibility
  extern const variable* var_backlink;        // [bool] target visibility

  extern const char var_extension[10];        // "extension"

  // Current action (meta/operation).
  //
  // The names unlike info are available during boot but may not yet be
  // lifted. The name is always for an outer operation (or meta operation
  // that hasn't been recognized as such yet).
  //
  extern string current_mname;
  extern string current_oname;

  extern const meta_operation_info* current_mif;
  extern const operation_info* current_inner_oif;
  extern const operation_info* current_outer_oif;
  extern size_t current_on; // Current operation number (1-based) in the
                            // meta-operation batch.

  extern execution_mode current_mode;

  // Some diagnostics (for example output directory creation/removal by the
  // fsdir rule) is just noise at verbosity level 1 unless it is the only
  // thing that is printed. So we can only suppress it in certain situations
  // (e.g., dist) where we know we have already printed something.
  //
  extern bool current_diag_noise;

  // Total number of dependency relationships and targets with non-noop
  // recipe in the current action.
  //
  // Together with target::dependents the dependency count is incremented
  // during the rule search & match phase and is decremented during execution
  // with the expectation of it reaching 0. Used as a sanity check.
  //
  // The target count is incremented after a non-noop recipe is matched and
  // decremented after such recipe has been executed. If such a recipe has
  // skipped executing the operation, then it should increment the skip count.
  // These two counters are used for progress monitoring and diagnostics.
  //
  extern atomic_count dependency_count;
  extern atomic_count target_count;
  extern atomic_count skip_count;

  inline void
  set_current_mif (const meta_operation_info& mif)
  {
    current_mname = mif.name;
    current_mif = &mif;
    current_on = 0; // Reset.
  }

  inline void
  set_current_oif (const operation_info& inner_oif,
                   const operation_info* outer_oif = nullptr,
                   bool diag_noise = true)
  {
    current_oname = (outer_oif == nullptr ? inner_oif : *outer_oif).name;
    current_inner_oif = &inner_oif;
    current_outer_oif = outer_oif;
    current_on++;
    current_mode = inner_oif.mode;
    current_diag_noise = diag_noise;

    // Reset counters (serial execution).
    //
    dependency_count.store (0, memory_order_relaxed);
    target_count.store (0, memory_order_relaxed);
    skip_count.store (0, memory_order_relaxed);
  }

  // Keep going flag.
  //
  // Note that setting it to false is not of much help unless we are running
  // serially. In parallel we queue most of the things up before we see any
  // failures.
  //
  extern bool keep_going;

  // Reset the build state. In particular, this removes all the targets,
  // scopes, and variables.
  //
  variable_overrides
  reset (const strings& cmd_vars);

  // Return the project name or empty string if unnamed.
  //
  inline const string&
  project (const scope& root)
  {
    auto l (root[var_project]);
    return l ? cast<string> (l) : empty_string;
  }

  // Return the src/out directory corresponding to the given out/src. The
  // passed directory should be a sub-directory of out/src_root.
  //
  dir_path
  src_out (const dir_path& out, const scope& root);

  dir_path
  src_out (const dir_path& out,
           const dir_path& out_root, const dir_path& src_root);

  dir_path
  out_src (const dir_path& src, const scope& root);

  dir_path
  out_src (const dir_path& src,
           const dir_path& out_root, const dir_path& src_root);

  // Action phrases, e.g., "configure update exe{foo}", "updating exe{foo}",
  // and "updating exe{foo} is configured". Use like this:
  //
  // info << "while " << diag_doing (a, t);
  //
  class target;

  struct diag_phrase
  {
    const action& a;
    const target& t;
    void (*f) (ostream&, const action&, const target&);
  };

  inline ostream&
  operator<< (ostream& os, const diag_phrase& p)
  {
    p.f (os, p.a, p.t);
    return os;
  }

  string
  diag_do (const action&);

  void
  diag_do (ostream&, const action&, const target&);

  inline diag_phrase
  diag_do (const action& a, const target& t)
  {
    return diag_phrase {a, t, &diag_do};
  }

  string
  diag_doing (const action&);

  void
  diag_doing (ostream&, const action&, const target&);

  inline diag_phrase
  diag_doing (const action& a, const target& t)
  {
    return diag_phrase {a, t, &diag_doing};
  }

  string
  diag_did (const action&);

  void
  diag_did (ostream&, const action&, const target&);

  inline diag_phrase
  diag_did (const action& a, const target& t)
  {
    return diag_phrase {a, t, &diag_did};
  }

  void
  diag_done (ostream&, const action&, const target&);

  inline diag_phrase
  diag_done (const action& a, const target& t)
  {
    return diag_phrase {a, t, &diag_done};
  }
}

#include <build2/context.ixx>

#endif // BUILD2_CONTEXT_HXX