aboutsummaryrefslogtreecommitdiff
path: root/libbuild2/context.hxx
blob: ce9a9961419a0b4c423377c4f456c43f9e44c928 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
// file      : libbuild2/context.hxx -*- C++ -*-
// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
// license   : MIT; see accompanying LICENSE file

#ifndef LIBBUILD2_CONTEXT_HXX
#define LIBBUILD2_CONTEXT_HXX

#include <libbuild2/types.hxx>
#include <libbuild2/utility.hxx>

#include <libbuild2/variable.hxx>
#include <libbuild2/operation.hxx>
#include <libbuild2/scheduler.hxx>

#include <libbuild2/export.hxx>

namespace build2
{
  class scope;

  // Main scheduler. Started up and shut down in main().
  //
  LIBBUILD2_SYMEXPORT extern scheduler sched;

  // In order to perform each operation the build system goes through the
  // following phases:
  //
  // load     - load the buildfiles
  // match    - search prerequisites and match rules
  // execute  - execute the matched rule
  //
  // The build system starts with a "serial load" phase and then continues
  // with parallel match and execute. Match, however, can be interrupted
  // both with load and execute.
  //
  // Match can be interrupted with "exclusive load" in order to load
  // additional buildfiles. Similarly, it can be interrupted with (parallel)
  // execute in order to build targetd required to complete the match (for
  // example, generated source code or source code generators themselves).
  //
  // Such interruptions are performed by phase change that is protected by
  // phase_mutex (which is also used to synchronize the state changes between
  // phases).
  //
  // Serial load can perform arbitrary changes to the build state. Exclusive
  // load, however, can only perform "island appends". That is, it can create
  // new "nodes" (variables, scopes, etc) but not (semantically) change
  // already existing nodes or invalidate any references to such (the idea
  // here is that one should be able to load additional buildfiles as long as
  // they don't interfere with the existing build state). The "islands" are
  // identified by the load_generation number (0 for the initial/serial
  // load). It is incremented in case of a phase switch and can be stored in
  // various "nodes" to verify modifications are only done "within the
  // islands".
  //
  LIBBUILD2_SYMEXPORT extern run_phase phase;
  LIBBUILD2_SYMEXPORT extern size_t load_generation;

  // A "tri-mutex" that keeps all the threads in one of the three phases. When
  // a thread wants to switch a phase, it has to wait for all the other
  // threads to do the same (or release their phase locks). The load phase is
  // exclusive.
  //
  // The interleaving match and execute is interesting: during match we read
  // the "external state" (e.g., filesystem entries, modifications times, etc)
  // and capture it in the "internal state" (our dependency graph). During
  // execute we are modifying the external state with controlled modifications
  // of the internal state to reflect the changes (e.g., update mtimes). If
  // you think about it, it's pretty clear that we cannot safely perform both
  // of these actions simultaneously. A good example would be running a code
  // generator and header dependency extraction simultaneously: the extraction
  // process may pick up headers as they are being generated. As a result, we
  // either have everyone treat the external state as read-only or write-only.
  //
  // There is also one more complication: if we are returning from a load
  // phase that has failed, then the build state could be seriously messed up
  // (things like scopes not being setup completely, etc). And once we release
  // the lock, other threads that are waiting will start relying on this
  // messed up state. So a load phase can mark the phase_mutex as failed in
  // which case all currently blocked and future lock()/relock() calls return
  // false. Note that in this case we still switch to the desired phase. See
  // the phase_{lock,switch,unlock} implementations for details.
  //
  class LIBBUILD2_SYMEXPORT run_phase_mutex
  {
  public:
    // Acquire a phase lock potentially blocking (unless already in the
    // desired phase) until switching to the desired phase is possible.
    //
    bool
    lock (run_phase);

    // Release the phase lock potentially allowing (unless there are other
    // locks on this phase) switching to a different phase.
    //
    void
    unlock (run_phase);

    // Switch from one phase to another. Semantically, just unlock() followed
    // by lock() but more efficient.
    //
    bool
    relock (run_phase unlock, run_phase lock);

  public:
    run_phase_mutex ()
        : fail_ (false), lc_ (0), mc_ (0), ec_ (0)
    {
      phase = run_phase::load;
    }

  private:
    friend struct phase_lock;
    friend struct phase_unlock;
    friend struct phase_switch;

    // We have a counter for each phase which represents the number of threads
    // in or waiting for this phase.
    //
    // We use condition variables to wait for a phase switch. The load phase
    // is exclusive so we have a separate mutex to serialize it (think of it
    // as a second level locking).
    //
    // When the mutex is unlocked (all three counters become zero, the phase
    // is always changed to load (this is also the initial state).
    //
    mutex m_;

    bool fail_;

    size_t lc_;
    size_t mc_;
    size_t ec_;

    condition_variable lv_;
    condition_variable mv_;
    condition_variable ev_;

    mutex lm_;
  };

  extern run_phase_mutex phase_mutex;

  // Grab a new phase lock releasing it on destruction. The lock can be
  // "owning" or "referencing" (recursive).
  //
  // On the referencing semantics: If there is already an instance of
  // phase_lock in this thread, then the new instance simply references it.
  //
  // The reason for this semantics is to support the following scheduling
  // pattern (in actual code we use wait_guard to RAII it):
  //
  // atomic_count task_count (0);
  //
  // {
  //   phase_lock l (run_phase::match);                    // (1)
  //
  //   for (...)
  //   {
  //     sched.async (task_count,
  //                  [] (...)
  //                  {
  //                    phase_lock pl (run_phase::match);  // (2)
  //                    ...
  //                  },
  //                  ...);
  //   }
  // }
  //
  // sched.wait (task_count);                              // (3)
  //
  // Here is what's going on here:
  //
  // 1. We first get a phase lock "for ourselves" since after the first
  //    iteration of the loop, things may become asynchronous (including
  //    attempts to switch the phase and modify the structure we are iteration
  //    upon).
  //
  // 2. The task can be queued or it can be executed synchronously inside
  //    async() (refer to the scheduler class for details on this semantics).
  //
  //    If this is an async()-synchronous execution, then the task will create
  //    a referencing phase_lock. If, however, this is a queued execution
  //    (including wait()-synchronous), then the task will create a top-level
  //    phase_lock.
  //
  //    Note that we only acquire the lock once the task starts executing
  //    (there is no reason to hold the lock while the task is sitting in the
  //    queue). This optimization assumes that whatever else we pass to the
  //    task (for example, a reference to a target) is stable (in other words,
  //    such a reference cannot become invalid).
  //
  // 3. Before calling wait(), we release our phase lock to allow switching
  //    the phase.
  //
  struct LIBBUILD2_SYMEXPORT phase_lock
  {
    explicit phase_lock (run_phase);
    ~phase_lock ();

    phase_lock (phase_lock&&) = delete;
    phase_lock (const phase_lock&) = delete;

    phase_lock& operator= (phase_lock&&) = delete;
    phase_lock& operator= (const phase_lock&) = delete;

    run_phase p;
  };

  // Assuming we have a lock on the current phase, temporarily release it
  // and reacquire on destruction.
  //
  struct LIBBUILD2_SYMEXPORT phase_unlock
  {
    phase_unlock (bool unlock = true);
    ~phase_unlock () noexcept (false);

    phase_lock* l;
  };

  // Assuming we have a lock on the current phase, temporarily switch to a
  // new phase and switch back on destruction.
  //
  struct LIBBUILD2_SYMEXPORT phase_switch
  {
    explicit phase_switch (run_phase);
    ~phase_switch () noexcept (false);

    run_phase o, n;
  };

  // Wait for a task count optionally and temporarily unlocking the phase.
  //
  struct wait_guard
  {
    ~wait_guard () noexcept (false);

    wait_guard (); // Empty.

    explicit
    wait_guard (atomic_count& task_count,
                bool phase = false);

    wait_guard (size_t start_count,
                atomic_count& task_count,
                bool phase = false);

    void
    wait ();

    // Note: move-assignable to empty only.
    //
    wait_guard (wait_guard&&);
    wait_guard& operator= (wait_guard&&);

    wait_guard (const wait_guard&) = delete;
    wait_guard& operator= (const wait_guard&) = delete;

    size_t start_count;
    atomic_count* task_count;
    bool phase;
  };

  // Current action (meta/operation).
  //
  // The names unlike info are available during boot but may not yet be
  // lifted. The name is always for an outer operation (or meta operation
  // that hasn't been recognized as such yet).
  //
  LIBBUILD2_SYMEXPORT extern string current_mname;
  LIBBUILD2_SYMEXPORT extern string current_oname;

  LIBBUILD2_SYMEXPORT extern const meta_operation_info* current_mif;
  LIBBUILD2_SYMEXPORT extern const operation_info* current_inner_oif;
  LIBBUILD2_SYMEXPORT extern const operation_info* current_outer_oif;

  // Current operation number (1-based) in the meta-operation batch.
  //
  LIBBUILD2_SYMEXPORT extern size_t current_on;

  LIBBUILD2_SYMEXPORT extern execution_mode current_mode;

  // Some diagnostics (for example output directory creation/removal by the
  // fsdir rule) is just noise at verbosity level 1 unless it is the only
  // thing that is printed. So we can only suppress it in certain situations
  // (e.g., dist) where we know we have already printed something.
  //
  LIBBUILD2_SYMEXPORT extern bool current_diag_noise;

  // Total number of dependency relationships and targets with non-noop
  // recipe in the current action.
  //
  // Together with target::dependents the dependency count is incremented
  // during the rule search & match phase and is decremented during execution
  // with the expectation of it reaching 0. Used as a sanity check.
  //
  // The target count is incremented after a non-noop recipe is matched and
  // decremented after such recipe has been executed. If such a recipe has
  // skipped executing the operation, then it should increment the skip count.
  // These two counters are used for progress monitoring and diagnostics.
  //
  LIBBUILD2_SYMEXPORT extern atomic_count dependency_count;
  LIBBUILD2_SYMEXPORT extern atomic_count target_count;
  LIBBUILD2_SYMEXPORT extern atomic_count skip_count;

  LIBBUILD2_SYMEXPORT void
  set_current_mif (const meta_operation_info&);

  LIBBUILD2_SYMEXPORT void
  set_current_oif (const operation_info& inner,
                   const operation_info* outer = nullptr,
                   bool diag_noise = true);

  // Keep going flag.
  //
  // Note that setting it to false is not of much help unless we are running
  // serially. In parallel we queue most of the things up before we see any
  // failures.
  //
  LIBBUILD2_SYMEXPORT extern bool keep_going;

  // Dry run flag (see --dry-run|-n).
  //
  // This flag is set only for the final execute phase (as opposed to those
  // that interrupt match) by the perform meta operation's execute() callback.
  //
  // Note that for this mode to function properly we have to use fake mtimes.
  // Specifically, a rule that pretends to update a target must set its mtime
  // to system_clock::now() and everyone else must use this cached value. In
  // other words, there should be no mtime re-query from the filesystem. The
  // same is required for "logical clean" (i.e., dry-run 'clean update' in
  // order to see all the command lines).
  //
  // At first, it may seem like we should also "dry-run" changes to depdb. But
  // that would be both problematic (some rules update it in apply() during
  // the match phase) and wasteful (why discard information). Also, depdb may
  // serve as an input to some commands (for example, to provide C++ module
  // mapping) which means that without updating it the commands we print might
  // not be runnable (think of the compilation database).
  //
  // One thing we need to be careful about if we are updating depdb is to not
  // render the target up-to-date. But in this case the depdb file will be
  // older than the target which in our model is treated as an interrupted
  // update (see depdb for details).
  //
  // Note also that sometimes it makes sense to do a bit more than absolutely
  // necessary or to discard information in order to keep the rule logic sane.
  // And some rules may choose to ignore this flag altogether. In this case,
  // however, the rule should be careful not to rely on functions (notably
  // from filesystem) that respect this flag in order not to end up with a
  // job half done.
  //
  LIBBUILD2_SYMEXPORT extern bool dry_run;

  // Reset the build state. In particular, this removes all the targets,
  // scopes, and variables.
  //
  LIBBUILD2_SYMEXPORT variable_overrides
  reset (const strings& cmd_vars);

  // Config module entry points.
  //
  LIBBUILD2_SYMEXPORT extern void (*config_save_variable) (
    scope&, const variable&, uint64_t flags);

  LIBBUILD2_SYMEXPORT extern const string& (*config_preprocess_create) (
    const variable_overrides&,
    values&,
    vector_view<opspec>&,
    bool lifted,
    const location&);

  // Cached variables.
  //

  // Note: consider printing in info meta-operation if adding anything here.
  //
  LIBBUILD2_SYMEXPORT extern const variable* var_src_root;
  LIBBUILD2_SYMEXPORT extern const variable* var_out_root;
  LIBBUILD2_SYMEXPORT extern const variable* var_src_base;
  LIBBUILD2_SYMEXPORT extern const variable* var_out_base;
  LIBBUILD2_SYMEXPORT extern const variable* var_forwarded;

  LIBBUILD2_SYMEXPORT extern const variable* var_project;
  LIBBUILD2_SYMEXPORT extern const variable* var_amalgamation;
  LIBBUILD2_SYMEXPORT extern const variable* var_subprojects;
  LIBBUILD2_SYMEXPORT extern const variable* var_version;

  // project.url
  //
  LIBBUILD2_SYMEXPORT extern const variable* var_project_url;

  // project.summary
  //
  LIBBUILD2_SYMEXPORT extern const variable* var_project_summary;

  // import.target
  //
  LIBBUILD2_SYMEXPORT extern const variable* var_import_target;

  // [bool] target visibility
  //
  LIBBUILD2_SYMEXPORT extern const variable* var_clean;

  // Forwarded configuration backlink mode. Valid values are:
  //
  // false     - no link.
  // true      - make a link using appropriate mechanism.
  // symbolic  - make a symbolic link.
  // hard      - make a hard link.
  // copy      - make a copy.
  // overwrite - copy over but don't remove on clean (committed gen code).
  //
  // Note that it can be set by a matching rule as a rule-specific variable.
  //
  // [string] target visibility
  //
  LIBBUILD2_SYMEXPORT extern const variable* var_backlink;

  // Prerequisite inclusion/exclusion. Valid values are:
  //
  // false  - exclude.
  // true   - include.
  // adhoc  - include but treat as an ad hoc input.
  //
  // If a rule uses prerequisites as inputs (as opposed to just matching them
  // with the "pass-through" semantics), then the adhoc value signals that a
  // prerequisite is an ad hoc input. A rule should match and execute such a
  // prerequisite (whether its target type is recognized as suitable input or
  // not) and assume that the rest will be handled by the user (e.g., it will
  // be passed via a command line argument or some such). Note that this
  // mechanism can be used to both treat unknown prerequisite types as inputs
  // (for example, linker scripts) as well as prevent treatment of known
  // prerequisite types as such while still matching and executing them (for
  // example, plugin libraries).
  //
  // A rule with the "pass-through" semantics should treat the adhoc value
  // the same as true.
  //
  // To query this value in rule implementations use the include() helpers
  // from <libbuild2/prerequisites.hxx>.
  //
  // [string] prereq visibility
  //
  LIBBUILD2_SYMEXPORT extern const variable* var_include;

  LIBBUILD2_SYMEXPORT extern const char var_extension[10]; // "extension"

  // The build.* namespace.
  //
  // .meta_operation
  //
  LIBBUILD2_SYMEXPORT extern const variable* var_build_meta_operation;
}

#include <libbuild2/context.ixx>

#endif // LIBBUILD2_CONTEXT_HXX