1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
|
// file : libbuild2/diagnostics.hxx -*- C++ -*-
// license : MIT; see accompanying LICENSE file
#ifndef LIBBUILD2_DIAGNOSTICS_HXX
#define LIBBUILD2_DIAGNOSTICS_HXX
#include <libbutl/diagnostics.hxx>
#include <libbuild2/types.hxx>
#include <libbuild2/forward.hxx>
#include <libbuild2/utility.hxx>
#include <libbuild2/export.hxx>
namespace build2
{
struct diag_record;
// Throw this exception to terminate the build. The handler should
// assume that the diagnostics has already been issued.
//
class failed: public std::exception {};
// Print process commmand line. If the number of elements is specified (or
// the const cstrings& version is used), then it will print the piped multi-
// process command line, if present. In this case, the expected format is as
// follows:
//
// name1 arg arg ... nullptr
// name2 arg arg ... nullptr
// ...
// nameN arg arg ... nullptr nullptr
//
LIBBUILD2_SYMEXPORT void
print_process (diag_record&,
const char* const* args, size_t n = 0);
LIBBUILD2_SYMEXPORT void
print_process (const char* const* args, size_t n = 0);
inline void
print_process (diag_record& dr,
const cstrings& args, size_t n = 0)
{
print_process (dr, args.data (), n != 0 ? n : args.size ());
}
inline void
print_process (const cstrings& args, size_t n = 0)
{
print_process (args.data (), n != 0 ? n : args.size ());
}
// As above but with process_env.
//
LIBBUILD2_SYMEXPORT void
print_process (diag_record&,
const process_env&, const char* const* args, size_t n = 0);
LIBBUILD2_SYMEXPORT void
print_process (const process_env&, const char* const* args, size_t n = 0);
inline void
print_process (diag_record& dr,
const process_env& pe, const cstrings& args, size_t n = 0)
{
print_process (dr, pe, args.data (), n != 0 ? n : args.size ());
}
inline void
print_process (const process_env& pe, const cstrings& args, size_t n = 0)
{
print_process (pe, args.data (), n != 0 ? n : args.size ());
}
// Program verbosity level (-v/--verbose plus --silent).
//
// 0 - disabled
// 1 - high-level information messages
// 2 - essential underlying commands that are being executed
// 3 - all underlying commands that are being executed
// 4 - information helpful to the user (e.g., why a rule did not match)
// 5 - information helpful to the developer
// 6 - even more detailed information
//
// If silent is true, then the level must be 0 (silent is level 0 that
// cannot be relaxed in certain contexts).
//
// While uint8 is more than enough, use uint16 for the ease of printing.
//
// Forward-declarated in <libbuild2/utility.hxx>.
//
// const uint16_t verb_never = 7;
// extern uint16_t verb;
// extern bool silent;
template <typename F> inline void l1 (const F& f) {if (verb >= 1) f ();}
template <typename F> inline void l2 (const F& f) {if (verb >= 2) f ();}
template <typename F> inline void l3 (const F& f) {if (verb >= 3) f ();}
template <typename F> inline void l4 (const F& f) {if (verb >= 4) f ();}
template <typename F> inline void l5 (const F& f) {if (verb >= 5) f ();}
template <typename F> inline void l6 (const F& f) {if (verb >= 6) f ();}
// Stream verbosity level. Determined by the diagnostic type (e.g., trace
// always has maximum verbosity) as well as the program verbosity. It is
// used to decide whether to print relative/absolute paths and default
// target extensions.
//
// Currently we have the following program to stream verbosity mapping:
//
// fail/error/warn/info <2:{0,0} 2:{0,1} >2:{1,2}
// trace *:{1,2}
//
// A stream that hasn't been (yet) assigned any verbosity explicitly (e.g.,
// ostringstream) defaults to maximum.
//
struct stream_verbosity
{
union
{
struct
{
// 0 - print relative.
// 1 - print absolute.
//
uint16_t path: 1;
// 0 - don't print.
// 1 - print if specified.
// 2 - print as 'foo.?' if unspecified and 'foo.' if specified as
// "no extension" (empty).
//
uint16_t extension: 2;
};
uint16_t value_;
};
constexpr
stream_verbosity (uint16_t p, uint16_t e): path (p), extension (e) {}
explicit
stream_verbosity (uint16_t v = 0): value_ (v) {}
};
constexpr stream_verbosity stream_verb_max = {1, 2};
// Default program to stream verbosity mapping, as outlined above.
//
inline stream_verbosity
stream_verb_map ()
{
return
verb < 2 ? stream_verbosity (0, 0) :
verb > 2 ? stream_verbosity (1, 2) :
/* */ stream_verbosity (0, 1);
}
LIBBUILD2_SYMEXPORT extern const int stream_verb_index;
inline stream_verbosity
stream_verb (ostream& os)
{
long v (os.iword (stream_verb_index));
return v == 0
? stream_verb_max
: stream_verbosity (static_cast<uint16_t> (v - 1));
}
inline void
stream_verb (ostream& os, stream_verbosity v)
{
os.iword (stream_verb_index) = static_cast<long> (v.value_) + 1;
}
// Progress reporting.
//
using butl::diag_progress;
using butl::diag_progress_lock;
// Return true if progress is to be shown. The max_verb argument is the
// maximum verbosity level that this type of progress should be shown by
// default.
//
inline bool
show_progress (uint16_t max_verb)
{
return diag_progress_option
? *diag_progress_option
: stderr_term && verb >= 1 && verb <= max_verb;
}
// Diagnostic facility.
//
// Note that this is the "complex" case we we derive from (rather than
// alias) a number of butl::diag_* types and provide custom operator<<
// "overrides" in order to make ADL look in the build2 rather than butl
// namespace.
//
using butl::diag_stream_lock;
using butl::diag_stream;
using butl::diag_epilogue;
using butl::diag_frame;
template <typename> struct diag_prologue;
template <typename> struct diag_mark;
struct diag_record: butl::diag_record
{
template <typename T>
const diag_record&
operator<< (const T& x) const
{
os << x;
return *this;
}
diag_record () = default;
template <typename B>
explicit
diag_record (const diag_prologue<B>& p): diag_record () { *this << p;}
template <typename B>
explicit
diag_record (const diag_mark<B>& m): diag_record () { *this << m;}
};
template <typename B>
struct diag_prologue: butl::diag_prologue<B>
{
using butl::diag_prologue<B>::diag_prologue;
template <typename T>
diag_record
operator<< (const T& x) const
{
diag_record r;
r.append (this->indent, this->epilogue);
B::operator() (r);
r << x;
return r;
}
friend const diag_record&
operator<< (const diag_record& r, const diag_prologue& p)
{
r.append (p.indent, p.epilogue);
p (r);
return r;
}
};
template <typename B>
struct diag_mark: butl::diag_mark<B>
{
using butl::diag_mark<B>::diag_mark;
template <typename T>
diag_record
operator<< (const T& x) const
{
return B::operator() () << x;
}
friend const diag_record&
operator<< (const diag_record& r, const diag_mark& m)
{
return r << m ();
}
};
template <typename B>
struct diag_noreturn_end: butl::diag_noreturn_end<B>
{
diag_noreturn_end () {} // For Clang 3.7 (const needs user default ctor).
using butl::diag_noreturn_end<B>::diag_noreturn_end;
[[noreturn]] friend void
operator<< (const diag_record& r, const diag_noreturn_end& e)
{
assert (r.full ());
e.B::operator() (r);
}
};
template <typename F>
struct diag_frame_impl: diag_frame
{
explicit
diag_frame_impl (F f): diag_frame (&thunk), func_ (move (f)) {}
private:
static void
thunk (const diag_frame& f, const butl::diag_record& r)
{
static_cast<const diag_frame_impl&> (f).func_ (
static_cast<const diag_record&> (r));
}
const F func_;
};
template <typename F>
inline diag_frame_impl<F>
make_diag_frame (F f)
{
return diag_frame_impl<F> (move (f));
}
struct LIBBUILD2_SYMEXPORT simple_prologue_base
{
explicit
simple_prologue_base (const char* type,
const char* mod,
const char* name,
stream_verbosity sverb)
: type_ (type), mod_ (mod), name_ (name), sverb_ (sverb) {}
void
operator() (const diag_record& r) const;
private:
const char* type_;
const char* mod_;
const char* name_;
const stream_verbosity sverb_;
};
struct LIBBUILD2_SYMEXPORT location_prologue_base
{
location_prologue_base (const char* type,
const char* mod,
const char* name,
const location& l,
stream_verbosity sverb)
: type_ (type), mod_ (mod), name_ (name),
loc_ (l),
sverb_ (sverb) {}
location_prologue_base (const char* type,
const char* mod,
const char* name,
const path_name_view& f,
stream_verbosity sverb)
: type_ (type), mod_ (mod), name_ (name),
loc_ (f),
sverb_ (sverb) {}
location_prologue_base (const char* type,
const char* mod,
const char* name,
path&& f,
stream_verbosity sverb)
: type_ (type), mod_ (mod), name_ (name),
file_ (move (f)), loc_ (file_),
sverb_ (sverb) {}
void
operator() (const diag_record& r) const;
private:
const char* type_;
const char* mod_;
const char* name_;
const path file_;
const location loc_;
const stream_verbosity sverb_;
};
struct basic_mark_base
{
using simple_prologue = diag_prologue<simple_prologue_base>;
using location_prologue = diag_prologue<location_prologue_base>;
explicit
basic_mark_base (const char* type,
const void* data = nullptr,
diag_epilogue* epilogue = &diag_frame::apply,
stream_verbosity (*sverb) () = &stream_verb_map,
const char* mod = nullptr,
const char* name = nullptr)
: sverb_ (sverb),
type_ (type), mod_ (mod), name_ (name), data_ (data),
epilogue_ (epilogue) {}
simple_prologue
operator() () const
{
return simple_prologue (epilogue_, type_, mod_, name_, sverb_ ());
}
location_prologue
operator() (const location& l) const
{
return location_prologue (epilogue_, type_, mod_, name_, l, sverb_ ());
}
location_prologue
operator() (const location_value& l) const
{
return location_prologue (epilogue_, type_, mod_, name_, l, sverb_ ());
}
location_prologue
operator() (const path_name& f) const
{
return location_prologue (epilogue_, type_, mod_, name_, f, sverb_ ());
}
location_prologue
operator() (const path_name_view& f) const
{
return location_prologue (epilogue_, type_, mod_, name_, f, sverb_ ());
}
location_prologue
operator() (const path_name_value& f) const
{
return location_prologue (epilogue_, type_, mod_, name_, f, sverb_ ());
}
// fail (relative (src)) << ...
//
location_prologue
operator() (path&& f) const
{
return location_prologue (
epilogue_, type_, mod_, name_, move (f), sverb_ ());
}
template <typename L>
location_prologue
operator() (const L& l) const
{
return location_prologue (
epilogue_, type_, mod_, name_, get_location (l, data_), sverb_ ());
}
protected:
stream_verbosity (*sverb_) ();
const char* type_;
const char* mod_;
const char* name_;
const void* data_;
diag_epilogue* const epilogue_;
};
using basic_mark = diag_mark<basic_mark_base>;
LIBBUILD2_SYMEXPORT extern const basic_mark error;
LIBBUILD2_SYMEXPORT extern const basic_mark warn;
LIBBUILD2_SYMEXPORT extern const basic_mark info;
LIBBUILD2_SYMEXPORT extern const basic_mark text;
// trace
//
struct trace_mark_base: basic_mark_base
{
explicit
trace_mark_base (const char* name, const void* data = nullptr)
: trace_mark_base (nullptr, name, data) {}
trace_mark_base (const char* mod,
const char* name,
const void* data = nullptr)
: basic_mark_base ("trace",
data,
nullptr, // No diag stack.
[]() {return stream_verb_max;},
mod,
name) {}
};
using trace_mark = diag_mark<trace_mark_base>;
using tracer = trace_mark;
// fail
//
struct fail_mark_base: basic_mark_base
{
explicit
fail_mark_base (const char* type,
const void* data = nullptr)
: basic_mark_base (type,
data,
[](const butl::diag_record& r, butl::diag_writer* w)
{
diag_frame::apply (r);
r.flush (w);
throw failed ();
},
&stream_verb_map,
nullptr,
nullptr) {}
};
using fail_mark = diag_mark<fail_mark_base>;
struct fail_end_base
{
[[noreturn]] void
operator() (const diag_record& r) const
{
// If we just throw then the record's destructor will see an active
// exception and will not flush the record.
//
r.flush ();
throw failed ();
}
};
using fail_end = diag_noreturn_end<fail_end_base>;
LIBBUILD2_SYMEXPORT extern const fail_mark fail;
LIBBUILD2_SYMEXPORT extern const fail_end endf;
// Diagnostics buffer.
//
// The purpose of this class is to handle diagnostics from child processes,
// where handle can mean:
//
// - Buffer it (to avoid interleaving in parallel builds).
//
// - Stream it (if the input can be split into diagnostic records).
//
// - Do nothing (in serial builds or if requested not to buffer).
//
// In the future this class will also be responsible for converting the
// diagnostics into the structured form (which means it may need to buffer
// even in serial builds).
//
class LIBBUILD2_SYMEXPORT diag_buffer
{
public:
explicit
diag_buffer (context& c): is (ifdstream::badbit), ctx_ (c) {}
public:
// If buffering is necessary or force is true, open a pipe and return the
// child process end of it. Otherwise, return stderr. If mode is
// non_blocking, then make reading from the parent end of the pipe
// non-blocking.
//
// The args0 argument is the child process program name for diagnostics.
// It is expected to remain valid until the call to close() and should
// normally be the same as args[0] passed to close().
//
// The force flag is normally used if custom diagnostics processing is
// required (filter, split, etc; see read() below).
//
// Note that the same buffer can go through multiple open-read-close
// sequences, for example, to execute multiple commands.
//
// All the below functions handle io errors, issue suitable diagnostics,
// and throw failed. If an exception is thrown from any of them, then the
// instance should not be used any further.
//
// Note that when reading from multiple streams in the non-blocking mode,
// only the last stream to be destroyed can normally have the skip mode
// since in case of an exception, skipping will be blocking.
//
process::pipe
open (const char* args0,
bool force = false,
fdstream_mode mode = fdstream_mode::skip);
// Open the buffer in the state as if after read() returned false, that
// is, the stream corresponding to the parent's end of the pipe reached
// EOF and has been closed. This is primarily useful when the diagnostics
// is being read in a custom way (for example, it has been merged to
// stdout) and all we want is to be able to call write() and close().
//
void
open_eof (const char* args0);
// Check whether the buffer has been opened with the open() call and
// hasn't yet been closed.
//
// Note that this function returning true does not mean that the pipe was
// opened (to check that, call is_open() on the stream member; see below).
//
bool
is_open () const
{
return state_ != state::closed;
}
// Read the diagnostics from the parent's end of the pipe if one was
// opened and buffer/stream it as necessary or forced. Return true if
// there could be more diagnostics to read (only possible in the non-
// blocking mode) and false otherwise, in which case also close the
// stream.
//
// Note that the force argument here (as well as in write() below) and
// in open() above are independent. Specifically, force in open() forces
// the opening of the pipe while force in read() and write() forces
// the buffering of the diagnostics.
//
// Instead of calling this function you can perform custom reading and, if
// necessary, buffering of the diagnostics by accessing the input stream
// (is) and underlying buffer (buf) directly. This can be used to filter,
// split the diagnostics into records according to a certain format, etc.
// Note that such custom processing implementation should maintain the
// overall semantics of diagnostics buffering in that it may only omit
// buffering in the serial case or if the diagnostics can be streamed in
// atomic records. See also write() below.
//
// The input stream is opened in the text mode and has the badbit but not
// failbit exception mask. The custom processing should also be compatible
// with the stream mode (blocking or non). If buffering is performed, then
// depending on the expected diagnostics the custom processing may want to
// reserve an appropriate initial buffer size to avoid unnecessary
// reallocation. As a convenience, in the blocking mode only, if the
// stream still contains some diagnostics, then it can be handled by
// calling read(). This is useful when needing to process only the inital
// part of the diagnostics. The custom processing may also close the
// stream manually before calling close().
//
bool
read (bool force = false);
// Close the parent end of the pipe if one was opened and write out any
// buffered diagnostics.
//
// If the child process exited abnormally or normally with non-0 code,
// then print the error diagnostics to this effect. Additionally, if the
// verbosity level is between 1 and the specified value, then print the
// command line as info after the error. If omit_normall is true, then
// don't print either for the normal exit (usually used when process
// failure can be tolerated).
//
// Normally the specified verbosity will be 1 and the command line args
// represent the verbosity level 2 (logical) command line. Note that args
// should only represent a single command in a pipe (see print_process()
// below for details).
//
// If the diag_buffer instance is destroyed before calling close(), then
// any buffered diagnostics is discarded.
//
// Note: see also run_finish(diag_buffer&).
//
// @@ TODO: need overload with process_env (see print_process).
//
void
close (const cstrings& args,
const process_exit& pe,
uint16_t verbosity = 1,
const location& loc = {},
bool omit_normall = false)
{
close (args.data (), pe, verbosity, loc, omit_normall);
}
void
close (const char* const* args,
const process_exit& pe,
uint16_t verbosity = 1,
const location& loc = {},
bool omit_normall = false);
// As above but with a custom diag record for the child exit diagnostics,
// if any. Note that if the diag record has the fail epilogue, then this
// function will throw.
//
void
close (diag_record&& = {});
// Direct access to the underlying stream and buffer for custom processing
// (see read() above for details).
//
// If serial is true, then we are running serially. If nobuf is true,
// then we are running in parallel but diagnostics buffering has been
// disabled (--no-diag-buffer). Note that there is a difference: during
// the serial execution we are free to hold the diag_stream_lock for as
// long as convenient, for example, for the whole duration of child
// process execution. Doing the same during parallel execution is very
// bad idea and we should read/write the diagnostics in chunks, normally
// one line at a time.
//
public:
ifdstream is;
vector<char> buf;
const char* args0;
bool serial;
bool nobuf;
// Buffer or stream a fragment of diagnostics as necessary or forced. If
// newline is true, also add a trailing newline.
//
// This function is normally called from a custom diagnostics processing
// implementation (see read() above for details). If nobuf is true, then
// the fragment should end on the line boundary to avoid interleaving.
//
void
write (const string&, bool newline, bool force = false);
private:
// Note that we don't seem to need a custom destructor to achieve the
// desired semantics: we can assume the process has exited before we are
// destroyed (because we supply stderr to its constructor) which means
// closing fdstream without reading any futher should be ok.
//
enum class state {closed, opened, eof};
context& ctx_;
state state_ = state::closed;
};
// Action phrases, e.g., "configure update exe{foo}", "updating exe{foo}",
// and "updating exe{foo} is configured". Use like this:
//
// info << "while " << diag_doing (a, t);
//
struct diag_phrase
{
const action& a;
const target& t;
void (*f) (ostream&, const action&, const target&);
};
inline ostream&
operator<< (ostream& os, const diag_phrase& p)
{
p.f (os, p.a, p.t);
return os;
}
LIBBUILD2_SYMEXPORT string
diag_do (context&, const action&);
LIBBUILD2_SYMEXPORT void
diag_do (ostream&, const action&, const target&);
inline diag_phrase
diag_do (const action& a, const target& t)
{
return diag_phrase {a, t, &diag_do};
}
LIBBUILD2_SYMEXPORT string
diag_doing (context&, const action&);
LIBBUILD2_SYMEXPORT void
diag_doing (ostream&, const action&, const target&);
inline diag_phrase
diag_doing (const action& a, const target& t)
{
return diag_phrase {a, t, &diag_doing};
}
LIBBUILD2_SYMEXPORT string
diag_did (context&, const action&);
LIBBUILD2_SYMEXPORT void
diag_did (ostream&, const action&, const target&);
inline diag_phrase
diag_did (const action& a, const target& t)
{
return diag_phrase {a, t, &diag_did};
}
LIBBUILD2_SYMEXPORT void
diag_done (ostream&, const action&, const target&);
inline diag_phrase
diag_done (const action& a, const target& t)
{
return diag_phrase {a, t, &diag_done};
}
}
#endif // LIBBUILD2_DIAGNOSTICS_HXX
|