aboutsummaryrefslogtreecommitdiff
path: root/libbuild2/utility.hxx
blob: 5fffc8cfcf888e0e59e8282bbb9fd7cacfaa3e7b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
// file      : libbuild2/utility.hxx -*- C++ -*-
// license   : MIT; see accompanying LICENSE file

#ifndef LIBBUILD2_UTILITY_HXX
#define LIBBUILD2_UTILITY_HXX

#include <tuple>       // make_tuple()
#include <memory>      // make_shared()
#include <string>      // to_string()
#include <utility>     // move(), forward(), declval(), make_pair(), swap()
#include <cassert>     // assert()
#include <iterator>    // make_move_iterator()
#include <algorithm>   // *
#include <functional>  // ref(), cref()
#include <type_traits>

#include <libbutl/ft/lang.hxx>

#include <libbutl/utility.hxx>  // combine_hash(), reverse_iterate(), etc
#include <libbutl/fdstream.hxx>
#include <libbutl/path-pattern.hxx>

#include <libbuild2/types.hxx>
#include <libbuild2/forward.hxx>

// "Fake" version values used during bootstrap.
//
#ifdef BUILD2_BOOTSTRAP
#  define LIBBUILD2_VERSION     9999999999999990000ULL
#  define LIBBUILD2_VERSION_STR "99999.99999.99999"
#  define LIBBUILD2_VERSION_ID  "99999.99999.99999"
#  define LIBBUTL_VERSION_STR   "99999.99999.99999"
#  define LIBBUTL_VERSION_ID    "99999.99999.99999"
#else
#  include <libbuild2/version.hxx>
#endif

#include <libbuild2/export.hxx>

namespace build2
{
  using std::move;
  using std::swap;
  using std::forward;
  using std::declval;

  using std::ref;
  using std::cref;

  using std::make_pair;
  using std::make_tuple;
  using std::make_shared;
  using std::make_move_iterator;
  using std::to_string;
  using std::stoul;
  using std::stoull;

  // <libbutl/utility.hxx>
  //
  using butl::reverse_iterate;
  using butl::compare_c_string;
  using butl::compare_pointer_target;
  //using butl::hash_pointer_target;
  using butl::combine_hash;
  using butl::icasecmp;
  using butl::icase_compare_string;
  using butl::icase_compare_c_string;
  using butl::lcase;
  using butl::ucase;
  using butl::alpha;
  using butl::alnum;
  using butl::digit;
  using butl::wspace;

  using butl::trim;
  using butl::next_word;
  using butl::sanitize_identifier;
  using butl::sanitize_strlit;

  using butl::make_guard;
  using butl::make_exception_guard;

  using butl::function_cast;

  using butl::getenv;
  using butl::auto_thread_env;

  using butl::throw_generic_error;
  using butl::throw_system_error;

  using butl::eof;

  // <libbutl/fdstream.hxx>
  //
  using butl::fdopen_null;
  using butl::open_file_or_stdin;
  using butl::open_file_or_stdout;

  // <libbutl/path-pattern.hxx>
  //
  using butl::path_pattern;

  // Perform process-wide initializations/adjustments/workarounds. Should be
  // called once early in main(). In particular, besides other things, this
  // functions does the following:
  //
  // - Sets PATH to include baseutils /bin on Windows.
  //
  // - Ignores SIGPIPE.
  //
  // - Calls tzset().
  //
  LIBBUILD2_SYMEXPORT void
  init_process ();

  // Diagnostics state (verbosity level, etc; see <libbuild2/diagnostics.hxx>).
  //
  // Note on naming of values (here and in the global state below) that come
  // from the command line options: if a value is not meant to be used
  // directly, then it has the _option suffix and a function or another
  // variable as its public interface.

  // Initialize the diagnostics state. Should be called once early in main().
  // Default values are for unit tests.
  //
  // If silent is true, verbosity should be 0.
  //
  LIBBUILD2_SYMEXPORT void
  init_diag (uint16_t verbosity,
             bool silent = false,
             optional<bool> progress = nullopt,
             bool no_lines = false,
             bool no_columns = false,
             bool stderr_term = false);

  const uint16_t verb_never = 7;
  LIBBUILD2_SYMEXPORT extern uint16_t verb;
  LIBBUILD2_SYMEXPORT extern bool silent;

  // --[no-]progress
  //
  LIBBUILD2_SYMEXPORT extern optional<bool> diag_progress_option;

  LIBBUILD2_SYMEXPORT extern bool diag_no_line;   // --no-line
  LIBBUILD2_SYMEXPORT extern bool diag_no_column; // --no-column

  LIBBUILD2_SYMEXPORT extern bool stderr_term; // True if stderr is a terminal.

  // Global state (verbosity, home/work directories, etc).

  // Initialize the global state. Should be called once early in main().
  // Default values are for unit tests.
  //
  LIBBUILD2_SYMEXPORT void
  init (void (*terminate) (bool),
        const char* argv0,
        bool serial_stop,
        optional<bool> mtime_check = nullopt,
        optional<path> config_sub = nullopt,
        optional<path> config_guess = nullopt);

  // Terminate function. If trace is false, then printing of the stack trace,
  // if any, should be omitted.
  //
  LIBBUILD2_SYMEXPORT extern void (*terminate) (bool trace);

  // Build system driver process path (argv0.initial is argv[0]).
  //
  LIBBUILD2_SYMEXPORT extern process_path argv0;

  // Build system core version and interface version.
  //
  LIBBUILD2_SYMEXPORT extern const standard_version build_version;
  LIBBUILD2_SYMEXPORT extern const string build_version_interface;

  // Whether running installed build and, if so, the library installation
  // directory (empty otherwise).
  //
  LIBBUILD2_SYMEXPORT extern const bool build_installed;
  LIBBUILD2_SYMEXPORT extern const dir_path build_install_lib; // $install.lib

  // --[no-]mtime-check
  //
  LIBBUILD2_SYMEXPORT extern optional<bool> mtime_check_option;

  LIBBUILD2_SYMEXPORT extern optional<path> config_sub;   // --config-sub
  LIBBUILD2_SYMEXPORT extern optional<path> config_guess; // --config-guess

  LIBBUILD2_SYMEXPORT void
  check_build_version (const standard_version_constraint&, const location&);

  // Work/home directories (must be initialized in main()) and relative path
  // calculation.
  //
  LIBBUILD2_SYMEXPORT extern dir_path work;
  LIBBUILD2_SYMEXPORT extern dir_path home;

  // By default this points to work. Setting this to something else should
  // only be done in tightly controlled, non-concurrent situations (e.g.,
  // state dump). If it is empty, then relative() below returns the original
  // path.
  //
  // Note: watch out for concurrent changes from multiple build contexts.
  //
  LIBBUILD2_SYMEXPORT extern const dir_path* relative_base;

  // If possible and beneficial, translate an absolute, normalized path into
  // relative to the relative_base directory, which is normally work. Note
  // that if the passed path is the same as relative_base, then this function
  // returns empty path.
  //
  template <typename K>
  basic_path<char, K>
  relative (const basic_path<char, K>&);

  class path_target;

  LIBBUILD2_SYMEXPORT path
  relative (const path_target&);

  // In addition to calling relative(), this function also uses shorter
  // notations such as '~/'. For directories the result includes the trailing
  // slash. If the path is the same as base, returns "./" if current is true
  // and empty string otherwise.
  //
  LIBBUILD2_SYMEXPORT string
  diag_relative (const path&, bool current = true);

  // Basic process utilities.
  //
  // The run*() functions with process_path/_env assume that you are printing
  // the process command line yourself.

  // Search for a process executable. Issue diagnostics and throw failed in
  // case of an error.
  //
  LIBBUILD2_SYMEXPORT process_path
  run_search (const char*& args0,
              bool path_only,
              const location& = location ());

  inline process_path
  run_search (const char*& args0, const location& l = location ())
  {
    return run_search (args0, false, l);
  }

  LIBBUILD2_SYMEXPORT process_path
  run_search (const path&,
              bool init = false,
              const dir_path& fallback = dir_path (),
              bool path_only = false,
              const location& = location ());

  LIBBUILD2_SYMEXPORT process_path
  run_try_search (const path&,
                  bool init = false,
                  const dir_path& fallback = dir_path (),
                  bool path_only = false,
                  const char* paths = nullptr);

  [[noreturn]] LIBBUILD2_SYMEXPORT void
  run_search_fail (const path&, const location& = location ());

  // Start a process with the specified arguments. Issue diagnostics and throw
  // failed in case of an error. If in is -1, then redirect stdin to a pipe
  // (can also be -2 to redirect it to /dev/null or equivalent). If out is -1,
  // then redirect stdout to a pipe. If stderr is redirected to stdout (can
  // be used to analyze diagnostics from the child process), then, in case of
  // an error, the last line read from stdout must be passed to run_finish()
  // below.
  //
  LIBBUILD2_SYMEXPORT process
  run_start (uint16_t verbosity,
             const process_env&, // Implicit-constructible from process_path.
             const char* const* args,
             int in = 0,
             int out = 1,
             process::pipe = {-1, 2},
             const location& = {});

  inline process
  run_start (uint16_t verbosity,
             const process_env& pe,
             const cstrings& args,
             int in = 0,
             int out = 1,
             process::pipe err = {-1, 2},
             const location& l = {})
  {
    return run_start (verbosity, pe, args.data (), in, out, move (err), l);
  }

  inline process
  run_start (const process_env& pe,
             const char* const* args,
             int in = 0,
             int out = 1,
             process::pipe err = {-1, 2},
             const location& l = {})
  {
    return run_start (verb_never, pe, args, in, out, move (err), l);
  }

  inline process
  run_start (const process_env& pe,
             const cstrings& args,
             int in = 0,
             int out = 1,
             process::pipe err = {-1, 2},
             const location& l = {})
  {
    return run_start (pe, args.data (), in, out, move (err), l);
  }

  // As above, but search for the process (including updating args[0]) and
  // print the process commands line at the specified verbosity level.
  //
  inline process
  run_start (uint16_t verbosity,
             const char* args[],
             int in = 0,
             int out = 1,
             process::pipe err = {-1, 2},
             const char* const* env = nullptr,
             const dir_path& cwd = {},
             const location& l = {})
  {
    process_path pp (run_search (args[0], l));
    return run_start (verbosity,
                      process_env (pp, cwd, env), args,
                      in, out, move (err),
                      l);
  }

  inline process
  run_start (uint16_t verbosity,
             cstrings& args,
             int in = 0,
             int out = 1,
             process::pipe err = {-1, 2},
             const char* const* env = nullptr,
             const dir_path& cwd = {},
             const location& l = {})
  {
    return run_start (verbosity,
                      args.data (),
                      in, out, move (err),
                      env, cwd, l);
  }

  // Wait for process termination returning true if the process exited
  // normally with a zero code and false otherwise. The latter case is
  // normally followed up with a call to run_finish().
  //
  LIBBUILD2_SYMEXPORT bool
  run_wait (const char* const* args, process&, const location& = location ());

  bool
  run_wait (const cstrings& args, process&, const location& = location ());

  // Wait for process termination, issues diagnostics, and throw failed.
  //
  // If the child process exited abnormally or normally with non-0 code, then
  // print the error diagnostics to this effect. Additionally, if the
  // verbosity level is between 1 and the specified value, then print the
  // command line as info after the error. If omit_normal is true, then don't
  // print either for the normal exit (usually used for custom diagnostics or
  // when process failure can be tolerated).
  //
  // Normally the specified verbosity will be 1 and the command line args
  // represent the verbosity level 2 (logical) command line. Or, to put it
  // another way, it should be 1 less than what gets passed to run_start().
  // Note that args should only represent a single command in a pipe (see
  // print_process() for details).
  //
  // See also diag_buffer::close().
  //
  // The line argument is used in cooperation with run_start() to diagnose a
  // failure to exec in case stderr is redirected to stdout (see the
  // implementation for details).
  //
  void
  run_finish (const char* const* args,
              process&,
              uint16_t verbosity,
              bool omit_normal = false,
              const location& = location ());

  void
  run_finish (const cstrings& args,
              process&,
              uint16_t verbosity,
              bool omit_normal = false,
              const location& = location ());

  void
  run_finish (const char* const* args,
              process&,
              const string& line,
              uint16_t verbosity,
              bool omit_normal = false,
              const location& = location ());

  // As above but if the process has exited normally with a non-zero code,
  // then return false rather than throwing.
  //
  // Note that the normal non-0 exit diagnostics is omitted by default
  // assuming appropriate custom diagnostics will be issued, if required.
  //
  bool
  run_finish_code (const char* const* args,
                   process&,
                   uint16_t verbosity,
                   bool omit_normal = true,
                   const location& = location ());

  bool
  run_finish_code (const cstrings& args,
                   process&,
                   uint16_t verbosity,
                   bool omit_normal = true,
                   const location& = location ());

  bool
  run_finish_code (const char* const* args,
                   process&,
                   const string&,
                   uint16_t verbosity,
                   bool omit_normal = true,
                   const location& = location ());

  // As above but with diagnostics buffering.
  //
  // Specifically, this version first waits for the process termination, then
  // calls diag_buffer::close(verbosity, omit_normal), and finally throws
  // failed if the process didn't exit with 0 code.
  //
  class diag_buffer;

  void
  run_finish (diag_buffer&,
              const char* const* args,
              process&,
              uint16_t verbosity,
              bool omit_normal = false,
              const location& = location ());

  void
  run_finish (diag_buffer&,
              const cstrings& args,
              process&,
              uint16_t verbosity,
              bool omit_normal = false,
              const location& = location ());

  // As above but if the process has exited normally with a non-zero code,
  // then return false rather than throwing.
  //
  // Note that the normal non-0 exit diagnostics is omitted by default
  // assuming appropriate custom diagnostics will be issued, if required.
  //
  bool
  run_finish_code (diag_buffer&,
                   const char* const* args,
                   process&,
                   uint16_t verbosity,
                   bool omit_normal = true,
                   const location& = location ());

  bool
  run_finish_code (diag_buffer&,
                   const cstrings& args,
                   process&,
                   uint16_t verbosity,
                   bool omit_normal = true,
                   const location& = location ());

  // Run the process with the specified arguments by calling the above start
  // and finish functions. Buffer diagnostics unless in the load phase.
  //
  LIBBUILD2_SYMEXPORT void
  run (context&,
       const process_env& pe, // Implicit-constructible from process_path.
       const char* const* args,
       uint16_t finish_verbosity);

  LIBBUILD2_SYMEXPORT void
  run (diag_buffer&,
       const process_env& pe,
       const char* const* args,
       uint16_t finish_verbosity);

  inline void
  run (context& ctx,
       const process_env& pe,
       const cstrings& args,
       uint16_t finish_verbosity)
  {
    run (ctx, pe, args.data (), finish_verbosity);
  }

  inline void
  run (diag_buffer& dbuf,
       const process_env& pe,
       const cstrings& args,
       uint16_t finish_verbosity)
  {
    run (dbuf, pe, args.data (), finish_verbosity);
  }

  // As above but pass cwd/env vars as arguments rather than as part of
  // process_env.
  //
  inline void
  run (context& ctx,
       const process_path& p,
       const char* const* args,
       uint16_t finish_verbosity,
       const char* const* env,
       const dir_path& cwd = {})
  {
    run (ctx, process_env (p, cwd, env), args, finish_verbosity);
  }

  inline void
  run (diag_buffer& dbuf,
       const process_path& p,
       const char* const* args,
       uint16_t finish_verbosity,
       const char* const* env,
       const dir_path& cwd = {})
  {
    run (dbuf, process_env (p, cwd, env), args, finish_verbosity);
  }

  inline void
  run (context& ctx,
       const process_path& p,
       const cstrings& args,
       uint16_t finish_verbosity,
       const char* const* env,
       const dir_path& cwd = {})
  {
    run (ctx, p, args.data (), finish_verbosity, env, cwd);
  }

  inline void
  run (diag_buffer& dbuf,
       const process_path& p,
       const cstrings& args,
       uint16_t finish_verbosity,
       const char* const* env,
       const dir_path& cwd = {})
  {
    run (dbuf, p, args.data (), finish_verbosity, env, cwd);
  }

  // Start the process as above and then call the specified function on each
  // trimmed line of the output until it returns a non-empty object T (tested
  // with T::empty()) which is then returned to the caller.
  //
  // If verbosity is specified, print the process commands line at that level
  // (with the verbosite-1 value passed run_finish()).
  //
  // If error is false, then redirecting stderr to stdout and don't fail if
  // the process exits normally but with non-0 code (can be used to suppress
  // and/or analyze diagnostics from the child process). Otherwise, buffer
  // diagnostics unless in the load phase.
  //
  // The predicate can move the value out of the passed string but, if error
  // is false, only in case of a "content match" (so that any diagnostics
  // lines are left intact). The function signature should be:
  //
  // T (string& line, bool last)
  //
  // If, in addition to error being false, ignore_exit is true, then the
  // program's normal exit status is ignored (if it is false and the program
  // exits with the non-zero status, then an empty T instance is returned).
  //
  // If checksum is not NULL, then feed it the content of each trimmed line
  // (including those that come after the callback returns non-empty object).
  //
  template <typename T, typename F>
  T
  run (context&,
       uint16_t verbosity,
       const process_env&, // Implicit-constructible from process_path.
       const char* const* args,
       F&&,
       bool error = true,
       bool ignore_exit = false,
       sha256* checksum = nullptr);

  template <typename T, typename F>
  inline T
  run (context& ctx,
       uint16_t verbosity,
       const process_env& pe,
       const cstrings& args,
       F&& f,
       bool error = true,
       bool ignore_exit = false,
       sha256* checksum = nullptr)
  {
    return run<T> (ctx,
                   verbosity,
                   pe, args.data (),
                   forward<F> (f),
                   error, ignore_exit, checksum);
  }

  template <typename T, typename F>
  T
  run (diag_buffer&,
       uint16_t verbosity,
       const process_env&,
       const char* const* args,
       F&&,
       sha256* checksum = nullptr);

  template <typename T, typename F>
  inline T
  run (diag_buffer& dbuf,
       uint16_t verbosity,
       const process_env& pe,
       const cstrings& args,
       F&& f,
       sha256* checksum = nullptr)
  {
    return run<T> (dbuf,
                   verbosity,
                   pe, args.data (),
                   forward<F> (f),
                   checksum);
  }

  template <typename T, typename F>
  inline T
  run (context&,
       const process_env&,
       const char* const* args,
       uint16_t finish_verbosity,
       F&&,
       bool error = true,
       bool ignore_exit = false,
       sha256* checksum = nullptr);

  template <typename T, typename F>
  inline T
  run (context& ctx,
       const process_env& pe,
       const cstrings& args,
       uint16_t finish_verbosity,
       F&& f,
       bool error = true,
       bool ignore_exit = false,
       sha256* checksum = nullptr)
  {
    return run<T> (ctx,
                   pe, args.data (),
                   finish_verbosity,
                   forward<F> (f),
                   error, ignore_exit, checksum);
  }

  template <typename T, typename F>
  inline T
  run (diag_buffer&,
       const process_env&,
       const char* const* args,
       uint16_t finish_verbosity,
       F&&,
       sha256* checksum = nullptr);

  template <typename T, typename F>
  inline T
  run (diag_buffer& dbuf,
       const process_env& pe,
       const cstrings& args,
       uint16_t finish_verbosity,
       F&& f,
       sha256* checksum = nullptr)
  {
    return run<T> (dbuf,
                   pe, args.data (),
                   finish_verbosity,
                   forward<F> (f),
                   checksum);
  }

  template <typename T, typename F>
  inline T
  run (context& ctx,
       uint16_t verbosity,
       const char* args[],
       F&& f,
       bool error = true,
       bool ignore_exit = false,
       sha256* checksum = nullptr)
  {
    process_path pp (run_search (args[0]));
    return run<T> (ctx,
                   verbosity,
                   pp, args,
                   forward<F> (f),
                   error, ignore_exit, checksum);
  }

  template <typename T, typename F>
  inline T
  run (context& ctx,
       uint16_t verbosity,
       cstrings& args,
       F&& f,
       bool error = true,
       bool ignore_exit = false,
       sha256* checksum = nullptr)
  {
    return run<T> (ctx,
                   verbosity,
                   args.data (),
                   forward<F> (f),
                   error, ignore_exit, checksum);
  }

  template <typename T, typename F>
  inline T
  run (diag_buffer& dbuf,
       uint16_t verbosity,
       const char* args[],
       F&& f,
       sha256* checksum = nullptr)
  {
    process_path pp (run_search (args[0]));
    return run<T> (dbuf,
                   verbosity,
                   pp, args,
                   forward<F> (f),
                   checksum);
  }

  template <typename T, typename F>
  inline T
  run (diag_buffer& dbuf,
       uint16_t verbosity,
       cstrings& args,
       F&& f,
       sha256* checksum = nullptr)
  {
    return run<T> (dbuf,
                   verbosity,
                   args.data (),
                   forward<F> (f),
                   checksum);
  }

  // As above but run a program without any arguments or with one argument.
  //
  // run <prog>
  //
  template <typename T, typename F>
  inline T
  run (context& ctx,
       uint16_t verbosity,
       const path& prog,
       F&& f,
       bool error = true,
       bool ignore_exit = false,
       sha256* checksum = nullptr)
  {
    const char* args[] = {prog.string ().c_str (), nullptr};
    return run<T> (ctx,
                   verbosity,
                   args,
                   forward<F> (f),
                   error, ignore_exit, checksum);
  }

  template <typename T, typename F>
  inline typename std::enable_if<
    (!std::is_same<typename std::decay<F>::type, const char**>::value &&
     !std::is_same<typename std::remove_reference<F>::type, cstrings>::value),
    T>::type
  run (context& ctx,
       uint16_t verbosity,
       const process_env& pe, // Implicit-constructible from process_path.
       F&& f,
       bool error = true,
       bool ignore_exit = false,
       sha256* checksum = nullptr)
  {
    const char* args[] = {pe.path->recall_string (), nullptr};
    return run<T> (ctx,
                   verbosity,
                   pe, args,
                   forward<F> (f),
                   error, ignore_exit, checksum);
  }

  // run <prog> <arg>
  //
  template <typename T, typename F>
  inline T
  run (context& ctx,
       uint16_t verbosity,
       const path& prog,
       const char* arg,
       F&& f,
       bool error = true,
       bool ignore_exit = false,
       sha256* checksum = nullptr)
  {
    const char* args[] = {prog.string ().c_str (), arg, nullptr};
    return run<T> (ctx,
                   verbosity,
                   args,
                   forward<F> (f),
                   error, ignore_exit, checksum);
  }

  template <typename T, typename F>
  inline T
  run (context& ctx,
       uint16_t verbosity,
       const process_env& pe, // Implicit-constructible from process_path.
       const char* arg,
       F&& f,
       bool error = true,
       bool ignore_exit = false,
       sha256* checksum = nullptr)
  {
    const char* args[] = {pe.path->recall_string (), arg, nullptr};
    return run<T> (ctx,
                   verbosity,
                   pe, args,
                   forward<F> (f),
                   error, ignore_exit, checksum);
  }

  // As above but a lower-level interface that erases T and F and can also be
  // used to suppress trimming.
  //
  // The passed function should return true if it should be called again
  // (i.e., the object is still empty in the T & F interface) and false
  // otherwise.
  //
  // The first version ruturn true if the result is usable and false
  // otherwise, depending on the process exit code and error/ignore_exit
  // values. (In the latter case, the T & F interface makes the resulting
  // object empty).
  //
  LIBBUILD2_SYMEXPORT bool
  run (context&,
       uint16_t verbosity,
       const process_env&,
       const char* const* args,
       uint16_t finish_verbosity,
       const function<bool (string& line, bool last)>&,
       bool trim = true,
       bool error = true,
       bool ignore_exit = false,
       sha256* checksum = nullptr);

  LIBBUILD2_SYMEXPORT void
  run (diag_buffer& dbuf,
       uint16_t verbosity,
       const process_env&,
       const char* const* args,
       uint16_t finish_verbosity,
       const function<bool (string& line, bool last)>&,
       bool trim = true,
       sha256* checksum = nullptr);

  // Concatenate the program path and arguments into a shallow NULL-terminated
  // vector of C-strings.
  //
  LIBBUILD2_SYMEXPORT cstrings
  process_args (const char* program, const strings& args);

  inline cstrings
  process_args (const string& program, const strings& args)
  {
    return process_args (program.c_str (), args);
  }

  // File descriptor streams.
  //
  fdpipe
  open_pipe ();

  auto_fd
  open_null ();

  // Empty/nullopt string, path, and project name.
  //
  LIBBUILD2_SYMEXPORT extern const string       empty_string;
  LIBBUILD2_SYMEXPORT extern const path         empty_path;
  LIBBUILD2_SYMEXPORT extern const dir_path     empty_dir_path;
  LIBBUILD2_SYMEXPORT extern const project_name empty_project_name;

  LIBBUILD2_SYMEXPORT extern const optional<string>       nullopt_string;
  LIBBUILD2_SYMEXPORT extern const optional<path>         nullopt_path;
  LIBBUILD2_SYMEXPORT extern const optional<dir_path>     nullopt_dir_path;
  LIBBUILD2_SYMEXPORT extern const optional<project_name> nullopt_project_name;

  // Hash a path potentially without the specific directory prefix.
  //
  // If prefix is not empty and is a super-path of the path to hash, then only
  // hash the suffix. Note that both paths are assumed to be normalized.
  //
  // This functionality is normally used to strip out_root from target paths
  // being hashed in order to avoid updates in case out_root was moved. Note
  // that this should only be done if the result of the update does not
  // include the out_root path in any form (as could be the case, for example,
  // for debug information, __FILE__ macro expansion, rpath, etc).
  //
  void
  hash_path (sha256&, const path&, const dir_path& prefix = dir_path ());

  // Append all the values from a variable to the C-string list. T is either
  // target or scope. The variable is expected to be of type strings.
  //
  // If excl is not NULL, then filter this option out (note: case sensitive).
  //
  template <typename T>
  void
  append_options (cstrings&, T&, const variable&, const char* excl = nullptr);

  template <typename T>
  void
  append_options (cstrings&, T&, const char*, const char* excl = nullptr);

  template <typename T>
  void
  append_options (strings&, T&, const variable&, const char* excl = nullptr);

  template <typename T>
  void
  append_options (strings&, T&, const char*, const char* excl = nullptr);

  template <typename T>
  void
  append_options (sha256&, T&, const variable&);

  template <typename T>
  void
  append_options (sha256&, T&, const char*);

  // As above but from the lookup directly.
  //
  LIBBUILD2_SYMEXPORT void
  append_options (cstrings&, const lookup&, const char* excl = nullptr);

  LIBBUILD2_SYMEXPORT void
  append_options (strings&, const lookup&, const char* excl = nullptr);

  LIBBUILD2_SYMEXPORT void
  append_options (sha256&, const lookup&);

  // As above but from the strings value directly.
  //
  void
  append_options (cstrings&, const strings&, const char* excl = nullptr);

  void
  append_options (strings&, const strings&, const char* excl = nullptr);

  void
  append_options (sha256&, const strings&);

  LIBBUILD2_SYMEXPORT void
  append_options (cstrings&,
                  const strings&, size_t,
                  const char* excl = nullptr);

  LIBBUILD2_SYMEXPORT void
  append_options (strings&,
                  const strings&, size_t,
                  const char* excl = nullptr);

  LIBBUILD2_SYMEXPORT void
  append_options (sha256&, const strings&, size_t);

  // As above but append/hash option values for the specified option (e.g.,
  // -I, -L).
  //
  template <typename I, typename F>
  void
  append_option_values (cstrings&,
                        const char* opt,
                        I begin, I end,
                        F&& get = [] (const string& s) {return s.c_str ();});

  template <typename I, typename F>
  void
  append_option_values (sha256&,
                        const char* opt,
                        I begin, I end,
                        F&& get = [] (const string& s) {return s;});

  // As above but append a single option (used for append/hash uniformity).
  //
  inline void
  append_option (cstrings& args, const char* o)
  {
    args.push_back (o);
  }

  inline void
  append_option (strings& args, const char* o)
  {
    args.push_back (o);
  }

  inline void
  append_option (sha256& csum, const char* o)
  {
    csum.append (o);
  }

  // Check if a specified option is present in the variable or value. T is
  // either target or scope. For the interator version use rbegin()/rend() to
  // search backwards.
  //
  template <typename T>
  bool
  find_option (const char* option,
               T&,
               const variable&,
               bool ignore_case = false);

  template <typename T>
  bool
  find_option (const char* option,
               T&,
               const char* variable,
               bool ignore_case = false);

  template <typename I>
  I
  find_option (const char* option, I begin, I end, bool ignore_case = false);

  LIBBUILD2_SYMEXPORT bool
  find_option (const char* option, const lookup&, bool ignore_case = false);

  LIBBUILD2_SYMEXPORT bool
  find_option (const char* option, const strings&, bool ignore_case = false);

  LIBBUILD2_SYMEXPORT bool
  find_option (const char* option, const cstrings&, bool ignore_case = false);

  // As above but look for several options returning true if any is present.
  //
  template <typename T>
  bool
  find_options (const initializer_list<const char*>&,
                T&,
                const variable&,
                bool = false);

  template <typename T>
  bool
  find_options (const initializer_list<const char*>&,
                T&,
                const char*,
                bool = false);

  LIBBUILD2_SYMEXPORT bool
  find_options (const initializer_list<const char*>&,
                const lookup&,
                bool = false);

  LIBBUILD2_SYMEXPORT bool
  find_options (const initializer_list<const char*>&,
                const strings&,
                bool = false);

  LIBBUILD2_SYMEXPORT bool
  find_options (const initializer_list<const char*>&,
                const cstrings&,
                bool = false);

  // As above but look for an option that has the specified prefix. Return the
  // pointer to option or NULL if not found (thus can be used as bool).
  // Search backward (which is normally consistent with how options override
  // each other). For the interator version use rbegin()/rend() to do the
  // same.
  //
  template <typename T>
  const string*
  find_option_prefix (const char* prefix, T&, const variable&, bool = false);

  template <typename T>
  const string*
  find_option_prefix (const char* prefix, T&, const char*, bool = false);

  template <typename I>
  I
  find_option_prefix (const char* prefix, I begin, I end, bool = false);

  LIBBUILD2_SYMEXPORT const string*
  find_option_prefix (const char* prefix, const lookup&, bool = false);

  LIBBUILD2_SYMEXPORT const string*
  find_option_prefix (const char* prefix, const strings&, bool = false);

  LIBBUILD2_SYMEXPORT const char*
  find_option_prefix (const char* prefix, const cstrings&, bool = false);

  // As above but look for several option prefixes.
  //
  template <typename T>
  const string*
  find_option_prefixes (const initializer_list<const char*>&,
                        T&,
                        const variable&,
                        bool = false);

  template <typename T>
  const string*
  find_option_prefixes (const initializer_list<const char*>&,
                        T&,
                        const char*,
                        bool = false);

  LIBBUILD2_SYMEXPORT const string*
  find_option_prefixes (const initializer_list<const char*>&,
                        const lookup&, bool = false);

  LIBBUILD2_SYMEXPORT const string*
  find_option_prefixes (const initializer_list<const char*>&,
                        const strings&,
                        bool = false);

  LIBBUILD2_SYMEXPORT const char*
  find_option_prefixes (const initializer_list<const char*>&,
                        const cstrings&,
                        bool = false);

  // Hash environment variable (its name and value) normally to be used as a
  // checksum. See also config::save_environment().
  //
  void
  hash_environment (sha256&, const char* name);

  void
  hash_environment (sha256&, const string& name);

  void
  hash_environment (sha256&, initializer_list<const char*> names);

  string
  hash_environment (initializer_list<const char*> names);

  void
  hash_environment (sha256&, const cstrings& names);

  string
  hash_environment (const cstrings& names);

  void
  hash_environment (sha256&, const strings& names);

  string
  hash_environment (const strings& names);

  // A NULL-terminated list of variables (may itself be NULL).
  //
  void
  hash_environment (sha256&, const char* const* names);

  string
  hash_environment (const char* const* names);

  // Find in the string the stem separated from other characters with the
  // specified separators or begin/end of the string. Return the stem's
  // position or npos if not found.
  //
  size_t
  find_stem (const string&, size_t pos, size_t n,
             const char* stem, const char* seps = "-_.");

  inline size_t
  find_stem (const string& s, const char* stem, const char* seps = "-_.")
  {
    return find_stem (s, 0, s.size (), stem, seps);
  }

  // Apply the specified substitution (stem) to a '*'-pattern. If pattern is
  // NULL or empty, then return the stem itself. Assume the pattern is valid,
  // i.e., contains a single '*' character.
  //
  LIBBUILD2_SYMEXPORT string
  apply_pattern (const char* stem, const char* pattern);

  inline string
  apply_pattern (const char* s, const string* p)
  {
    return apply_pattern (s, p != nullptr ? p->c_str () : nullptr);
  }

  inline string
  apply_pattern (const char* s, const string& p)
  {
    return apply_pattern (s, p.c_str ());
  }

  // Try to parse a string as a non-negative number returning nullopt if the
  // argument is not a valid number or the number is greater than the
  // specified maximum.
  //
  optional<uint64_t>
  parse_number (const string&, uint64_t max = UINT64_MAX);
}

#include <libbuild2/utility.ixx>
#include <libbuild2/utility.txx>

#endif // LIBBUILD2_UTILITY_HXX