aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2017-03-17 16:57:49 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2017-03-17 16:57:49 +0200
commitb5960ebbe7236eef62620b0a07df8de93dffc81e (patch)
treec95f4f54ca67a9886211d2c27b0a2a2fef5012e6
parentfdb3931c1948fbf9d56c98165f411aa5abf1b02d (diff)
Shorten scheduler queue depth, make it customizable via command line
-rw-r--r--build2/b-options8
-rw-r--r--build2/b-options.cxx15
-rw-r--r--build2/b-options.ixx12
-rw-r--r--build2/b.cli10
-rw-r--r--build2/b.cxx2
-rw-r--r--build2/scheduler.cxx11
6 files changed, 54 insertions, 4 deletions
diff --git a/build2/b-options b/build2/b-options
index c692cae..e674798 100644
--- a/build2/b-options
+++ b/build2/b-options
@@ -422,6 +422,12 @@ namespace build2
bool
max_jobs_specified () const;
+ const size_t&
+ queue_depth () const;
+
+ bool
+ queue_depth_specified () const;
+
const bool&
serial_stop () const;
@@ -495,6 +501,8 @@ namespace build2
bool jobs_specified_;
size_t max_jobs_;
bool max_jobs_specified_;
+ size_t queue_depth_;
+ bool queue_depth_specified_;
bool serial_stop_;
bool no_column_;
bool no_line_;
diff --git a/build2/b-options.cxx b/build2/b-options.cxx
index d89adcf..d31868c 100644
--- a/build2/b-options.cxx
+++ b/build2/b-options.cxx
@@ -578,6 +578,8 @@ namespace build2
jobs_specified_ (false),
max_jobs_ (),
max_jobs_specified_ (false),
+ queue_depth_ (4),
+ queue_depth_specified_ (false),
serial_stop_ (),
no_column_ (),
no_line_ (),
@@ -706,6 +708,13 @@ namespace build2
<< " scheduler implementation for details." << ::std::endl;
os << std::endl
+ << "\033[1m--queue-depth\033[0m|\033[1m-Q\033[0m \033[4mnum\033[0m The queue depth as a multiplier over the number of active" << ::std::endl
+ << " jobs. Normally we want a deeper queue if the jobs take" << ::std::endl
+ << " long (for example, compilation) and shorter if they are" << ::std::endl
+ << " quick (for example, simple tests). The default is 4. See" << ::std::endl
+ << " the build system scheduler implementation for details." << ::std::endl;
+
+ os << std::endl
<< "\033[1m--serial-stop\033[0m|\033[1m-s\033[0m Run serially and stop at the first error. This mode is" << ::std::endl
<< " useful to investigate build failures that are caused by" << ::std::endl
<< " build system errors rather than compilation errors. Note" << ::std::endl
@@ -801,6 +810,12 @@ namespace build2
_cli_options_map_["-J"] =
&::build2::cl::thunk< options, size_t, &options::max_jobs_,
&options::max_jobs_specified_ >;
+ _cli_options_map_["--queue-depth"] =
+ &::build2::cl::thunk< options, size_t, &options::queue_depth_,
+ &options::queue_depth_specified_ >;
+ _cli_options_map_["-Q"] =
+ &::build2::cl::thunk< options, size_t, &options::queue_depth_,
+ &options::queue_depth_specified_ >;
_cli_options_map_["--serial-stop"] =
&::build2::cl::thunk< options, bool, &options::serial_stop_ >;
_cli_options_map_["-s"] =
diff --git a/build2/b-options.ixx b/build2/b-options.ixx
index c230f7c..6c49ede 100644
--- a/build2/b-options.ixx
+++ b/build2/b-options.ixx
@@ -270,6 +270,18 @@ namespace build2
return this->max_jobs_specified_;
}
+ inline const size_t& options::
+ queue_depth () const
+ {
+ return this->queue_depth_;
+ }
+
+ inline bool options::
+ queue_depth_specified () const
+ {
+ return this->queue_depth_specified_;
+ }
+
inline const bool& options::
serial_stop () const
{
diff --git a/build2/b.cli b/build2/b.cli
index f1fd7dc..726f04e 100644
--- a/build2/b.cli
+++ b/build2/b.cli
@@ -380,6 +380,16 @@ namespace build2
on 64-bit. See the build system scheduler implementation for details."
}
+ size_t --queue-depth|-Q = 4
+ {
+ "<num>",
+ "The queue depth as a multiplier over the number of active jobs.
+ Normally we want a deeper queue if the jobs take long (for example,
+ compilation) and shorter if they are quick (for example, simple tests).
+ The default is 4. See the build system scheduler implementation for
+ details."
+ }
+
bool --serial-stop|-s
{
"Run serially and stop at the first error. This mode is useful to
diff --git a/build2/b.cxx b/build2/b.cxx
index 17a6645..60ac2f0 100644
--- a/build2/b.cxx
+++ b/build2/b.cxx
@@ -356,7 +356,7 @@ main (int argc, char* argv[])
fail << "invalid --max-jobs|-J value";
}
- sched.startup (jobs, 1, max_jobs);
+ sched.startup (jobs, 1, max_jobs, jobs * ops.queue_depth ());
variable_cache_mutex_shard_size = sched.shard_size ();
variable_cache_mutex_shard.reset (
diff --git a/build2/scheduler.cxx b/build2/scheduler.cxx
index 05816b8..9a3003b 100644
--- a/build2/scheduler.cxx
+++ b/build2/scheduler.cxx
@@ -255,12 +255,17 @@ namespace build2
max_threads_ = max_threads;
// This value should be proportional to the amount of hardware concurrency
- // we have (no use queing things if helpers cannot keep up). Note that the
- // queue entry is quite sizable.
+ // we have (no use queing things up if helpers cannot keep up). Note that
+ // the queue entry is quite sizable.
+ //
+ // The relationship is as follows: we want to have a deeper queue if the
+ // tasks take long (e.g., compilation) and shorter if they are quick (e.g,
+ // test execution). If the tasks are quick then the synchronization
+ // overhead required for queuing/dequeuing things starts to dominate.
//
task_queue_depth_ = queue_depth != 0
? queue_depth
- : max_active * sizeof (void*) * 4;
+ : max_active * 4;
queued_task_count_.store (0, memory_order_relaxed);