aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--INSTALL8
-rw-r--r--INSTALL-DEV4
-rw-r--r--etc/systemd/brep-load.service2
-rw-r--r--load/load.cli6
-rw-r--r--load/load.cxx154
-rw-r--r--tests/load/buildfile2
-rw-r--r--tests/load/driver.cxx2
-rw-r--r--tests/load/loadtab (renamed from tests/load/r.conf)0
8 files changed, 86 insertions, 92 deletions
diff --git a/INSTALL b/INSTALL
index de25c4d..b738683 100644
--- a/INSTALL
+++ b/INSTALL
@@ -114,10 +114,10 @@ To troubleshoot, see PostgreSQL logs.
5. Create Database Schema and Load Repositories
$ mkdir config
-$ edit config/brep-load.conf # Loader configuration, see brep-load(1).
+$ edit config/loadtab # Loader configuration, see brep-load(1).
$ install/bin/brep-migrate
-$ install/bin/brep-load config/brep-load.conf
+$ install/bin/brep-load config/loadtab
To verify:
@@ -250,7 +250,7 @@ The following crontab entry will execute the loader every five minutes:
$ crontab -
MAILTO=<brep-admin-email>
PATH=/usr/local/bin:/bin:/usr/bin
-*/5 * * * * $HOME/install/bin/brep-load $HOME/config/brep-load.conf
+*/5 * * * * $HOME/install/bin/brep-load $HOME/config/loadtab
^D
Note that here we assume that bpkg (which is executed by brep-load) is in one
@@ -360,7 +360,7 @@ If using cron, then simply wait for the next run.
You can also do a manual load:
-$ install/bin/brep-load config/brep-load.conf
+$ install/bin/brep-load config/loadtab
Next, start apache:
diff --git a/INSTALL-DEV b/INSTALL-DEV
index 66d554c..8ad9ae7 100644
--- a/INSTALL-DEV
+++ b/INSTALL-DEV
@@ -52,7 +52,7 @@ $ migrate/brep-migrate
# Or use some other loader config.
#
-$ load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/r.conf
+$ load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/loadtab
To verify:
@@ -121,7 +121,7 @@ To do a "complete reload" (i.e., recreate database schema, load the repository
data, and reload the Apache2 plugin), execute the following from brep/:
migrate/brep-migrate --recreate
-load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/r.conf
+load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/loadtab
sudo /etc/init.d/apache2 restart
sudo systemctl restart apache2
diff --git a/etc/systemd/brep-load.service b/etc/systemd/brep-load.service
index bc12e4b..f9d4e31 100644
--- a/etc/systemd/brep-load.service
+++ b/etc/systemd/brep-load.service
@@ -5,7 +5,7 @@ Description=brep repository loader service
Type=oneshot
#User=brep
#Group=brep
-ExecStart=/home/brep/install/bin/brep-load /home/brep/config/brep-load.conf
+ExecStart=/home/brep/install/bin/brep-load /home/brep/config/loadtab
[Install]
WantedBy=default.target
diff --git a/load/load.cli b/load/load.cli
index a3fcdb4..7850d86 100644
--- a/load/load.cli
+++ b/load/load.cli
@@ -13,18 +13,18 @@ include <brep/types>;
"\summary=load build2 repositories into database"
{
- "<options> <file>",
+ "<options> <loadtab-file>",
"\h|SYNOPSIS|
\cb{brep-load --help}\n
\cb{brep-load --version}\n
- \c{\b{brep-load} [<options>] <file>}
+ \c{\b{brep-load} [<options>] <loadtab-file>}
\h|DESCRIPTION|
\cb{brep-load} reads the list of repositories from the specified
- configuration <file>, fetches their manifest files, and loads the
+ configuration <loadtab-file>, fetches their manifest files, and loads the
repository and package information into the database, suitable for
consumption by the \cb{brep} web module.
diff --git a/load/load.cxx b/load/load.cxx
index 926f173..8856e38 100644
--- a/load/load.cxx
+++ b/load/load.cxx
@@ -22,6 +22,7 @@
#include <butl/process>
#include <butl/fdstream>
#include <butl/filesystem>
+#include <butl/tab-parser>
#include <butl/manifest-parser>
#include <bpkg/manifest>
@@ -72,6 +73,12 @@ struct internal_repository
using internal_repositories = vector<internal_repository>;
+ // Parse loadtab file.
+ //
+ // loadtab consists of lines in the following format:
+ //
+ // <remote-repository-location> <display-name> cache:<local-repository-location> [fingerprint:<fingerprint>]
+ //
static internal_repositories
load_repositories (path p)
{
@@ -82,124 +89,121 @@ load_repositories (path p)
try
{
- ifdstream ifs (p, ifdstream::in, ifdstream::badbit);
+ ifdstream ifs (p);
+ tab_parser parser (ifs, p.string ());
- string s;
- for (uint64_t l (1); getline (ifs, s); ++l)
+ tab_fields tl;
+ while (!(tl = parser.next ()).empty ())
{
- auto b (s.cbegin ());
- auto i (b);
- auto e (s.cend ());
+ size_t n (tl.size ()); // Fields count.
+ size_t i (0); // The field currently being processed.
- // Skip until first non-space (true) or space (false).
+ // Report an error for the field currently being processed. If i == n
+ // then we refer to the end-of-line column (presumably reporting a missed
+ // field).
//
- auto skip = [&i, &e] (bool s = true) -> decltype (i)
+ auto bad_line = [&p, &tl, &i, n] (const string& d, size_t offset = 0)
{
- for (; i != e && space (*i) == s; ++i) ;
- return i;
- };
-
- skip (); // Skip leading spaces.
-
- if (i == e || *i == '#') // Empty line or comment.
- continue;
-
- // From now on pb will track the begining of the next part
- // while i -- the end.
- //
- auto pb (i); // Location begin.
- skip (false); // Find end of location.
+ // Offset beyond the end-of-line is meaningless.
+ //
+ assert (i < n || (i == n && offset == 0));
- auto bad_line = [&p, l, &pb, &b] (const string& d)
- {
- cerr << p << ':' << l << ':' << pb - b + 1 << ": error: " << d
- << endl;
+ cerr << p << ':' << tl.line << ':'
+ << (i == n
+ ? tl.end_column
+ : tl[i].column + offset)
+ << ": error: " << d << endl;
throw failed ();
};
- repository_location location;
+ internal_repository r;
try
{
- location = repository_location (string (pb, i));
+ r.location = repository_location (tl[i].value);
}
catch (const invalid_argument& e)
{
bad_line (e.what ());
}
- if (location.local ())
+ if (r.location.local ())
bad_line ("local repository location");
- for (const auto& r: repos)
- if (r.location.canonical_name () == location.canonical_name ())
+ for (const auto& rp: repos)
+ if (rp.location.canonical_name () == r.location.canonical_name ())
bad_line ("duplicate canonical name");
- pb = skip (); // Find begin of display name.
-
- if (pb == e)
+ // Display name field is a required one.
+ //
+ if (++i == n)
bad_line ("no display name found");
- skip (false); // Find end of display name.
-
- string name (pb, i);
- repository_location cache_location;
- optional<string> fingerprint;
+ r.display_name = move (tl[i++].value);
// Parse options, that have <name>:<value> form. Currently defined
// options are cache (mandatory for now) and fingerprint.
//
- while ((pb = skip ()) != e)
+ for (; i < n; ++i)
{
- skip (false); // Find end of the option (no spaces allowed).
-
- string nv (pb, i);
+ string nv (tl[i].value);
size_t vp;
if (strncmp (nv.c_str (), "cache:", vp = 6) == 0)
{
- if (!cache_location.empty ())
+ if (!r.cache_location.empty ())
bad_line ("cache option redefinition");
- // If the internal repository cache path is relative, then calculate
- // its absolute path. Such path is considered to be relative to the
- // configuration file directory path so result is independent from
- // whichever directory is current for the loader process. Note that
- // the resulting absolute path should be a valid repository location.
- //
- dir_path cache_path = dir_path (string (nv, vp));
- if (cache_path.relative ())
- cache_path = p.directory () / cache_path;
-
try
{
- cache_location = repository_location (cache_path.string ());
+ // If the internal repository cache path is relative, then
+ // calculate its absolute path. Such path is considered to be
+ // relative to the configuration file directory path so result is
+ // independent from whichever directory is current for the loader
+ // process. Note that the resulting absolute path should be a valid
+ // repository location.
+ //
+ dir_path cache_path = dir_path (string (nv, vp));
+ if (cache_path.relative ())
+ cache_path = p.directory () / cache_path;
+
+ r.cache_location = repository_location (cache_path.string ());
// Created from the absolute path repository location can not be
// other than absolute.
//
- assert (cache_location.absolute ());
+ assert (r.cache_location.absolute ());
+ }
+ catch (const invalid_path& e) // Thrown by dir_path().
+ {
+ bad_line (string ("invalid cache path: ") + e.what ());
}
- catch (const invalid_argument&)
+ catch (const invalid_argument& e) // Thrown by repository_location().
{
- bad_line ("invalid cache path");
+ bad_line (string ("invalid cache path: ") + e.what ());
}
+
+ if (!file_exists (r.packages_path ()))
+ bad_line ("'packages' file does not exist");
+
+ if (!file_exists (r.repositories_path ()))
+ bad_line ("'repositories' file does not exist");
}
else if (strncmp (nv.c_str (), "fingerprint:", vp = 12) == 0)
{
- if (fingerprint)
+ if (r.fingerprint)
bad_line ("fingerprint option redefinition");
- fingerprint = string (nv, vp);
+ r.fingerprint = string (nv, vp);
// Sanity check.
//
- if (!fingerprint->empty ())
+ if (!r.fingerprint->empty ())
{
try
{
- fingerprint_to_sha256 (*fingerprint);
+ fingerprint_to_sha256 (*r.fingerprint);
}
catch (const invalid_argument&)
{
@@ -211,29 +215,19 @@ load_repositories (path p)
bad_line ("invalid option '" + nv + "'");
}
- if (cache_location.empty ()) // For now cache option is mandatory.
+ // For now cache option is mandatory.
+ //
+ if (r.cache_location.empty ())
bad_line ("no cache option found");
- internal_repository r {
- move (location),
- move (name),
- move (cache_location),
- move (fingerprint)};
-
- if (!file_exists (r.packages_path ()))
- bad_line ("'packages' file does not exist");
-
- if (!file_exists (r.repositories_path ()))
- bad_line ("'repositories' file does not exist");
-
repos.emplace_back (move (r));
-
- // Check that there is no non-whitespace junk at the end.
- //
- if (skip () != e)
- bad_line ("junk after filesystem path");
}
}
+ catch (const tab_parsing& e)
+ {
+ cerr << e << endl;
+ throw failed ();
+ }
catch (const io_error& e)
{
cerr << "error: unable to read " << p << ": " << e << endl;
diff --git a/tests/load/buildfile b/tests/load/buildfile
index a4b0ffb..2088ee1 100644
--- a/tests/load/buildfile
+++ b/tests/load/buildfile
@@ -18,4 +18,4 @@ exe{driver}: test = false
# precondition: PostgreSQL server running port 8432 with brep schema created.
# test:
-# ./driver ../../load/brep-load --db-host localhost --db-port 8432 ./r.conf
+# ./driver ../../load/brep-load --db-host localhost --db-port 8432 ./loadtab
diff --git a/tests/load/driver.cxx b/tests/load/driver.cxx
index 396d89f..a0ab10f 100644
--- a/tests/load/driver.cxx
+++ b/tests/load/driver.cxx
@@ -52,7 +52,7 @@ main (int argc, char* argv[])
auto print_usage = [argv]()
{
cerr << "usage: " << argv[0]
- << " <loader_path> [loader_options] <loader_conf_file>" << endl;
+ << " <loader_path> [loader_options] <loadtab_file>" << endl;
};
if (argc < 3)
diff --git a/tests/load/r.conf b/tests/load/loadtab
index fe06984..fe06984 100644
--- a/tests/load/r.conf
+++ b/tests/load/loadtab