Implement brep-migrate utility

4 files changed, 0 insertions, 920 deletions

diff --git a/loader/.gitignore b/loader/.gitignore
deleted file mode 100644
index 820b183..0000000
--- a/loader/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-options
-options.?xx
-brep-loader
diff --git a/loader/buildfile b/loader/buildfile
deleted file mode 100644
index 477cebd..0000000
--- a/loader/buildfile
+++ /dev/null
@@ -1,20 +0,0 @@
-# file      : loader/buildfile
-# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
-# license   : MIT; see accompanying LICENSE file
-
-import libs += libbpkg%lib{bpkg}
-import libs += libbutl%lib{butl}
-import libs += libodb-pgsql%lib{odb-pgsql}
-import libs += libodb%lib{odb}
-
-include ../brep/
-
-exe{brep-loader}:            \
-{            cxx}{ loader  } \
-{hxx ixx     cxx}{ options } \
-../brep/lib{brep} $libs
-
-cli.options += -I $src_root --include-with-brackets --include-prefix loader \
---guard-prefix LOADER
-
-{hxx ixx cxx}{options}: cli{options}
diff --git a/loader/loader.cxx b/loader/loader.cxx
deleted file mode 100644
index b1b7706..0000000
--- a/loader/loader.cxx
+++ /dev/null
@@ -1,851 +0,0 @@
-// file      : loader/loader.cxx -*- C++ -*-
-// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
-// license   : MIT; see accompanying LICENSE file
-
-#include <sstream>
-#include <fstream>
-#include <iostream>
-#include <stdexcept> // runtime_error, invalid_argument
-#include <algorithm> // find(), find_if()
-
-#include <odb/session.hxx>
-#include <odb/database.hxx>
-#include <odb/transaction.hxx>
-
-#include <odb/pgsql/database.hxx>
-#include <odb/pgsql/exceptions.hxx>
-#include <odb/pgsql/connection.hxx>
-#include <odb/pgsql/transaction.hxx>
-
-#include <butl/filesystem>
-
-#include <bpkg/manifest-parser> // manifest_parsing
-
-#include <brep/types>
-#include <brep/utility>
-#include <brep/version>
-
-#include <brep/package>
-#include <brep/package-odb>
-
-#include <loader/options>
-
-using namespace std;
-using namespace odb::core;
-using namespace butl;
-using namespace bpkg;
-using namespace brep;
-
-namespace pgsql = odb::pgsql;
-
-static void
-usage (ostream& os)
-{
-  os << "Usage: brep-loader [options] <file>" << endl
-     << "File lists internal repositories." << endl
-     << "Options:" << endl;
-
-  options::print_usage (os);
-}
-
-static inline bool
-space (char c) noexcept
-{
-  return c == ' ' || c == '\t';
-}
-
-struct internal_repository
-{
-  repository_location location;
-  string display_name;
-  dir_path local_path;
-
-  path
-  packages_path () const {return local_path / path ("packages");}
-
-  path
-  repositories_path () const {return local_path / path ("repositories");}
-};
-
-using internal_repositories = vector<internal_repository>;
-
-static internal_repositories
-load_repositories (path p)
-{
-  internal_repositories repos;
-
-  if (p.relative ())
-    p.complete ();
-
-  ifstream ifs (p.string ());
-  if (!ifs.is_open ())
-    throw ifstream::failure (p.string () + ": unable to open");
-
-  ifs.exceptions (ifstream::badbit);
-
-  try
-  {
-    string s;
-    for (uint64_t l (1); getline (ifs, s); ++l)
-    {
-      auto b (s.cbegin ());
-      auto i (b);
-      auto e (s.cend ());
-
-      // Skip until first non-space (true) or space (false).
-      //
-      auto skip = [&i, &e](bool s = true) -> decltype (i)
-      {
-        for (; i != e && space (*i) == s; ++i)
-          ;
-        return i;
-      };
-
-      skip (); // Skip leading spaces.
-
-      if (i == e || *i == '#') // Empty line or comment.
-        continue;
-
-      // From now on pb will track the begining of the next part
-      // while i -- the end.
-      //
-      auto pb (i);  // Location begin.
-      skip (false); // Find end of location.
-
-      auto bad_line ([&p, l, &pb, &b](const string& d) {
-          ostringstream os;
-          os << p << ':' << l << ':' << pb - b + 1 << ": error: " << d;
-          throw runtime_error (os.str ());
-        });
-
-      repository_location location;
-
-      try
-      {
-        location = repository_location (string (pb, i));
-      }
-      catch (const invalid_argument& e)
-      {
-        bad_line (e.what ());
-      }
-
-      if (location.local ())
-        bad_line ("local repository location");
-
-      for (const auto& r: repos)
-        if (r.location.canonical_name () == location.canonical_name ())
-          bad_line ("duplicate canonical name");
-
-      pb = skip (); // Find begin of display name.
-
-      if (pb == e)
-        bad_line ("no display name found");
-
-      skip (false); // Find end of display name.
-
-      string name (pb, i);
-      pb = skip (); // Find begin of filesystem path.
-
-      if (pb == e) // For now filesystem path is mandatory.
-        bad_line ("no filesystem path found");
-
-      skip (false); // Find end of filesystem path (no spaces allowed).
-
-      internal_repository r {
-        move (location),
-        move (name),
-        dir_path (string (pb, i))};
-
-      // If the internal repository local path is relative, then
-      // calculate its absolute local path. Such path is considered to be
-      // relative to configuration file directory path so result is
-      // independent from whichever directory is current for the loader
-      // process.
-      //
-      if (r.local_path.relative ())
-        r.local_path = p.directory () / r.local_path;
-
-      try
-      {
-        r.local_path.normalize ();
-      }
-      catch (const invalid_path&)
-      {
-        bad_line ("can't normalize local path");
-      }
-
-      if (!file_exists (r.packages_path ()))
-        bad_line ("'packages' file does not exist");
-
-      if (!file_exists (r.repositories_path ()))
-        bad_line ("'repositories' file does not exist");
-
-      repos.emplace_back (move (r));
-
-      // Check that there is no non-whitespace junk at the end.
-      //
-      if (skip () != e)
-        bad_line ("junk after filesystem path");
-    }
-  }
-  catch (const ifstream::failure&)
-  {
-    throw ifstream::failure (p.string () + ": io failure");
-  }
-
-  return repos;
-}
-
-// Check if repositories persistent state is outdated. If any repository
-// differes from its persistent state or there is a persistent repository
-// which is not listed in configuration file then the whole persistent
-// state will be recreated. Will consider optimization later when the
-// package model, including search related objects, settles down.
-//
-static bool
-changed (const internal_repositories& repos, database& db)
-{
-  strings names;
-  for (auto& r: repos)
-  {
-    shared_ptr<repository> pr (
-      db.find<repository> (r.location.canonical_name ()));
-
-    if (pr == nullptr || r.location.string () != pr->location.string () ||
-        r.display_name != pr->display_name || r.local_path != pr->local_path ||
-        file_mtime (r.packages_path ()) != pr->packages_timestamp ||
-        file_mtime (r.repositories_path ()) != pr->repositories_timestamp ||
-        !pr->internal)
-      return true;
-
-    names.emplace_back (r.location.canonical_name ());
-  }
-
-  using query = query<repository>;
-
-  // Check if there is an internal repository not being listed in the
-  // configuration file.
-  //
-  return
-    !db.query<repository> (
-      query::internal && !query::name.in_range (names.begin (), names.end ())).
-    empty ();
-}
-
-static timestamp
-manifest_stream (const path& p, ifstream& f)
-{
-  f.open (p.string ());
-  if (!f.is_open ())
-    throw ifstream::failure (p.string () + ": unable to open");
-
-  f.exceptions (ifstream::badbit | ifstream::failbit);
-  return file_mtime (p);
-}
-
-// Load the repository packages from the 'packages' file and persist the
-// repository. Should be called once per repository.
-//
-static void
-load_packages (const shared_ptr<repository>& rp, database& db)
-{
-  // packages_timestamp other than timestamp_nonexistent signals the
-  // repository packages are already loaded.
-  //
-  assert (rp->packages_timestamp == timestamp_nonexistent);
-
-  // Only locally accessible repositories allowed until package manager API is
-  // ready.
-  //
-  assert (!rp->local_path.empty ());
-
-  package_manifests pkm;
-
-  {
-    ifstream ifs;
-    path p (rp->local_path / path ("packages"));
-    rp->packages_timestamp = manifest_stream (p, ifs);
-
-    manifest_parser mp (ifs, p.string ());
-    pkm = package_manifests (mp);
-  }
-
-  for (auto& pm: pkm)
-  {
-    shared_ptr<package> p (db.find<package> (package_id (pm.name, pm.version)));
-
-    if (p == nullptr)
-    {
-      if (rp->internal)
-      {
-        // Create internal package object.
-        //
-        optional<string> dsc;
-        if (pm.description)
-        {
-          if (pm.description->file)
-          {
-            // @@ Pull description from the file when package manager API
-            // is ready.
-          }
-          else
-            dsc = move (*pm.description);
-        }
-
-        string chn;
-        for (auto& c: pm.changes)
-        {
-          if (c.file)
-          {
-            // @@ Pull change notes from the file when package manager
-            //    API is ready.
-          }
-          else
-          {
-            if (chn.empty ())
-              chn = move (c);
-            else
-              chn += "\n" + c;
-          }
-        }
-
-        dependencies ds;
-
-        for (auto& pda: pm.dependencies)
-        {
-          ds.emplace_back (pda.conditional, move (pda.comment));
-
-          for (auto& pd: pda)
-            // Proper version will be assigned during dependency resolution
-            // procedure. Here we rely on the fact the foreign key constraint
-            // check is deferred until the current transaction commit.
-            //
-            ds.back ().push_back ({
-                lazy_shared_ptr<package> (
-                  db, package_id (move (pd.name), version ())),
-                move (pd.constraint)});
-        }
-
-        p = make_shared<package> (
-          move (pm.name),
-          move (pm.version),
-          pm.priority ? move (*pm.priority) : priority (),
-          move (pm.summary),
-          move (pm.license_alternatives),
-          move (pm.tags),
-          move (dsc),
-          move (chn),
-          move (pm.url),
-          move (pm.package_url),
-          move (pm.email),
-          move (pm.package_email),
-          move (ds),
-          move (pm.requirements),
-          move (pm.location),
-          rp);
-      }
-      else
-        // Create external package object.
-        //
-        p = make_shared<package> (move (pm.name), move (pm.version), rp);
-
-      db.persist (p);
-    }
-    else
-    {
-      // @@ Need to ensure that the same packages coming from different
-      //    repositories are equal. Probably will invent hashsum at some point
-      //    for this purpose.
-      //
-
-      // As soon as internal repositories get loaded first, the internal
-      // package can duplicate an internal package only.
-      //
-      assert (!rp->internal || p->internal ());
-
-      p->other_repositories.push_back (rp);
-      db.update (p);
-    }
-  }
-
-  db.persist (rp); // Save the repository state.
-}
-
-// Load the repository manifest values, prerequsite repositories, and their
-// complements state from the 'repositories' file. Update the repository
-// persistent state to save changed members. Should be called once per
-// persisted internal repository.
-//
-static void
-load_repositories (const shared_ptr<repository>& rp, database& db)
-{
-  // repositories_timestamp other than timestamp_nonexistent signals that
-  // repository prerequisites are already loaded.
-  //
-  assert (rp->repositories_timestamp == timestamp_nonexistent);
-
-  // Only locally accessible repositories allowed until package manager API is
-  // ready.
-  //
-  assert (!rp->local_path.empty ());
-
-  // Repository is already persisted by the load_packages() function call.
-  //
-  assert (db.find<repository> (rp->name) != nullptr);
-
-  repository_manifests rpm;
-
-  {
-    ifstream ifs;
-    path p (rp->local_path / path ("repositories"));
-    rp->repositories_timestamp = manifest_stream (p, ifs);
-
-    manifest_parser mp (ifs, p.string ());
-    rpm = repository_manifests (mp);
-  }
-
-  for (auto& rm: rpm)
-  {
-    if (rm.effective_role () == repository_role::prerequisite && !rp->internal)
-      continue; // Ignore the external repository prerequisite entry.
-
-    if (rm.effective_role () == repository_role::base)
-    {
-      assert (rp->location.remote () && !rp->url);
-
-      // Update the base repository with manifest values.
-      //
-      rp->url = rm.effective_url (rp->location);
-
-      // @@ Should we throw if url is not available for external repository ?
-      //    Can, basically, repository be available on the web but have no web
-      //    interface associated ?
-      //
-      //    Yes, there can be no web interface. So we should just not form
-      //    links to packages from such repos.
-      //
-      if (rp->url)
-      {
-        // Normalize web interface url adding trailing '/' if not present.
-        //
-        auto& u (*rp->url);
-        assert (!u.empty ());
-        if (u.back () != '/')
-          u += '/';
-      }
-
-      if (rp->internal)
-      {
-        rp->email = move (rm.email);
-        rp->summary = move (rm.summary);
-        rp->description = move (rm.description);
-      }
-
-      continue;
-    }
-
-    // Load prerequisite or complement repository.
-    //
-    assert (!rm.location.empty ());
-
-    repository_location rl;
-
-    auto bad_location (
-      [&rp, &rm]()
-      {
-        ostringstream o;
-        o << "invalid location '" << rm.location.string ()
-          << "' of the prerequisite repository for internal "
-          "repository '" << rp->location.string () << "'";
-
-        throw runtime_error (o.str ());
-      });
-
-    try
-    {
-      // Absolute path location make no sense for the web interface.
-      //
-      if (rm.location.absolute ())
-        bad_location ();
-
-      // Convert the relative repository location to remote one, leave remote
-      // location unchanged.
-      //
-      rl = repository_location (rm.location.string (), rp->location);
-    }
-    catch (const invalid_argument&)
-    {
-      bad_location ();
-    }
-
-    const auto& cn (rl.canonical_name ());
-
-    // Add repository to prerequisites or complements member of the dependent
-    // repository.
-    //
-    auto& rs (rm.effective_role () == repository_role::prerequisite
-              ? rp->prerequisites
-              : rp->complements);
-
-    rs.emplace_back (db, cn);
-
-    shared_ptr<repository> pr (db.find<repository> (cn));
-
-    if (pr != nullptr)
-      // The prerequisite repository is already loaded.
-      //
-      continue;
-
-    pr = make_shared<repository> (move (rl));
-
-    // If the prerequsite repository location is a relative path, then
-    // calculate its absolute local path.
-    //
-    if (rm.location.relative ())
-    {
-      dir_path& lp (pr->local_path);
-      lp = rp->local_path / rm.location.path ();
-
-      try
-      {
-        lp.normalize ();
-      }
-      catch (const invalid_path&)
-      {
-        ostringstream o;
-        o << "can't normalize local path'" << lp.string ()
-          << "' of the prerequisite repository for internal "
-          "repository '" << rp->location.string () << "'";
-
-        throw runtime_error (o.str ());
-      }
-    }
-
-    load_packages (pr, db);
-    load_repositories (pr, db);
-  }
-
-  db.update (rp);
-}
-
-// Check if the package is available from the specified repository,
-// its prerequisite repositories, or one of their complements,
-// recursively.
-//
-static bool
-find (const lazy_shared_ptr<repository>& r,
-      const package& p,
-      bool prereq = true)
-{
-  assert (r != nullptr);
-
-  const auto& o (p.other_repositories);
-  if (r == p.internal_repository || find (o.begin (), o.end (), r) != o.end ())
-    return true;
-
-  auto rp (r.load ());
-  for (const auto& cr: rp->complements)
-  {
-    if (find (lazy_shared_ptr<repository> (cr), p, false))
-      return true;
-  }
-
-  if (prereq)
-  {
-    for (auto pr: rp->prerequisites)
-    {
-      if (find (lazy_shared_ptr<repository> (pr), p, false))
-        return true;
-    }
-  }
-
-  return false;
-}
-
-// Resolve package dependencies. Ensure that the best matching dependency
-// belongs to the package repositories, their immediate prerequisite
-// repositories, or their complements, recursively. Should be called once per
-// internal package.
-//
-static void
-resolve_dependencies (package& p, database& db)
-{
-  // Resolve dependencies for internal packages only.
-  //
-  assert (p.internal ());
-
-  if (p.dependencies.empty ())
-    return;
-
-  for (auto& da: p.dependencies)
-  {
-    for (auto& d: da)
-    {
-      // Dependency should not be resolved yet.
-      //
-      assert (d.package.object_id ().version.empty ());
-
-      using query = query<package>;
-      query q (query::id.name == d.name ());
-      const auto& vm (query::id.version);
-
-      if (d.constraint)
-      {
-        auto c (*d.constraint);
-
-        if (c.min_version)
-        {
-          if (c.min_open)
-            q = q && vm > *c.min_version;
-          else
-            q = q && vm >= *c.min_version;
-        }
-
-        if (c.max_version)
-        {
-          if (c.max_open)
-            q = q && vm < *c.max_version;
-          else
-            q = q && vm <= *c.max_version;
-        }
-      }
-
-      for (const auto& pp: db.query<package> (q + order_by_version_desc (vm)))
-      {
-        if (find (p.internal_repository, pp))
-        {
-          d.package.reset (db, pp.id);
-          break;
-        }
-      }
-
-      if (d.package.object_id ().version.empty ())
-      {
-        ostringstream o;
-        o << "can't resolve dependency " << d << " of the package "
-          << p.id.name << " " << p.version.string ()
-          << " (" << p.internal_repository.load ()->name << ")";
-
-        // Practically it is enough to resolve at least one dependency
-        // alternative to build a package. Meanwhile here we consider an error
-        // specifying in the manifest file an alternative which can't be
-        // resolved.
-        //
-        throw runtime_error (o.str ());
-      }
-    }
-  }
-
-  db.update (p); // Update the package state.
-}
-
-using package_ids = vector<package_id>;
-
-// Ensure the package dependency chain do not contain the package id. Throw
-// runtime_error otherwise. Continue the chain with the package id and call
-// itself recursively for each prerequisite of the package. Should be called
-// once per internal package.
-//
-// @@ This should probably be eventually moved to bpkg.
-//
-static void
-detect_dependency_cycle (const package_id& id, package_ids& chain, database& db)
-{
-  // Package of one version depending on the same package of another version
-  // is something obscure. So the comparison is made up to a package name.
-  //
-  auto pr ([&id](const package_id& i) -> bool {return i.name == id.name;});
-  auto i (find_if (chain.begin (), chain.end (), pr));
-
-  if (i != chain.end ())
-  {
-    ostringstream o;
-    o << "package dependency cycle: ";
-
-    auto prn (
-      [&o, &db](const package_id& id)
-      {
-        shared_ptr<package> p (db.load<package> (id));
-        assert (p->internal () || !p->other_repositories.empty ());
-
-        shared_ptr<repository> r (
-          p->internal ()
-          ? p->internal_repository.load ()
-          : p->other_repositories[0].load ());
-
-        o << id.name << " " << p->version.string () << " (" << r->name << ")";
-      });
-
-    for (; i != chain.end (); ++i)
-    {
-      prn (*i);
-      o << " -> ";
-    }
-
-    prn (id);
-    throw runtime_error (o.str ());
-  }
-
-  chain.push_back (id);
-
-  shared_ptr<package> p (db.load<package> (id));
-  for (const auto& da: p->dependencies)
-  {
-    for (const auto& d: da)
-      detect_dependency_cycle (d.package.object_id (), chain, db);
-  }
-
-  chain.pop_back ();
-}
-
-int
-main (int argc, char* argv[])
-{
-  try
-  {
-    cli::argv_scanner scan (argc, argv, true);
-    options ops (scan);
-
-    // Version.
-    //
-    if (ops.version ())
-    {
-      cout << "brep-loader " << BREP_VERSION_STR << endl
-           << "libbrep " << LIBBREP_VERSION_STR << endl
-           << "libbpkg " << LIBBPKG_VERSION_STR << endl
-           << "libbutl " << LIBBUTL_VERSION_STR << endl
-           << "Copyright (c) 2014-2016 Code Synthesis Ltd" << endl
-           << "MIT; see accompanying LICENSE file" << endl;
-
-      return 0;
-    }
-
-    // Help.
-    //
-    if (ops.help ())
-    {
-      usage (cout);
-      return 0;
-    }
-
-    if (argc < 2)
-    {
-      cerr << "<file> argument not provided" << endl;
-      usage (cerr);
-      return 1;
-    }
-
-    if (argc > 2)
-    {
-      cerr << "unexpected argument encountered" << endl;
-      usage (cerr);
-      return 1;
-    }
-
-    pgsql::database db (ops.db_user (),
-                        ops.db_password (),
-                        ops.db_name (),
-                        ops.db_host (),
-                        ops.db_port ());
-
-    // Prevent several loader instances from updating DB simultaneously.
-    //
-    {
-      transaction t (db.begin ());
-      db.execute ("CREATE TABLE IF NOT EXISTS loader_mutex ()");
-      t.commit ();
-    }
-
-    pgsql::connection_ptr synch_c (db.connection ());
-
-    // Don't make current.
-    //
-    pgsql::transaction synch_t (synch_c->begin (), false);
-
-    try
-    {
-      synch_c->execute ("LOCK TABLE loader_mutex NOWAIT");
-    }
-    catch (const pgsql::database_exception& e)
-    {
-      if (e.sqlstate () == "55P03")
-        return 2; // Other loader instance acquired the mutex.
-
-      throw;
-    }
-
-    // Load the description of all the internal repositories from the
-    // configuration file.
-    //
-    internal_repositories irs (load_repositories (path (argv[1])));
-
-    transaction t (db.begin ());
-
-    if (changed (irs, db))
-    {
-      // Rebuild repositories persistent state from scratch.
-      //
-      db.erase_query<package> ();
-      db.erase_query<repository> ();
-
-      // On the first pass over the internal repositories we load their
-      // packages.
-      //
-      uint16_t priority (1);
-      for (const auto& ir: irs)
-      {
-        shared_ptr<repository> r (
-          make_shared<repository> (ir.location,
-                                   move (ir.display_name),
-                                   move (ir.local_path),
-                                   priority++));
-
-        load_packages (r, db);
-      }
-
-      // On the second pass over the internal repositories we load their
-      // (not yet loaded) manifest values, complement, and prerequisite
-      // repositories.
-      //
-      for (const auto& ir: irs)
-      {
-        shared_ptr<repository> r (
-          db.load<repository> (ir.location.canonical_name ()));
-
-        load_repositories (r, db);
-      }
-
-      session s;
-      using query = query<package>;
-
-      // Resolve internal packages dependencies.
-      //
-      for (auto& p:
-             db.query<package> (query::internal_repository.is_not_null ()))
-        resolve_dependencies (p, db);
-
-      // Ensure there is no package dependency cycles.
-      //
-      package_ids chain;
-      for (const auto& p:
-             db.query<package> (query::internal_repository.is_not_null ()))
-        detect_dependency_cycle (p.id, chain, db);
-    }
-
-    t.commit ();
-    synch_t.commit (); // Release the mutex.
-  }
-  catch (const cli::exception& e)
-  {
-    cerr << e << endl;
-    usage (cerr);
-    return 1;
-  }
-  // Fully qualified to avoid ambiguity with odb exception.
-  //
-  catch (const std::exception& e)
-  {
-    cerr << e.what () << endl;
-    return 1;
-  }
-}
diff --git a/loader/options.cli b/loader/options.cli
deleted file mode 100644
index 0899156..0000000
--- a/loader/options.cli
+++ /dev/null
@@ -1,46 +0,0 @@
-// file      : loader/options.cli
-// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
-// license   : MIT; see accompanying LICENSE file
-
-include <string>;
-include <cstdint>; // uint16_t
-
-class options
-{
-  bool --help {"Print usage information and exit."}
-  bool --version {"Print version and exit."}
-
-  std::string --db-user|-u
-  {
-    "<user>",
-    "Database user name. If not specified, then operating system (login)
-     name is used."
-  }
-
-  std::string --db-password
-  {
-    "<pass>",
-    "Database password. If not specified, then login without password is
-     expected to work."
-  }
-
-  std::string --db-name|-n = "brep"
-  {
-    "<name>",
-    "Database name. If not specified, then '\cb{brep}' is used by default."
-  }
-
-  std::string --db-host|-h
-  {
-    "<host>",
-    "Database host name, address, or socket. If not specified, then connect
-     to \cb{localhost} using the operating system-default mechanism
-     (Unix-domain socket, etc)."
-  }
-
-  std::uint16_t --db-port|-p = 0
-  {
-    "<port>",
-    "Database port number. If not specified, the default port is used."
-  }
-};