From 8e866579cb459c5104c532d5e41d562d45236ea5 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Tue, 30 Jun 2015 19:20:16 +0200 Subject: Implement loader --- brep/.gitignore | 5 + brep/buildfile | 24 ++++ brep/diagnostics.cxx | 2 +- brep/module.cxx | 8 +- brep/options.cli | 2 +- brep/package | 302 ++++++++++++++++++++++++++++++++++++++++----------- brep/package.cxx | 120 +++++++++++++++++++- brep/search.cxx | 74 +++++++------ brep/services.cxx | 21 ++++ brep/view.cxx | 92 ++++++++-------- 10 files changed, 496 insertions(+), 154 deletions(-) create mode 100644 brep/.gitignore create mode 100644 brep/buildfile create mode 100644 brep/services.cxx (limited to 'brep') diff --git a/brep/.gitignore b/brep/.gitignore new file mode 100644 index 0000000..2c23e23 --- /dev/null +++ b/brep/.gitignore @@ -0,0 +1,5 @@ +options +options.?xx +package-odb* +package.sql + diff --git a/brep/buildfile b/brep/buildfile new file mode 100644 index 0000000..13e1d20 --- /dev/null +++ b/brep/buildfile @@ -0,0 +1,24 @@ +# file : brep/buildfile +# copyright : Copyright (c) 2014-2015 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +using cli + +.: libso{brep brep-apache} + +import libs += libbpkg%lib{bpkg} +import libs += libodb-pgsql%lib{odb-pgsql} +import libs += libodb%lib{odb} + +brep = cxx{package package-odb} +libso{brep}: $brep $libs +libso{brep}: cxx.export.poptions = -I$out_root -I$src_root + +brep = cxx{diagnostics module services search view} cli.cxx{options} +web = ../web/apache/cxx{request service} +libso{brep-apache}: $brep $web libso{brep} $libs + +cli.options += -I $src_root --include-with-brackets --include-prefix brep \ +--guard-prefix BREP --generate-file-scanner --suppress-usage + +cli.cxx{options}: cli{options} diff --git a/brep/diagnostics.cxx b/brep/diagnostics.cxx index e04c214..0278792 100644 --- a/brep/diagnostics.cxx +++ b/brep/diagnostics.cxx @@ -22,7 +22,7 @@ namespace brep // once C++17 uncaught_exceptions() becomes available. // if (!data_.empty () && - (!std::uncaught_exception () /*|| exception_unwinding_dtor*/)) + (!uncaught_exception () /*|| exception_unwinding_dtor*/)) { data_.back ().msg = os_.str (); // Save last message. diff --git a/brep/module.cxx b/brep/module.cxx index 32988e0..ceadc23 100644 --- a/brep/module.cxx +++ b/brep/module.cxx @@ -4,8 +4,8 @@ #include -#include -#include +#include +#include #include #include @@ -89,7 +89,7 @@ namespace brep argv.push_back (nv.value.c_str ()); } - int argc (argv.size()); + int argc (argv.size ()); try { @@ -140,7 +140,7 @@ namespace brep // using A = B (*)(int,int); // A func(B (*)(char),B (*)(wchar_t)); // __PRETTY_FUNCTION__ looks like this: -// virtual std::string (* (* brep::search::func(std::string (* (*)(char))(int)\ +// virtual std::string (* (* brep::search::func(std::string (* (*)(char))(int) // ,std::string (* (*)(wchar_t))(int)) const)(int, int))(int) // string module:: diff --git a/brep/options.cli b/brep/options.cli index 606898c..b7c571e 100644 --- a/brep/options.cli +++ b/brep/options.cli @@ -15,7 +15,7 @@ namespace brep class db_options { std::string db-host = "localhost"; - std::uint16_t db-port = 3306; + std::uint16_t db-port = 5432; }; class search_options: module_options, db_options diff --git a/brep/package b/brep/package index 62ad8d6..c0c9fc9 100644 --- a/brep/package +++ b/brep/package @@ -2,30 +2,60 @@ // copyright : Copyright (c) 2014-2015 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file -#ifndef BPKG_PACKAGE -#define BPKG_PACKAGE +#ifndef BREP_PACKAGE +#define BREP_PACKAGE #include #include #include +#include #include // shared_ptr -#include // pair +#include // size_t +#include // move() #include // uint16 #include #include // database #include -#include + +#include +#include +#include +#include + +namespace brep +{ + // Use an image type to map bpkg::version to the database since there + // is no way to modify individual components directly. + // + #pragma db value + struct _version + { + std::uint16_t epoch; + std::string upstream; + std::uint16_t revision; + std::string canonical_upstream; + }; +} #include +// We have to keep this mapping at the global scope instead of inside +// the brep namespace because it needs to be also effective in the +// bpkg namespace from which we "borrow" types (and some of them use +// version). +// +#pragma db map type(bpkg::version) as(brep::_version) \ + to(brep::_version{(?).epoch (), \ + (?).upstream (), \ + (?).revision (), \ + (?).canonical_upstream ()}) \ + from(bpkg::version ((?).epoch, std::move ((?).upstream), (?).revision)) + namespace brep { // @@ If namespace, then should probably call it 'repo'. // - // @@ Should probably use optional from libbutl instead of odb::nullable - // for consistency. Create butl profile? - // // @@ Might make sense to put some heavy members (e.g., description, // containers) into a separate section. // @@ -36,38 +66,63 @@ namespace brep // Forward declarations. // - struct package; - struct package_version; + class repository; + class package; + class package_version; using strings = std::vector; - using version = bpkg::version; - #pragma db value(version) definition + template + using optional = butl::optional; + + using path = butl::path; + + #pragma db map type(path) as(std::string) \ + to((?).string ()) from(brep::path (?)) + + using dir_path = butl::dir_path; + + #pragma db map type(dir_path) as(std::string) \ + to((?).string ()) from(brep::dir_path (?)) + + using timestamp = butl::timestamp; - using version_type = brep::version; + #pragma db map type(timestamp) as(std::uint64_t) \ + to(std::chrono::system_clock::to_time_t (?)) \ + from(std::chrono::system_clock::from_time_t (?)) + + using version = bpkg::version; + using repository_location = bpkg::repository_location; #pragma db value struct package_version_id { + std::string repository; std::string package; std::uint16_t epoch; - std::string canonical; + std::string canonical_upstream; // Database mapping. // - #pragma db member(package) points_to(package) + #pragma db member(repository) points_to(repository) //on_delete(cascade) + #pragma db member(package) points_to(package) //on_delete(cascade) }; inline bool operator< (const package_version_id& x, const package_version_id& y) { - int r (x.package.compare (y.package)); + int r (x.repository.compare (y.repository)); + + if (r != 0) + return r < 0; + + r = x.package.compare (y.package); if (r != 0) return r < 0; return x.epoch < y.epoch || - x.epoch == y.epoch && x.canonical < y.canonical; + (x.epoch == y.epoch && x.canonical_upstream < y.canonical_upstream); } using priority = bpkg::priority; @@ -76,7 +131,7 @@ namespace brep using url = bpkg::url; #pragma db value(url) definition - #pragma db member(url::value) virtual(std::string) before access(this) \ + #pragma db member(url::value) virtual(std::string) before access(this) \ column("") using email = bpkg::email; @@ -153,31 +208,110 @@ namespace brep std::size_t first; std::size_t second; - index_pair (std::size_t f = 0, std::size_t s = 0): first (f), second (s) {} + index_pair () = default; + index_pair (std::size_t f, std::size_t s): first (f), second (s) {} bool operator< (const index_pair& v) const { - return first < v.first || first == v.first && second < v.second; + return first < v.first || (first == v.first && second < v.second); } }; #pragma db object pointer(std::shared_ptr) session - struct package + class repository { - // Manifest data. + public: + using path_type = brep::path; + using timestamp_type = brep::timestamp; + using package_versions_type = + std::vector>; + using prerequisite_repositories_type = + std::vector>; + + // Create internal repository. + // + repository (repository_location, + std::string display_name, + dir_path local_path); + + // Create external repository. + // + explicit + repository (repository_location l) + : location (std::move (l)), internal (false) {} + + repository_location location; + std::string display_name; + + // Non empty for internal repositories and external ones with a filesystem + // path location. + // + dir_path local_path; + + // Initialized with timestamp_nonexistent by default. + // + timestamp_type timestamp; + + bool internal; + package_versions_type package_versions; + prerequisite_repositories_type prerequisite_repositories; + + // Database mapping. // + #pragma db value + struct _id_type + { + std::string canonical_name; + std::string location; + }; + + _id_type + _id () const; + + void + _id (_id_type&&); + + #pragma db member(location) transient + + #pragma db member(id) virtual(_id_type) before id(canonical_name) \ + get(_id) set(_id (std::move (?))) column("") + + #pragma db member(package_versions) inverse(id.data.repository) + #pragma db member(prerequisite_repositories) id_column("repository") \ + value_column("prerequisite_repository") value_not_null + + private: + friend class odb::access; + repository () = default; + }; + + #pragma db object pointer(std::shared_ptr) session + class package + { + public: using url_type = brep::url; using email_type = brep::email; + package (std::string name, + std::string summary, + strings tags, + optional description, + url_type, + optional package_url, + email_type, + optional package_email); + + // Manifest data. + // std::string name; std::string summary; strings tags; - std::string description; + optional description; url_type url; - odb::nullable package_url; + optional package_url; email_type email; - odb::nullable package_email; + optional package_email; std::vector> versions; // Additional data. @@ -187,55 +321,71 @@ namespace brep // #pragma db member(name) id #pragma db member(tags) id_column("package") value_column("tag") - #pragma db member(versions) inverse(id.package) + #pragma db member(versions) inverse(id.data.package) + + private: + friend class odb::access; + package () = default; }; #pragma db object pointer(std::shared_ptr) session - struct package_version + class package_version { - // Manifest data. - // + public: + using repository_type = brep::repository; using package_type = brep::package; + using version_type = brep::version; using priority_type = brep::priority; using license_alternatives_type = brep::license_alternatives; using dependencies_type = brep::dependencies; using requirements_type = brep::requirements; + package_version (odb::lazy_shared_ptr, + odb::lazy_shared_ptr, + version_type, + priority_type, + license_alternatives_type, + std::string changes, + dependencies_type, + requirements_type); + + // Manifest data. + // + odb::lazy_shared_ptr repository; + odb::lazy_shared_ptr package; version_type version; - std::shared_ptr package; priority_type priority; license_alternatives_type license_alternatives; std::string changes; dependencies_type dependencies; requirements_type requirements; - // Additional data. - // - std::string repository; // E.g., "stable", "testing". - // Database mapping. // // id // - package_version_id - id () const + #pragma db value + struct _id_type { - return package_version_id - {package->name, version.epoch, version.canonical}; - } + #pragma db column("") + package_version_id data; + std::string upstream; + std::uint16_t revision; + }; + + _id_type + _id () const; void - id (const package_version_id&, odb::database&); + _id (_id_type&&, odb::database&); #pragma db member(version) transient #pragma db member(package) transient - #pragma db member(id) virtual(package_version_id) before id \ - get(id) set(id ((?), (!))) column("") - #pragma db member(upstream) virtual(std::string) after(id) \ - get(version.upstream) set(version.upstream=(?)) - #pragma db member(revision) virtual(std::uint16_t) after(upstream) \ - get(version.revision) set(version.revision = (?)) + #pragma db member(repository) transient + + #pragma db member(id) virtual(_id_type) before id(data) \ + get(_id) set(_id (std::move (?), (!))) column("") // license // @@ -247,11 +397,11 @@ namespace brep #pragma db member(_license_key::second) column("index") #pragma db member(license_alternatives) id_column("") value_column("") - #pragma db member(licenses) \ - virtual(_licenses_type) \ - after(license_alternatives) \ - get(_get (this.license_alternatives)) \ - set(_set (this.license_alternatives, (?))) \ + #pragma db member(licenses) \ + virtual(_licenses_type) \ + after(license_alternatives) \ + get(_get (this.license_alternatives)) \ + set(_set (this.license_alternatives, (?))) \ id_column("") key_column("") value_column("license") // dependencies @@ -265,11 +415,11 @@ namespace brep #pragma db member(_dependency_key::second) column("index") #pragma db member(dependencies) id_column("") value_column("") - #pragma db member(dependency_alternatives) \ - virtual(_dependency_alternatives_type) \ - after(dependencies) \ - get(_get (this.dependencies)) \ - set(_set (this.dependencies, (?))) \ + #pragma db member(dependency_alternatives) \ + virtual(_dependency_alternatives_type) \ + after(dependencies) \ + get(_get (this.dependencies)) \ + set(_set (this.dependencies, (?))) \ id_column("") key_column("") value_column("dep_") // requirements @@ -283,12 +433,40 @@ namespace brep #pragma db member(_requirement_key::second) column("index") #pragma db member(requirements) id_column("") value_column("") - #pragma db member(requirement_alternatives) \ - virtual(_requirement_alternatives_type) \ - after(requirements) \ - get(_get (this.requirements)) \ - set(_set (this.requirements, (?))) \ + #pragma db member(requirement_alternatives) \ + virtual(_requirement_alternatives_type) \ + after(requirements) \ + get(_get (this.requirements)) \ + set(_set (this.requirements, (?))) \ id_column("") key_column("") value_column("id") + + private: + friend class odb::access; + package_version () = default; + }; + + #pragma db view object(package_version) \ + query((?) + "ORDER BY" + package_version::id.data.epoch + "DESC," + \ + package_version::id.data.canonical_upstream + "DESC," + \ + package_version::id.revision + "DESC LIMIT 1") + struct max_package_version + { + using version_type = brep::version; + + version_type version; + + void + _id (package_version::_id_type&&); + + // Database mapping. + // + #pragma db member(version) transient + + // Can't specify column expression using aggregate max function for a + // data member of a composite value type. + // + #pragma db member(id) virtual(package_version::_id_type) \ + get() set(_id (std::move (?))) }; } @@ -306,7 +484,7 @@ namespace brep #include #include #include // size_t -#include // pair, declval() +#include // declval() #include #include // remove_reference @@ -355,4 +533,4 @@ namespace odb } } -#endif // BPKG_PACKAGE +#endif // BREP_PACKAGE diff --git a/brep/package.cxx b/brep/package.cxx index 11cd974..585a0b5 100644 --- a/brep/package.cxx +++ b/brep/package.cxx @@ -4,19 +4,129 @@ #include +#include // move() +#include + #include -#include +#include + +using namespace std; +using namespace odb::core; namespace brep { + // Utility functions + // + static inline bool + alpha (char c) + { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + + static inline bool + digit (char c) + { + return c >= '0' && c <= '9'; + } + + // package + // + package:: + package (string n, + string s, + strings t, + optional d, + url_type u, + optional pu, + email_type e, + optional pe) + : name (move (n)), + summary (move (s)), + tags (move (t)), + description (move (d)), + url (move (u)), + package_url (move (pu)), + email (move (e)), + package_email (move (pe)) + { + } + // package_version // + package_version:: + package_version (lazy_shared_ptr rp, + lazy_shared_ptr pk, + version_type vr, + priority_type pr, + license_alternatives_type la, + string ch, + dependencies_type dp, + requirements_type rq) + : repository (move (rp)), + package (move (pk)), + version (move (vr)), + priority (move (pr)), + license_alternatives (move (la)), + changes (move (ch)), + dependencies (move (dp)), + requirements (move (rq)) + { + } + + package_version::_id_type package_version:: + _id () const + { + return _id_type { + { + repository.object_id (), + package.object_id (), + version.epoch (), + version.canonical_upstream () + }, + version.upstream (), + version.revision ()}; + } + void package_version:: - id (const package_version_id& v, odb::database& db) + _id (_id_type&& v, database& db) + { + repository = lazy_shared_ptr (db, v.data.repository); + package = lazy_shared_ptr (db, v.data.package); + version = version_type (v.data.epoch, move (v.upstream), v.revision); + assert (version.canonical_upstream () == v.data.canonical_upstream); + } + + // max_package_version + // + void max_package_version:: + _id (package_version::_id_type&& v) + { + version = version_type (v.data.epoch, move (v.upstream), v.revision); + assert (version.canonical_upstream () == v.data.canonical_upstream); + } + + // repository + // + repository:: + repository (repository_location l, string d, dir_path p) + : location (move (l)), + display_name (move (d)), + local_path (move (p)), + internal (true) + { + } + + repository::_id_type repository:: + _id () const + { + return _id_type {location.canonical_name (), location.string ()}; + } + + void repository:: + _id (_id_type&& l) { - version.epoch = v.epoch; - version.canonical = v.canonical; - package = db.load (v.package); + location = repository_location (move (l.location)); + assert (location.canonical_name () == l.canonical_name); } } diff --git a/brep/search.cxx b/brep/search.cxx index 16e9f23..fc86284 100644 --- a/brep/search.cxx +++ b/brep/search.cxx @@ -14,10 +14,12 @@ #include +#include + #include #include -#include +#include using namespace std; using namespace odb::core; @@ -33,11 +35,11 @@ namespace brep cli::unknown_mode::fail, cli::unknown_mode::fail); - db_ = make_shared("", - "", - "brep", - options_->db_host (), - options_->db_port ()); + db_ = make_shared ("", + "", + "brep", + options_->db_host (), + options_->db_port ()); if (options_->results_on_page () > 30) fail << "too many search results on page: " @@ -52,13 +54,21 @@ namespace brep { MODULE_DIAG; - std::shared_ptr cli (make_shared ()); - - cli->name = "cli"; - cli->summary = "CLI is ..."; - cli->description = "This is CLI"; - cli->tags.push_back ("compiler"); - cli->tags.push_back ("C++"); + shared_ptr cli ( + make_shared ("cli", + "CLI is ...", + strings ({"compiler", "c++"}), + string ("This is CLI"), + url (), + url (), + email (), + email ())); + + shared_ptr stable ( + make_shared ( + repository_location ("http://pkg.cpp.org/1/stable"), + "Stable", + dir_path ("/var/pkg/1/stable"))); licenses l; l.comment = "License\"A'"; @@ -67,13 +77,6 @@ namespace brep l.push_back ("BBB"); l.push_back ("CCC"); - std::shared_ptr v (make_shared ()); - - v->version = version ("1.1"); - v->package = cli; - - v->license_alternatives.push_back (l); - dependency_alternatives da; da.push_back ( {"icl", version_comparison{version ("1.3.3"), comparison::gt}}); @@ -81,30 +84,29 @@ namespace brep da.push_back ( {"ocl", version_comparison{version ("1.5.5"), comparison::lt}}); - v->dependencies.push_back (da); - - { - requirement_alternatives ra; - ra.push_back ("TAO"); - ra.push_back ("ORBacus"); - - v->requirements.push_back (ra); - } + requirement_alternatives ra1; + ra1.push_back ("TAO"); + ra1.push_back ("ORBacus"); - { - requirement_alternatives ra; - ra.push_back ("Xerces"); - - v->requirements.push_back (ra); - } + requirement_alternatives ra2; + ra2.push_back ("Xerces"); - cli->versions.push_back (v); + shared_ptr v ( + make_shared (stable, + cli, + version ("1.1"), + priority (), + license_alternatives ({l}), + "some changes 1\nsome changes 2", + dependencies ({da}), + requirements ({ra1, ra2}))); transaction t (db_->begin ()); // t.tracer (odb::stderr_full_tracer); { db_->persist (cli); + db_->persist (stable); db_->persist (v); } diff --git a/brep/services.cxx b/brep/services.cxx new file mode 100644 index 0000000..86b27b7 --- /dev/null +++ b/brep/services.cxx @@ -0,0 +1,21 @@ +// file : services.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include +#include + +#include + +using namespace brep; +using web::apache::service; + +static search search_mod; +service AP_MODULE_DECLARE_DATA search_srv ("search", + search_mod, + {"db-host", "db-port", "conf"}); + +static view view_mod; +service AP_MODULE_DECLARE_DATA view_srv ("view", + view_mod, + {"db-host", "db-port", "conf"}); diff --git a/brep/view.cxx b/brep/view.cxx index 222558a..08e2592 100644 --- a/brep/view.cxx +++ b/brep/view.cxx @@ -16,7 +16,7 @@ #include #include -#include +#include using namespace std; using namespace odb::core; @@ -31,11 +31,11 @@ namespace brep cli::unknown_mode::fail, cli::unknown_mode::fail); - db_ = make_shared("", - "", - "brep", - options_->db_host (), - options_->db_port ()); + db_ = make_shared ("", + "", + "brep", + options_->db_host (), + options_->db_port ()); } void view:: @@ -46,9 +46,11 @@ namespace brep shared_ptr p (db_->load ("cli")); - for (auto& vp : p->versions) + for (auto& vp: p->versions) { - s.cache_insert (*db_, vp.object_id (), vp.load ()); + shared_ptr v (vp.load ()); + v->repository.load (); + v->package.load (); } t.commit (); @@ -71,61 +73,61 @@ namespace brep o << "

\n" << p->name << ": " << p->versions.size (); - for (const auto& vp : p->versions) + for (const auto& vp: p->versions) { - shared_ptr v ( - s.cache_find (*db_, vp.object_id ())); + // Just finds package_version object in session cache. + // + shared_ptr v (vp.load ()); - if (!v) - { - o << "
no version in cache !"; - } - else + assert (v != nullptr); + assert (v->repository.get_eager () != nullptr); + assert (v->package.get_eager () != nullptr); + + o << "
version:" << v->version.string () + << "
package:" << v->package->name + << "
repo:" << v->repository->display_name + << "
changes:" << v->changes + << "
licenses:" << v->license_alternatives.size (); + + for (const auto& la: v->license_alternatives) { - o << "
version:" << v->version.string() - << "
licenses:" << v->license_alternatives.size (); + o << "
"; - for (const auto& la : v->license_alternatives) + for (const auto& l: la) { - o << "
"; - - for (const auto& l : la) - { - o << " |" << l << "|"; - } + o << " |" << l << "|"; } + } + + o << "
deps:" << v->dependencies.size (); - o << "
deps:" << v->dependencies.size (); + for (const auto& da: v->dependencies) + { + o << "
"; - for (const auto& da : v->dependencies) + for (const auto& d: da) { - o << "
"; + o << " |" << d.package; - for (const auto& d : da) + if (d.version) { - o << " |" << d.package; - - if (d.version) - { - o << "," << d.version->value.string () << "," - << static_cast (d.version->operation) << "|"; - } + o << "," << d.version->value.string () << "," + << static_cast (d.version->operation) << "|"; } } + } - o << "
requirements:" << v->requirements.size (); + o << "
requirements:" << v->requirements.size (); - for (const auto& ra : v->requirements) - { - o << "
"; + for (const auto& ra: v->requirements) + { + o << "
"; - for (const auto& r : ra) - { - o << " |" << r << "|"; - } + for (const auto& r: ra) + { + o << " |" << r << "|"; } } - } o << "

Search" -- cgit v1.1