diff options
39 files changed, 1871 insertions, 203 deletions
diff --git a/brep/.gitignore b/brep/.gitignore new file mode 100644 index 0000000..2c23e23 --- /dev/null +++ b/brep/.gitignore @@ -0,0 +1,5 @@ +options +options.?xx +package-odb* +package.sql + diff --git a/brep/buildfile b/brep/buildfile new file mode 100644 index 0000000..13e1d20 --- /dev/null +++ b/brep/buildfile @@ -0,0 +1,24 @@ +# file : brep/buildfile +# copyright : Copyright (c) 2014-2015 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +using cli + +.: libso{brep brep-apache} + +import libs += libbpkg%lib{bpkg} +import libs += libodb-pgsql%lib{odb-pgsql} +import libs += libodb%lib{odb} + +brep = cxx{package package-odb} +libso{brep}: $brep $libs +libso{brep}: cxx.export.poptions = -I$out_root -I$src_root + +brep = cxx{diagnostics module services search view} cli.cxx{options} +web = ../web/apache/cxx{request service} +libso{brep-apache}: $brep $web libso{brep} $libs + +cli.options += -I $src_root --include-with-brackets --include-prefix brep \ +--guard-prefix BREP --generate-file-scanner --suppress-usage + +cli.cxx{options}: cli{options} diff --git a/brep/diagnostics.cxx b/brep/diagnostics.cxx index e04c214..0278792 100644 --- a/brep/diagnostics.cxx +++ b/brep/diagnostics.cxx @@ -22,7 +22,7 @@ namespace brep // once C++17 uncaught_exceptions() becomes available. // if (!data_.empty () && - (!std::uncaught_exception () /*|| exception_unwinding_dtor*/)) + (!uncaught_exception () /*|| exception_unwinding_dtor*/)) { data_.back ().msg = os_.str (); // Save last message. diff --git a/brep/module.cxx b/brep/module.cxx index 32988e0..ceadc23 100644 --- a/brep/module.cxx +++ b/brep/module.cxx @@ -4,8 +4,8 @@ #include <brep/module> -#include <httpd/httpd.h> -#include <httpd/http_log.h> +#include <httpd.h> +#include <http_log.h> #include <vector> #include <string> @@ -89,7 +89,7 @@ namespace brep argv.push_back (nv.value.c_str ()); } - int argc (argv.size()); + int argc (argv.size ()); try { @@ -140,7 +140,7 @@ namespace brep // using A = B (*)(int,int); // A func(B (*)(char),B (*)(wchar_t)); // __PRETTY_FUNCTION__ looks like this: -// virtual std::string (* (* brep::search::func(std::string (* (*)(char))(int)\ +// virtual std::string (* (* brep::search::func(std::string (* (*)(char))(int) // ,std::string (* (*)(wchar_t))(int)) const)(int, int))(int) // string module:: diff --git a/brep/options.cli b/brep/options.cli index 606898c..b7c571e 100644 --- a/brep/options.cli +++ b/brep/options.cli @@ -15,7 +15,7 @@ namespace brep class db_options { std::string db-host = "localhost"; - std::uint16_t db-port = 3306; + std::uint16_t db-port = 5432; }; class search_options: module_options, db_options diff --git a/brep/package b/brep/package index 62ad8d6..c0c9fc9 100644 --- a/brep/package +++ b/brep/package @@ -2,30 +2,60 @@ // copyright : Copyright (c) 2014-2015 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file -#ifndef BPKG_PACKAGE -#define BPKG_PACKAGE +#ifndef BREP_PACKAGE +#define BREP_PACKAGE #include <map> #include <string> #include <vector> +#include <chrono> #include <memory> // shared_ptr -#include <utility> // pair +#include <cstddef> // size_t +#include <utility> // move() #include <cstdint> // uint16 #include <odb/core.hxx> #include <odb/forward.hxx> // database #include <odb/lazy-ptr.hxx> -#include <odb/nullable.hxx> + +#include <butl/path> +#include <butl/path-io> +#include <butl/optional> +#include <butl/timestamp> + +namespace brep +{ + // Use an image type to map bpkg::version to the database since there + // is no way to modify individual components directly. + // + #pragma db value + struct _version + { + std::uint16_t epoch; + std::string upstream; + std::uint16_t revision; + std::string canonical_upstream; + }; +} #include <bpkg/manifest> +// We have to keep this mapping at the global scope instead of inside +// the brep namespace because it needs to be also effective in the +// bpkg namespace from which we "borrow" types (and some of them use +// version). +// +#pragma db map type(bpkg::version) as(brep::_version) \ + to(brep::_version{(?).epoch (), \ + (?).upstream (), \ + (?).revision (), \ + (?).canonical_upstream ()}) \ + from(bpkg::version ((?).epoch, std::move ((?).upstream), (?).revision)) + namespace brep { // @@ If namespace, then should probably call it 'repo'. // - // @@ Should probably use optional from libbutl instead of odb::nullable - // for consistency. Create butl profile? - // // @@ Might make sense to put some heavy members (e.g., description, // containers) into a separate section. // @@ -36,38 +66,63 @@ namespace brep // Forward declarations. // - struct package; - struct package_version; + class repository; + class package; + class package_version; using strings = std::vector<std::string>; - using version = bpkg::version; - #pragma db value(version) definition + template <typename T> + using optional = butl::optional<T>; + + using path = butl::path; + + #pragma db map type(path) as(std::string) \ + to((?).string ()) from(brep::path (?)) + + using dir_path = butl::dir_path; + + #pragma db map type(dir_path) as(std::string) \ + to((?).string ()) from(brep::dir_path (?)) + + using timestamp = butl::timestamp; - using version_type = brep::version; + #pragma db map type(timestamp) as(std::uint64_t) \ + to(std::chrono::system_clock::to_time_t (?)) \ + from(std::chrono::system_clock::from_time_t (?)) + + using version = bpkg::version; + using repository_location = bpkg::repository_location; #pragma db value struct package_version_id { + std::string repository; std::string package; std::uint16_t epoch; - std::string canonical; + std::string canonical_upstream; // Database mapping. // - #pragma db member(package) points_to(package) + #pragma db member(repository) points_to(repository) //on_delete(cascade) + #pragma db member(package) points_to(package) //on_delete(cascade) }; inline bool operator< (const package_version_id& x, const package_version_id& y) { - int r (x.package.compare (y.package)); + int r (x.repository.compare (y.repository)); + + if (r != 0) + return r < 0; + + r = x.package.compare (y.package); if (r != 0) return r < 0; return x.epoch < y.epoch || - x.epoch == y.epoch && x.canonical < y.canonical; + (x.epoch == y.epoch && x.canonical_upstream < y.canonical_upstream); } using priority = bpkg::priority; @@ -76,7 +131,7 @@ namespace brep using url = bpkg::url; #pragma db value(url) definition - #pragma db member(url::value) virtual(std::string) before access(this) \ + #pragma db member(url::value) virtual(std::string) before access(this) \ column("") using email = bpkg::email; @@ -153,31 +208,110 @@ namespace brep std::size_t first; std::size_t second; - index_pair (std::size_t f = 0, std::size_t s = 0): first (f), second (s) {} + index_pair () = default; + index_pair (std::size_t f, std::size_t s): first (f), second (s) {} bool operator< (const index_pair& v) const { - return first < v.first || first == v.first && second < v.second; + return first < v.first || (first == v.first && second < v.second); } }; #pragma db object pointer(std::shared_ptr) session - struct package + class repository { - // Manifest data. + public: + using path_type = brep::path; + using timestamp_type = brep::timestamp; + using package_versions_type = + std::vector<odb::lazy_weak_ptr<package_version>>; + using prerequisite_repositories_type = + std::vector<odb::lazy_weak_ptr<repository>>; + + // Create internal repository. + // + repository (repository_location, + std::string display_name, + dir_path local_path); + + // Create external repository. + // + explicit + repository (repository_location l) + : location (std::move (l)), internal (false) {} + + repository_location location; + std::string display_name; + + // Non empty for internal repositories and external ones with a filesystem + // path location. + // + dir_path local_path; + + // Initialized with timestamp_nonexistent by default. + // + timestamp_type timestamp; + + bool internal; + package_versions_type package_versions; + prerequisite_repositories_type prerequisite_repositories; + + // Database mapping. // + #pragma db value + struct _id_type + { + std::string canonical_name; + std::string location; + }; + + _id_type + _id () const; + + void + _id (_id_type&&); + + #pragma db member(location) transient + + #pragma db member(id) virtual(_id_type) before id(canonical_name) \ + get(_id) set(_id (std::move (?))) column("") + + #pragma db member(package_versions) inverse(id.data.repository) + #pragma db member(prerequisite_repositories) id_column("repository") \ + value_column("prerequisite_repository") value_not_null + + private: + friend class odb::access; + repository () = default; + }; + + #pragma db object pointer(std::shared_ptr) session + class package + { + public: using url_type = brep::url; using email_type = brep::email; + package (std::string name, + std::string summary, + strings tags, + optional<std::string> description, + url_type, + optional<url_type> package_url, + email_type, + optional<email_type> package_email); + + // Manifest data. + // std::string name; std::string summary; strings tags; - std::string description; + optional<std::string> description; url_type url; - odb::nullable<url_type> package_url; + optional<url_type> package_url; email_type email; - odb::nullable<email_type> package_email; + optional<email_type> package_email; std::vector<odb::lazy_weak_ptr<package_version>> versions; // Additional data. @@ -187,55 +321,71 @@ namespace brep // #pragma db member(name) id #pragma db member(tags) id_column("package") value_column("tag") - #pragma db member(versions) inverse(id.package) + #pragma db member(versions) inverse(id.data.package) + + private: + friend class odb::access; + package () = default; }; #pragma db object pointer(std::shared_ptr) session - struct package_version + class package_version { - // Manifest data. - // + public: + using repository_type = brep::repository; using package_type = brep::package; + using version_type = brep::version; using priority_type = brep::priority; using license_alternatives_type = brep::license_alternatives; using dependencies_type = brep::dependencies; using requirements_type = brep::requirements; + package_version (odb::lazy_shared_ptr<repository_type>, + odb::lazy_shared_ptr<package_type>, + version_type, + priority_type, + license_alternatives_type, + std::string changes, + dependencies_type, + requirements_type); + + // Manifest data. + // + odb::lazy_shared_ptr<repository_type> repository; + odb::lazy_shared_ptr<package_type> package; version_type version; - std::shared_ptr<package_type> package; priority_type priority; license_alternatives_type license_alternatives; std::string changes; dependencies_type dependencies; requirements_type requirements; - // Additional data. - // - std::string repository; // E.g., "stable", "testing". - // Database mapping. // // id // - package_version_id - id () const + #pragma db value + struct _id_type { - return package_version_id - {package->name, version.epoch, version.canonical}; - } + #pragma db column("") + package_version_id data; + std::string upstream; + std::uint16_t revision; + }; + + _id_type + _id () const; void - id (const package_version_id&, odb::database&); + _id (_id_type&&, odb::database&); #pragma db member(version) transient #pragma db member(package) transient - #pragma db member(id) virtual(package_version_id) before id \ - get(id) set(id ((?), (!))) column("") - #pragma db member(upstream) virtual(std::string) after(id) \ - get(version.upstream) set(version.upstream=(?)) - #pragma db member(revision) virtual(std::uint16_t) after(upstream) \ - get(version.revision) set(version.revision = (?)) + #pragma db member(repository) transient + + #pragma db member(id) virtual(_id_type) before id(data) \ + get(_id) set(_id (std::move (?), (!))) column("") // license // @@ -247,11 +397,11 @@ namespace brep #pragma db member(_license_key::second) column("index") #pragma db member(license_alternatives) id_column("") value_column("") - #pragma db member(licenses) \ - virtual(_licenses_type) \ - after(license_alternatives) \ - get(_get (this.license_alternatives)) \ - set(_set (this.license_alternatives, (?))) \ + #pragma db member(licenses) \ + virtual(_licenses_type) \ + after(license_alternatives) \ + get(_get (this.license_alternatives)) \ + set(_set (this.license_alternatives, (?))) \ id_column("") key_column("") value_column("license") // dependencies @@ -265,11 +415,11 @@ namespace brep #pragma db member(_dependency_key::second) column("index") #pragma db member(dependencies) id_column("") value_column("") - #pragma db member(dependency_alternatives) \ - virtual(_dependency_alternatives_type) \ - after(dependencies) \ - get(_get (this.dependencies)) \ - set(_set (this.dependencies, (?))) \ + #pragma db member(dependency_alternatives) \ + virtual(_dependency_alternatives_type) \ + after(dependencies) \ + get(_get (this.dependencies)) \ + set(_set (this.dependencies, (?))) \ id_column("") key_column("") value_column("dep_") // requirements @@ -283,12 +433,40 @@ namespace brep #pragma db member(_requirement_key::second) column("index") #pragma db member(requirements) id_column("") value_column("") - #pragma db member(requirement_alternatives) \ - virtual(_requirement_alternatives_type) \ - after(requirements) \ - get(_get (this.requirements)) \ - set(_set (this.requirements, (?))) \ + #pragma db member(requirement_alternatives) \ + virtual(_requirement_alternatives_type) \ + after(requirements) \ + get(_get (this.requirements)) \ + set(_set (this.requirements, (?))) \ id_column("") key_column("") value_column("id") + + private: + friend class odb::access; + package_version () = default; + }; + + #pragma db view object(package_version) \ + query((?) + "ORDER BY" + package_version::id.data.epoch + "DESC," + \ + package_version::id.data.canonical_upstream + "DESC," + \ + package_version::id.revision + "DESC LIMIT 1") + struct max_package_version + { + using version_type = brep::version; + + version_type version; + + void + _id (package_version::_id_type&&); + + // Database mapping. + // + #pragma db member(version) transient + + // Can't specify column expression using aggregate max function for a + // data member of a composite value type. + // + #pragma db member(id) virtual(package_version::_id_type) \ + get() set(_id (std::move (?))) }; } @@ -306,7 +484,7 @@ namespace brep #include <map> #include <vector> #include <cstddef> // size_t -#include <utility> // pair, declval() +#include <utility> // declval() #include <cassert> #include <type_traits> // remove_reference @@ -355,4 +533,4 @@ namespace odb } } -#endif // BPKG_PACKAGE +#endif // BREP_PACKAGE diff --git a/brep/package.cxx b/brep/package.cxx index 11cd974..585a0b5 100644 --- a/brep/package.cxx +++ b/brep/package.cxx @@ -4,19 +4,129 @@ #include <brep/package> +#include <utility> // move() +#include <cassert> + #include <odb/database.hxx> -#include <brep/package-odb.hxx> +#include <brep/package-odb> + +using namespace std; +using namespace odb::core; namespace brep { + // Utility functions + // + static inline bool + alpha (char c) + { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + + static inline bool + digit (char c) + { + return c >= '0' && c <= '9'; + } + + // package + // + package:: + package (string n, + string s, + strings t, + optional<string> d, + url_type u, + optional<url_type> pu, + email_type e, + optional<email_type> pe) + : name (move (n)), + summary (move (s)), + tags (move (t)), + description (move (d)), + url (move (u)), + package_url (move (pu)), + email (move (e)), + package_email (move (pe)) + { + } + // package_version // + package_version:: + package_version (lazy_shared_ptr<repository_type> rp, + lazy_shared_ptr<package_type> pk, + version_type vr, + priority_type pr, + license_alternatives_type la, + string ch, + dependencies_type dp, + requirements_type rq) + : repository (move (rp)), + package (move (pk)), + version (move (vr)), + priority (move (pr)), + license_alternatives (move (la)), + changes (move (ch)), + dependencies (move (dp)), + requirements (move (rq)) + { + } + + package_version::_id_type package_version:: + _id () const + { + return _id_type { + { + repository.object_id (), + package.object_id (), + version.epoch (), + version.canonical_upstream () + }, + version.upstream (), + version.revision ()}; + } + void package_version:: - id (const package_version_id& v, odb::database& db) + _id (_id_type&& v, database& db) + { + repository = lazy_shared_ptr<repository_type> (db, v.data.repository); + package = lazy_shared_ptr<package_type> (db, v.data.package); + version = version_type (v.data.epoch, move (v.upstream), v.revision); + assert (version.canonical_upstream () == v.data.canonical_upstream); + } + + // max_package_version + // + void max_package_version:: + _id (package_version::_id_type&& v) + { + version = version_type (v.data.epoch, move (v.upstream), v.revision); + assert (version.canonical_upstream () == v.data.canonical_upstream); + } + + // repository + // + repository:: + repository (repository_location l, string d, dir_path p) + : location (move (l)), + display_name (move (d)), + local_path (move (p)), + internal (true) + { + } + + repository::_id_type repository:: + _id () const + { + return _id_type {location.canonical_name (), location.string ()}; + } + + void repository:: + _id (_id_type&& l) { - version.epoch = v.epoch; - version.canonical = v.canonical; - package = db.load<package_type> (v.package); + location = repository_location (move (l.location)); + assert (location.canonical_name () == l.canonical_name); } } diff --git a/brep/search.cxx b/brep/search.cxx index 16e9f23..fc86284 100644 --- a/brep/search.cxx +++ b/brep/search.cxx @@ -14,10 +14,12 @@ #include <odb/pgsql/database.hxx> +#include <butl/path> + #include <web/module> #include <brep/package> -#include <brep/package-odb.hxx> +#include <brep/package-odb> using namespace std; using namespace odb::core; @@ -33,11 +35,11 @@ namespace brep cli::unknown_mode::fail, cli::unknown_mode::fail); - db_ = make_shared<odb::pgsql::database>("", - "", - "brep", - options_->db_host (), - options_->db_port ()); + db_ = make_shared<odb::pgsql::database> ("", + "", + "brep", + options_->db_host (), + options_->db_port ()); if (options_->results_on_page () > 30) fail << "too many search results on page: " @@ -52,13 +54,21 @@ namespace brep { MODULE_DIAG; - std::shared_ptr<package> cli (make_shared<package> ()); - - cli->name = "cli"; - cli->summary = "CLI is ..."; - cli->description = "This is CLI"; - cli->tags.push_back ("compiler"); - cli->tags.push_back ("C++"); + shared_ptr<package> cli ( + make_shared<package> ("cli", + "CLI is ...", + strings ({"compiler", "c++"}), + string ("This is CLI"), + url (), + url (), + email (), + email ())); + + shared_ptr<repository> stable ( + make_shared<repository> ( + repository_location ("http://pkg.cpp.org/1/stable"), + "Stable", + dir_path ("/var/pkg/1/stable"))); licenses l; l.comment = "License\"A'"; @@ -67,13 +77,6 @@ namespace brep l.push_back ("BBB"); l.push_back ("CCC"); - std::shared_ptr<package_version> v (make_shared<package_version> ()); - - v->version = version ("1.1"); - v->package = cli; - - v->license_alternatives.push_back (l); - dependency_alternatives da; da.push_back ( {"icl", version_comparison{version ("1.3.3"), comparison::gt}}); @@ -81,30 +84,29 @@ namespace brep da.push_back ( {"ocl", version_comparison{version ("1.5.5"), comparison::lt}}); - v->dependencies.push_back (da); - - { - requirement_alternatives ra; - ra.push_back ("TAO"); - ra.push_back ("ORBacus"); - - v->requirements.push_back (ra); - } + requirement_alternatives ra1; + ra1.push_back ("TAO"); + ra1.push_back ("ORBacus"); - { - requirement_alternatives ra; - ra.push_back ("Xerces"); - - v->requirements.push_back (ra); - } + requirement_alternatives ra2; + ra2.push_back ("Xerces"); - cli->versions.push_back (v); + shared_ptr<package_version> v ( + make_shared<package_version> (stable, + cli, + version ("1.1"), + priority (), + license_alternatives ({l}), + "some changes 1\nsome changes 2", + dependencies ({da}), + requirements ({ra1, ra2}))); transaction t (db_->begin ()); // t.tracer (odb::stderr_full_tracer); { db_->persist (cli); + db_->persist (stable); db_->persist (v); } diff --git a/services.cxx b/brep/services.cxx index 86b27b7..86b27b7 100644 --- a/services.cxx +++ b/brep/services.cxx diff --git a/brep/view.cxx b/brep/view.cxx index 222558a..08e2592 100644 --- a/brep/view.cxx +++ b/brep/view.cxx @@ -16,7 +16,7 @@ #include <web/module> #include <brep/package> -#include <brep/package-odb.hxx> +#include <brep/package-odb> using namespace std; using namespace odb::core; @@ -31,11 +31,11 @@ namespace brep cli::unknown_mode::fail, cli::unknown_mode::fail); - db_ = make_shared<odb::pgsql::database>("", - "", - "brep", - options_->db_host (), - options_->db_port ()); + db_ = make_shared<odb::pgsql::database> ("", + "", + "brep", + options_->db_host (), + options_->db_port ()); } void view:: @@ -46,9 +46,11 @@ namespace brep shared_ptr<package> p (db_->load<package> ("cli")); - for (auto& vp : p->versions) + for (auto& vp: p->versions) { - s.cache_insert<package_version> (*db_, vp.object_id (), vp.load ()); + shared_ptr<package_version> v (vp.load ()); + v->repository.load (); + v->package.load (); } t.commit (); @@ -71,61 +73,61 @@ namespace brep o << "<p>\n" << p->name << ": " << p->versions.size (); - for (const auto& vp : p->versions) + for (const auto& vp: p->versions) { - shared_ptr<package_version> v ( - s.cache_find<package_version> (*db_, vp.object_id ())); + // Just finds package_version object in session cache. + // + shared_ptr<package_version> v (vp.load ()); - if (!v) - { - o << "<br>no version in cache !"; - } - else + assert (v != nullptr); + assert (v->repository.get_eager () != nullptr); + assert (v->package.get_eager () != nullptr); + + o << "<br>version:" << v->version.string () + << "<br>package:" << v->package->name + << "<br>repo:" << v->repository->display_name + << "<br>changes:" << v->changes + << "<br>licenses:" << v->license_alternatives.size (); + + for (const auto& la: v->license_alternatives) { - o << "<br>version:" << v->version.string() - << "<br>licenses:" << v->license_alternatives.size (); + o << "<br>"; - for (const auto& la : v->license_alternatives) + for (const auto& l: la) { - o << "<br>"; - - for (const auto& l : la) - { - o << " |" << l << "|"; - } + o << " |" << l << "|"; } + } + + o << "<br>deps:" << v->dependencies.size (); - o << "<br>deps:" << v->dependencies.size (); + for (const auto& da: v->dependencies) + { + o << "<br>"; - for (const auto& da : v->dependencies) + for (const auto& d: da) { - o << "<br>"; + o << " |" << d.package; - for (const auto& d : da) + if (d.version) { - o << " |" << d.package; - - if (d.version) - { - o << "," << d.version->value.string () << "," - << static_cast<int> (d.version->operation) << "|"; - } + o << "," << d.version->value.string () << "," + << static_cast<int> (d.version->operation) << "|"; } } + } - o << "<br>requirements:" << v->requirements.size (); + o << "<br>requirements:" << v->requirements.size (); - for (const auto& ra : v->requirements) - { - o << "<br>"; + for (const auto& ra: v->requirements) + { + o << "<br>"; - for (const auto& r : ra) - { - o << " |" << r << "|"; - } + for (const auto& r: ra) + { + o << " |" << r << "|"; } } - } o << "<p><a href='search?a=1&b&c=2&d=&&x=a+b'>Search</a>" @@ -1,14 +1,62 @@ DEBUG="-g -ggdb -fno-inline" -cd ./brep; cli --generate-file-scanner --suppress-usage --hxx-suffix "" \ - --option-prefix "" ./options.cli; cd .. +cd ./brep -cd ./brep; odb -d pgsql --std c++11 --generate-query --generate-schema \ +echo "odb package" + +odb -d pgsql --std c++11 --generate-query --generate-schema \ --odb-epilogue '#include <brep/wrapper-traits>' \ --hxx-prologue '#include <brep/wrapper-traits>' \ -I .. -I ../../libbpkg -I ../../libbutl \ - package; cd .. + --hxx-suffix "" --include-with-brackets \ + --include-prefix brep --guard-prefix BREP \ + package +e=$? +if test $e -ne 0; then exit $e; fi + +echo "g++ libbrep.so" + +s="package.cxx package-odb.cxx" + +g++ -shared $DEBUG -std=c++11 -I.. -I../../libbpkg \ + -I../../libbutl -L../../libbpkg/bpkg -L../../libbutl/butl \ + -fPIC -o libbrep.so $s -lbpkg -lbutl -lodb-pgsql -lodb + +echo "cli brep-apache options" + +cli --include-with-brackets --include-prefix brep --hxx-suffix "" \ + --guard-prefix BREP --generate-file-scanner --suppress-usage ./options.cli + +echo "g++ libbrep-apache.so" + +s="search.cxx view.cxx module.cxx diagnostics.cxx services.cxx options.cxx \ +../web/apache/request.cxx ../web/apache/service.cxx" + +g++ -shared $DEBUG -std=c++11 -I. -I/usr/include/apr-1 -I/usr/include/httpd \ + -I.. -I../../libbpkg -I../../libbutl -L. -L../../libbpkg/bpkg \ + -fPIC -o libbrep-apache.so $s -lbrep -lbpkg -lodb-pgsql -lodb + +cd ../loader + +echo "cli loader options" + +cli --hxx-suffix "" ./options.cli + +echo "g++ brep-loader" + +s="loader.cxx options.cxx" + +g++ $DEBUG -std=c++11 -I. -I.. -I../../libbpkg \ + -I../../libbutl -L../brep -L../../libbpkg/bpkg -L../../libbutl/butl \ + -o brep-loader $s -lbrep -lbpkg -lbutl -lodb-pgsql -lodb + +cd ../tests/loader + +echo "g++ tests/loader" + +s="driver.cxx" -g++ -shared $DEBUG -std=c++11 -I. -I/usr/include/apr-1 -I ../libbpkg \ - -I ../libbutl -L ../libbpkg/bpkg \ - -fPIC -o libbrep.so `find . -name '*.cxx'` -lbpkg -lodb-pgsql -lodb +g++ $DEBUG -std=c++11 -I. -I../.. -I../../../libbpkg \ + -I../../../libbutl -L../../brep -L../../../libbpkg/bpkg \ + -L../../../libbutl/butl \ + -o driver $s -lbrep -lbpkg -lbutl -lodb-pgsql -lodb diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 0000000..225c27f --- /dev/null +++ b/build/.gitignore @@ -0,0 +1 @@ +config.build diff --git a/build/bootstrap.build b/build/bootstrap.build index 350c8cc..fda9151 100644 --- a/build/bootstrap.build +++ b/build/bootstrap.build @@ -1,2 +1,3 @@ -project_name = brep +project = brep using config +using test diff --git a/build/root.build b/build/root.build index e14129a..66ca55f 100644 --- a/build/root.build +++ b/build/root.build @@ -1,3 +1,16 @@ using cxx +hxx.ext = +ixx.ext = ixx +txx.ext = txx +cxx.ext = cxx + cxx.std = 11 +cxx.poptions += -I$out_root -I$src_root + +# All exe{} in tests/ are, well, tests. +# +tests/: +{ + test.exe = true +} @@ -1,8 +1,7 @@ -brep=brep/{diagnostics module options package package-odb search view} -web=web/apache/{request service} +# file : buildfile +# copyright : Copyright (c) 2014-2015 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file -import libs += libbpkg - -libso{brep}: cxx{$brep $web services} $libs -cxx.poptions += -I$src_root -cxx.libs += -lodb-pgsql -lodb +d = brep/ loader/ tests/ +.: $d +include $d diff --git a/etc/apachectl b/etc/apachectl index 43e39ee..e282d9f 100755 --- a/etc/apachectl +++ b/etc/apachectl @@ -88,6 +88,11 @@ case $ARGV in ERROR=$? if test $ERROR -eq 0; then + $LYNX $STATUSURL 1>/dev/null 2>&1 + ERROR=$? + fi + + if test $ERROR -eq 0; then echo "server started" else echo "server starting failed" @@ -3,6 +3,7 @@ SCRIPT_DIR=`dirname $0` CONFIG_DIR=`cd $SCRIPT_DIR; pwd` PROJECT_DIR="$CONFIG_DIR/.." WORKSPACE_DIR="$PROJECT_DIR/var" +LIB_DIRS="$PROJECT_DIR/brep:$PROJECT_DIR/../libbutl/butl:$PROJECT_DIR/../libbpkg/bpkg" # PostgreSQL settings (used in pgctl) PG_PORT=8432 @@ -18,9 +19,16 @@ AP_ADMIN_EMAIL=admin@cppget.org AP_LOG_LEVEL=trace1 AP_DB_HOST="$PG_WORKSPACE_DIR" AP_DB_PORT=$PG_PORT -AP_LIB_DIRS="$PROJECT_DIR/../libbutl/butl:$PROJECT_DIR/../libbpkg/bpkg" -AP_MODULE_DIR="$PROJECT_DIR" +AP_LIB_DIRS="$LIB_DIRS" +AP_MODULE_DIR="$PROJECT_DIR/brep" AP_WWW_DIR="$PROJECT_DIR/www" AP_CONFIG_DIR="$CONFIG_DIR" AP_LOG_DIR="$WORKSPACE_DIR/log/httpd" AP_WORKSPACE_DIR="$WORKSPACE_DIR/run/httpd" + +# brep-loader settings (used in loader) +LD_DB_HOST="$PG_WORKSPACE_DIR" +LD_DB_PORT=$PG_PORT +LD_REPOSITORIES="$CONFIG_DIR/repositories.conf" +LD_LIB_DIRS="$LIB_DIRS" +LD_EXE_DIRS="$PROJECT_DIR/loader" diff --git a/etc/httpd.conf b/etc/httpd.conf index bf64517..71ac903 100644 --- a/etc/httpd.conf +++ b/etc/httpd.conf @@ -40,7 +40,7 @@ LoadModule authz_host_module /usr/lib64/httpd/modules/mod_authz_host.so LoadModule expires_module /usr/lib64/httpd/modules/mod_expires.so LoadModule dir_module /usr/lib64/httpd/modules/mod_dir.so -LoadModule search_srv ${AP_MODULE_DIR}/libbrep.so +LoadModule search_srv ${AP_MODULE_DIR}/libbrep-apache.so <IfModule search_srv> search-db-host ${AP_DB_HOST} @@ -48,7 +48,7 @@ LoadModule search_srv ${AP_MODULE_DIR}/libbrep.so search-conf "${AP_CONFIG_DIR}/search.conf" </IfModule> -LoadModule view_srv ${AP_MODULE_DIR}/libbrep.so +LoadModule view_srv ${AP_MODULE_DIR}/libbrep-apache.so <IfModule view_srv> view-db-host ${AP_DB_HOST} diff --git a/etc/loader b/etc/loader new file mode 100755 index 0000000..1ae3b48 --- /dev/null +++ b/etc/loader @@ -0,0 +1,18 @@ +#!/bin/sh +# file : etc/loader +# copyright : Copyright (c) 2014-2015 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file +# +# Designed to simplify running brep-loader utility. + +. `dirname $0`/config + +if test -n "$LD_LIB_DIRS"; then + export LD_LIBRARY_PATH=$LD_LIB_DIRS:$LD_LIBRARY_PATH +fi + +if test -n "$LD_EXE_DIRS"; then + export PATH=$LD_EXE_DIRS:$PATH +fi + +brep-loader --db-host "$LD_DB_HOST" --db-port $PG_PORT "$LD_REPOSITORIES" diff --git a/loader/.gitignore b/loader/.gitignore new file mode 100644 index 0000000..820b183 --- /dev/null +++ b/loader/.gitignore @@ -0,0 +1,3 @@ +options +options.?xx +brep-loader diff --git a/loader/buildfile b/loader/buildfile new file mode 100644 index 0000000..d4e4678 --- /dev/null +++ b/loader/buildfile @@ -0,0 +1,20 @@ +# file : loader/buildfile +# copyright : Copyright (c) 2014-2015 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +using cli + +import libs += libbpkg%lib{bpkg} +import libs += libbutl%lib{butl} +import libs += libodb-pgsql%lib{odb-pgsql} +import libs += libodb%lib{odb} + +include ../brep/ + +loader = cxx{loader} cli.cxx{options} +exe{brep-loader}: $loader ../brep/libso{brep} $libs + +cli.options += -I $src_root --include-with-brackets --include-prefix loader \ +--guard-prefix LOADER + +cli.cxx{options}: cli{options} diff --git a/loader/loader.cxx b/loader/loader.cxx new file mode 100644 index 0000000..f13349d --- /dev/null +++ b/loader/loader.cxx @@ -0,0 +1,577 @@ +// file : loader/loader.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <vector> +#include <memory> // shared_ptr, make_shared() +#include <string> +#include <utility> // move() +#include <cstdint> // uint64_t +#include <sstream> +#include <fstream> +#include <iostream> +#include <stdexcept> // runtime_error, invalid_argument + +#include <odb/session.hxx> +#include <odb/database.hxx> +#include <odb/transaction.hxx> + +#include <odb/pgsql/database.hxx> +#include <odb/pgsql/exceptions.hxx> +#include <odb/pgsql/connection.hxx> +#include <odb/pgsql/transaction.hxx> + +#include <butl/timestamp> // timestamp_nonexistent +#include <butl/filesystem> + +#include <bpkg/manifest-parser> // manifest_parsing + +#include <brep/package> +#include <brep/package-odb> + +#include <loader/options> + +using namespace std; +using namespace odb::core; +using namespace butl; +using namespace bpkg; +using namespace brep; + +namespace pgsql = odb::pgsql; + +static void +usage () +{ + cout << "Usage: brep-loader [options] <file>" << endl + << "File lists internal repositories." << endl + << "Options:" << endl; + + options::print_usage (cout); +} + +static inline bool +space (char c) noexcept +{ + return c == ' ' || c == '\t'; +} + +struct internal_repository +{ + repository_location location; + string display_name; + dir_path local_path; + + path + packages_path () const {return local_path / path ("packages");} +}; + +using internal_repositories = vector<internal_repository>; + +static internal_repositories +load_repositories (path p) +{ + internal_repositories repos; + + if (p.relative ()) + p.complete (); + + ifstream ifs (p.string ()); + if (!ifs.is_open ()) + throw ifstream::failure (p.string () + ": unable to open"); + + ifs.exceptions (ifstream::badbit); + + try + { + string s; + for (uint64_t l (1); getline (ifs, s); ++l) + { + auto b (s.cbegin ()); + auto i (b); + auto e (s.cend ()); + + // Skip until first non-space (true) or space (false). + // + auto skip ([&i, &e](bool s = true) -> decltype (i) { + for (; i != e && space (*i) == s; ++i); return i;}); + + skip (); // Skip leading spaces. + + if (i == e || *i == '#') // Empty line or comment. + continue; + + // From now on pb will track the begining of the next part + // while i -- the end. + // + auto pb (i); // Location begin. + skip (false); // Find end of location. + + auto bad_line ([&p, l, &pb, &b](const string& d) { + ostringstream os; + os << p << ':' << l << ':' << pb - b + 1 << ": error: " << d; + throw runtime_error (os.str ()); + }); + + repository_location location; + + try + { + location = repository_location (string (pb, i)); + } + catch (const invalid_argument& e) + { + bad_line (e.what ()); + } + + if (location.local ()) + bad_line ("local repository location"); + + for (const auto& r: repos) + if (r.location.canonical_name () == location.canonical_name ()) + bad_line ("duplicate canonical name"); + + pb = skip (); // Find begin of display name. + + if (pb == e) + bad_line ("no display name found"); + + skip (false); // Find end of display name. + + string name (pb, i); + pb = skip (); // Find begin of filesystem path. + + if (pb == e) // For now filesystem path is mandatory. + bad_line ("no filesystem path found"); + + skip (false); // Find end of filesystem path (no spaces allowed). + + internal_repository r { + move (location), + move (name), + dir_path (string (pb, i))}; + + // If the internal repository local path is relative, then + // calculate its absolute local path. Such path is considered to be + // relative to configuration file directory path so result is + // independent from whichever directory is current for the loader + // process. + // + if (r.local_path.relative ()) + { + r.local_path = p.directory () / r.local_path; + } + + try + { + r.local_path.normalize (); + } + catch (const invalid_path&) + { + bad_line ("can't normalize local path"); + } + + if (!file_exists (r.packages_path ())) + bad_line ("'packages' file does not exist"); + + repos.emplace_back (move (r)); + + // Check that there is no non-whitespace junk at the end. + // + if (skip () != e) + bad_line ("junk after filesystem path"); + } + } + catch (const ifstream::failure&) + { + throw ifstream::failure (p.string () + ": io failure"); + } + + return repos; +} + +// Check if repositories persistent state is outdated. If any repository +// differes from its persistent state or there is a persistent repository +// which is not listed in configuration file then the whole persistent +// state will be recreated. Will consider optimization later when the +// package model, including search related objects, settles down. +// +static bool +changed (const internal_repositories& repos, database& db) +{ + strings names; + for (auto& r: repos) + { + shared_ptr<repository> pr ( + db.find<repository> (r.location.canonical_name ())); + + if (pr == nullptr || r.location.string () != pr->location.string () || + r.display_name != pr->display_name || r.local_path != pr->local_path || + file_mtime (r.packages_path ()) != pr->timestamp || !pr->internal) + return true; + + names.emplace_back (r.location.canonical_name ()); + } + + using query = query<repository>; + + // Check if there is an internal repository not being listed in the + // configuration file. + // + auto rs ( + db.query<repository> (query::internal && + !query::id.canonical_name.in_range (names.begin (), + names.end ()))); + + return !rs.empty (); +} + +// Load the repository state (including of its prerequsite repositories) +// from the 'packages' file. +// +static void +load_repository (const shared_ptr<repository>& rp, database& db) +{ + if (rp->timestamp != timestamp_nonexistent) + return; // The repository is already loaded. + + // Only locally accessible repositories allowed until package manager API is + // ready. + // + assert (!rp->local_path.empty ()); + + path p (rp->local_path / path ("packages")); + + ifstream ifs (p.string ()); + if (!ifs.is_open ()) + throw ifstream::failure (p.string () + ": unable to open"); + ifs.exceptions (ifstream::badbit | ifstream::failbit); + + // Mark as loaded. This is important in case we try to load this + // repository again recursively. + // + rp->timestamp = file_mtime (p); + + manifest_parser mp (ifs, p.string ()); + manifests ms (mp); + + // Close to avoid unpredictable number of files being simultaneously + // opened due to load_repository() recursive calls. + // + ifs.close (); + + // Don't add prerequisite repositories for external repositories. + // + if (rp->internal) + { + for (auto& rm: ms.repositories) + { + if (rm.location.empty ()) + continue; // Ignore entry for this repository. + + repository_location rl; + + auto bad_location ( + [&rp, &rm]() + { + ostringstream o; + o << "invalid location '" << rm.location.string () + << "' of the prerequisite repository for internal " + "repository '" << rp->location.string () << "'"; + + throw runtime_error (o.str ()); + }); + + try + { + // Absolute path location make no sense for the web interface. + // + if (rm.location.absolute ()) + bad_location (); + + // Convert the relative repository location to remote one, leave remote + // location unchanged. + // + rl = repository_location (rm.location.string (), rp->location); + } + catch (const invalid_argument&) + { + bad_location (); + } + + shared_ptr<repository> pr (db.find<repository> (rl.canonical_name ())); + + if (pr == nullptr) + { + pr = make_shared<repository> (move (rl)); + + // If the prerequsite repository location is a relative path, then + // calculate its absolute local path. + // + if (rm.location.relative ()) + { + dir_path& lp (pr->local_path); + lp = rp->local_path / rm.location.path (); + + try + { + lp.normalize (); + } + catch (const invalid_path&) + { + ostringstream o; + o << "can't normalize local path'" << lp.string () + << "' of the prerequisite repository for internal " + "repository '" << rp->location.string () << "'"; + + throw runtime_error (o.str ()); + } + } + + db.persist (pr); + } + + load_repository (pr, db); + + rp->prerequisite_repositories.emplace_back (pr); + } + } + + // Temporary reset ODB session for the current thread while persisting + // package and package_version objects to decrease memory consumption. + // + session& s (session::current ()); + session::reset_current (); + + for (auto& pm: ms.packages) + { + max_package_version mv; + + // If there are no package_version objects persisted yet for this + // package, then query_one() will leave mv unchanged in which case + // the version member remains empty. The empty version value is + // less than any non-empty one so the condition below evaluates + // to true and the package object gets persisted. + // + db.query_one<max_package_version> ( + query<max_package_version>::id.data.package == pm.name, mv); + + if (mv.version < pm.version) + { + // Create the package object. + // + brep::optional<string> desc; // Ambiguity with butl::optional. + + // Don't add description for external repository packages. + // + if (rp->internal && pm.description) + { + if (pm.description->file) + { + // @@ Pull description from the file when package manager API + // is ready. + } + else + desc = move (*pm.description); + } + + package p (pm.name, + move (pm.summary), + move (pm.tags), + move (desc), + move (pm.url), + move (pm.package_url), + move (pm.email), + move (pm.package_email)); + + if (mv.version.empty ()) + db.persist (p); + else + db.update (p); + } + + // Create package version object. + // + dependencies dep; + requirements req; + string chn; + + // Don't add dependencies, requirements and changes for external + // repository packages. + // + if (rp->internal) + { + dep = move (pm.dependencies); + req = move (pm.requirements); + + for (auto& c: pm.changes) + { + if (c.file) + { + // @@ Pull change notes from the file when package manager + // API is ready. + } + else + { + if (chn.empty ()) + chn = move (c); + else + chn += "\n" + c; + } + } + } + + package_version pv (rp, + lazy_shared_ptr<package> (db, pm.name), + move (pm.version), + pm.priority ? move (*pm.priority) : priority (), + move (pm.license_alternatives), + move (chn), + move (dep), + move (req)); + + db.persist (pv); + } + + session::current (s); // Restore current session. + + db.update (rp); // Save the repository state. +} + +int +main (int argc, char* argv[]) +{ + try + { + cli::argv_scanner scan (argc, argv, true); + options ops (scan); + + // Version. + // + if (ops.version ()) + { + cout << "brep-loader 0.0.0" << endl + << "Copyright (c) 2014-2015 Code Synthesis Ltd" << endl + << "MIT; see accompanying LICENSE file" << endl; + + return 0; + } + + // Help. + // + if (ops.help ()) + { + usage (); + return 0; + } + + if (argc < 2) + { + cout << "<file> argument not provided" << endl; + usage (); + return 1; + } + + if (argc > 2) + { + cout << "unexpected argument encountered" << endl; + usage (); + return 1; + } + + pgsql::database db ("", "", "brep", ops.db_host (), ops.db_port ()); + + // Prevent several loader instances from updating DB simultaneously. + // + { + transaction t (db.begin ()); + db.execute ("CREATE TABLE IF NOT EXISTS loader_mutex ()"); + t.commit (); + } + + pgsql::connection_ptr synch_c (db.connection ()); + + // Don't make current. + // + pgsql::transaction synch_t (synch_c->begin (), false); + + try + { + synch_c->execute ("LOCK TABLE loader_mutex NOWAIT"); + } + catch (const pgsql::database_exception& e) + { + if (e.sqlstate () == "55P03") + return 2; // Other loader instance acquired the mutex. + + throw; + } + + // Load the description of all the internal repositories from + // the configuration file. + // + internal_repositories irs (load_repositories (path (argv[1]))); + + transaction t (db.begin ()); + + if (changed (irs, db)) + { + // Rebuild repositories persistent state from scratch. + // + db.erase_query<repository> (); + db.erase_query<package> (); + db.erase_query<package_version> (); + + // We use repository object timestamp as a flag to signal that + // we have already loaded this repo. The easiest way to make + // it work in case of cycles is to use a session. This way, + // the repository object on which we updated the timestamp + // will be the same as the one we may check down the call + // stack. + // + session s; + + // On the first pass over the internal repositories list we + // persist empty repository objects, setting the interal flag + // to true and timestamp to non-existent. The idea is to + // establish the "final" list of internal repositories. + // + for (auto& ir: irs) + { + shared_ptr<repository> r ( + make_shared<repository> (ir.location, + move (ir.display_name), + move (ir.local_path))); + + db.persist (r); + } + + // On the second pass over the internal repositories we + // load them and all their (not yet loaded) prerequisite + // repositories. + // + for (const auto& ir: irs) + { + shared_ptr<repository> r ( + db.load<repository> (ir.location.canonical_name ())); + + load_repository (r, db); + } + } + + t.commit (); + synch_t.commit (); // Release the mutex. + } + catch (const cli::exception& e) + { + cerr << e << endl; + usage (); + return 1; + } + // Fully qualified to avoid ambiguity with odb exception. + // + catch (const std::exception& e) + { + cerr << e.what () << endl; + return 1; + } +} diff --git a/loader/options.cli b/loader/options.cli new file mode 100644 index 0000000..9e96585 --- /dev/null +++ b/loader/options.cli @@ -0,0 +1,24 @@ +// file : loader/options.cli +// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +include <string>; +include <cstdint>; + +class options +{ + bool --help {"Print usage information and exit."}; + bool --version {"Print version and exit."}; + + std::string --db-host = "localhost" + { + "<host>", + "Connect database server using specified host or socket directory." + }; + + std::uint16_t --db-port = 5432 + { + "<port>", + "Connect database server using specified port." + }; +}; diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..e54525b --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +driver diff --git a/tests/buildfile b/tests/buildfile new file mode 100644 index 0000000..41a9bcc --- /dev/null +++ b/tests/buildfile @@ -0,0 +1,7 @@ +# file : tests/buildfile +# copyright : Copyright (c) 2014-2015 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +d = loader/ +.: $d +include $d diff --git a/tests/loader/buildfile b/tests/loader/buildfile new file mode 100644 index 0000000..1ce249f --- /dev/null +++ b/tests/loader/buildfile @@ -0,0 +1,21 @@ +# file : tests/loader/buildfile +# copyright : Copyright (c) 2014-2015 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +import libs += libbpkg%lib{bpkg} +import libs += libbutl%lib{butl} +import libs += libodb-pgsql%lib{odb-pgsql} +import libs += libodb%lib{odb} + +include ../../brep/ + +exe{driver}: cxx{driver} ../../brep/libso{brep} $libs + +# Disable for now until build2 test module supports running custom +# commands, pre/post steps. +# +exe{driver}: test = false + +# precondition: PostgreSQL server running port 8432 with brep schema created. +# test: +# ./driver ../../loader/brep-loader --db-host localhost --db-port 8432 ./r.conf diff --git a/tests/loader/driver.cxx b/tests/loader/driver.cxx new file mode 100644 index 0000000..1dd011f --- /dev/null +++ b/tests/loader/driver.cxx @@ -0,0 +1,487 @@ +// file : tests/loader/driver.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <vector> +#include <memory> // shared_ptr +#include <string> +#include <cassert> +#include <iostream> +#include <exception> +#include <algorithm> // sort(), find() + +#include <odb/session.hxx> +#include <odb/transaction.hxx> + +#include <odb/pgsql/database.hxx> + +#include <butl/process> +#include <butl/timestamp> // timestamp_nonexistent +#include <butl/filesystem> + +#include <brep/package> +#include <brep/package-odb> + +using namespace std; +using namespace odb::core; +using namespace butl; +using namespace brep; + +static inline bool +operator== (const dependency& a, const dependency& b) +{ + return a.package == b.package && !a.version == !b.version && + (!a.version || (a.version->value == b.version->value && + a.version->operation == b.version->operation)); +} + +int +main (int argc, char* argv[]) +{ + if (argc != 7) + { + cerr << "usage: " << argv[0] + << " <loader_path> --db-host <host> --db-port <port>" + << " <loader_conf_file>" << endl; + + return 1; + } + + try + { + path cp (argv[6]); + + // Make configuration file path absolute to use it's directory as base for + // internal repositories relative local paths. + // + if (cp.relative ()) + cp.complete (); + + // Update packages file timestamp to enforce loader to update + // persistent state. + // + path p (cp.directory () / path ("internal/1/stable/packages")); + char const* args[] = {"touch", p.string ().c_str (), nullptr}; + assert (process (args).wait ()); + + timestamp srt (file_mtime (p)); + + // Run the loader. + // + char const** ld_args (const_cast<char const**> (argv + 1)); + assert (process (ld_args).wait ()); + + // Check persistent objects validity. + // + odb::pgsql::database db ("", "", "brep", argv[3], stoul (argv[5])); + + { + session s; + transaction t (db.begin ()); + + assert (db.query<repository> ().size () == 3); + assert (db.query<package> ().size () == 4); + assert (db.query<package_version> ().size () == 7); + + shared_ptr<repository> sr (db.load<repository> ("cppget.org/stable")); + shared_ptr<repository> mr (db.load<repository> ("cppget.org/math")); + shared_ptr<repository> cr (db.load<repository> ("cppget.org/misc")); + + // Verify 'stable' repository. + // + assert (sr->location.canonical_name () == "cppget.org/stable"); + assert (sr->location.string () == + "http://pkg.cppget.org/internal/1/stable"); + assert (sr->display_name == "stable"); + + dir_path srp (cp.directory () / dir_path ("internal/1/stable")); + assert (sr->local_path == srp.normalize ()); + + assert (sr->timestamp == srt); + assert (sr->internal); + assert (sr->prerequisite_repositories.size () == 2); + + vector<shared_ptr<repository>> pr {mr, cr}; + + auto i (find (pr.begin (), + pr.end (), + sr->prerequisite_repositories[0].load ())); + + assert (i != pr.end ()); + pr.erase (i); + + assert (find (pr.begin (), + pr.end (), + sr->prerequisite_repositories[1].load ()) != pr.end ()); + + auto& srv (sr->package_versions); + assert (srv.size () == 4); + + using lv_t = decltype (srv[0]); + auto vc ([](const lv_t& a, const lv_t& b){ + auto v1 (a.load ()); + auto v2 (b.load ()); + + int r (v1->package.object_id ().compare (v2->package.object_id ())); + + if (r) + return r < 0; + + return v1->version < v2->version; + }); + + sort (srv.begin (), srv.end (), vc); + + version fv1 ("1.2.2"); + shared_ptr<package_version> fpv1 ( + db.load<package_version> ( + package_version_id { + "cppget.org/stable", + "libfoo", + fv1.epoch (), + fv1.canonical_upstream ()})); + assert (srv[0].load () == fpv1); + + version fv2 ("1.2.3-4"); + shared_ptr<package_version> fpv2 ( + db.load<package_version> ( + package_version_id { + "cppget.org/stable", + "libfoo", + fv2.epoch (), + fv2.canonical_upstream ()})); + assert (srv[1].load () == fpv2); + + version fv3 ("1.2.4"); + shared_ptr<package_version> fpv3 ( + db.load<package_version> ( + package_version_id { + "cppget.org/stable", + "libfoo", + fv3.epoch (), + fv3.canonical_upstream ()})); + assert (srv[2].load () == fpv3); + + version xv ("1.0.0-1"); + shared_ptr<package_version> xpv ( + db.load<package_version> ( + package_version_id { + "cppget.org/stable", + "libstudxml", + xv.epoch (), + xv.canonical_upstream ()})); + assert (srv[3].load () == xpv); + + // Verify libstudxml package. + // + shared_ptr<package> px (db.load<package> ("libstudxml")); + assert (px->name == "libstudxml"); + assert (px->summary == "Modern C++ XML API"); + assert (px->tags == strings ({"c++", "xml", "parser", "serializer", + "pull", "streaming", "modern"})); + assert (!px->description); + assert (px->url == "http://www.codesynthesis.com/projects/libstudxml/"); + assert (!px->package_url); + assert (px->email == email ("studxml-users@codesynthesis.com", + "Public mailing list, posts by non-members " + "are allowed but moderated.")); + assert (px->package_email && + *px->package_email == email ("boris@codesynthesis.com", + "Direct email to the author.")); + + auto& xpvs (px->versions); + assert (xpvs.size () == 1); + assert (xpvs[0].load () == xpv); + + // Verify libstudxml package version. + // + assert (xpv->repository.load () == sr); + assert (xpv->package.load () == px); + assert (xpv->version == version ("1.0.0-1")); + assert (xpv->priority == priority::low); + assert (xpv->changes.empty ()); + + assert (xpv->license_alternatives.size () == 1); + assert (xpv->license_alternatives[0].size () == 1); + assert (xpv->license_alternatives[0][0] == "MIT"); + + assert (xpv->dependencies.size () == 2); + assert (xpv->dependencies[0].size () == 1); + assert (xpv->dependencies[0][0] == + (dependency { + "libexpat", + brep::optional<version_comparison> ( + version_comparison{version ("2.0.0"), comparison::ge})})); + + assert (xpv->dependencies[1].size () == 1); + assert (xpv->dependencies[1][0] == + (dependency {"libgenx", brep::optional<version_comparison> ()})); + + assert (xpv->requirements.empty ()); + + // Verify libfoo package. + // + shared_ptr<package> pf (db.load<package> ("libfoo")); + assert (pf->name == "libfoo"); + assert (pf->summary == "The Foo Math Library"); + assert (pf->tags == strings ({"c++", "foo", "math"})); + assert (*pf->description == "Very good math library."); + assert (pf->url == "http://www.example.com/foo/"); + assert (!pf->package_url); + assert (pf->email == email ("foo-users@example.com")); + assert (!pf->package_email); + + auto& fpv (pf->versions); + assert (fpv.size () == 4); + sort (fpv.begin (), fpv.end (), vc); + assert (fpv[0].load () == fpv1); + assert (fpv[1].load () == fpv2); + assert (fpv[2].load () == fpv3); + // Asserting fpv[3].load () == fpv4 goes later when fpv4, belonging to + // another repository, get introduced. + // + + // Verify libfoo package versions. + // + assert (fpv1->repository.load () == sr); + assert (fpv1->package.load () == pf); + assert (fpv1->version == version ("1.2.2")); + assert (fpv1->priority == priority::low); + assert (fpv1->changes.empty ()); + + assert (fpv1->license_alternatives.size () == 1); + assert (fpv1->license_alternatives[0].size () == 1); + assert (fpv1->license_alternatives[0][0] == "MIT"); + + assert (fpv1->dependencies.size () == 2); + assert (fpv1->dependencies[0].size () == 1); + assert (fpv1->dependencies[1].size () == 1); + + assert (fpv1->dependencies[0][0] == + (dependency { + "libbar", + brep::optional<version_comparison> ( + version_comparison{version ("2.4.0"), comparison::le})})); + + assert (fpv1->dependencies[1][0] == + (dependency { + "libexp", + brep::optional<version_comparison> ( + version_comparison{version ("1+1.2"), comparison::eq})})); + + requirements& fpvr1 (fpv1->requirements); + assert (fpvr1.size () == 4); + + assert (fpvr1[0] == strings ({"linux", "windows", "macosx"})); + assert (!fpvr1[0].conditional); + assert (fpvr1[0].comment.empty ()); + + assert (fpvr1[1] == strings ({"c++11"})); + assert (!fpvr1[1].conditional); + assert (fpvr1[1].comment.empty ()); + + assert (fpvr1[2].empty ()); + assert (fpvr1[2].conditional); + assert (fpvr1[2].comment == "VC++ 12.0 or later if targeting Windows."); + + assert (fpvr1[3].empty ()); + assert (fpvr1[3].conditional); + assert (fpvr1[3].comment == + "libc++ standard library if using Clang on Mac OS X."); + + assert (fpv2->repository.load () == sr); + assert (fpv2->package.load () == pf); + assert (fpv2->version == version ("1.2.3-4")); + assert (fpv2->priority == priority::low); + assert (fpv2->changes.empty ()); + + assert (fpv2->license_alternatives.size () == 1); + assert (fpv2->license_alternatives[0].size () == 1); + assert (fpv2->license_alternatives[0][0] == "MIT"); + + assert (fpv2->dependencies.size () == 1); + assert (fpv2->dependencies[0].size () == 1); + assert (fpv2->dependencies[0][0] == + (dependency { + "libmisc", + brep::optional<version_comparison> ( + version_comparison{version ("2.0.0"), comparison::ge})})); + + assert (fpv3->repository.load () == sr); + assert (fpv3->package.load () == pf); + assert (fpv3->version == version ("1.2.4")); + assert (fpv3->priority == priority::low); + assert (fpv3->changes == "some changes 1\nsome changes 2"); + + assert (fpv3->license_alternatives.size () == 1); + assert (fpv3->license_alternatives[0].size () == 1); + assert (fpv3->license_alternatives[0][0] == "MIT"); + + assert (fpv3->dependencies.size () == 1); + assert (fpv3->dependencies[0].size () == 1); + assert (fpv3->dependencies[0][0] == + (dependency { + "libmisc", + brep::optional<version_comparison> ( + version_comparison{version ("2.0.0"), comparison::ge})})); + + // Verify 'math' repository. + // + assert (mr->location.canonical_name () == "cppget.org/math"); + assert (mr->location.string () == + "http://pkg.cppget.org/internal/1/math"); + assert (mr->display_name == "math"); + + dir_path mrp (cp.directory () / dir_path ("internal/1/math")); + assert (mr->local_path == mrp.normalize ()); + + assert (mr->timestamp == + file_mtime (dir_path (mr->local_path) / path ("packages"))); + assert (mr->internal); + assert (mr->prerequisite_repositories.size () == 1); + assert (mr->prerequisite_repositories[0].load () == cr); + + auto& mrv (mr->package_versions); + assert (mrv.size () == 2); + sort (mrv.begin (), mrv.end (), vc); + + version ev ("1+1.2"); + shared_ptr<package_version> epv ( + db.load<package_version> ( + package_version_id { + "cppget.org/math", + "libexp", + ev.epoch (), + ev.canonical_upstream ()})); + assert (mrv[0].load () == epv); + + version fv4 ("1.2.4-1"); + shared_ptr<package_version> fpv4 ( + db.load<package_version> ( + package_version_id { + "cppget.org/math", + "libfoo", + fv4.epoch (), + fv4.canonical_upstream ()})); + assert (mrv[1].load () == fpv4); + assert (fpv[3].load () == fpv4); + + // Verify libexp package. + // + shared_ptr<package> pe (db.load<package> ("libexp")); + assert (pe->name == "libexp"); + assert (pe->summary == "The exponent"); + assert (pe->tags == strings ({"c++", "exponent"})); + assert (!pe->description); + assert (pe->url == "http://www.exp.com"); + assert (!pe->package_url); + assert (pe->email == email ("users@exp.com")); + assert (!pe->package_email); + + auto& epvs (pe->versions); + assert (epvs.size () == 1); + assert (epvs[0].load () == epv); + + // Verify libexp package version. + // + assert (epv->repository.load () == mr); + assert (epv->package.load () == pe); + assert (epv->version == version ("1+1.2")); + assert (epv->priority == priority (priority::low)); + assert (epv->changes.empty ()); + + assert (epv->license_alternatives.size () == 1); + assert (epv->license_alternatives[0].size () == 1); + assert (epv->license_alternatives[0][0] == "MIT"); + + assert (epv->dependencies.size () == 1); + assert (epv->dependencies[0].size () == 1); + assert (epv->dependencies[0][0] == + (dependency {"libmisc", brep::optional<version_comparison> ()})); + + assert (epv->requirements.empty ()); + + // Verify 'misc' repository. + // + assert (cr->location.canonical_name () == "cppget.org/misc"); + assert (cr->location.string () == + "http://pkg.cppget.org/external/1/misc"); + assert (cr->display_name.empty ()); + + dir_path crp (cp.directory () / dir_path ("external/1/misc")); + assert (cr->local_path == crp.normalize ()); + + assert (cr->timestamp == + file_mtime (dir_path (cr->local_path) / path ("packages"))); + assert (!cr->internal); + assert (cr->prerequisite_repositories.empty ()); + + auto& crv (cr->package_versions); + assert (crv.size () == 1); + + version bv ("2.3.5"); + shared_ptr<package_version> bpv ( + db.load<package_version> ( + package_version_id { + "cppget.org/misc", + "libbar", + bv.epoch (), + bv.canonical_upstream ()})); + assert (crv[0].load () == bpv); + + // Verify libbar package. + // + shared_ptr<package> pb (db.load<package> ("libbar")); + assert (pb->name == "libbar"); + assert (pb->summary == "The Bar library"); + assert (pb->tags == strings ({"c++", "bar"})); + assert (!pb->description); + assert (pb->url == "http://www.example.com/bar/"); + assert (!pb->package_url); + assert (pb->email == email ("bar-users@example.com")); + assert (!pb->package_email); + + auto& bpvs (pb->versions); + assert (bpvs.size () == 1); + assert (bpvs[0].load () == bpv); + + // Verify libbar package version. + // + assert (bpv->repository.load () == cr); + assert (bpv->package.load () == pb); + assert (bpv->version == version ("2.3.5")); + + assert (bpv->priority == priority (priority::security, + "Very important to install.")); + assert (bpv->changes.empty ()); + + assert (bpv->license_alternatives.size () == 1); + assert (bpv->license_alternatives[0].size () == 1); + assert (bpv->license_alternatives[0][0] == "GPLv2"); + + assert (bpv->dependencies.empty ()); + assert (bpv->requirements.empty ()); + + // Update package summary, update package persistent state, rerun loader + // and ensure the model were not rebuilt. + // + pf->summary = "test"; + db.update (pf); + + t.commit (); + } + + assert (process (ld_args).wait ()); + + transaction t (db.begin ()); + assert (db.load<package> ("libfoo")->summary == "test"); + t.commit (); + } + // Fully qualified to avoid ambiguity with odb exception. + // + catch (const std::exception& e) + { + cerr << e.what () << endl; + return 1; + } +} diff --git a/tests/loader/external/1/misc/packages b/tests/loader/external/1/misc/packages new file mode 100644 index 0000000..d60d7fe --- /dev/null +++ b/tests/loader/external/1/misc/packages @@ -0,0 +1,25 @@ +: 1 +# Foreign repository manifest. +# +location: http://pkg.example.org/1/misc +: +# Foreign repository manifest. +# +location: http://pkg.example.org/1/math +: +# Local repository manifest (this repository). +# +: +name: libbar +version: 2.3.5 +priority: security; Very important to install. +summary: The Bar library +description: very very good library. +license: GPLv2 +tags: c++, bar +url: http://www.example.com/bar/ +email: bar-users@example.com +depends: libfoo +depends: libmath >= 2.0.0 +requires: linux | windows | macosx +changes: some changes diff --git a/tests/loader/internal/1/math/packages b/tests/loader/internal/1/math/packages new file mode 100644 index 0000000..2722f6e --- /dev/null +++ b/tests/loader/internal/1/math/packages @@ -0,0 +1,26 @@ +: 1 +# Foreign repository manifest. +# +location: ../../../external/1/misc +: +# Local repository manifest (this repository). +# +: +name: libexp +version: 1+1.2 +summary: The exponent +license: MIT +tags: c++, exponent +url: http://www.exp.com +email: users@exp.com +depends: libmisc +: +name: libfoo +version: 1.2.4-1 +summary: The Foo Math Library +description: Very good math library. +license: MIT +tags: c++, foo, math +url: http://www.example.com/foo/ +email: foo-users@example.com +depends: libmisc >= 2.3.0 diff --git a/tests/loader/internal/1/stable/packages b/tests/loader/internal/1/stable/packages new file mode 100644 index 0000000..a1aa839 --- /dev/null +++ b/tests/loader/internal/1/stable/packages @@ -0,0 +1,60 @@ +: 1 +# Foreign repository manifest. +# +location: http://pkg.cppget.org/1/math +: +# Foreign repository manifest. +# +location: ../../../external/1/misc +: +# Local repository manifest (this repository). +# +: +name: libfoo +version: 1.2.3-4 +summary: The Foo library +license: MIT +tags: c++, foo +url: http://www.example.com/foo/ +email: foo-users@example.com +depends: libmisc >= 2.0.0 +: +name: libstudxml +version: 1.0.0-1 +summary: Modern C++ XML API +license: MIT +tags: c++, xml, parser, serializer, pull, streaming, modern +description-file: README +changes-file: NEWS +url: http://www.codesynthesis.com/projects/libstudxml/ +email: studxml-users@codesynthesis.com; Public mailing list, posts by\ + non-members are allowed but moderated. +package-email: boris@codesynthesis.com; Direct email to the author. +depends: libexpat >= 2.0.0 +depends: libgenx +: +name: libfoo +version: 1.2.2 +summary: The Foo library +license: MIT +tags: c++, foo +url: http://www.example.com/foo/ +email: foo-users@example.com +depends: libbar <= 2.4.0 +depends: libexp == 1+1.2 +requires: linux | windows | macosx +requires: c++11 +requires: ? ; VC++ 12.0 or later if targeting Windows. +requires: ? ; libc++ standard library if using Clang on Mac OS X. +: +name: libfoo +version: 1.2.4 +summary: The Foo Library +description: Very good foo library. +license: MIT +tags: c++, foo +url: http://www.example.com/foo/ +email: foo-users@example.com +depends: libmisc >= 2.0.0 +changes: some changes 1 +changes: some changes 2 diff --git a/tests/loader/r.conf b/tests/loader/r.conf new file mode 100644 index 0000000..fea477a --- /dev/null +++ b/tests/loader/r.conf @@ -0,0 +1,2 @@ +http://pkg.cppget.org/internal/1/stable stable internal/1/stable +http://pkg.cppget.org/internal/1/math math internal/1/math diff --git a/web/apache/log b/web/apache/log index c976a65..c568ef5 100644 --- a/web/apache/log +++ b/web/apache/log @@ -5,8 +5,8 @@ #ifndef WEB_APACHE_LOG #define WEB_APACHE_LOG -#include <httpd/httpd.h> // request_rec -#include <httpd/http_log.h> +#include <httpd.h> // request_rec +#include <http_log.h> #include <cstdint> // uint64_t #include <algorithm> // min() diff --git a/web/apache/request b/web/apache/request index 75e9add..59d4600 100644 --- a/web/apache/request +++ b/web/apache/request @@ -7,9 +7,9 @@ #include <apr_strings.h> -#include <httpd/httpd.h> -#include <httpd/http_core.h> -#include <httpd/util_script.h> +#include <httpd.h> +#include <http_core.h> +#include <util_script.h> #include <ios> #include <chrono> @@ -18,10 +18,10 @@ #include <cassert> #include <istream> #include <ostream> +#include <utility> // move() #include <streambuf> #include <stdexcept> #include <exception> -#include <algorithm> // move #include <web/module> #include <web/apache/stream> diff --git a/web/apache/request.cxx b/web/apache/request.cxx index f89f0e7..7727b35 100644 --- a/web/apache/request.cxx +++ b/web/apache/request.cxx @@ -15,9 +15,9 @@ #include <sstream> #include <ostream> #include <cstring> +#include <utility> // move() #include <stdexcept> #include <streambuf> -#include <algorithm> // move() using namespace std; @@ -80,7 +80,7 @@ namespace web for (auto h (reinterpret_cast<const apr_table_entry_t *> (ha->elts)); n--; ++h) { - if (!::strcasecmp (h->key, "Cookie")) + if (::strcasecmp (h->key, "Cookie") == 0) { for (const char* n (h->val); n != 0; ) { @@ -121,8 +121,8 @@ namespace web content (status_code status, const string& type, bool buffer) { if (out_ && status == rec_->status && buffer == buffer_ && - !::strcasecmp (rec_->content_type ? rec_->content_type : "", - type.c_str ())) + ::strcasecmp (rec_->content_type ? rec_->content_type : "", + type.c_str ()) == 0) { return *out_; } diff --git a/web/apache/request.ixx b/web/apache/request.ixx index 9fa9e6d..0d3aefc 100644 --- a/web/apache/request.ixx +++ b/web/apache/request.ixx @@ -55,7 +55,8 @@ namespace web form_data_.reset (new std::string ()); const char* ct (apr_table_get (rec_->headers_in, "Content-Type")); - if (ct && !strncasecmp ("application/x-www-form-urlencoded", ct, 33)) + if (ct && + strncasecmp ("application/x-www-form-urlencoded", ct, 33) == 0) { std::istream& istr (content ()); std::getline (istr, *form_data_); diff --git a/web/apache/service b/web/apache/service index fd5f817..7ac01a2 100644 --- a/web/apache/service +++ b/web/apache/service @@ -5,12 +5,12 @@ #ifndef WEB_APACHE_SERVICE #define WEB_APACHE_SERVICE -#include <httpd/httpd.h> +#include <httpd.h> #include <string> #include <vector> #include <cassert> -#include <algorithm> // move() +#include <utility> // move() #include <web/module> #include <web/apache/log> @@ -45,7 +45,7 @@ namespace web exemplar_ (exemplar), option_names_ (std::move (opts)) { - init_directives(); + init_directives (); // instance<M> () is invented to delegate processing from apache // request handler C function to the service non static member @@ -89,7 +89,7 @@ namespace web worker_initializer (apr_pool_t*, server_rec* server) noexcept { log l (server); - instance<M> ()->init_worker(l); + instance<M> ()->init_worker (l); } template <typename M> @@ -101,21 +101,21 @@ namespace web request req (r); log l (r->server); - return srv->handle<M>(req, l); + return srv->handle<M> (req, l); } private: void - init_directives(); + init_directives (); void - init_worker(log& l) noexcept; + init_worker (log& l) noexcept; static const char* add_option (cmd_parms *parms, void *mconfig, const char *value) noexcept; template <typename M> - int handle(request& r, log& l) noexcept; + int handle (request& r, log& l) noexcept; private: std::string name_; diff --git a/web/apache/service.cxx b/web/apache/service.cxx index dc06a12..69bb874 100644 --- a/web/apache/service.cxx +++ b/web/apache/service.cxx @@ -7,8 +7,8 @@ #include <unistd.h> // getppid() #include <signal.h> // kill() -#include <httpd/httpd.h> -#include <httpd/http_config.h> +#include <httpd.h> +#include <http_config.h> #include <memory> // unique_ptr #include <string> @@ -21,9 +21,9 @@ namespace web namespace apache { void service:: - init_directives() + init_directives () { - assert(cmds == nullptr); + assert (cmds == nullptr); // Fill apache module directive definitions. Directives share // common name space in apache configuration file, so to prevent name @@ -52,13 +52,13 @@ namespace web }; } - *d = {}; + *d = {nullptr, nullptr, nullptr, 0, RAW_ARGS, nullptr}; cmds = directives.release (); } const char* service:: - add_option (cmd_parms *parms, void *mconfig, const char *value) noexcept + add_option (cmd_parms* parms, void*, const char* value) noexcept { service& srv (*reinterpret_cast<service*> (parms->cmd->cmd_data)); string name (parms->cmd->name + srv.name_.length () + 1); @@ -75,7 +75,7 @@ namespace web } void service:: - init_worker(log& l) noexcept + init_worker (log& l) noexcept { static const string func_name ( "web::apache::service<" + name_ + ">::init_worker"); diff --git a/web/apache/service.txx b/web/apache/service.txx index 38fa8ad..179980c 100644 --- a/web/apache/service.txx +++ b/web/apache/service.txx @@ -2,7 +2,7 @@ // copyright : Copyright (c) 2014-2015 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file -#include <httpd/http_log.h> +#include <http_log.h> #include <exception> @@ -12,7 +12,7 @@ namespace web { template <typename M> int service:: - handle(request& r, log& l) noexcept + handle (request& r, log& l) noexcept { static const std::string func_name ( "web::apache::service<" + name_ + ">::handle"); diff --git a/web/apache/stream b/web/apache/stream index 93d6855..5a8b4c7 100644 --- a/web/apache/stream +++ b/web/apache/stream @@ -5,8 +5,8 @@ #ifndef WEB_APACHE_STREAM #define WEB_APACHE_STREAM -#include <httpd/httpd.h> -#include <httpd/http_protocol.h> +#include <httpd.h> +#include <http_protocol.h> #include <ios> // streamsize #include <vector> |