diff options
Diffstat (limited to 'libbrep/package.hxx')
-rw-r--r-- | libbrep/package.hxx | 434 |
1 files changed, 363 insertions, 71 deletions
diff --git a/libbrep/package.hxx b/libbrep/package.hxx index 33444a9..45008d4 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -18,9 +18,9 @@ // Used by the data migration entries. // -#define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 19 +#define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 27 -#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 19, closed) +#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 33, closed) namespace brep { @@ -49,9 +49,12 @@ namespace brep using bpkg::text_type; using bpkg::to_text_type; + // Note that here we assume that the saved string representation of a type + // is always recognized later. + // #pragma db map type(text_type) as(string) \ to(to_string (?)) \ - from(brep::to_text_type (?)) + from(*brep::to_text_type (?)) using optional_text_type = optional<text_type>; @@ -69,26 +72,15 @@ namespace brep set(this = brep::manifest_url ((?), "" /* comment */)) \ column("") - // email - // - using bpkg::email; - - #pragma db value(email) definition - #pragma db member(email::value) virtual(string) before access(this) column("") - // licenses // using bpkg::licenses; - using license_alternatives = vector<licenses>; + using license_alternatives = small_vector<licenses, 1>; #pragma db value(licenses) definition // dependencies // - using bpkg::version_constraint; - - #pragma db value(version_constraint) definition - // Notes: // // 1. Will the package be always resolvable? What if it is in @@ -160,49 +152,73 @@ namespace brep operator!= (const dependency&, const dependency&); #pragma db value - class dependency_alternatives: public vector<dependency> + class dependency_alternative: public small_vector<dependency, 1> + { + public: + // While we currently don't use the reflect, prefer, accept, and require + // values, let's save them for completeness. + // + optional<string> enable; + optional<string> reflect; + optional<string> prefer; + optional<string> accept; + optional<string> require; + + dependency_alternative () = default; + dependency_alternative (optional<string> e, + optional<string> r, + optional<string> p, + optional<string> a, + optional<string> q) + : enable (move (e)), + reflect (move (r)), + prefer (move (p)), + accept (move (a)), + require (move (q)) {} + }; + + #pragma db value + class dependency_alternatives: public small_vector<dependency_alternative, 1> { public: - bool conditional; bool buildtime; string comment; dependency_alternatives () = default; - dependency_alternatives (bool d, bool b, string c) - : conditional (d), buildtime (b), comment (move (c)) {} + dependency_alternatives (bool b, string c) + : buildtime (b), comment (move (c)) {} }; using dependencies = vector<dependency_alternatives>; - // requirements - // - using bpkg::requirement_alternatives; - using requirements = vector<requirement_alternatives>; - - #pragma db value(requirement_alternatives) definition - // tests // - using bpkg::test_dependency_type; - using bpkg::to_test_dependency_type; - - #pragma db map type(test_dependency_type) as(string) \ - to(to_string (?)) \ - from(brep::to_test_dependency_type (?)) - #pragma db value struct test_dependency: dependency { test_dependency_type type; + bool buildtime; + optional<string> enable; + optional<string> reflect; test_dependency () = default; test_dependency (package_name n, test_dependency_type t, - optional<version_constraint> c) - : dependency {std::move (n), std::move (c), nullptr /* package */}, - type (t) + bool b, + optional<version_constraint> c, + optional<string> e, + optional<string> r) + : dependency {move (n), move (c), nullptr /* package */}, + type (t), + buildtime (b), + enable (move (e)), + reflect (move (r)) { } + + // Database mapping. + // + #pragma db member(buildtime) }; // certificate @@ -225,17 +241,82 @@ namespace brep // Create the tenant object with the timestamp set to now and the archived // flag set to false. // - explicit - tenant (string id); + tenant (string id, + bool private_, + optional<string> interactive, + optional<tenant_service>); string id; + // If this flag is true, then display the packages in the web interface + // only in the tenant view mode. + // + bool private_; // Note: foreign-mapped in build. + + // Interactive package build breakpoint. + // + // If present, then packages from this tenant will only be built + // interactively and only non-interactively otherwise. + // + optional<string> interactive; // Note: foreign-mapped in build. + timestamp creation_timestamp; - bool archived = false; // Note: foreign-mapped in build. + bool archived = false; // Note: foreign-mapped in build. + + optional<tenant_service> service; // Note: foreign-mapped in build. + + // Note that due to the implementation complexity and performance + // considerations, the service notifications are not synchronized. This + // leads to a potential race, so that before we have sent the `queued` + // notification for a package build, some other thread (potentially in a + // different process) could have already sent the `building` notification + // for it. It feels like there is no easy way to reliably fix that. + // Instead, we just decrease the probability of such a notifications + // sequence failure by delaying builds of the freshly queued packages for + // some time. Specifically, whenever the `queued` notification is ought + // to be sent (normally out of the database transaction, since it likely + // sends an HTTP request, etc) the tenant's queued_timestamp member is set + // to the current time. During the configured time interval since that + // time point the build tasks may not be issued for the tenant's packages. + // + // Also note that while there are similar potential races for other + // notification sequences, their probability is rather low due to the + // natural reasons (non-zero build task execution time, etc) and thus we + // just ignore them. + // + optional<timestamp> queued_timestamp; // Note: foreign-mapped in build. + + // Note that after the package tenant is created but before the first + // build object is created, there is no easy way to produce a list of + // unbuilt package configurations. That would require to know the build + // toolchain(s), which are normally extracted from the build objects. + // Thus, the empty unbuilt package configurations list is ambiguous and + // can either mean that no more package configurations can be built or + // that we have not enough information to produce the list. To + // disambiguate the empty list in the interface, in the latter case we + // want to display the question mark instead of 0 as an unbuilt package + // configurations count. To achieve this we will stash the build toolchain + // in the tenant when a package from this tenant is considered for a build + // for the first time but no configuration is picked for the build (the + // target configurations are excluded, an auxiliary machine is not + // available, etc). We will also use the stashed toolchain as a fallback + // until we are able to retrieve the toolchain(s) from the tenant builds + // to produce the unbuilt package configurations list. + // + // Note: foreign-mapped in build. + // + optional<brep::build_toolchain> build_toolchain; // Database mapping. // #pragma db member(id) id + #pragma db member(private_) + + #pragma db index("tenant_service_i") \ + unique \ + members(service.id, service.type) + + #pragma db index member(service.id) private: friend class odb::access; @@ -366,6 +447,67 @@ namespace brep string d; }; + #pragma db value + struct typed_text + { + string text; + text_type type; + + #pragma db member(text) column("") + }; + + // Tweak public_key_id mapping to include a constraint (this only affects the + // database schema). + // + #pragma db member(public_key_id::tenant) points_to(tenant) + + #pragma db object pointer(shared_ptr) session + class public_key: public string + { + public: + public_key (string tenant, string fingerprint, string key) + : string (move (key)), id (move (tenant), move (fingerprint)) {} + + public_key_id id; + + // Database mapping. + // + #pragma db member(id) id column("") + + #pragma db member(data) virtual(string) access(this) + + private: + friend class odb::access; + public_key () = default; + }; + + // package_build_config + // + using package_build_config = + build_package_config_template<lazy_shared_ptr<public_key>>; + + using package_build_configs = + build_package_configs_template<lazy_shared_ptr<public_key>>; + + #pragma db value(package_build_config) definition + + #pragma db member(package_build_config::builds) transient + #pragma db member(package_build_config::constraints) transient + #pragma db member(package_build_config::auxiliaries) transient + #pragma db member(package_build_config::bot_keys) transient + + // package_build_bot_keys + // + using package_build_bot_keys = vector<lazy_shared_ptr<public_key>>; + using package_build_bot_key_key = odb::nested_key<package_build_bot_keys>; + + using package_build_bot_keys_map = std::map<package_build_bot_key_key, + lazy_shared_ptr<public_key>>; + + #pragma db value(package_build_bot_key_key) + #pragma db member(package_build_bot_key_key::outer) column("config_index") + #pragma db member(package_build_bot_key_key::inner) column("index") + // Tweak package_id mapping to include a constraint (this only affects the // database schema). // @@ -384,9 +526,12 @@ namespace brep using dependencies_type = brep::dependencies; using requirements_type = brep::requirements; using build_constraints_type = brep::build_constraints; + using build_auxiliaries_type = brep::build_auxiliaries; // Create internal package object. // + // Note: the default build package config is expected to always be present. + // package (package_name, version_type, optional<string> upstream_version, @@ -396,9 +541,9 @@ namespace brep license_alternatives_type, small_vector<string, 5> topics, small_vector<string, 5> keywords, - optional<string> description, - optional<text_type> description_type, - string changes, + optional<typed_text> description, + optional<typed_text> package_description, + optional<typed_text> changes, optional<manifest_url> url, optional<manifest_url> doc_url, optional<manifest_url> src_url, @@ -413,6 +558,9 @@ namespace brep small_vector<test_dependency, 1> tests, build_class_exprs, build_constraints_type, + build_auxiliaries_type, + package_build_bot_keys, + package_build_configs, optional<path> location, optional<string> fragment, optional<string> sha256sum, @@ -427,14 +575,20 @@ namespace brep // // External package can also be a separate test for some primary package // (and belong to a complement but yet external repository), and so we may - // need its build class expressions and constraints to decide if to build - // it together with the primary package or not (see test-exclude task - // manifest value for details). + // need its build class expressions, constraints, and configurations to + // decide if to build it together with the primary package or not (see + // test-exclude task manifest value for details). Additionally, when the + // test package is being built the auxiliary machines may also be + // required. + // + // Note: the default build package config is expected to always be present. // package (package_name name, version_type, build_class_exprs, build_constraints_type, + build_auxiliaries_type, + package_build_configs, shared_ptr<repository_type>); bool @@ -460,31 +614,53 @@ namespace brep // Matches the package name if the project name is not specified in // the manifest. // - package_name project; + package_name project; // Note: foreign-mapped in build. priority_type priority; string summary; license_alternatives_type license_alternatives; small_vector<string, 5> topics; small_vector<string, 5> keywords; - optional<string> description; // Absent if type is unknown. - optional<text_type> description_type; // Present if description is present. - string changes; + + // Note that the descriptions and changes are absent if the respective + // type is unknown. + // + optional<typed_text> description; + optional<typed_text> package_description; + optional<typed_text> changes; + optional<manifest_url> url; optional<manifest_url> doc_url; optional<manifest_url> src_url; optional<manifest_url> package_url; optional<email_type> email; optional<email_type> package_email; - optional<email_type> build_email; - optional<email_type> build_warning_email; - optional<email_type> build_error_email; + optional<email_type> build_email; // Note: foreign-mapped in build. + optional<email_type> build_warning_email; // Note: foreign-mapped in build. + optional<email_type> build_error_email; // Note: foreign-mapped in build. dependencies_type dependencies; - requirements_type requirements; + requirements_type requirements; // Note: foreign-mapped in build. small_vector<test_dependency, 1> tests; // Note: foreign-mapped in build. + // Common build classes, constraints, auxiliaries, and bot keys that apply + // to all configurations unless overridden. + // build_class_exprs builds; // Note: foreign-mapped in build. build_constraints_type build_constraints; // Note: foreign-mapped in build. + build_auxiliaries_type build_auxiliaries; // Note: foreign-mapped in build. + package_build_bot_keys build_bot_keys; // Note: foreign-mapped in build. + package_build_configs build_configs; // Note: foreign-mapped in build. + + // Group the build_configs, builds, and build_constraints members of this + // object together with their respective nested configs entries into the + // separate section for an explicit load. + // + // Note that while the build auxiliaries and bot keys are persisted via + // the newly created package objects, they are only used via the + // foreign-mapped build_package objects (see build-package.hxx for + // details). Thus, we add them to the never-loaded unused_section (see + // below). + // odb::section build_section; // Note that it is foreign-mapped in build. @@ -515,6 +691,18 @@ namespace brep bool buildable; // Note: foreign-mapped in build. optional<brep::unbuildable_reason> unbuildable_reason; + // If this flag is true, then all the package configurations are buildable + // with the custom build bots. If false, then all configurations are + // buildable with the default bots. If nullopt, then some configurations + // are buildable with the custom and some with the default build bots. + // + // Note: meaningless if buildable is false. + // + optional<bool> custom_bot; // Note: foreign-mapped in build. + + private: + odb::section unused_section; + // Database mapping. // #pragma db member(id) id column("") @@ -549,38 +737,75 @@ namespace brep // dependencies // - using _dependency_key = odb::nested_key<dependency_alternatives>; + // Note that this is a 2-level nested container which is mapped to three + // container tables each containing data of each dimension. + + // Container of the dependency_alternatives values. + // + #pragma db member(dependencies) id_column("") value_column("") + + // Container of the dependency_alternative values. + // + using _dependency_alternative_key = + odb::nested_key<dependency_alternatives>; + using _dependency_alternatives_type = - std::map<_dependency_key, dependency>; + std::map<_dependency_alternative_key, dependency_alternative>; - #pragma db value(_dependency_key) - #pragma db member(_dependency_key::outer) column("dependency_index") - #pragma db member(_dependency_key::inner) column("index") + #pragma db value(_dependency_alternative_key) + #pragma db member(_dependency_alternative_key::outer) column("dependency_index") + #pragma db member(_dependency_alternative_key::inner) column("index") - #pragma db member(dependencies) id_column("") value_column("") #pragma db member(dependency_alternatives) \ virtual(_dependency_alternatives_type) \ after(dependencies) \ get(odb::nested_get (this.dependencies)) \ set(odb::nested_set (this.dependencies, std::move (?))) \ + id_column("") key_column("") value_column("") + + // Container of the dependency values. + // + using _dependency_key = odb::nested2_key<dependency_alternatives>; + using _dependency_alternative_dependencies_type = + std::map<_dependency_key, dependency>; + + #pragma db value(_dependency_key) + #pragma db member(_dependency_key::outer) column("dependency_index") + #pragma db member(_dependency_key::middle) column("alternative_index") + #pragma db member(_dependency_key::inner) column("index") + + #pragma db member(dependency_alternative_dependencies) \ + virtual(_dependency_alternative_dependencies_type) \ + after(dependency_alternatives) \ + get(odb::nested2_get (this.dependencies)) \ + set(odb::nested2_set (this.dependencies, std::move (?))) \ id_column("") key_column("") value_column("dep_") // requirements // - using _requirement_key = odb::nested_key<requirement_alternatives>; - using _requirement_alternatives_type = - std::map<_requirement_key, string>; - - #pragma db value(_requirement_key) - #pragma db member(_requirement_key::outer) column("requirement_index") - #pragma db member(_requirement_key::inner) column("index") + // Note that this is a 2-level nested container which is mapped to three + // container tables each containing data of each dimension. + // Container of the requirement_alternatives values. + // #pragma db member(requirements) id_column("") value_column("") + + // Container of the requirement_alternative values. + // #pragma db member(requirement_alternatives) \ - virtual(_requirement_alternatives_type) \ + virtual(requirement_alternatives_map) \ after(requirements) \ get(odb::nested_get (this.requirements)) \ set(odb::nested_set (this.requirements, std::move (?))) \ + id_column("") key_column("") value_column("") + + // Container of the requirement (string) values. + // + #pragma db member(requirement_alternative_requirements) \ + virtual(requirement_alternative_requirements_map) \ + after(requirement_alternatives) \ + get(odb::nested2_get (this.requirements)) \ + set(odb::nested2_set (this.requirements, std::move (?))) \ id_column("") key_column("") value_column("id") // tests @@ -597,7 +822,74 @@ namespace brep #pragma db member(build_constraints) id_column("") value_column("") \ section(build_section) - #pragma db member(build_section) load(lazy) update(always) + // build_auxiliaries + // + #pragma db member(build_auxiliaries) id_column("") value_column("") \ + section(unused_section) + + // build_bot_keys + // + #pragma db member(build_bot_keys) \ + id_column("") value_column("key_") value_not_null \ + section(unused_section) + + // build_configs + // + // Note that package_build_config::{builds,constraints,auxiliaries, + // bot_keys} are persisted/loaded via the separate nested containers (see + // commons.hxx for details). + // + #pragma db member(build_configs) id_column("") value_column("config_") \ + section(build_section) + + #pragma db member(build_config_builds) \ + virtual(build_class_exprs_map) \ + after(build_configs) \ + get(odb::nested_get ( \ + brep::build_package_config_builds (this.build_configs))) \ + set(brep::build_package_config_builds bs; \ + odb::nested_set (bs, std::move (?)); \ + move (bs).to_configs (this.build_configs)) \ + id_column("") key_column("") value_column("") \ + section(build_section) + + #pragma db member(build_config_constraints) \ + virtual(build_constraints_map) \ + after(build_config_builds) \ + get(odb::nested_get ( \ + brep::build_package_config_constraints (this.build_configs))) \ + set(brep::build_package_config_constraints cs; \ + odb::nested_set (cs, std::move (?)); \ + move (cs).to_configs (this.build_configs)) \ + id_column("") key_column("") value_column("") \ + section(build_section) + + #pragma db member(build_config_auxiliaries) \ + virtual(build_auxiliaries_map) \ + after(build_config_constraints) \ + get(odb::nested_get ( \ + brep::build_package_config_auxiliaries (this.build_configs))) \ + set(brep::build_package_config_auxiliaries as; \ + odb::nested_set (as, std::move (?)); \ + move (as).to_configs (this.build_configs)) \ + id_column("") key_column("") value_column("") \ + section(unused_section) + + #pragma db member(build_config_bot_keys) \ + virtual(package_build_bot_keys_map) \ + after(build_config_auxiliaries) \ + get(odb::nested_get ( \ + brep::build_package_config_bot_keys< \ + lazy_shared_ptr<brep::public_key>> (this.build_configs))) \ + set(brep::build_package_config_bot_keys< \ + lazy_shared_ptr<brep::public_key>> bks; \ + odb::nested_set (bks, std::move (?)); \ + move (bks).to_configs (this.build_configs)) \ + id_column("") key_column("") value_column("key_") value_not_null \ + section(unused_section) + + #pragma db member(build_section) load(lazy) update(always) + #pragma db member(unused_section) load(lazy) update(manual) // other_repositories // @@ -615,9 +907,9 @@ namespace brep friend class odb::access; package (): tenant (id.tenant), name (id.name) {} - // Save keywords, summary, description, and changes to weighted_text - // a, b, c, d members, respectively. So a word found in keywords will - // have a higher weight than if it's found in the summary. + // Save keywords, summary, descriptions, and changes to weighted_text a, + // b, c, d members, respectively. So a word found in keywords will have a + // higher weight than if it's found in the summary. // weighted_text search_text () const; |