From 699d0dfe6769ca949808bf78606a689aeff117df Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Tue, 28 Mar 2023 10:12:17 +0200 Subject: Add support for JSON structured result output in pkg-bindist --- bpkg/common.cli | 15 ++- bpkg/pkg-bindist.cli | 104 ++++++++++++++++++ bpkg/pkg-bindist.cxx | 185 ++++++++++++++++++++++++++------ bpkg/pkg-status.cli | 2 +- bpkg/system-package-manager-archive.cxx | 15 +-- bpkg/system-package-manager-archive.hxx | 2 +- bpkg/system-package-manager-debian.cxx | 30 +++--- bpkg/system-package-manager-debian.hxx | 2 +- bpkg/system-package-manager-fedora.cxx | 45 ++++---- bpkg/system-package-manager-fedora.hxx | 2 +- bpkg/system-package-manager.cxx | 21 ++-- bpkg/system-package-manager.hxx | 22 +++- 12 files changed, 353 insertions(+), 92 deletions(-) diff --git a/bpkg/common.cli b/bpkg/common.cli index 0d97631..c7d28bc 100644 --- a/bpkg/common.cli +++ b/bpkg/common.cli @@ -98,9 +98,6 @@ namespace bpkg \cb{test}, etc." } - // In the future we may also have --structured-result, similar to the - // build system. - // bool --no-result { "Don't print informational messages about the outcome of performing @@ -109,6 +106,18 @@ namespace bpkg instead, unless suppressed with \cb{--no-progress}." } + string --structured-result + { + "", + "Write the result of performing a command in a structured form. In + this mode, instead of printing to \cb{stderr} informational messages + about the outcome of performing a command or some of its parts, + \cb{bpkg} writes to \cb{stdout} a machine-readable result description + in the specified format. Not all commands support producing structured + result and valid values are command-specific. Consult the command + documentation for details." + } + // When it comes to external programs (such as curl, git, etc), if stderr // is not a terminal, the logic is actually tri-state: With --no-progress // we suppress any progress. With --progress we request full progress. diff --git a/bpkg/pkg-bindist.cli b/bpkg/pkg-bindist.cli index fd2a197..71497f5 100644 --- a/bpkg/pkg-bindist.cli +++ b/bpkg/pkg-bindist.cli @@ -728,6 +728,110 @@ namespace bpkg } }; + " + \h|STRUCTURED RESULT| + + Instead of printing to \cb{stderr} the list of generated binary packages in + a format more suitable for human consumption, the \cb{pkg-bindist} command + can be instructed to write it to \cb{stdout} in a machine-readable form by + specifying the \cb{--structured-result} option. Currently, the only + recognized format value for this option is \cb{json} with the output being + a JSON object that is a serialized representation of the following C++ + struct \cb{bindist_result}: + + \ + struct os_release + { + string name_id; // ID + vector like_ids; // ID_LIKE + optional version_id; // VERSION_ID + optional variant_id; // VARIANT_ID + + optional name; // NAME + optional version_codename; // VERSION_CODENAME + optional variant; // VARIANT + }; + + struct file + { + string path; + string type; + }; + + struct package + { + string name; + string version; + optional system_name; + optional system_version; + vector files; + }; + + struct bindist_result + { + string distribution; // --distribution or auto-detected + string architecture; // --architecture or auto-detected + os_release os_release; // --os-release-* or auto-detected + optional recursive; // --recursive + bool private; // --private + bool dependent_config; // See --allow-dependent-config + + package package; + vector dependencies; // Only in --recursive=separate + }; + \ + + For example: + + \ + { + \"distribution\": \"debian\", + \"architecture\": \"amd64\", + \"os_release\": { + \"name_id\": \"debian\", + \"version_id\": \"11\", + \"name\": \"Debian GNU/Linux\" + }, + \"package\": { + \"name\": \"libfoo\", + \"version\": \"2.5.0-b.23\", + \"system_name\": \"libfoo\", + \"system_version\": \"2.5.0~b.23-0~debian11\", + \"files\": [ + { + \"path\": \"/tmp/libfoo_2.5.0~b.23-0~debian11_amd64.deb\", + \"type\": \"main.deb\" + }, + { + \"path\": \"/tmp/libfoo-dev_2.5.0~b.23-0~debian11_amd64.deb\", + \"type\": \"dev.deb\" + }, + ... + ] + } + } + \ + + See the JSON OUTPUT section in \l{bpkg-common-options(1)} for details on + the overall properties of this format and the semantics of the \cb{struct} + serialization. + + The \cb{file::type} member is a distribution-specific value that classifies + the file. For the \cb{debian} distribution the possible values are + \cb{main.deb}, \cb{dev.deb}, \cb{doc.deb}, \cb{common.deb}, + \cb{dbgsym.deb}, \cb{changes} (\cb{.changes} file), and \cb{buildid} + (\cb{.buildid} file); see \l{bpkg#bindist-mapping-debian-produce Debian + Package Mapping for Production} for background. For the \cb{fedora} + distribution the possible values are \cb{main.rpm}, \cb{devel.rpm}, + \cb{static.rpm}, \cb{doc.rpm}, \cb{common.rpm}, and \cb{debuginfo.rpm}; see + \l{bpkg#bindist-mapping-fedora-produce Fedora Package Mapping for + Production} for background. For the \cb{archive} distribution this is the + archive type (\cb{--archive-type}), for example, \cb{tar.xz} or \cb{zip}. + + The \cb{system_name} and \cb{system_version} members in \cb{package} are + absent if not applicable to the distribution (for example, \cb{archive}). + " + // NOTE: remember to add the corresponding `--class-doc ...=exclude-base` // (both in bpkg/ and doc/) if adding a new base class. // diff --git a/bpkg/pkg-bindist.cxx b/bpkg/pkg-bindist.cxx index cb29af1..5038d12 100644 --- a/bpkg/pkg-bindist.cxx +++ b/bpkg/pkg-bindist.cxx @@ -4,6 +4,9 @@ #include #include +#include // cout + +#include #include #include @@ -235,7 +238,7 @@ namespace bpkg else if (m == "full") rec = recursive_mode::full; else if (m == "separate") rec = recursive_mode::separate; else - dr << fail << "unknown mode '" << m << "' specified with --recursive"; + dr << fail << "unknown --recursive mode '" << m << "'"; } if (o.private_ ()) @@ -254,6 +257,16 @@ namespace bpkg dr << info << "run 'bpkg help pkg-bindist' for more information"; } + if (o.structured_result_specified ()) + { + if (o.no_result ()) + fail << "both --structured-result and --no-result specified"; + + if (o.structured_result () != "json") + fail << "unknown --structured-result format '" + << o.structured_result () << "'"; + } + // Sort arguments into package names and configuration variables. // vector pns; @@ -297,12 +310,12 @@ namespace bpkg // Note that we shouldn't need to install anything or use sudo. // - unique_ptr spm ( + pair, string> spm ( make_production_system_package_manager (o, host_triplet, o.distribution (), o.architecture ())); - if (spm == nullptr) + if (spm.first == nullptr) { fail << "no standard distribution package manager for this host " << "or it is not yet supported" << @@ -338,17 +351,23 @@ namespace bpkg // Generate one binary package. // + using binary_file = system_package_manager::binary_file; + using binary_files = system_package_manager::binary_files; + struct result { - paths bins; - packages deps; + binary_files bins; + packages deps; shared_ptr pkg; }; + bool dependent_config (false); + auto generate = [&o, &vars, rec, &spm, - &c, &db] (const vector& pns, - bool first) -> result + &c, &db, + &dependent_config] (const vector& pns, + bool first) -> result { // Resolve package names to selected packages and verify they are all // configured. While at it collect their available packages and @@ -377,24 +396,30 @@ namespace bpkg // binary package in a configuration that is specific to some // dependents. // - if (!o.allow_dependent_config ()) + for (const config_variable& v: p->config_variables) { - for (const config_variable& v: p->config_variables) + switch (v.source) { - switch (v.source) + case config_source::dependent: { - case config_source::dependent: + if (!o.allow_dependent_config ()) { fail << "configuration variable " << v.name << " is imposed " << " by dependent package" << info << "specify it as user configuration to allow" << - info << "or specify --allow-dependent-config" << endf; + info << "or specify --allow-dependent-config"; } - case config_source::user: - case config_source::reflect: + + dependent_config = true; break; } + case config_source::user: + case config_source::reflect: + break; } + + if (dependent_config) + break; } // Load the available package for type/languages as well as the @@ -462,14 +487,14 @@ namespace bpkg // an option to specify/override it (along with languages). Note that // there will probably be no way to override type for dependencies. // - paths r (spm->generate (pkgs, - deps, - vars, - db.config, - pm, - type, langs, - recursive_full, - first)); + binary_files r (spm.first->generate (pkgs, + deps, + vars, + db.config, + pm, + type, langs, + recursive_full, + first)); return result {move (r), move (deps), move (pkgs.front ().selected)}; }; @@ -522,25 +547,117 @@ namespace bpkg if (rs.front ().bins.empty ()) return 0; // Assume prepare-only mode or similar. - if (verb && !o.no_result ()) + if (o.no_result ()) + ; + else if (!o.structured_result_specified ()) + { + if (verb) + { + const string& d (o.distribution_specified () + ? o.distribution () + : spm.first->os_release.name_id); + + for (auto b (rs.begin ()), i (b); i != rs.end (); ++i) + { + const selected_package& p (*i->pkg); + + string ver (p.version.string (false /* ignore_revision */, + true /* ignore_iteration */)); + + diag_record dr (text); + + dr << "generated " << d << " package for " + << (i != b ? "dependency " : "") + << p.name << '/' << ver << ':'; + + for (const binary_file& f: i->bins) + dr << "\n " << f.path; + } + } + } + else { - const string& d (o.distribution_specified () - ? o.distribution () - : spm->os_release.name_id); + json::stream_serializer s (cout); + + auto member = [&s] (const char* n, const string& v) + { + if (!v.empty ()) + s.member (n, v); + }; - for (auto b (rs.begin ()), i (b); i != rs.end (); ++i) + auto package = [&s, &member] (const result& r) { - const selected_package& p (*i->pkg); + const selected_package& p (*r.pkg); + const binary_files& bfs (r.bins); + + string ver (p.version.string (false /* ignore_revision */, + true /* ignore_iteration */)); + + s.begin_object (); // package + { + member ("name", p.name.string ()); + member ("version", ver); + member ("system_name", bfs.system_name); + member ("system_version", bfs.system_version); + s.member_begin_array ("files"); + for (const binary_file& bf: bfs) + { + s.begin_object (); // file + { + member ("path", bf.path.string ()); + member ("type", bf.type); + } + s.end_object (); // file + }; + s.end_array (); + } + s.end_object (); // package + }; + + s.begin_object (); // bindist_result + { + member ("distribution", spm.second); + member ("architecture", spm.first->arch); + + s.member_begin_object ("os_release"); + { + const auto& r (spm.first->os_release); + + member ("name_id", r.name_id); + + if (!r.like_ids.empty ()) + { + s.member_begin_array ("like_ids"); + for (const string& id: r.like_ids) s.value (id); + s.end_array (); + } - diag_record dr (text); + member ("version_id", r.version_id); + member ("variant_id", r.variant_id); - dr << "generated " << d << " package for " - << (i != b ? "dependency " : "") - << p.name << '/' << p.version << ':'; + member ("name", r.name); + member ("version_codename", r.version_codename); + member ("variant", r.variant); + } + s.end_object (); // os_release - for (const path& p: i->bins) - dr << "\n " << p; + member ("recursive", o.recursive ()); + if (o.private_ ()) s.member ("private", true); + if (dependent_config) s.member ("dependent_config", true); + + s.member_name ("package"); + package (rs.front ()); + + if (rs.size () > 1) + { + s.member_begin_array ("dependencies"); + for (auto i (rs.begin ()); ++i != rs.end (); ) package (*i); + s.end_array (); + } } + s.end_object (); // bindist_result + + cout << endl; } return 0; diff --git a/bpkg/pkg-status.cli b/bpkg/pkg-status.cli index 59319bf..084b7a3 100644 --- a/bpkg/pkg-status.cli +++ b/bpkg/pkg-status.cli @@ -218,7 +218,7 @@ namespace bpkg ] \ - See the JSON OUTPUT section in \l{bdep-common-options(1)} for details on + See the JSON OUTPUT section in \l{bpkg-common-options(1)} for details on the overall properties of this format and the semantics of the \cb{struct} serialization. diff --git a/bpkg/system-package-manager-archive.cxx b/bpkg/system-package-manager-archive.cxx index 16e635d..8496bb3 100644 --- a/bpkg/system-package-manager-archive.cxx +++ b/bpkg/system-package-manager-archive.cxx @@ -45,7 +45,7 @@ namespace bpkg else target = host; - arch = target.string (); // Set in case queried by someone else. + arch = target.string (); // Set since queried (e.g., JSON value). } // env --chdir= tar|zip ... . @@ -288,7 +288,7 @@ namespace bpkg // directory as a chroot. Then tar/zip this directory to produce one or more // binary package archives. // - paths system_package_manager_archive:: + auto system_package_manager_archive:: generate (const packages& pkgs, const packages& deps, const strings& vars, @@ -297,7 +297,7 @@ namespace bpkg const string& pt, const small_vector& langs, optional recursive_full, - bool first) + bool first) -> binary_files { tracer trace ("system_package_manager_archive::generate"); @@ -730,7 +730,7 @@ namespace bpkg if (verb >= 1) text << "prepared " << dst; - return paths {}; + return binary_files {}; } // Create the archive. @@ -743,7 +743,7 @@ namespace bpkg // We don't do anything for source archives, not sure why we should // do something here. // - paths r; + binary_files r; { const strings& ts ( ops->archive_type_specified () @@ -758,7 +758,10 @@ namespace bpkg if (t.size () > 1 && t.front () == '.') t.erase (0, 1); - r.push_back (archive (out, base, t)); + // Using archive type as file type seems appropriate. + // + path f (archive (out, base, t)); + r.push_back (binary_file {move (f), move (t)}); } } diff --git a/bpkg/system-package-manager-archive.hxx b/bpkg/system-package-manager-archive.hxx index c5b7b70..01c4a2a 100644 --- a/bpkg/system-package-manager-archive.hxx +++ b/bpkg/system-package-manager-archive.hxx @@ -17,7 +17,7 @@ namespace bpkg class system_package_manager_archive: public system_package_manager { public: - virtual paths + virtual binary_files generate (const packages&, const packages&, const strings&, diff --git a/bpkg/system-package-manager-debian.cxx b/bpkg/system-package-manager-debian.cxx index 53d3a07..19f6391 100644 --- a/bpkg/system-package-manager-debian.cxx +++ b/bpkg/system-package-manager-debian.cxx @@ -1958,7 +1958,7 @@ namespace bpkg // Note: this setup requires dpkg-dev (or build-essential) and debhelper // packages. // - paths system_package_manager_debian:: + auto system_package_manager_debian:: generate (const packages& pkgs, const packages& deps, const strings& vars, @@ -1967,7 +1967,7 @@ namespace bpkg const string& pt, const small_vector& langs, optional recursive_full, - bool first) + bool first) -> binary_files { tracer trace ("system_package_manager_debian::generate"); @@ -3487,7 +3487,7 @@ namespace bpkg print_process (dr, args); } - return paths {}; + return binary_files {}; } try @@ -3542,13 +3542,17 @@ namespace bpkg // Collect and return the binary package paths. // - paths r; - auto add = [&out, &r] (const string& n, bool opt = false) + binary_files r; + + r.system_name = gen_main ? st.main : st.dev; + r.system_version = st.system_version; + + auto add = [&out, &r] (const string& n, const char* t, bool opt = false) { path p (out / n); if (exists (p)) - r.push_back (move (p)); + r.push_back (binary_file {move (p), t}); else if (!opt) fail << "expected output file " << p << " does not exist"; }; @@ -3559,19 +3563,19 @@ namespace bpkg // const string& ver (st.system_version); - if (gen_main) add (st.main + '_' + ver + '_' + arch + ".deb"); - if (!binless) add (st.main + "-dbgsym_" + ver + '_' + arch + ".deb", true); + if (gen_main) add (st.main + '_' + ver + '_' + arch + ".deb", "main.deb"); + if (!binless) add (st.main + "-dbgsym_" + ver + '_' + arch + ".deb", "dbgsym.deb", true); - if (!st.dev.empty ()) add (st.dev + '_' + ver + '_' + arch + ".deb"); - if (!st.doc.empty ()) add (st.doc + '_' + ver + "_all.deb"); - if (!st.common.empty ()) add (st.common + '_' + ver + "_all.deb"); + if (!st.dev.empty ()) add (st.dev + '_' + ver + '_' + arch + ".deb", "dev.deb"); + if (!st.doc.empty ()) add (st.doc + '_' + ver + "_all.deb", "doc.deb"); + if (!st.common.empty ()) add (st.common + '_' + ver + "_all.deb", "common.deb"); // Besides the binary packages (.deb) we also get the .buildinfo and // .changes files, which could be useful. Note that their names are based // on the source package name. // - add (pn.string () + '_' + ver + '_' + arch + ".buildinfo"); - add (pn.string () + '_' + ver + '_' + arch + ".changes"); + add (pn.string () + '_' + ver + '_' + arch + ".buildinfo", "buildinfo"); + add (pn.string () + '_' + ver + '_' + arch + ".changes", "changes"); return r; } diff --git a/bpkg/system-package-manager-debian.hxx b/bpkg/system-package-manager-debian.hxx index eb8b214..336f7a7 100644 --- a/bpkg/system-package-manager-debian.hxx +++ b/bpkg/system-package-manager-debian.hxx @@ -135,7 +135,7 @@ namespace bpkg virtual void install (const vector&) override; - virtual paths + virtual binary_files generate (const packages&, const packages&, const strings&, diff --git a/bpkg/system-package-manager-fedora.cxx b/bpkg/system-package-manager-fedora.cxx index a504035..dcd953d 100644 --- a/bpkg/system-package-manager-fedora.cxx +++ b/bpkg/system-package-manager-fedora.cxx @@ -2299,7 +2299,7 @@ namespace bpkg // Note: this setup requires rpmdevtools (rpmdev-setuptree) and its // dependency rpm-build and rpm packages. // - paths system_package_manager_fedora:: + auto system_package_manager_fedora:: generate (const packages& pkgs, const packages& deps, const strings& vars, @@ -2308,7 +2308,7 @@ namespace bpkg const string& pt, const small_vector& langs, optional recursive_full, - bool /* first */) + bool /* first */) -> binary_files { tracer trace ("system_package_manager_fedora::generate"); @@ -4244,7 +4244,7 @@ namespace bpkg print_process (dr, args); } - return paths {}; + return binary_files {}; } try @@ -4295,7 +4295,9 @@ namespace bpkg // // Here we will use `rpm --eval` to resolve the RPM sub-package paths. // - paths r; + binary_files r; + r.system_name = gen_main ? st.main : st.devel; + r.system_version = st.system_version; { string expressions; @@ -4309,34 +4311,37 @@ namespace bpkg const string& package_arch (!build_arch.empty () ? build_arch : arch); - size_t np (0); - auto add_package = [&expressions, &rpmfile, &np, &add_macro] - (const string& name, const string& arch) -> size_t + auto add_package = [&r, &expressions, &rpmfile, &add_macro] + (const string& name, + const string& arch, + const char* type) -> size_t { add_macro ("NAME", name); add_macro ("ARCH", arch); expressions += rpmfile + '\n'; - return np++; + r.push_back (binary_file {path (), type}); // Reserve. + return r.size () - 1; }; if (gen_main) - add_package (st.main, package_arch); + add_package (st.main, package_arch, "main.rpm"); if (!st.devel.empty ()) - add_package (st.devel, package_arch); + add_package (st.devel, package_arch, "devel.rpm"); if (!st.static_.empty ()) - add_package (st.static_, package_arch); + add_package (st.static_, package_arch, "static.rpm"); if (!st.doc.empty ()) - add_package (st.doc, "noarch"); + add_package (st.doc, "noarch", "doc.rpm"); if (!st.common.empty ()) - add_package (st.common, "noarch"); + add_package (st.common, "noarch", "common.rpm"); - optional di (!binless - ? add_package (st.main + "-debuginfo", arch) - : optional ()); + optional di ( + !binless + ? add_package (st.main + "-debuginfo", arch, "debuginfo.rpm") + : optional ()); // Strip the trailing newline since rpm adds one. // @@ -4344,13 +4349,11 @@ namespace bpkg strings expansions (eval (cstrings ({expressions.c_str ()}))); - if (expansions.size () != np) + if (expansions.size () != r.size ()) fail << "number of RPM file path expansions differs from number " << "of path expressions"; - r.reserve (np); - - for (size_t i (0); i != expansions.size(); ++i) + for (size_t i (0); i != r.size(); ++i) { try { @@ -4364,7 +4367,7 @@ namespace bpkg // etc). // if (exists (p)) - r.push_back (move (p)); + r[i].path = move (p); else if (!di || i != *di) // Not a -debuginfo sub-package? fail << "expected output file " << p << " does not exist"; } diff --git a/bpkg/system-package-manager-fedora.hxx b/bpkg/system-package-manager-fedora.hxx index 672b2a1..3e68b98 100644 --- a/bpkg/system-package-manager-fedora.hxx +++ b/bpkg/system-package-manager-fedora.hxx @@ -203,7 +203,7 @@ namespace bpkg virtual void install (const vector&) override; - virtual paths + virtual binary_files generate (const packages&, const packages&, const strings&, diff --git a/bpkg/system-package-manager.cxx b/bpkg/system-package-manager.cxx index 2ffb1bb..977b000 100644 --- a/bpkg/system-package-manager.cxx +++ b/bpkg/system-package-manager.cxx @@ -136,7 +136,7 @@ namespace bpkg return r; } - unique_ptr + pair, string> make_production_system_package_manager (const pkg_bindist_options& o, const target_triplet& host, const string& name, @@ -163,7 +163,7 @@ namespace bpkg if (o.os_release_version_id_specified ()) oos->version_id = o.os_release_version_id (); - unique_ptr r; + pair, string> r; if (oos) { os_release& os (*oos); @@ -173,8 +173,9 @@ namespace bpkg // if (name == "archive") { - r.reset (new system_package_manager_archive ( - move (os), host, arch, progress, &o)); + r.first.reset (new system_package_manager_archive ( + move (os), host, arch, progress, &o)); + r.second = "archive"; } else if (host.class_ == "linux") { @@ -188,8 +189,9 @@ namespace bpkg if (os.name_id != "debian" && !is_or_like (os, "debian")) os.like_ids.push_back ("debian"); - r.reset (new system_package_manager_debian ( - move (os), host, arch, progress, &o)); + r.first.reset (new system_package_manager_debian ( + move (os), host, arch, progress, &o)); + r.second = "debian"; } else if (is_or_like (os, "fedora") || is_or_like (os, "rhel") || @@ -204,15 +206,16 @@ namespace bpkg if (os.name_id != "fedora" && !is_or_like (os, "fedora")) os.like_ids.push_back ("fedora"); - r.reset (new system_package_manager_fedora ( - move (os), host, arch, progress, &o)); + r.first.reset (new system_package_manager_fedora ( + move (os), host, arch, progress, &o)); + r.second = "fedora"; } // NOTE: remember to update the --distribution pkg-bindist option // documentation if adding support for another package manager. } } - if (r == nullptr) + if (r.first == nullptr) { if (!name.empty ()) fail << "unsupported package manager '" << name << "' for host " diff --git a/bpkg/system-package-manager.hxx b/bpkg/system-package-manager.hxx index 736a53a..372730d 100644 --- a/bpkg/system-package-manager.hxx +++ b/bpkg/system-package-manager.hxx @@ -181,6 +181,7 @@ namespace bpkg // // Return the list of paths to binary packages and any other associated // files (build metadata, etc) that could be useful for their consumption. + // Each returned file has a distribution-specific type that classifies it. // If the result is empty, assume the prepare-only mode (or similar) with // appropriate result diagnostics having been already issued. // @@ -198,7 +199,21 @@ namespace bpkg using packages = vector; - virtual paths + struct binary_file + { + bpkg::path path; + string type; + }; + + struct binary_files: public vector + { + // Empty if not applicable. + // + string system_name; + string system_version; + }; + + virtual binary_files generate (const packages& pkgs, const packages& deps, const strings& vars, @@ -432,12 +447,15 @@ namespace bpkg bool yes, const string& sudo); + // Create for production. The second half of the result is the effective + // distribution name. + // // Note that the reference to options is expected to outlive the returned // instance. // class pkg_bindist_options; - unique_ptr + pair, string> make_production_system_package_manager (const pkg_bindist_options&, const target_triplet&, const string& name, -- cgit v1.1