diff options
| -rw-r--r-- | bpkg/pkg-bindist.cli | 13 | ||||
| -rw-r--r-- | bpkg/system-package-manager-debian.cxx | 542 | ||||
| -rw-r--r-- | bpkg/system-package-manager-debian.hxx | 13 | ||||
| -rw-r--r-- | bpkg/system-package-manager.cxx | 2 | ||||
| -rw-r--r-- | bpkg/system-package-manager.hxx | 3 |
5 files changed, 446 insertions, 127 deletions
diff --git a/bpkg/pkg-bindist.cli b/bpkg/pkg-bindist.cli index 5686ed6..ad743d9 100644 --- a/bpkg/pkg-bindist.cli +++ b/bpkg/pkg-bindist.cli @@ -98,6 +98,19 @@ namespace bpkg documentation in the build system manual for details. This option only makes sense together with \cb{--recursive}." } + + bool --wipe-out + { + "Wipe the output directory (\ci{out-dir}) clean before using it to + generate the binary package." + } + + bool --keep-out + { + "Keep intermediate files in the output directory (\ci{out-dir}) that + were used to generate the binary package. This is primarily useful + for troubleshooting." + } }; " diff --git a/bpkg/system-package-manager-debian.cxx b/bpkg/system-package-manager-debian.cxx index 47df65d..37844f1 100644 --- a/bpkg/system-package-manager-debian.cxx +++ b/bpkg/system-package-manager-debian.cxx @@ -3,8 +3,12 @@ #include <bpkg/system-package-manager-debian.hxx> +#include <libbutl/filesystem.hxx> // permissions + #include <bpkg/diagnostics.hxx> +#include <bpkg/pkg-bindist-options.hxx> + using namespace butl; namespace bpkg @@ -1455,6 +1459,311 @@ namespace bpkg } } + // Map non-system bpkg package to system package name(s) and version. + // + // This is used both to map the package being generated and its + // dependencies. What should we do with extras returned in package_status? + // We can't really generate any of them (which files would we place in + // them?) nor can we list them as dependencies (we don't know their system + // versions). So it feels like the only sensible choice is to ignore extras. + // + // In a sense, we have a parallel arrangement going on here: binary packages + // that we generate don't have extras (i.e., they include everything + // necessary in the "standard" packages from the main group) and when we + // punch a system dependency based on a non-system bpkg package, we assume + // it was generated by us and thus doesn't have any extras. Or, to put it + // another way, if you want the system dependency to refer to a "native" + // system package with extras you need to configure it as a system bpkg + // package. + // + // In fact, this extends to package names. For example, unless custom + // mapping is specified, we will generate libsqlite3 and libsqlite3-dev + // while native names are libsqlite3-0 and libsqlite3-dev. While this + // duality is not ideal, presumably we will normally only be producing our + // binary packages if there are no suitable native packages. And for a few + // exception (e.g., our package is "better" in some way, such as configured + // differently or fixes a critical bug), we will just have to provide + // appropriate manual mapping that makes sure the names match (the extras is + // still a potential problem though -- we will only have them as + // dependencies if we build against a native system package). + // + // @@ TODO: test, especially distribution version logic. + // + package_status system_package_manager_debian:: + map_package (const package_name& pn, + const version& pv, + const available_packages& aps) + { + // We should only have one available package corresponding to this package + // name/version. + // + assert (aps.size () == 1); + + strings ns (system_package_names (aps, + os_release.name_id, + os_release.version_id, + os_release.like_ids)); + package_status r; + if (ns.empty ()) + { + // Automatically translate our package name similar to the consumption + // case above. Except here we don't attempt to deduce main from -dev, + // naturally. + // + const string& n (pn.string ()); + + // The best we can do in trying to detect whether this is a library is + // to check for the lib prefix. Libraries without the lib prefix and + // non-libraries with the lib prefix (both of which we do not + // recomment) will have to provide a manual mapping. + // + if (n.compare (0, 3, "lib") == 0 && n.size () > 3) + { + r = package_status (n, n + "-dev"); + } + else + r = package_status (n); + } + else + { + // Even though we only pass one available package, we may still end up + // with multiple mappings. In this case we take the first, per the + // documentation. + // + r = parse_name_value (pn, + ns.front (), + false /* need_doc */, + false /* need_dbg */); + } + + // Map the version. + // + // To recap, a Debian package version has the following form: + // + // [<epoch>:]<upstream>[-<revision>] + // + // For details on the ordering semantics, see the Version control file + // field documentation in the Debian Policy Manual. While overall + // unsurprising, one notable exception is `~`, which sorts before anything + // else and is commonly used for upstream pre-releases. For example, + // 1.0~beta1~svn1245 sorts earlier than 1.0~beta1, which sorts earlier + // than 1.0. + // + // There are also various special version conventions (such as all the + // revision components in 1.4-5+deb10u1~bpo9u1) but they all appear to + // express relationships between native packages and/or their upstream and + // thus do not apply to our case. + // + // Ok, so how do we map our version to that? To recap, the bpkg version + // has the following form: + // + // [+<epoch>-]<upstream>[-<prerel>][+<revision>] + // + // Let's start with the case where neither distribution nor upstream + // version is specified and we need to derive everything from the bpkg + // version. + // + // <epoch> + // + // On one hand, if we keep the epoch, it won't necessarily match + // Debian's native package epoch. But on the other it will allow our + // binary packages form different epochs to co-exist. Seeing that this + // can be easily overridden with a custom distribution version, let's + // keep it. + // + // Note that while the Debian start/default epoch is 0, ours is 1 (we + // use the 0 epoch for stub packages). So we will need to shift this + // value range. + // + // + // <upstream>[-<prerel>] + // + // Our upstream version maps naturally to Debian's. That is, our + // upstream version format/semantics is a subset of Debian's. + // + // If this is a pre-release, then we could fail (that is, don't allow + // pre-releases) but then we won't be able to test on pre-release + // packages, for example, to make sure the name mapping is correct. + // Plus sometimes it's useful to publish pre-releases. We could ignore + // it, but then such packages will be indistinguishable from each other + // and the final release, which is not ideal. On the other hand, Debian + // has the mechanism (`~`) which is essentially meant for this, so let's + // use it. We will use <prerel> as is since its format is the same as + // upstream and thus should map naturally. + // + // + // <revision> + // + // Similar to epoch, our revision won't necessarily match Debian's + // native package revision. But on the other hand it will allow us to + // establish a correspondence between source and binary packages. Plus, + // upgrades between binary package revisions will be handled naturally. + // Seeing that we allow overriding the revision with a custom + // distribution version (see below), let's keep it. + // + // Note also that both Debian and our revision start/default is 0. + // However, it is Debian's convention to start revision from 1. But it + // doesn't seem worth it for us to do any shifting here and so we will + // use our revision as is. + // + // Another related question is whether we should also include some + // metadata that identifies the distribution and its version that this + // package is for. The strongest precedent here is probably Ubuntu's + // PPA. While there doesn't appear to be a consistent approach, one can + // often see versions like these: + // + // 2.1.0-1~ppa0~ubuntu14.04.1, + // 1.4-5-1.2.1~ubuntu20.04.1~ppa1 + // 22.12.2-0ubuntu1~ubuntu23.04~ppa1 + // + // Seeing that this is a non-sortable component (what in semver would be + // called "build metadata"), using `~` is probably not the worst choice. + // + // So we follow this lead and add the ~<name_id><version_id> component + // to revision. Note that this also means we will have to make the 0 + // revision explicit. For example: + // + // 1.2.3-0~ubuntu20.04 + // 1.2.3-1~debian10 + // + // The next case to consider is when we have the upstream version + // (upstream-version manifest value). After some rumination it feels + // correct to use it instead of the <epoch>-<upstream> components in the + // above mapping (upstream version itself cannot have epoch). In other + // words, we will add the pre-release and revision components from the + // bpkg version. If this is not the desired semantics, then it can always + // be overrided with the distribution version. + // + // Finally, we have the distribution version. The <epoch> and <upstream> + // components are straightforward: they should be specified by the + // distribution version as required. This leaves pre-release and + // revision. It feels like in most cases we would want these copied over + // from the bpkg version automatically -- it's too tedious and error- + // prone to maintain them manually. However, we want the user to have the + // full override ability. So instead, if empty revision is specified, as + // in 1.2.3-, then we automatically add bpkg revision. Similarly, if empty + // pre-release is specified, as in 1.2.3~, then we add bpkg pre-release. + // To add both automatically, we would specify 1.2.3~- (other combinations + // are 1.2.3~b.1- and 1.2.3~-1). + // + // Note also that per the Debian version specification, if upstream + // contains `:` and/or `-`, then epoch and/or revision must be specified + // explicitly, respectively. Note that the bpkg upstream version may not + // contain either. + // + const shared_ptr<available_package>& ap (aps.front ().first); + const lazy_shared_ptr<repository_fragment>& rf (aps.front ().second); + + string& sv (r.system_version); + + if (optional<string> ov = system_package_version (ap, + rf, + os_release.name_id, + os_release.version_id, + os_release.like_ids)) + { + string& dv (*ov); + size_t n (dv.size ()); + + // Find the revision and pre-release positions, if any. + // + size_t rp (dv.rfind ('-')); + size_t pp (dv.rfind ('~', rp)); + + // Copy over the [<epoch>:]<upstream> part. + // + sv.assign (dv, 0, pp < rp ? pp : rp); + + // Add pre-release copying over the bpkg version value if empty. + // + if (pp != string::npos) + { + if (size_t pn = (rp != string::npos ? rp : n) - (pp + 1)) + { + sv.append (dv, pp, pn + 1); + } + else + { + if (pv.release) + { + assert (!pv.release->empty ()); // Cannot be earliest special. + sv += '~'; + sv += *pv.release; + } + } + } + + // Add revision copying over the bpkg version value if empty. + // + if (rp != string::npos) + { + if (size_t rn = n - (rp + 1)) + { + sv.append (dv, rp, rn + 1); + } + else + { + sv += '-'; + sv += to_string (pv.revision ? *pv.revision : 0); + } + } + else + sv += "-0"; // Default revision (for build metadata; see below). + } + else + { + if (ap->upstream_version) + { + const string& uv (*ap->upstream_version); + + // Add explicit epoch if upstream contains `:`. + // + // Note that we don't need to worry about `-` since we always add + // revision (see below). + // + if (uv.find (':') != string::npos) + sv = "0:"; + + sv += uv; + } + else + { + // Add epoch unless maps to 0. + // + assert (pv.epoch != 0); // Cannot be a stub. + if (pv.epoch != 1) + { + sv = to_string (pv.epoch - 1); + sv += ':'; + } + + sv += pv.upstream; + } + + // Add pre-release. + // + if (pv.release) + { + assert (!pv.release->empty ()); // Cannot be earliest special. + sv += '~'; + sv += *pv.release; + } + + // Add revision. + // + sv += '-'; + sv += to_string (pv.revision ? *pv.revision : 0); + } + + // Add build matadata. + // + sv += '~'; + sv += os_release.name_id; + sv += os_release.version_id; // Could be empty. + + return r; + } + // Some background on creating Debian packages (for a bit more detailed // overview see the Debian Packaging Tutorial). // @@ -1524,125 +1833,9 @@ namespace bpkg generate (packages&& pkgs, packages&& deps, strings&&, - const dir_path&, + const dir_path& out, optional<recursive_mode>) { - // Map non-system bpkg package to system package name(s) and version. - // - // This is used both to map the package being generated and its - // dependencies. What should we do with extras returned in package_status? - // We can't really generate any of them (which files would we place in - // them?) and we can't list them as dependencies (we don't know their - // system versions). So it feels like the only sensible choice is to - // ignore extras. - // - // In a sense, we have a parallel arrangement going on here: binary - // packages that we generate don't have extras (i.e., they include - // everything necessary in the "standard" packages from the main group) - // and when we punch a system dependency based on a non-system bpkg - // package, we assume it was generated by us and thus doesn't have any - // extras. Or, to put it another way, if you want the system dependency to - // refer to a "native" system package with extras you need to configure it - // as a system bpkg package. - // - // In fact, this extends to package names. For example, unless custom - // mapping is specified, we will generate libsqlite3 and libsqlite3-dev - // while native names are libsqlite3-0 and libsqlite3-dev. While this - // duality is not ideal, presumably we will normally only be producing - // our binary packages if there are no suitable native packages. And for a - // few exception (e.g., our package is "better" in some way, such as - // configured differently or fixes a critical bug), we will just have to - // provide appropriate manual mapping that makes sure the names match (the - // extras is still a potential problem though -- we will only have them as - // dependencies if we build against a native system package). - // - auto map_package = [this] (const shared_ptr<selected_package>& sp, - const available_packages& aps) -> package_status - { - // We should only have one available package corresponding to the - // selected package. - // - assert (sp->substate != package_substate::system && aps.size () == 1); - - strings ns (system_package_names (aps, - os_release.name_id, - os_release.version_id, - os_release.like_ids)); - package_status r; - if (ns.empty ()) - { - // Automatically translate our package name similar to the consumption - // case above. Except here we don't attempt to deduce main from -dev, - // naturally. - // - const string& pn (sp->name.string ()); - - // The best we can do in trying to detect whether this is a library is - // to check for the lib prefix. Libraries without the lib prefix and - // non-libraries with the lib prefix (both of which we do not - // recomment) will have to provide a manual mapping. - // - if (pn.compare (0, 3, "lib") == 0 && pn.size () > 3) - { - r = package_status (pn, pn + "-dev"); - } - else - r = package_status (pn); - } - else - { - // Even though we only pass one available package, we may still end up - // with multiple mappings. In this case we take the first, per the - // documentation. - // - r = parse_name_value (sp->name, - ns.front (), - false /* need_doc */, - false /* need_dbg */); - } - - // Map the version. - // - // @@ Should we fail if either is a pre-release (we cannot represent - // it in Debian version). But then won't be able to test on - // pre-release packages. maybe just strip/ignore? Can we translate - // it to something appropriate in Debian version (something in the - // revision)? - // - // @@ Should we include our revision as Debian revistion even if the - // version is mapped? Feels natural. - // - const shared_ptr<available_package>& ap (aps.front ().first); - const lazy_shared_ptr<repository_fragment>& rf (aps.front ().second); - - - if (optional<string> v = system_package_version (ap, - rf, - os_release.name_id, - os_release.version_id, - os_release.like_ids)) - { - r.system_version = move (*v); - } - else if (ap->upstream_version) - { - r.system_version = *ap->upstream_version; - } - else - { - // Strip or keep epoch? On one hand it won't necessarily match Debian - // but on the other it will allow packages form different epochs to - // co-exist. - // - // @@ Also need to make sure have explicit epoch/revision if upstream - // contains `:` or `-`. Can our upstream contain them? - // - r.system_version = sp->version.upstream; - } - - return r; - }; - // As a first step, figure out the system names and version of the package // we are generating and all the dependencies, diagnosing anything fishy. // @@ -1650,8 +1843,9 @@ namespace bpkg // the status cache. // const shared_ptr<selected_package>& sp (pkgs.front ().first); - const available_packages& aps (pkgs.front ().second); - package_status s (map_package (sp, aps)); + const package_name& pn (sp->name); + const version& pv (sp->version); + package_status s (map_package (pn, pv, pkgs.front ().second)); vector<package_status> sdeps; sdeps.reserve (deps.size ()); @@ -1665,18 +1859,118 @@ namespace bpkg { optional<package_status> os (status (sp->name, aps)); - if (!os) - fail << "bad boy"; + if (!os || os->status != package_status::installed) + fail << os_release.name_id << " package for " << sp->name + << " system package is no longer installed"; - // @@ We should confirm configured version matches mapped back. - // Can be `*`! + // For good measure verify the mapped back version still matches + // configured. Note that besides the normal case (queried by the + // system package manager), it could have also been specified by the + // user as an actual version or a wildcard. Ignoring this check for a + // wildcard feels consistent with the overall semantics. + // + if (sp->version != wildcard_version && sp->version != os->version) + { + fail << "current " << os_release.name_id << " package version for " + << sp->name << " system package does not match configured" << + info << "configured version: " << sp->version << + info << "current version: " << os->version << " (" + << os->system_version << ')'; + } s = move (*os); } else - s = map_package (sp, aps); + s = map_package (sp->name, sp->version, aps); sdeps.push_back (move (s)); } + + { + auto print_status = [] (diag_record& dr, const package_status& s) + { + dr << s.main + << (s.dev.empty () ? "" : " ") << s.dev + << (s.doc.empty () ? "" : " ") << s.doc + << (s.dbg.empty () ? "" : " ") << s.dbg + << (s.common.empty () ? "" : " ") << s.common + << ' ' << s.system_version; + }; + + diag_record dr (text); + print_status (dr, s); + + for (const package_status& ds: sdeps) + { + dr << "\n "; + print_status (dr, ds); + } + } + + // Start assembling the package "source" directory. + // + // It's hard to predict all the files that will be generated (and + // potentially read), so we will just require a clean output directory. + // + // Also, by default, we are going to keep all the intermediate files on + // failure for troubleshooting. + // + if (exists (out)) + { + if (!empty (out)) + { + if (!ops_->wipe_out ()) + fail << "directory " << out << " is not empty" << + info << "use --wipe to clean it up but be careful"; + + rm_r (out, false); + } + } + + // Normally the source directory is called <name>-<upstream-version> + // (e.g., as unpacked from the source archive). + // + dir_path src (out / dir_path (pn.string () + '-' + pv.string ())); + dir_path deb (src / dir_path ("debian")); + mk_p (deb); + + // The rules makefile. Note that it must be executable. + // + path rules (deb / "rules"); + try + { + // See fdopen() for details (umask, etc). + // + permissions ps (permissions::ru | permissions::wu | permissions::xu | + permissions::rg | permissions::wg | permissions::xg | + permissions::ro | permissions::wo | permissions::xo); + ofdstream os (fdopen (rules, + fdopen_mode::out | fdopen_mode::create, + ps)); + + os << "#!/usr/bin/make -f\n" + << "# -*- makefile -*-\n" + << '\n'; + + // See debhelper(7) for details on these. + // + if (verb == 0) + os << "export DH_QUIET=1\n"; + else if (verb >= 2) + os << "export DH_VERBOSE=1\n"; + + os.close (); + } + catch (const io_error& e) + { + fail << "unable to write to " << rules << ": " << e; + } + + // Cleanup intermediate files unless requested not to. + // + if (!ops_->keep_out ()) + { + rm_r (src); + } } } diff --git a/bpkg/system-package-manager-debian.hxx b/bpkg/system-package-manager-debian.hxx index 1e53e38..8fccb55 100644 --- a/bpkg/system-package-manager-debian.hxx +++ b/bpkg/system-package-manager-debian.hxx @@ -164,11 +164,13 @@ namespace bpkg system_package_manager_debian (bpkg::os_release&& osr, const target_triplet& h, string a, - optional<bool> progress) + optional<bool> progress, + const pkg_bindist_options& ops) : system_package_manager (move (osr), h, a.empty () ? arch_from_target (h) : move (a), - progress) {} + progress), + ops_ (&ops) {} // Implementation details exposed for testing (see definitions for // documentation). @@ -201,6 +203,11 @@ namespace bpkg static string arch_from_target (const target_triplet&); + package_status + map_package (const package_name&, + const version&, + const available_packages&); + // If simulate is not NULL, then instead of executing the actual apt-cache // and apt-get commands simulate their execution: (1) for apt-cache by // printing their command lines and reading the results from files @@ -242,6 +249,8 @@ namespace bpkg bool installed_ = false; // True if already installed. std::map<package_name, optional<system_package_status_debian>> status_cache_; + + const pkg_bindist_options* ops_ = nullptr; // Only for production. }; } diff --git a/bpkg/system-package-manager.cxx b/bpkg/system-package-manager.cxx index 899d390..a04c213 100644 --- a/bpkg/system-package-manager.cxx +++ b/bpkg/system-package-manager.cxx @@ -154,7 +154,7 @@ namespace bpkg os.like_ids.push_back ("debian"); r.reset (new system_package_manager_debian ( - move (os), host, arch, progress)); + move (os), host, arch, progress, o)); } else if (is_or_like (os, "fedora") || is_or_like (os, "rhel") || diff --git a/bpkg/system-package-manager.hxx b/bpkg/system-package-manager.hxx index 305a00e..91f9a49 100644 --- a/bpkg/system-package-manager.hxx +++ b/bpkg/system-package-manager.hxx @@ -346,6 +346,9 @@ namespace bpkg bool yes, const string& sudo); + // Note that the reference to options is expected to outlive the returned + // instance. + // class pkg_bindist_options; unique_ptr<system_package_manager> |