Add support for upload handlers and implement brep-upload-bindist handler

7 files changed, 206 insertions, 10 deletions

diff --git a/tests/ci/data.testscript b/tests/ci/data.testscript
index e269d0d..6f44c85 100644
--- a/tests/ci/data.testscript
+++ b/tests/ci/data.testscript
@@ -3,11 +3,11 @@
 
 # Pre-created CI request data directory that will be copied by subsequent
 # tests and scope setup commands. The common approach will be that group
-# scopes copy and modify the parent scope submission directory as required
+# scopes copy and modify the parent scope request data directory as required
 # by the nested tests and scopes. Tests will also clone the parent scope
-# submission data directory to optionally modify it, use and cleanup at the
-# end. Note that configuration can not be shared between multiple submission
-# handler processes. Also we need to make sure that submission data
+# request data data directory to optionally modify it, use and cleanup at the
+# end. Note that request data directory can not be shared between multiple
+# submission handler processes. Also we need to make sure that request data
 # directories are not cloned while being used by submission handler scripts.
 #
 data_dir = $regex.replace($path_search('*/request.manifest', $src_base), \
diff --git a/tests/submit/data.testscript b/tests/submit/data.testscript
index 29b607f..b0fe8f0 100644
--- a/tests/submit/data.testscript
+++ b/tests/submit/data.testscript
@@ -3,12 +3,13 @@
 
 # Pre-created submission data directory that will be copied by subsequent
 # tests and scope setup commands. The common approach will be that group
-# scopes copy and modify the parent scope submission directory as required by
-# the nested tests and scopes. Tests will also clone the parent scope
-# submission data directory to optionally modify it, use and cleanup at the
-# end. Note that configuration can not be shared between multiple submission
-# handler processes. Also we need to make sure that submission data
-# directories are not cloned while being used by submission handler scripts.
+# scopes copy and modify the parent scope submission data directory as
+# required by the nested tests and scopes. Tests will also clone the parent
+# scope submission data directory to optionally modify it, use and cleanup at
+# the end. Note that submission data directory can not be shared between
+# multiple submission handler processes. Also we need to make sure that
+# submission data directories are not cloned while being used by submission
+# handler scripts.
 #
 data_dir = $regex.replace($path_search('*/request.manifest', $src_base), \
                           '(.*)/.*', \
diff --git a/tests/upload/545f1f78-63ea-4acf-bcb8-37b2513a78c8/archive.tar b/tests/upload/545f1f78-63ea-4acf-bcb8-37b2513a78c8/archive.tar
new file mode 100644
index 0000000..d3b5b17
--- /dev/null
+++ b/tests/upload/545f1f78-63ea-4acf-bcb8-37b2513a78c8/archive.tar
diff --git a/tests/upload/545f1f78-63ea-4acf-bcb8-37b2513a78c8/request.manifest b/tests/upload/545f1f78-63ea-4acf-bcb8-37b2513a78c8/request.manifest
new file mode 100644
index 0000000..c59303b
--- /dev/null
+++ b/tests/upload/545f1f78-63ea-4acf-bcb8-37b2513a78c8/request.manifest
@@ -0,0 +1,22 @@
+: 1
+id: 545f1f78-63ea-4acf-bcb8-37b2513a78c8
+session: 4d4c8b36-56c5-42e0-91d1-58bfd1228002/libhello/1.1.0+10/x86_64-linux-\
+gnu/linux_fedora_37-gcc_12.2-bindist/default/queue/0.3.0/1683122318585120886
+instance: archive
+archive: archive.tar
+sha256sum: 4fa79e4e11a03db321514800806a2b0a3a8eef9375dc22963f4e5a16764c0d5e
+timestamp: 2023-05-08T09:18:20Z
+name: libhello
+version: 1.1.0+10
+project: hello
+target-config: linux_fedora_37-gcc_12.2-bindist
+package-config: default
+target: x86_64-linux-gnu
+tenant: 4d4c8b36-56c5-42e0-91d1-58bfd1228002
+toolchain-name: queue
+toolchain-version: 0.3.0
+repository-name: git:build2.org/var/scm/hello/libhello#master@7f62790591b66bd\
+a248140013bdbd12bf078c2a2
+machine-name: linux_fedora_37-bindist-gcc_12.2
+machine-summary: Fedora Linux 37 with system-default GCC 12.2.1 and bpkg-pkg-\
+bindist prerequisites
diff --git a/tests/upload/buildfile b/tests/upload/buildfile
new file mode 100644
index 0000000..32d7720
--- /dev/null
+++ b/tests/upload/buildfile
@@ -0,0 +1,13 @@
+# file      : tests/upload/buildfile
+# license   : MIT; see accompanying LICENSE file
+
+dir = ../../brep/handler/upload/
+
+include $dir
+
+commons = data
+
+./: testscript{* -{$commons}} common_testscript{$commons} {*/ -test/}{**} \
+    $dir/exe{brep-upload-bindist}
+
+testscript{upload-bindist}@./: test = $out_base/$dir/brep-upload-bindist
diff --git a/tests/upload/data.testscript b/tests/upload/data.testscript
new file mode 100644
index 0000000..3d3eede
--- /dev/null
+++ b/tests/upload/data.testscript
@@ -0,0 +1,34 @@
+# file      : tests/upload/data.testscript
+# license   : MIT; see accompanying LICENSE file
+
+# Pre-created upload data directory that will be copied by subsequent tests
+# and scope setup commands. The common approach will be that group scopes copy
+# and modify the parent scope upload data directory as required by the nested
+# tests and scopes. Tests will also clone the parent scope upload data
+# directory to optionally modify it, use and cleanup at the end. Note that
+# upload data directory can not be shared between multiple upload handler
+# processes. Also we need to make sure that upload data directories are not
+# cloned while being used by upload handler scripts.
+#
+data_dir = $regex.replace($path_search('*/request.manifest', $src_base), \
+                          '(.*)/.*', \
+                          '\1')
+
+request_id = "$data_dir"
+
+# Copy the original upload data directory to the root scope.
+#
++cp -r $src_base/$data_dir ./
+
+root_data_dir = $~/$data_dir
+
+# The most commonly used upload data directory cloning command that copies it
+# from the parent scope working directory.
+#
+clone_data       = [cmdline] cp --no-cleanup -r ../$data_dir ./
+clone_data_clean = [cmdline] cp --no-cleanup -r ../$data_dir ./ &$data_dir/***
+
+# Clones the original upload data directory.
+#
+clone_root_data       = [cmdline] cp --no-cleanup -r $root_data_dir ./
+clone_root_data_clean = [cmdline] cp --no-cleanup -r $root_data_dir ./ &$data_dir/***
diff --git a/tests/upload/upload-bindist.testscript b/tests/upload/upload-bindist.testscript
new file mode 100644
index 0000000..d43c567
--- /dev/null
+++ b/tests/upload/upload-bindist.testscript
@@ -0,0 +1,126 @@
+# file      : tests/upload/upload-bindist.testscript
+# license   : MIT; see accompanying LICENSE file
+
+.include data.testscript
+
+: args
+{
+  : no-dir
+  :
+  $* 2>>~%EOE% != 0
+  %\[.+\] \[brep:error\] \[ref \] \[brep-upload-bindist\]: usage: .+brep-upload-bindist \[<options>\] <root> <dir>%
+  EOE
+
+  : no-root
+  :
+  $* $~/dir 2>>~%EOE% != 0
+  %\[.+\] \[brep:error\] \[ref dir\] \[brep-upload-bindist\]: usage: .+brep-upload-bindist \[<options>\] <root> <dir>%
+  EOE
+
+  : root-not-exist
+  :
+  : While at it, also test that the trailing slash is stripped from the
+  : directory paths.
+  :
+  $* $~/root/ $~/dir/ 2>>~%EOE% != 0
+    %\[.+\] \[brep:error\] \[ref dir\] \[brep-upload-bindist\]: '.+root' does not exist or is not a directory%
+    EOE
+
+  : data-not-exist
+  :
+  mkdir root;
+  $* $~/root $~/dir 2>>~%EOE% != 0
+    %\[.+\] \[brep:error\] \[ref dir\] \[brep-upload-bindist\]: '.+dir' does not exist or is not a directory%
+    EOE
+}
+
+: success
+:
+{
+  mkdir --no-cleanup bindist-root/ &bindist-root/***;
+
+  # Test the first upload.
+  #
+  $clone_data;
+
+  $* $~/bindist-root/ $~/$data_dir >>"EOO";
+    : 1
+    status: 200
+    message: binary distribution packages are published
+    reference: $request_id
+    EOO
+
+  timestamp = '2023-05-08T09:18:20Z';
+  tenant    = '4d4c8b36-56c5-42e0-91d1-58bfd1228002';
+  dir = [dir_path] bindist-root/$tenant/archive/fedora35/hello/libhello/1.1.0+10/;
+
+  test -f $dir/default/libhello-1.0.0+10.tar.xz;
+  test -f $dir/default-$timestamp/libhello-1.0.0+10.tar.xz;
+
+  # Repeat the upload using the same timestamp to make sure that we properly
+  # handle this situation (by adding the retry number as a suffix to the
+  # package configuration directory name).
+  #
+  $clone_data;
+
+  $* $~/bindist-root/ $~/$data_dir >>"EOO" &bindist-root/***;
+    : 1
+    status: 200
+    message: binary distribution packages are published
+    reference: $request_id
+    EOO
+
+  test -f $dir/default/libhello-1.0.0+10.tar.xz;
+  test -f $dir/default-$timestamp-0/libhello-1.0.0+10.tar.xz;
+  test -d $dir/default-$timestamp/ != 0;
+
+  # Test the second upload without --keep-previous option.
+  #
+  data_dir2   = 22222222-2222-2222-2222-222222222222;
+  request_id2 = $data_dir2;
+  timestamp2  = '2023-05-09T09:18:20Z';
+
+  cp --no-cleanup -r ../$data_dir ./$data_dir2;
+
+  sed -i -e "s%^\(id:\) .+\$%\\1 $request_id2%" \
+      $data_dir2/request.manifest;
+
+  sed -i -e "s%^\(timestamp:\) .+\$%\\1 $timestamp2%" \
+      $data_dir2/request.manifest;
+
+  $* $~/bindist-root/ $~/$data_dir2 >>"EOO";
+    : 1
+    status: 200
+    message: binary distribution packages are published
+    reference: $request_id2
+    EOO
+
+  test -f $dir/default/libhello-1.0.0+10.tar.xz;
+  test -f $dir/default-$timestamp2/libhello-1.0.0+10.tar.xz;
+  test -d $dir/default-$timestamp.0/ != 0;
+
+  # Test the third upload with --keep-previous option.
+  #
+  data_dir3   = 33333333-3333-3333-3333-333333333333;
+  request_id3 = $data_dir3;
+  timestamp3  = '2023-05-10T09:18:20Z';
+
+  cp --no-cleanup -r ../$data_dir ./$data_dir3;
+
+  sed -i -e "s%^\(id:\) .+\$%\\1 $request_id3%" \
+      $data_dir3/request.manifest;
+
+  sed -i -e "s%^\(timestamp:\) .+\$%\\1 $timestamp3%" \
+      $data_dir3/request.manifest;
+
+  $* --keep-previous $~/bindist-root/ $~/$data_dir3 >>"EOO";
+    : 1
+    status: 200
+    message: binary distribution packages are published
+    reference: $request_id3
+    EOO
+
+  test -f $dir/default/libhello-1.0.0+10.tar.xz;
+  test -f $dir/default-$timestamp3/libhello-1.0.0+10.tar.xz;
+  test -f $dir/default-$timestamp2/libhello-1.0.0+10.tar.xz
+}