Retry periodically failing rmdir in bootstrapt scripts for Windows

4 files changed, 184 insertions, 44 deletions

diff --git a/etc/bootstrap/bbot-bootstrap-clang.bat b/etc/bootstrap/bbot-bootstrap-clang.bat
index 09bd0e7..79c4009 100644
--- a/etc/bootstrap/bbot-bootstrap-clang.bat
+++ b/etc/bootstrap/bbot-bootstrap-clang.bat
@@ -5,6 +5,45 @@ rem license   : TBC; see accompanying LICENSE file
 
 setlocal EnableExtensions EnableDelayedExpansion
 
+goto start
+
+@rem Remove a directory recursively. Noop for non-existent directory.
+@rem
+@rem Note that the underlying rmdir call periodically fails with the
+@rem 'directory not empty' diagnostics, in which case we make up to 100
+@rem removal attempts.
+@rem
+:rmdir_S
+
+  @if not exist %1\ goto :eof
+
+  @setlocal EnableDelayedExpansion
+
+  @set /A "i=0"
+
+  :rmdir_S_try
+
+  @rem Note that rmdir doesn't properly set errorlevel, so we check for the
+  @rem directory existence to detect the failure.
+  @rem
+  rmdir /S /Q %1\
+
+  @if exist %1\ (
+    if %i% neq 99 (
+      set /A "i=%i%+1"
+      goto rmdir_S_try
+    )
+
+    echo error: unable to remove %1\
+    endlocal
+    exit /b 1
+  )
+
+  @endlocal
+@goto :eof
+
+:start
+
 rem If the MSVC variable is set, then set up the environment via the MSVC
 rem command prompt rather than letting Clang find some default (note that
 rem in the latter case, clang++ should be in PATH).
@@ -60,18 +99,14 @@ rem
 @rem
 @rem Bootstrap the toolchain and then build bbot.
 @rem
-@if exist %BUILD%\bootstrap\ (
-  rmdir /S /Q %BUILD%\bootstrap
-  @if errorlevel 1 goto error
-)
+@call :rmdir_S %BUILD%\bootstrap
+@if errorlevel 1 goto error
 
 mkdir %BUILD%\bootstrap
 @if errorlevel 1 goto error
 
-@if exist %INSTALL% (
-  rmdir /S /Q %INSTALL%
-  @if errorlevel 1 goto error
-)
+@call :rmdir_S %INSTALL%
+@if errorlevel 1 goto error
 
 cd %BUILD%\bootstrap
 
@@ -147,14 +182,14 @@ cmd /C build-clang.bat --make mingw32-make --make -j%NUMBER_OF_PROCESSORS%^
 @if errorlevel 1 goto error
 
 cd ..
-rmdir /S /Q %bstrap%
+@call :rmdir_S %bstrap%
 @if errorlevel 1 goto error
 
 
 @rem
 @rem Build and install the bbot worker.
 @rem
-@for /D %%d in (build2-toolchain-*) do set "config=%%d"
+@for /D %%d in (build2-toolchain-*) do @set "config=%%d"
 cd %config%
 
 bpkg --fetch-timeout %TIMEOUT% build --yes bbot
@@ -164,7 +199,7 @@ bpkg install bbot
 @if errorlevel 1 goto error
 
 cd ..
-rmdir /S /Q %config%
+@call :rmdir_S %config%
 @if errorlevel 1 goto error
 
 @rem
diff --git a/etc/bootstrap/bbot-bootstrap-mingw.bat b/etc/bootstrap/bbot-bootstrap-mingw.bat
index a8bfc80..bea22d9 100644
--- a/etc/bootstrap/bbot-bootstrap-mingw.bat
+++ b/etc/bootstrap/bbot-bootstrap-mingw.bat
@@ -5,6 +5,45 @@ rem license   : TBC; see accompanying LICENSE file
 
 setlocal EnableExtensions EnableDelayedExpansion
 
+goto start
+
+@rem Remove a directory recursively. Noop for non-existent directory.
+@rem
+@rem Note that the underlying rmdir call periodically fails with the
+@rem 'directory not empty' diagnostics, in which case we make up to 100
+@rem removal attempts.
+@rem
+:rmdir_S
+
+  @if not exist %1\ goto :eof
+
+  @setlocal EnableDelayedExpansion
+
+  @set /A "i=0"
+
+  :rmdir_S_try
+
+  @rem Note that rmdir doesn't properly set errorlevel, so we check for the
+  @rem directory existence to detect the failure.
+  @rem
+  rmdir /S /Q %1\
+
+  @if exist %1\ (
+    if %i% neq 99 (
+      set /A "i=%i%+1"
+      goto rmdir_S_try
+    )
+
+    echo error: unable to remove %1\
+    endlocal
+    exit /b 1
+  )
+
+  @endlocal
+@goto :eof
+
+:start
+
 set "BUILD=C:\tmp"
 set "INSTALL=C:\build2"
 set "BOOTSTRAP=C:\bootstrap"
@@ -35,18 +74,14 @@ rem
 @rem
 @rem Bootstrap the toolchain and then build bbot.
 @rem
-@if exist %BUILD%\bootstrap\ (
-  rmdir /S /Q %BUILD%\bootstrap
-  @if errorlevel 1 goto error
-)
+@call :rmdir_S %BUILD%\bootstrap
+@if errorlevel 1 goto error
 
 mkdir %BUILD%\bootstrap
 @if errorlevel 1 goto error
 
-@if exist %INSTALL% (
-  rmdir /S /Q %INSTALL%
-  @if errorlevel 1 goto error
-)
+@call :rmdir_S %INSTALL%
+@if errorlevel 1 goto error
 
 cd %BUILD%\bootstrap
 
@@ -127,7 +162,7 @@ del trust
 @rem
 @rem Bootstrap and install the toolchain.
 @rem
-@for /D %%d in (build2-toolchain-*) do set "bstrap=%%d"
+@for /D %%d in (build2-toolchain-*) do @set "bstrap=%%d"
 cd %bstrap%
 
 @rem Bootstrap in parallel using GNU make (which is part of baseutils).
@@ -139,7 +174,7 @@ cmd /C build-mingw.bat --make mingw32-make --make -j%NUMBER_OF_PROCESSORS%^
 @if errorlevel 1 goto error
 
 cd ..
-rmdir /S /Q %bstrap%
+@call :rmdir_S %bstrap%
 @if errorlevel 1 goto error
 
 
@@ -156,7 +191,7 @@ bpkg install bbot
 @if errorlevel 1 goto error
 
 cd ..
-rmdir /S /Q %config%
+@call :rmdir_S %config%
 @if errorlevel 1 goto error
 
 @rem
diff --git a/etc/bootstrap/bbot-bootstrap-msvc-14.bat b/etc/bootstrap/bbot-bootstrap-msvc-14.bat
index 8d8bb89..efa1ecd 100644
--- a/etc/bootstrap/bbot-bootstrap-msvc-14.bat
+++ b/etc/bootstrap/bbot-bootstrap-msvc-14.bat
@@ -5,6 +5,45 @@ rem license   : TBC; see accompanying LICENSE file
 
 setlocal EnableExtensions EnableDelayedExpansion
 
+goto start
+
+@rem Remove a directory recursively. Noop for non-existent directory.
+@rem
+@rem Note that the underlying rmdir call periodically fails with the
+@rem 'directory not empty' diagnostics, in which case we make up to 100
+@rem removal attempts.
+@rem
+:rmdir_S
+
+  @if not exist %1\ goto :eof
+
+  @setlocal EnableDelayedExpansion
+
+  @set /A "i=0"
+
+  :rmdir_S_try
+
+  @rem Note that rmdir doesn't properly set errorlevel, so we check for the
+  @rem directory existence to detect the failure.
+  @rem
+  rmdir /S /Q %1\
+
+  @if exist %1\ (
+    if %i% neq 99 (
+      set /A "i=%i%+1"
+      goto rmdir_S_try
+    )
+
+    echo error: unable to remove %1\
+    endlocal
+    exit /b 1
+  )
+
+  @endlocal
+@goto :eof
+
+:start
+
 set "MSVC=C:\Program Files (x86)\Microsoft Visual Studio 14.0"
 set "VCVARS=%MSVC%\VC\vcvarsall.bat"
 set "VCARCH=amd64"
@@ -45,18 +84,14 @@ rem
 @rem
 @rem Bootstrap the toolchain and then build bbot.
 @rem
-@if exist %BUILD%\bootstrap\ (
-  rmdir /S /Q %BUILD%\bootstrap
-  @if errorlevel 1 goto error
-)
+@call :rmdir_S %BUILD%\bootstrap
+@if errorlevel 1 goto error
 
 mkdir %BUILD%\bootstrap
 @if errorlevel 1 goto error
 
-@if exist %INSTALL% (
-  rmdir /S /Q %INSTALL%
-  @if errorlevel 1 goto error
-)
+@call :rmdir_S %INSTALL%
+@if errorlevel 1 goto error
 
 cd %BUILD%\bootstrap
 
@@ -124,7 +159,7 @@ cmd /C build-msvc.bat --timeout %TIMEOUT% --install-dir %INSTALL%^
 @if errorlevel 1 goto error
 
 cd ..
-rmdir /S /Q %bstrap%
+@call :rmdir_S %bstrap%
 @if errorlevel 1 goto error
 
 
@@ -137,7 +172,7 @@ rmdir /S /Q %bstrap%
 @rem
 @rem Build and install the bbot worker.
 @rem
-@for /D %%d in (build2-toolchain-*) do set "config=%%d"
+@for /D %%d in (build2-toolchain-*) do @set "config=%%d"
 cd %config%
 
 bpkg --fetch-timeout %TIMEOUT% build --yes bbot
@@ -147,7 +182,7 @@ bpkg install bbot
 @if errorlevel 1 goto error
 
 cd ..
-rmdir /S /Q %config%
+@call :rmdir_S %config%
 @if errorlevel 1 goto error
 
 @rem
diff --git a/etc/bootstrap/bbot-bootstrap-msvc.bat b/etc/bootstrap/bbot-bootstrap-msvc.bat
index 6dd9579..ba08fd5 100644
--- a/etc/bootstrap/bbot-bootstrap-msvc.bat
+++ b/etc/bootstrap/bbot-bootstrap-msvc.bat
@@ -5,6 +5,45 @@ rem license   : TBC; see accompanying LICENSE file
 
 setlocal EnableExtensions EnableDelayedExpansion
 
+goto start
+
+@rem Remove a directory recursively. Noop for non-existent directory.
+@rem
+@rem Note that the underlying rmdir call periodically fails with the
+@rem 'directory not empty' diagnostics, in which case we make up to 100
+@rem removal attempts.
+@rem
+:rmdir_S
+
+  @if not exist %1\ goto :eof
+
+  @setlocal EnableDelayedExpansion
+
+  @set /A "i=0"
+
+  :rmdir_S_try
+
+  @rem Note that rmdir doesn't properly set errorlevel, so we check for the
+  @rem directory existence to detect the failure.
+  @rem
+  rmdir /S /Q %1\
+
+  @if exist %1\ (
+    if %i% neq 99 (
+      set /A "i=%i%+1"
+      goto rmdir_S_try
+    )
+
+    echo error: unable to remove %1\
+    endlocal
+    exit /b 1
+  )
+
+  @endlocal
+@goto :eof
+
+:start
+
 rem set "MSVC=C:\Program Files (x86)\Microsoft Visual Studio\2017\Community"
 set "MSVC=C:\Program Files (x86)\Microsoft Visual Studio\2019\Community"
 set "VCVARS=VC\Auxiliary\Build\vcvars64.bat"
@@ -44,18 +83,14 @@ rem
 @rem
 @rem Bootstrap the toolchain and then build bbot.
 @rem
-@if exist %BUILD%\bootstrap\ (
-  rmdir /S /Q %BUILD%\bootstrap
-  @if errorlevel 1 goto error
-)
+@call :rmdir_S %BUILD%\bootstrap
+@if errorlevel 1 goto error
 
 mkdir %BUILD%\bootstrap
 @if errorlevel 1 goto error
 
-@if exist %INSTALL% (
-  rmdir /S /Q %INSTALL%
-  @if errorlevel 1 goto error
-)
+@call :rmdir_S %INSTALL%
+@if errorlevel 1 goto error
 
 cd %BUILD%\bootstrap
 
@@ -123,7 +158,7 @@ cmd /C build-msvc.bat --timeout %TIMEOUT% --install-dir %INSTALL%^
 @if errorlevel 1 goto error
 
 cd ..
-rmdir /S /Q %bstrap%
+@call :rmdir_S %bstrap%
 @if errorlevel 1 goto error
 
 
@@ -136,7 +171,7 @@ rmdir /S /Q %bstrap%
 @rem
 @rem Build and install the bbot worker.
 @rem
-@for /D %%d in (build2-toolchain-*) do set "config=%%d"
+@for /D %%d in (build2-toolchain-*) do @set "config=%%d"
 cd %config%
 
 bpkg --fetch-timeout %TIMEOUT% build --yes bbot
@@ -146,7 +181,7 @@ bpkg install bbot
 @if errorlevel 1 goto error
 
 cd ..
-rmdir /S /Q %config%
+@call :rmdir_S %config%
 @if errorlevel 1 goto error
 
 @rem