Jenkins: Make the pipeline more resilient to node dropouts

.ci/Jenkinsfile

@@ -12,7 +12,7 @@
  *
  * Authors:	RichardG, <richardg867@gmail.com>
  *
- *		Copyright 2021 RichardG.
+ *		Copyright 2021-2022 RichardG.
  */
 
 def repository = 'https://github.com/86Box/86Box.git'
@@ -105,9 +105,9 @@ def gitClone(repository, branch) {
 		} else if (env.GIT_COMMIT != scmVars.GIT_COMMIT) {
 			/* Checkout the commit read from the polling log. */
 			if (isUnix())
-				sh "git checkout ${env.GIT_COMMIT} || exit 0"
+				sh returnStatus: true, script: "git checkout ${env.GIT_COMMIT}"
 			else
-				bat "git checkout ${env.GIT_COMMIT} || exit /b 0"
+				bat returnStatus: true, script: "git checkout ${env.GIT_COMMIT}"
 		}
 		println "[-] Using git commit [${env.GIT_COMMIT}]"
 
@@ -122,16 +122,26 @@ def gitClone(repository, branch) {
 
 def removeDir(dir) {
 	if (isUnix())
-		sh "rm -rf \"$dir\" || exit 0"
+		return sh(returnStatus: true, script: "rm -rf \"$dir\"")
 	else
-		bat "if exist \"$dir\" rd /s /q \"$dir\" & exit /b 0"
+		return bat(returnStatus: true, script: "rd /s /q \"$dir\"")
 }
 
 def runBuild(args) {
 	if (isUnix())
-		sh "chmod u+x \"$WORKSPACE/.ci/build.sh\" && exec \"$WORKSPACE/.ci/build.sh\" $args"
+		return sh(returnStatus: true, script: "chmod u+x \"$WORKSPACE/.ci/build.sh\" && exec \"$WORKSPACE/.ci/build.sh\" $args")
 	else
-		bat "C:\\msys64\\msys2_shell.cmd -msys2 -defterm -here -no-start -c 'exec \"\$(cygpath -u \\'%WORKSPACE%\\')/.ci/build.sh\" $args'"
+		return bat(returnStatus: true, script: "C:\\msys64\\msys2_shell.cmd -msys2 -defterm -here -no-start -c 'exec \"\$(cygpath -u \\'%WORKSPACE%\\')/.ci/build.sh\" $args'")
+}
+
+def failStage() {
+	/* Mark that a failure occurred. */
+	anyFailure = true
+
+	/* Force this stage to fail. */
+	catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+		def x = 1 / 0
+	}
 }
 
 pipeline {
@@ -142,7 +152,6 @@ pipeline {
 	}
 
 	options {
-		disableConcurrentBuilds()
 		quietPeriod(0)
 	}
 
@@ -155,6 +164,7 @@ pipeline {
 	stages {
 		stage('Source Tarball') {
 			agent none
+			failFast false
 
 			steps {
 				script {
@@ -178,17 +188,20 @@ pipeline {
 
 					/* Adding to the above, run a git clone as soon as possible on any node
 					   to further avoid race conditions caused by busy node executor delays. */
-					node {
+					retry(10) {
+						node('citadel') {
 							/* Run git clone. */
 							gitClone(repository, branch)
 
 							/* Clean workspace, in case this is running in a non-build node. */
 							cleanWs()
 						}
+					}
 
 					/* Create source tarball. */
-					node('Linux') {
 					try {
+						retry(10) {
+							node('Linux') {
 								/* Run git clone. */
 								gitClone(repository, branch)
 
@@ -196,36 +209,39 @@ pipeline {
 								dir("${env.WORKSPACE_TMP}/output") {
 									/* Run source tarball creation process. */
 									def packageName = "${env.JOB_BASE_NAME}-Source-b${env.BUILD_NUMBER}"
-								runBuild("-s \"$packageName\"")
+									if (runBuild("-s \"$packageName\"") == 0) {
-
 										/* Archive resulting artifacts. */
 										archiveArtifacts artifacts: "$packageName*"
+									} else {
+										/* Fail this stage. */
+										failStage()
+									}
 								}
 
 								/* Clean up. */
 								removeDir("${env.WORKSPACE_TMP}/output")
+							}
+						}
 					} catch (e) {
-							/* Mark that a failure occurred. */
+						/* Fail this stage. */
-							anyFailure = true
+						failStage()
-
-							/* Force this stage to fail. */
-							catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-								throw e;
-							}
-						}
 					}
 
 					/* Build here to avoid creating a redundant parent stage on the stage view. */
 					osArchs.each { os, thisOsArchs ->
+						def combinations = [:]
 						thisOsArchs.each { arch ->
 							def thisArchDynarecs = dynarecArchs[arch]
 							if (!thisArchDynarecs)
 								thisArchDynarecs = ['NoDR']
 							thisArchDynarecs.each { dynarec ->
 								presets.each { preset ->
-									node(os) {
+									def combination = "$os $arch $dynarec $preset"
-										stage("$os $arch $dynarec $preset") {
+									combinations[combination] = {
 										try {
+											retry(10) {
+												node(os) {
+													stage(combination) {
 														/* Run git clone. */
 														gitClone(repository, branch)
 
@@ -233,30 +249,34 @@ pipeline {
 														dir("${env.WORKSPACE_TMP}/output") {
 															/* Run build process. */
 															def packageName = "${env.JOB_BASE_NAME}${dynarecSlugs[dynarec]}${presetSlugs[preset]}-$os-$arch-b${env.BUILD_NUMBER}"
+															def ret = -1
 															dir("${dynarecNames[dynarec]}/$os - ${archNames[arch]}") {
-														runBuild("-b \"$packageName\" \"$arch\" ${presetFlags[preset]} ${dynarecFlags[dynarec]} ${osFlags[os]} -D \"BUILD_TYPE=$BUILD_TYPE\" -D \"EMU_BUILD=build ${env.BUILD_NUMBER}\" -D \"EMU_BUILD_NUM=${env.BUILD_NUMBER}\"")
+																ret = runBuild("-b \"$packageName\" \"$arch\" ${presetFlags[preset]} ${dynarecFlags[dynarec]} ${osFlags[os]} -D \"BUILD_TYPE=$BUILD_TYPE\" -D \"EMU_BUILD=build ${env.BUILD_NUMBER}\" -D \"EMU_BUILD_NUM=${env.BUILD_NUMBER}\"")
 															}
 
+															if (ret == 0) {
 																/* Archive resulting artifacts. */
 																archiveArtifacts artifacts: "**/**/$packageName*"
+															} else {
+																/* Fail this stage. */
+																failStage()
+															}
 														}
 
 														/* Clean up. */
 														removeDir("${env.WORKSPACE_TMP}/output")
+													}
+												}
+											}
 										} catch (e) {
-												/* Mark that a failure occurred. */
+											/* Fail this stage. */
-												anyFailure = true
+											failStage()
-
-												/* Force this stage to fail. */
-												catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-													throw e;
-												}
-											}
 										}
 									}
 								}
 							}
 						}
+						parallel combinations
 					}
 				}
 			}
@@ -287,7 +307,7 @@ pipeline {
 							    scmWebUrl: commitBrowser
 
 						/* Notify IRC, which needs a node for whatever reason. */
-						node {
+						node('citadel') {
 							ircNotify()
 						}
 					} catch (e) {