Jenkins: Make the pipeline more resilient to node dropouts

.ci/Jenkinsfile

@@ -12,7 +12,7 @@
  *
  * Authors:	RichardG, <richardg867@gmail.com>
  *
- *		Copyright 2021 RichardG.
+ *		Copyright 2021-2022 RichardG.
  */
 
 def repository = 'https://github.com/86Box/86Box.git'
@@ -105,9 +105,9 @@ def gitClone(repository, branch) {
 		} else if (env.GIT_COMMIT != scmVars.GIT_COMMIT) {
 			/* Checkout the commit read from the polling log. */
 			if (isUnix())
-				sh "git checkout ${env.GIT_COMMIT} || exit 0"
+				sh returnStatus: true, script: "git checkout ${env.GIT_COMMIT}"
 			else
-				bat "git checkout ${env.GIT_COMMIT} || exit /b 0"
+				bat returnStatus: true, script: "git checkout ${env.GIT_COMMIT}"
 		}
 		println "[-] Using git commit [${env.GIT_COMMIT}]"
 
@@ -122,16 +122,26 @@ def gitClone(repository, branch) {
 
 def removeDir(dir) {
 	if (isUnix())
-		sh "rm -rf \"$dir\" || exit 0"
+		return sh(returnStatus: true, script: "rm -rf \"$dir\"")
 	else
-		bat "if exist \"$dir\" rd /s /q \"$dir\" & exit /b 0"
+		return bat(returnStatus: true, script: "rd /s /q \"$dir\"")
 }
 
 def runBuild(args) {
 	if (isUnix())
-		sh "chmod u+x \"$WORKSPACE/.ci/build.sh\" && exec \"$WORKSPACE/.ci/build.sh\" $args"
+		return sh(returnStatus: true, script: "chmod u+x \"$WORKSPACE/.ci/build.sh\" && exec \"$WORKSPACE/.ci/build.sh\" $args")
 	else
-		bat "C:\\msys64\\msys2_shell.cmd -msys2 -defterm -here -no-start -c 'exec \"\$(cygpath -u \\'%WORKSPACE%\\')/.ci/build.sh\" $args'"
+		return bat(returnStatus: true, script: "C:\\msys64\\msys2_shell.cmd -msys2 -defterm -here -no-start -c 'exec \"\$(cygpath -u \\'%WORKSPACE%\\')/.ci/build.sh\" $args'")
+}
+
+def failStage() {
+	/* Mark that a failure occurred. */
+	anyFailure = true
+
+	/* Force this stage to fail. */
+	catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+		def x = 1 / 0
+	}
 }
 
 pipeline {
@@ -142,7 +152,6 @@ pipeline {
 	}
 
 	options {
-		disableConcurrentBuilds()
 		quietPeriod(0)
 	}
 
@@ -155,6 +164,7 @@ pipeline {
 	stages {
 		stage('Source Tarball') {
 			agent none
+			failFast false
 
 			steps {
 				script {
@@ -178,85 +188,95 @@ pipeline {
 
 					/* Adding to the above, run a git clone as soon as possible on any node
 					   to further avoid race conditions caused by busy node executor delays. */
-					node {
+					retry(10) {
-						/* Run git clone. */
+						node('citadel') {
-						gitClone(repository, branch)
-
-						/* Clean workspace, in case this is running in a non-build node. */
-						cleanWs()
-					}
-
-					/* Create source tarball. */
-					node('Linux') {
-						try {
 							/* Run git clone. */
 							gitClone(repository, branch)
 
-							/* Switch to temp directory. */
+							/* Clean workspace, in case this is running in a non-build node. */
-							dir("${env.WORKSPACE_TMP}/output") {
+							cleanWs()
-								/* Run source tarball creation process. */
+						}
-								def packageName = "${env.JOB_BASE_NAME}-Source-b${env.BUILD_NUMBER}"
+					}
-								runBuild("-s \"$packageName\"")
 
-								/* Archive resulting artifacts. */
+					/* Create source tarball. */
-								archiveArtifacts artifacts: "$packageName*"
+					try {
-							}
+						retry(10) {
+							node('Linux') {
+								/* Run git clone. */
+								gitClone(repository, branch)
 
-							/* Clean up. */
+								/* Switch to temp directory. */
-							removeDir("${env.WORKSPACE_TMP}/output")
+								dir("${env.WORKSPACE_TMP}/output") {
-						} catch (e) {
+									/* Run source tarball creation process. */
-							/* Mark that a failure occurred. */
+									def packageName = "${env.JOB_BASE_NAME}-Source-b${env.BUILD_NUMBER}"
-							anyFailure = true
+									if (runBuild("-s \"$packageName\"") == 0) {
+										/* Archive resulting artifacts. */
+										archiveArtifacts artifacts: "$packageName*"
+									} else {
+										/* Fail this stage. */
+										failStage()
+									}
+								}
 
-							/* Force this stage to fail. */
+								/* Clean up. */
-							catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+								removeDir("${env.WORKSPACE_TMP}/output")
-								throw e;
 							}
 						}
+					} catch (e) {
+						/* Fail this stage. */
+						failStage()
 					}
 
 					/* Build here to avoid creating a redundant parent stage on the stage view. */
 					osArchs.each { os, thisOsArchs ->
+						def combinations = [:]
 						thisOsArchs.each { arch ->
 							def thisArchDynarecs = dynarecArchs[arch]
 							if (!thisArchDynarecs)
 								thisArchDynarecs = ['NoDR']
 							thisArchDynarecs.each { dynarec ->
 								presets.each { preset ->
-									node(os) {
+									def combination = "$os $arch $dynarec $preset"
-										stage("$os $arch $dynarec $preset") {
+									combinations[combination] = {
-											try {
+										try {
-												/* Run git clone. */
+											retry(10) {
-												gitClone(repository, branch)
+												node(os) {
+													stage(combination) {
+														/* Run git clone. */
+														gitClone(repository, branch)
 
-												/* Switch to output directory. */
+														/* Switch to output directory. */
-												dir("${env.WORKSPACE_TMP}/output") {
+														dir("${env.WORKSPACE_TMP}/output") {
-													/* Run build process. */
+															/* Run build process. */
-													def packageName = "${env.JOB_BASE_NAME}${dynarecSlugs[dynarec]}${presetSlugs[preset]}-$os-$arch-b${env.BUILD_NUMBER}"
+															def packageName = "${env.JOB_BASE_NAME}${dynarecSlugs[dynarec]}${presetSlugs[preset]}-$os-$arch-b${env.BUILD_NUMBER}"
-													dir("${dynarecNames[dynarec]}/$os - ${archNames[arch]}") {
+															def ret = -1
-														runBuild("-b \"$packageName\" \"$arch\" ${presetFlags[preset]} ${dynarecFlags[dynarec]} ${osFlags[os]} -D \"BUILD_TYPE=$BUILD_TYPE\" -D \"EMU_BUILD=build ${env.BUILD_NUMBER}\" -D \"EMU_BUILD_NUM=${env.BUILD_NUMBER}\"")
+															dir("${dynarecNames[dynarec]}/$os - ${archNames[arch]}") {
+																ret = runBuild("-b \"$packageName\" \"$arch\" ${presetFlags[preset]} ${dynarecFlags[dynarec]} ${osFlags[os]} -D \"BUILD_TYPE=$BUILD_TYPE\" -D \"EMU_BUILD=build ${env.BUILD_NUMBER}\" -D \"EMU_BUILD_NUM=${env.BUILD_NUMBER}\"")
+															}
+
+															if (ret == 0) {
+																/* Archive resulting artifacts. */
+																archiveArtifacts artifacts: "**/**/$packageName*"
+															} else {
+																/* Fail this stage. */
+																failStage()
+															}
+														}
+
+														/* Clean up. */
+														removeDir("${env.WORKSPACE_TMP}/output")
 													}
-
-													/* Archive resulting artifacts. */
-													archiveArtifacts artifacts: "**/**/$packageName*"
-												}
-
-												/* Clean up. */
-												removeDir("${env.WORKSPACE_TMP}/output")
-											} catch (e) {
-												/* Mark that a failure occurred. */
-												anyFailure = true
-
-												/* Force this stage to fail. */
-												catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-													throw e;
 												}
 											}
+										} catch (e) {
+											/* Fail this stage. */
+											failStage()
 										}
 									}
 								}
 							}
 						}
+						parallel combinations
 					}
 				}
 			}
@@ -287,7 +307,7 @@ pipeline {
 							    scmWebUrl: commitBrowser
 
 						/* Notify IRC, which needs a node for whatever reason. */
-						node {
+						node('citadel') {
 							ircNotify()
 						}
 					} catch (e) {