updated 6-provision-project-jobs.sh to limit concurrent number of jobs to improved reliability

1 files changed, 116 insertions, 86 deletions

diff --git a/auto/helper-logic b/auto/helper-logic
index 525365d..200098d 100644
--- a/auto/helper-logic
+++ b/auto/helper-logic
@@ -324,9 +324,6 @@ function ProvisionJob {
   # Extract the job name.
   JOB_NAME=$(echo "$1" | jq -r .name)
 
-  # Print the job name.
-  PrintEmptyIndicator "$JOB_NAME"
-
   # Parse the json.
   DESCRIPTION=$(echo "$1" | jq -r .description)
   GIT_URL=$(echo "$1" | jq -r .gitUrl)
@@ -377,10 +374,7 @@ function ProvisionJob {
     UrlPush "/createItem?name=$JOB_NAME" "$JOBPATH"
   fi
 
-  if [ $? -eq 0 ]; then
-    ClearPreviousLine
-    PrintOkIndicator "$JOB_NAME"
-  else
+  if [ $? -ne 0 ]; then
     ClearPreviousLine
     PrintErrorIndicator "Failed to create job: $JOB_NAME"
   fi
@@ -390,29 +384,19 @@ function ProvisionJobs {
   # Create the jobs.
   echo "Provisioning jobs in Jenkins..."
 
-  # Declare the JOBS list.
-  declare -a JOBS
-
   # Make sure the work directory exists.
   mkdir -p "$WORKDIR"
 
+  # Declare the JOBS list.
+  declare -a JOBS
+
   # Read one job at a time from the json file.
+  # Store the single jobs in the JOBS array.
   while read item
   do
     # Find the name of the job.
     JOB_NAME=$(echo "$item" | jq -r .name)
 
-    # Find the branches for the job.
-    BRANCH_REGEX=$(echo "$item" | jq -r .branchRegex)
-
-    # Split branch regex
-    BRANCHES=$(sed -e 's/(//g' -e 's/)//g' -e 's/|/\n/g' <<< "${BRANCH_REGEX}")
-
-    for BRANCH in $BRANCHES
-    do
-      JOBS+=( "$JOB_NAME|$BRANCH" )
-    done
-
     # Branch filter
     DESCRIPTION=$(echo "$item" | jq -r .description)
     if [ "$DESCRIPTION" == "$EXCLUDED_DESCRIPTION" ]; then
@@ -420,89 +404,135 @@ function ProvisionJobs {
       continue
     fi
 
-    # Provision the job.
-    ProvisionJob $item
+    JOBS+=( "$item" )
   done < <(cat $1 | jq -c '.[]')
 
-  echo
-  echo "Waiting for jobs to be provisioned in Jenkins..."
-  echo
-  echo "Total number of jobs: ${#JOBS[@]}"
-  echo
+  BUILD_FAILED="false"
 
-  for job in "${JOBS[@]}"; do
-    jobSplit=(${job//|/ })
-    JOB_NAME="${jobSplit[0]}"
-    JOB_BRANCH="${jobSplit[1]}"
+  echo "Total number of pipelines: ${#JOBS[@]}"
 
-    PrintJobNotStarted $JOB_NAME $JOB_BRANCH
-  done
+  # Has the CONCURRENT_JOBS_COUNT environment variable been set?
+  if [ "$(printenv CONCURRENT_JOBS_COUNT)" ]; then
+    # Yes, so we use that value to define the number of concurrent jobs.
+    CONCURRENT_JOBS_COUNT=$(printenv CONCURRENT_JOBS_COUNT)
+    echo "Concurrent jobs: $CONCURRENT_JOBS_COUNT (Overridden by the CONCURRENT_JOBS_COUNT environment variable)"
+  else
+    # No, so we read the number of CPU cores, and use that.
+    CONCURRENT_JOBS_COUNT="$(nproc)"
 
-  while :
+    # If less than 4 CPU cores is found, we set it to 4.
+    CONCURRENT_JOBS_COUNT=$(( $CONCURRENT_JOBS_COUNT < 4 ? 4 : $CONCURRENT_JOBS_COUNT))
+    echo "Concurrent jobs: $CONCURRENT_JOBS_COUNT (Can be overridden by setting the CONCURRENT_JOBS_COUNT environment variable)"
+  fi
+
+  echo
+
+  declare -A JOB_RESULTS
+
+  # While there are more jobs to build.
+  while [ ${#JOBS[@]} -gt 0 ]
   do
-    # Define the current state, and set it to true.
-    # If anything isn't ready, we set it to false.
-    FINISHED_THIS_RUN="true"
+    # Create a list to store the current "chunk" of jobs in.
+    declare -a CURRENT_JOBS
 
-    # Move the cursor up to the top of the list on screen.
-    for job in "${JOBS[@]}"; do
-      tput cuu1
-    done
+    # Make sure it is empty.
+    CURRENT_JOBS=()
 
-    # Check the status for each job
-    for job in "${JOBS[@]}"; do
-      jobSplit=(${job//|/ })
-      JOB_NAME="${jobSplit[0]}"
-      JOB_BRANCH="${jobSplit[1]}"
+    # While the number of jobs in the list is less than the concurrent job count.
+    while [ ${#CURRENT_JOBS[@]} -lt $CONCURRENT_JOBS_COUNT ]
+    do
+      # Are there any jobs left in the main list?
+      if [ ${#JOBS[@]} -gt 0 ]; then
+        # Yes, so we take the first element.
+        item=${JOBS[0]}
+        # Remove it from the main list.
+        JOBS=("${JOBS[@]:1}")
 
-      # Branch filter
-      if ! echo $JOB_BRANCH | grep -E "$SELECTED_BRANCH_REGEX" > /dev/null; then
-        PrintJobExcluded $JOB_NAME $JOB_BRANCH
-        continue
-      fi
+        ProvisionJob $item
 
-      completed=false
-      started=true
-      url="${JENKINS_URL}/job/${JOB_NAME}/job/${JOB_BRANCH}/api/json"
-      nextBuildNumber=$(curl -s -g "$url" | jq -r .nextBuildNumber 2> /dev/null)
-      if [ "$nextBuildNumber" != "" ] && [ $nextBuildNumber -gt 2 ]; then
-        completed=true
+        # Find the branches for the job.
+        BRANCH_REGEX=$(echo "$item" | jq -r .branchRegex)
+        # Split branch regex
+        BRANCHES=$(sed -e 's/(//g' -e 's/)//g' -e 's/|/\n/g' <<< "${BRANCH_REGEX}")
+        for BRANCH in $BRANCHES
+        do
+          # Branch filter
+          if ! echo $BRANCH | grep -E "$SELECTED_BRANCH_REGEX" > /dev/null; then
+            continue
+          fi
+
+          PrintJobNotStarted $JOB_NAME $BRANCH
+
+          # And add it to our current list.
+          CURRENT_JOBS+=( "$JOB_NAME|$BRANCH" )
+        done
       else
-        url="${JENKINS_URL}/job/${JOB_NAME}/job/${JOB_BRANCH}/1/api/json"
-        if [ "$(curl -o /dev/null -s -w "%{http_code}\n" "${url}")" -eq 404 ]; then
-          started=false
-        fi
+        # No more jobs, so we break out of the loop.
+        break
       fi
+    done
 
-      if ! $started; then
-        # The indexing hasn't started for this branch yet.
-        PrintJobNotStarted $JOB_NAME $JOB_BRANCH
-        FINISHED_THIS_RUN="false"
-      else
-        # Indexing has started.
-        if ! $completed; then
-          RESULT=$(curl -Ss -g --fail-with-body "$url" | jq -r .result)
-          if [[ "${RESULT}" != "SUCCESS" ]]; then
-            # But it hasn't finished yet.
-            PrintJobRunning $JOB_NAME $JOB_BRANCH
-            FINISHED_THIS_RUN="false"
-          else
-            completed=true
+    while :
+    do
+      # Define the current state, and set it to true.
+      # If anything isn't ready, we set it to false.
+      FINISHED_THIS_RUN="true"
+
+      # Move the cursor up one line for each job in the current job list.
+      for job in "${CURRENT_JOBS[@]}"; do
+        tput cuu1
+      done
+
+      # For each job in the current job list.
+      for job in "${CURRENT_JOBS[@]}"; do
+        # Split into name and branch.
+        jobSplit=(${job//|/ })
+        JOB_NAME="${jobSplit[0]}"
+        JOB_BRANCH="${jobSplit[1]}"
+
+        completed=false
+        started=true
+        url="${JENKINS_URL}/job/${JOB_NAME}/job/${JOB_BRANCH}/api/json"
+        nextBuildNumber=$(curl -s -g "$url" | jq -r .nextBuildNumber 2> /dev/null)
+        if [ "$nextBuildNumber" != "" ] && [ $nextBuildNumber -gt 2 ]; then
+          completed=true
+        else
+          url="${JENKINS_URL}/job/${JOB_NAME}/job/${JOB_BRANCH}/1/api/json"
+          if [ "$(curl -o /dev/null -s -w "%{http_code}\n" "${url}")" -eq 404 ]; then
+            started=false
           fi
         fi
-        if $completed; then
-          PrintJobCompleted $JOB_NAME $JOB_BRANCH
+
+        if ! $started; then
+          # The indexing hasn't started for this branch yet.
+          PrintJobNotStarted $JOB_NAME $JOB_BRANCH
+          FINISHED_THIS_RUN="false"
+        else
+          # Indexing has started.
+          if ! $completed; then
+            RESULT=$(curl -Ss -g --fail-with-body "$url" | jq -r .result)
+            if [[ "${RESULT}" != "SUCCESS" ]]; then
+              # But it hasn't finished yet.
+              PrintJobRunning $JOB_NAME $JOB_BRANCH
+              FINISHED_THIS_RUN="false"
+            else
+              completed=true
+            fi
+          fi
+          if $completed; then
+            PrintJobCompleted $JOB_NAME $JOB_BRANCH
+          fi
         fi
+      done
+
+      # When we get here, if FINISHED_THIS_RUN is true, everything is ready
+      if [[ $FINISHED_THIS_RUN == "true" ]]; then
+        break;
+      else
+        # Sleep 5 seconds
+        sleep 5
       fi
     done
-
-    # When we get here, if FINISHED_THIS_RUN is true, everything is ready
-    if [[ $FINISHED_THIS_RUN == "true" ]]; then
-      break;
-    else
-      # Sleep 5 seconds
-      sleep 5
-    fi
   done
 }