SL-15572 - retry logic for codeticket.py commands

build.sh

@@ -16,6 +16,29 @@
 # * The special style in which python is invoked is intentional to permit
 #   use of a native python install on windows - which requires paths in DOS form
 
+retry_cmd()
+{
+    max_attempts="$1"; shift
+    initial_wait="$1"; shift
+    attempt_num=1
+    echo "trying" "$@"
+    until "$@"
+    do
+        if ((attempt_num==max_attempts))
+        then
+            echo "Last attempt $attempt_num failed"
+            return 1
+        else
+            wait_time=$(($attempt_num*$initial_wait))
+            echo "Attempt $attempt_num failed. Trying again in $wait_time seconds..."
+            sleep $wait_time
+            attempt_num=$(($attempt_num+1))
+        fi
+    done
+    echo "succeeded"
+    return 0
+}
+
 build_dir_Darwin()
 {
   echo build-darwin-x86_64
@@ -463,7 +486,7 @@ then
       succeeded=$build_coverity
     else
       # Upload base package.
-      python_cmd "$helpers/codeticket.py" addoutput Installer "$package"  \
+      retry_cmd 4 30 python_cmd "$helpers/codeticket.py" addoutput Installer "$package"  \
           || fatal "Upload of installer failed"
       wait_for_codeticket
 
@@ -473,7 +496,7 @@ then
         package=$(installer_$arch "$package_id")
         if [ x"$package" != x ]
         then
-          python_cmd "$helpers/codeticket.py" addoutput "Installer $package_id" "$package" \
+          retry_cmd 4 30 python_cmd "$helpers/codeticket.py" addoutput "Installer $package_id" "$package" \
               || fatal "Upload of installer $package_id failed"
           wait_for_codeticket
         else
@@ -487,7 +510,7 @@ then
           if [ "${RELEASE_CRASH_REPORTING:-}" != "OFF" ]
           then
               # Upload crash reporter file
-              python_cmd "$helpers/codeticket.py" addoutput "Symbolfile" "$VIEWER_SYMBOL_FILE" \
+              retry_cmd 4 30 python_cmd "$helpers/codeticket.py" addoutput "Symbolfile" "$VIEWER_SYMBOL_FILE" \
                   || fatal "Upload of symbolfile failed"
               wait_for_codeticket
           fi
@@ -497,10 +520,7 @@ then
           if [ -r "$build_dir/llphysicsextensions_package" ]
           then
               llphysicsextensions_package=$(cat $build_dir/llphysicsextensions_package)
-              # This next upload is a frequent failure; see if giving the last one some time helps
-              # JJ is making changes to Codeticket that we hope will eliminate this failure soon
-              sleep 300
-              python_cmd "$helpers/codeticket.py" addoutput "Physics Extensions Package" "$llphysicsextensions_package" --private \
+              retry_cmd 4 30 python_cmd "$helpers/codeticket.py" addoutput "Physics Extensions Package" "$llphysicsextensions_package" --private \
                   || fatal "Upload of physics extensions package failed"
           fi
       fi