Feat: Added retry functionality to benchmarks (#667)

Resolves #668 

## The problem
Benchmarks jobs run concurrently for the different operating systems.
This means that when it comes time to push the benchmark results to [the
assigned benchmark results
repo](https://github.com/luau-lang/benchmark-data), there can be two
different jobs trying to push changes at the same time. In such a case,
one of the pushes will fail and we end up missing some benchmark results
data from the workflow run.

## The solution
Whenever a push fails, we need to retry the steps leading up to the push
(checking out the benchmark results repo, storing benchmark results,
pushing the results to [a specific
repo](https://github.com/luau-lang/benchmark-data)).

### Note
There are 3 push attempts before submitting to failure.

## TL;DR
This PR retries pushing benchmark results when they fail to get pushed
(often due to pushing from multiple jobs concurrently)

Co-authored-by: Jamie Kuppens <reshurum@gmail.com>
Co-authored-by: Ignacio Falk <flakolefluk@gmail.com>

.github/workflows/benchmark-dev.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+
     paths-ignore:
       - "docs/**"
       - "papers/**"
@@ -61,34 +62,49 @@ jobs:
         run: |
           python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt
 
-      - name: Checkout Benchmark Results repository
+      - name: Push benchmark results
-        uses: actions/checkout@v3
+        id: pushBenchmarkAttempt1
+        continue-on-error: true
+        uses: ./.github/workflows/push-results
         with:
           repository: ${{ matrix.benchResultsRepo.name }}
-          ref: ${{ matrix.benchResultsRepo.branch }}
+          branch: ${{ matrix.benchResultsRepo.branch }}
           token: ${{ secrets.BENCH_GITHUB_TOKEN }}
           path: "./gh-pages"
+          bench_name: "${{ matrix.bench.title }} (Windows ${{matrix.arch}})"
+          bench_tool: "benchmarkluau"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
 
-      - name: Store ${{ matrix.bench.title }} result
+      - name: Push benchmark results (Attempt 2)
-        uses: Roblox/rhysd-github-action-benchmark@v-luau
+        id: pushBenchmarkAttempt2
+        continue-on-error: true
+        if: steps.pushBenchmarkAttempt1.outcome == 'failure'
+        uses: ./.github/workflows/push-results
         with:
-          name: ${{ matrix.bench.title }} (Windows ${{matrix.arch}})
+          repository: ${{ matrix.benchResultsRepo.name }}
-          tool: "benchmarkluau"
+          branch: ${{ matrix.benchResultsRepo.branch }}
-          output-file-path: ./${{ matrix.bench.script }}-output.txt
+          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
-          external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json
+          path: "./gh-pages"
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          bench_name: "${{ matrix.bench.title }} (Windows ${{matrix.arch}})"
+          bench_tool: "benchmarkluau"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
 
-      - name: Push benchmark results
+      - name: Push benchmark results (Attempt 3)
-        if: github.event_name == 'push'
+        id: pushBenchmarkAttempt3
-        run: |
+        continue-on-error: true
-          echo "Pushing benchmark results..."
+        if: steps.pushBenchmarkAttempt2.outcome == 'failure'
-          cd gh-pages
+        uses: ./.github/workflows/push-results
-          git config user.name github-actions
+        with:
-          git config user.email github@users.noreply.github.com
+          repository: ${{ matrix.benchResultsRepo.name }}
-          git add ./dev/bench/data-${{ matrix.os }}.json
+          branch: ${{ matrix.benchResultsRepo.branch }}
-          git commit -m "Add benchmarks results for ${{ github.sha }}"
+          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
-          git push
+          path: "./gh-pages"
-          cd ..
+          bench_name: "${{ matrix.bench.title }} (Windows ${{matrix.arch}})"
+          bench_tool: "benchmarkluau"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
 
   unix:
     name: ${{matrix.os}}
@@ -142,44 +158,94 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt
 
-      - name: Checkout Benchmark Results repository
+      - name: Push benchmark results
-        uses: actions/checkout@v3
+        id: pushBenchmarkAttempt1
+        continue-on-error: true
+        uses: ./.github/workflows/push-results
         with:
           repository: ${{ matrix.benchResultsRepo.name }}
-          ref: ${{ matrix.benchResultsRepo.branch }}
+          branch: ${{ matrix.benchResultsRepo.branch }}
           token: ${{ secrets.BENCH_GITHUB_TOKEN }}
           path: "./gh-pages"
+          bench_name: ${{ matrix.bench.title }}
+          bench_tool: "benchmarkluau"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
 
-      - name: Store ${{ matrix.bench.title }} result
+      - name: Push benchmark results (Attempt 2)
-        uses: Roblox/rhysd-github-action-benchmark@v-luau
+        id: pushBenchmarkAttempt2
+        continue-on-error: true
+        if: steps.pushBenchmarkAttempt1.outcome == 'failure'
+        uses: ./.github/workflows/push-results
         with:
-          name: ${{ matrix.bench.title }}
+          repository: ${{ matrix.benchResultsRepo.name }}
-          tool: "benchmarkluau"
+          branch: ${{ matrix.benchResultsRepo.branch }}
-          output-file-path: ./${{ matrix.bench.script }}-output.txt
+          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
-          external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json
+          path: "./gh-pages"
-          github-token: ${{ secrets.BENCH_GITHUB_TOKEN }}
+          bench_name: ${{ matrix.bench.title }}
+          bench_tool: "benchmarkluau"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
 
-      - name: Store ${{ matrix.bench.title }} result (CacheGrind)
+      - name: Push benchmark results (Attempt 3)
+        id: pushBenchmarkAttempt3
+        continue-on-error: true
+        if: steps.pushBenchmarkAttempt2.outcome == 'failure'
+        uses: ./.github/workflows/push-results
+        with:
+          repository: ${{ matrix.benchResultsRepo.name }}
+          branch: ${{ matrix.benchResultsRepo.branch }}
+          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
+          path: "./gh-pages"
+          bench_name: ${{ matrix.bench.title }}
+          bench_tool: "benchmarkluau"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
+
+      - name: Push Cachegrind benchmark results
         if: matrix.os == 'ubuntu-latest'
-        uses: Roblox/rhysd-github-action-benchmark@v-luau
+        id: pushBenchmarkCachegrindAttempt1
+        continue-on-error: true
+        uses: ./.github/workflows/push-results
         with:
-          name: ${{ matrix.bench.title }} (CacheGrind)
+          repository: ${{ matrix.benchResultsRepo.name }}
-          tool: "roblox"
+          branch: ${{ matrix.benchResultsRepo.branch }}
-          output-file-path: ./${{ matrix.bench.script }}-output.txt
+          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
-          external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json
+          path: "./gh-pages"
-          github-token: ${{ secrets.BENCH_GITHUB_TOKEN }}
+          bench_name: ${{ matrix.bench.title }} (CacheGrind)
+          bench_tool: "roblox"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
 
-      - name: Push benchmark results
+      - name: Push Cachegrind benchmark results (Attempt 2)
-        if: github.event_name == 'push'
+        if: matrix.os == 'ubuntu-latest' && steps.pushBenchmarkCachegrindAttempt1.outcome == 'failure'
-        run: |
+        id: pushBenchmarkCachegrindAttempt2
-          echo "Pushing benchmark results..."
+        continue-on-error: true
-          cd gh-pages
+        uses: ./.github/workflows/push-results
-          git config user.name github-actions
+        with:
-          git config user.email github@users.noreply.github.com
+          repository: ${{ matrix.benchResultsRepo.name }}
-          git add ./dev/bench/data-${{ matrix.os }}.json
+          branch: ${{ matrix.benchResultsRepo.branch }}
-          git commit -m "Add benchmarks results for ${{ github.sha }}"
+          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
-          git push
+          path: "./gh-pages"
-          cd ..
+          bench_name: ${{ matrix.bench.title }} (CacheGrind)
+          bench_tool: "roblox"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
+
+      - name: Push Cachegrind benchmark results (Attempt 3)
+        if: matrix.os == 'ubuntu-latest' && steps.pushBenchmarkCachegrindAttempt2.outcome == 'failure'
+        id: pushBenchmarkCachegrindAttempt3
+        continue-on-error: true
+        uses: ./.github/workflows/push-results
+        with:
+          repository: ${{ matrix.benchResultsRepo.name }}
+          branch: ${{ matrix.benchResultsRepo.branch }}
+          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
+          path: "./gh-pages"
+          bench_name: ${{ matrix.bench.title }} (CacheGrind)
+          bench_tool: "roblox"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
 
   static-analysis:
     name: luau-analyze
@@ -197,6 +263,7 @@ jobs:
             }
         benchResultsRepo:
           - { name: "luau-lang/benchmark-data", branch: "main" }
+
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
@@ -228,43 +295,91 @@ jobs:
       - name: Run ${{ matrix.bench.title }} (Warm Cachegrind)
         run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}" 1  ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt
 
-      - name: Checkout Benchmark Results repository
+      - name: Push static analysis results
-        uses: actions/checkout@v3
+        id: pushStaticAnalysisAttempt1
+        continue-on-error: true
+        uses: ./.github/workflows/push-results
         with:
           repository: ${{ matrix.benchResultsRepo.name }}
-          ref: ${{ matrix.benchResultsRepo.branch }}
+          branch: ${{ matrix.benchResultsRepo.branch }}
           token: ${{ secrets.BENCH_GITHUB_TOKEN }}
           path: "./gh-pages"
+          bench_name: ${{ matrix.bench.title }}
+          bench_tool: "roblox"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
 
-      - name: Store ${{ matrix.bench.title }} result
+      - name: Push static analysis results (Attempt 2)
-        uses: Roblox/rhysd-github-action-benchmark@v-luau
+        if: steps.pushStaticAnalysisAttempt1.outcome == 'failure'
+        id: pushStaticAnalysisAttempt2
+        continue-on-error: true
+        uses: ./.github/workflows/push-results
         with:
-          name: ${{ matrix.bench.title }}
+          repository: ${{ matrix.benchResultsRepo.name }}
-          tool: "benchmarkluau"
+          branch: ${{ matrix.benchResultsRepo.branch }}
+          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
+          path: "./gh-pages"
+          bench_name: ${{ matrix.bench.title }}
+          bench_tool: "roblox"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
 
-          gh-pages-branch: "main"
+      - name: Push static analysis results (Attempt 3)
-          output-file-path: ./${{ matrix.bench.script }}-output.txt
+        if: steps.pushStaticAnalysisAttempt2.outcome == 'failure'
-          external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json
+        id: pushStaticAnalysisAttempt3
-          github-token: ${{ secrets.BENCH_GITHUB_TOKEN }}
+        continue-on-error: true
-
+        uses: ./.github/workflows/push-results
-      - name: Store ${{ matrix.bench.title }} result (CacheGrind)
-        uses: Roblox/rhysd-github-action-benchmark@v-luau
         with:
-          name: ${{ matrix.bench.title }}
+          repository: ${{ matrix.benchResultsRepo.name }}
-          tool: "roblox"
+          branch: ${{ matrix.benchResultsRepo.branch }}
-          gh-pages-branch: "main"
+          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
-          output-file-path: ./${{ matrix.bench.script }}-output.txt
+          path: "./gh-pages"
-          external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json
+          bench_name: ${{ matrix.bench.title }}
-          github-token: ${{ secrets.BENCH_GITHUB_TOKEN }}
+          bench_tool: "roblox"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
 
-      - name: Push benchmark results
+      - name: Push static analysis Cachegrind results
-        if: github.event_name == 'push'
+        if: matrix.os == 'ubuntu-latest'
-        run: |
+        id: pushStaticAnalysisCachegrindAttempt1
-          echo "Pushing benchmark results..."
+        continue-on-error: true
-          cd gh-pages
+        uses: ./.github/workflows/push-results
-          git config user.name github-actions
+        with:
-          git config user.email github@users.noreply.github.com
+          repository: ${{ matrix.benchResultsRepo.name }}
-          git add ./dev/bench/data-${{ matrix.os }}.json
+          branch: ${{ matrix.benchResultsRepo.branch }}
-          git commit -m "Add benchmarks results for ${{ github.sha }}"
+          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
-          git push
+          path: "./gh-pages"
-          cd ..
+          bench_name: ${{ matrix.bench.title }}
+          bench_tool: "roblox"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
+
+      - name: Push static analysis Cachegrind results (Attempt 2)
+        if: matrix.os == 'ubuntu-latest' && steps.pushStaticAnalysisCachegrindAttempt1.outcome == 'failure'
+        id: pushStaticAnalysisCachegrindAttempt2
+        continue-on-error: true
+        uses: ./.github/workflows/push-results
+        with:
+          repository: ${{ matrix.benchResultsRepo.name }}
+          branch: ${{ matrix.benchResultsRepo.branch }}
+          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
+          path: "./gh-pages"
+          bench_name: ${{ matrix.bench.title }}
+          bench_tool: "roblox"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"
+
+      - name: Push static analysis Cachegrind results (Attempt 2)
+        if: matrix.os == 'ubuntu-latest' && steps.pushStaticAnalysisCachegrindAttempt2.outcome == 'failure'
+        id: pushStaticAnalysisCachegrindAttempt3
+        continue-on-error: true
+        uses: ./.github/workflows/push-results
+        with:
+          repository: ${{ matrix.benchResultsRepo.name }}
+          branch: ${{ matrix.benchResultsRepo.branch }}
+          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
+          path: "./gh-pages"
+          bench_name: ${{ matrix.bench.title }}
+          bench_tool: "roblox"
+          bench_output_file_path: "./${{ matrix.bench.script }}-output.txt"
+          bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json"

.github/workflows/push-results/action.yml

@@ -0,0 +1,63 @@
+name: Checkout & push results
+description: Checkout a given repo and push results to GitHub
+inputs:
+  repository:
+    required: true
+    type: string
+    description: The benchmark results repository to check out
+  branch:
+    required: true
+    type: string
+    description: The benchmark results repository's branch to check out
+  token:
+    required: true
+    type: string
+    description: The GitHub token to use for pushing results
+  path:
+    required: true
+    type: string
+    description: The path to check out the results repository to
+  bench_name:
+    required: true
+    type: string
+  bench_tool:
+    required: true
+    type: string
+  bench_output_file_path:
+    required: true
+    type: string
+  bench_external_data_json_path:
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      with:
+        repository: ${{ inputs.repository }}
+        ref: ${{ inputs.branch }}
+        token: ${{ inputs.token }}
+        path: ${{ inputs.path }}
+
+    - name: Store results
+      uses: Roblox/rhysd-github-action-benchmark@v-luau
+      with:
+        name: ${{ inputs.bench_name }}
+        tool: ${{ inputs.bench_tool }}
+        gh-pages-branch: ${{ inputs.branch }}
+        output-file-path: ${{ inputs.bench_output_file_path }}
+        external-data-json-path: ${{ inputs.bench_external_data_json_path }}
+
+    - name: Push benchmark results
+      shell: bash
+      run: |
+        echo "Pushing benchmark results..."
+        cd gh-pages
+        git config user.name github-actions
+        git config user.email github@users.noreply.github.com
+        git add *.json
+        git commit -m "Add benchmarks results for ${{ github.sha }}"
+        git push
+        cd ..