luau/.github/workflows/benchmark.yml

name: Luau Benchmarks

on:
  push:
    branches:
      - feat/benchmarks-multi-os
    paths-ignore:
      - "docs/**"
      - "papers/**"
      - "rfcs/**"
      - "*.md"
      - "prototyping/**"

jobs:
  windows:
    name: Run ${{ matrix.bench.title }} (Windows ${{matrix.arch}})
    strategy:
      fail-fast: false
      matrix:
        os: [windows-latest]
        arch: [Win32, x64]
        engine:
          - { channel: stable, version: latest }
        bench:
          - {
              script: "run-benchmarks",
              timeout: 12,
              title: "Luau Benchmarks",
              cachegrindTitle: "Performance",
              cachegrindIterCount: 20,
            }
        benchResultsRepo:
          - { name: "AllanJeremy/luau-benchmark-results", branch: "main" }

    runs-on: ${{ matrix.os }}
    steps:
      - name: Checkout Luau repository
        uses: actions/checkout@v3

      # - name: Build Luau
      #   shell: bash # necessary for fail-fast
      #   run: |
      #     mkdir build && cd build
      #     cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo
      #     cmake --build . --target Luau.Repl.CLI --config RelWithDebInfo
      #     cmake --build . --target Luau.Analyze.CLI --config RelWithDebInfo
      - name: cmake configure
        run: cmake . -A ${{matrix.arch}} -DLUAU_WERROR=ON

      - name: cmake test
        shell: bash # necessary for fail-fast
        run: |
          cmake --build . --target Luau.UnitTest Luau.Conformance --config Debug
          Debug/Luau.UnitTest.exe
          Debug/Luau.Conformance.exe
      - name: cmake test w/flags
        shell: bash # necessary for fail-fast
        run: |
          Debug/Luau.UnitTest.exe --fflags=true
          Debug/Luau.Conformance.exe --fflags=true
      - name: cmake cli
        shell: bash # necessary for fail-fast
        run: |
          cmake --build . --target Luau.Repl.CLI Luau.Analyze.CLI --config Debug # match config with tests to improve build time
          Debug/luau tests/conformance/assert.lua
          Debug/luau-analyze tests/conformance/assert.lua

      - name: Check dir structure
        run: |
          ls build/Luau.Repl.CLI.dir

      - uses: actions/setup-python@v3
        with:
          python-version: "3.9"
          architecture: "x64"

      - name: Install python dependencies
        run: |
          python -m pip install requests
          python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose

      - name: Run benchmark
        run: |
          python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt

      - name: Checkout Benchmark Results repository
        uses: actions/checkout@v3
        with:
          repository: ${{ matrix.benchResultsRepo.name }}
          ref: ${{ matrix.benchResultsRepo.branch }}
          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
          path: "./gh-pages"

      - name: Store ${{ matrix.bench.title }} result
        uses: Roblox/rhysd-github-action-benchmark@v-luau
        with:
          name: ${{ matrix.bench.title }}
          tool: "benchmarkluau"
          output-file-path: ./${{ matrix.bench.script }}-output.txt
          external-data-json-path: ./gh-pages/dev/bench/data.json
          alert-threshold: 150%
          fail-threshold: 200%
          fail-on-alert: true
          comment-on-alert: true
          comment-always: true
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Push benchmark results
        if: github.event_name == 'push'
        run: |
          echo "Pushing benchmark results..."
          cd gh-pages
          git config user.name github-actions
          git config user.email github@users.noreply.github.com
          git add ./dev/bench/data.json
          git commit -m "Add benchmarks results for ${{ github.sha }}"
          git push
          cd ..

  # unix:
  #   name: Run ${{ matrix.bench.title }} (Unix)
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       os: [ubuntu-latest]
  #       engine:
  #         - { channel: stable, version: latest }
  #       bench:
  #         - {
  #             script: "run-benchmarks",
  #             timeout: 12,
  #             title: "Luau Benchmarks",
  #             cachegrindTitle: "Performance",
  #             cachegrindIterCount: 20,
  #           }
  #       benchResultsRepo:
  #         - { name: "luau-lang/benchmark-data", branch: "main" }

  #   runs-on: ${{ matrix.os }}
  #   steps:
  #     - name: Checkout Luau repository
  #       uses: actions/checkout@v3

  #     - name: Build Luau
  #       run: make config=release luau luau-analyze

  #     - uses: actions/setup-python@v3
  #       with:
  #         python-version: "3.9"
  #         architecture: "x64"

  #     - name: Install python dependencies
  #       run: |
  #         python -m pip install requests
  #         python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose
  #     - name: Run benchmark
  #       run: |
  #         python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt
  #     - name: Checkout Benchmark Results repository
  #       uses: actions/checkout@v3
  #       with:
  #         repository: ${{ matrix.benchResultsRepo.name }}
  #         ref: ${{ matrix.benchResultsRepo.branch }}
  #         token: ${{ secrets.BENCH_GITHUB_TOKEN }}
  #         path: "./gh-pages"

  #     - name: Store ${{ matrix.bench.title }} result
  #       uses: Roblox/rhysd-github-action-benchmark@v-luau
  #       with:
  #         name: ${{ matrix.bench.title }}
  #         tool: "benchmarkluau"
  #         output-file-path: ./${{ matrix.bench.script }}-output.txt
  #         external-data-json-path: ./gh-pages/dev/bench/data.json
  #         alert-threshold: 150%
  #         fail-threshold: 200%
  #         fail-on-alert: true
  #         comment-on-alert: true
  #         comment-always: true
  #         github-token: ${{ secrets.GITHUB_TOKEN }}

  #     - name: Push benchmark results
  #       if: github.event_name == 'push'
  #       run: |
  #         echo "Pushing benchmark results..."
  #         cd gh-pages
  #         git config user.name github-actions
  #         git config user.email github@users.noreply.github.com
  #         git add ./dev/bench/data.json
  #         git commit -m "Add benchmarks results for ${{ github.sha }}"
  #         git push
  #         cd ..