luau/.github/workflows/benchmark.yml

name: Luau Benchmarks

on:
  push:
    branches:
      - feat/benchmarks-multi-os
    paths-ignore:
      - "docs/**"
      - "papers/**"
      - "rfcs/**"
      - "*.md"
      - "prototyping/**"

jobs:
  # windows:
  #   name: Run ${{ matrix.bench.title }} (Windows ${{matrix.arch}})
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       os: [windows-latest]
  #       arch: [Win32, x64]
  #       engine:
  #         - { channel: stable, version: latest }
  #       bench:
  #         - {
  #             script: "run-benchmarks",
  #             timeout: 12,
  #             title: "Luau Benchmarks",
  #             cachegrindTitle: "Performance",
  #             cachegrindIterCount: 20,
  #           }
  #       benchResultsRepo:
  #         - { name: "AllanJeremy/luau-benchmark-results", branch: "main" }

  #   runs-on: ${{ matrix.os }}
  #   steps:
  #     - name: Checkout Luau repository
  #       uses: actions/checkout@v3

  #     - name: Build Luau
  #       shell: bash # necessary for fail-fast
  #       run: |
  #         mkdir build && cd build
  #         cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo
  #         cmake --build . --target Luau.Repl.CLI --config RelWithDebInfo
  #         cmake --build . --target Luau.Analyze.CLI --config RelWithDebInfo

  #     - name: Move build files to root
  #       run: |
  #         move build/RelWithDebInfo/* .

  #     - name: Check dir structure
  #       run: |
  #         ls build/RelWithDebInfo
  #         ls
  #     - uses: actions/setup-python@v3
  #       with:
  #         python-version: "3.9"
  #         architecture: "x64"

  #     - name: Install python dependencies
  #       run: |
  #         python -m pip install requests
  #         python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose

  #     - name: Run benchmark
  #       run: |
  #         python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt

  #     - name: Checkout Benchmark Results repository
  #       uses: actions/checkout@v3
  #       with:
  #         repository: ${{ matrix.benchResultsRepo.name }}
  #         ref: ${{ matrix.benchResultsRepo.branch }}
  #         token: ${{ secrets.BENCH_GITHUB_TOKEN }}
  #         path: "./gh-pages"

  #     - name: Store ${{ matrix.bench.title }} result
  #       uses: Roblox/rhysd-github-action-benchmark@v-luau
  #       with:
  #         name: ${{ matrix.bench.title }}
  #         tool: "benchmarkluau"
  #         output-file-path: ./${{ matrix.bench.script }}-output.txt
  #         external-data-json-path: ./gh-pages/dev/bench/data.json
  #         alert-threshold: 150%
  #         fail-threshold: 200%
  #         fail-on-alert: true
  #         comment-on-alert: true
  #         comment-always: true
  #         github-token: ${{ secrets.GITHUB_TOKEN }}

  # - name: Push benchmark results
  #   if: github.event_name == 'push'
  #   run: |
  #     echo "Pushing benchmark results..."
  #     cd gh-pages
  #     git config user.name github-actions
  #     git config user.email github@users.noreply.github.com
  #     git add ./dev/bench/data.json
  #     git commit -m "Add benchmarks results for ${{ github.sha }}"
  #     git push
  #     cd ..

  unix:
    name: Run ${{ matrix.bench.title }} (${{ matrix.os}})
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest]
        bench:
          - {
              script: "run-benchmarks",
              timeout: 12,
              title: "Luau Benchmarks",
              cachegrindTitle: "Performance",
              cachegrindIterCount: 20,
            }
        benchResultsRepo:
          - { name: "AllanJeremy/luau-benchmark-results", branch: "main" }

    runs-on: ${{ matrix.os }}
    steps:
      - name: Checkout Luau repository
        uses: actions/checkout@v3

      - name: Something that should run on ubuntu-latest only
        if: ${{matrix.os}} == 'ubuntu-latest'
        run: |
          echo "Running on ubuntu-latest"
      # - name: Build Luau
      #   run: make config=release luau luau-analyze

      # - uses: actions/setup-python@v3
      #   with:
      #     python-version: "3.9"
      #     architecture: "x64"

      # - name: Install python dependencies
      #   run: |
      #     python -m pip install requests
      #     python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose

      # - name: Run benchmark
      #   run: |
      #     python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt

      # - name: Install valgrind
      #   if: ${{ matrix.os }} == ubuntu-latest
      #   run: |
      #     sudo apt-get install valgrind

      # - name: Run ${{ matrix.bench.title }} (Cold Cachegrind)
      #   if: ${{ matrix.os }} == ubuntu-latest
      #   run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 | tee -a ${{ matrix.bench.script }}-output.txt

      # - name: Run ${{ matrix.bench.title }} (Warm Cachegrind)
      #   if: ${{ matrix.os }} == ubuntu-latest
      #   run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt

      # - name: Checkout Benchmark Results repository
      #   uses: actions/checkout@v3
      #   with:
      #     repository: ${{ matrix.benchResultsRepo.name }}
      #     ref: ${{ matrix.benchResultsRepo.branch }}
      #     token: ${{ secrets.BENCH_GITHUB_TOKEN }}
      #     path: "./gh-pages"

      # - name: Store ${{ matrix.bench.title }} result
      #   uses: Roblox/rhysd-github-action-benchmark@v-luau
      #   with:
      #     name: ${{ matrix.bench.title }}
      #     tool: "benchmarkluau"
      #     output-file-path: ./${{ matrix.bench.script }}-output.txt
      #     external-data-json-path: ./gh-pages/dev/bench/data.json
      #     alert-threshold: 150%
      #     fail-threshold: 200%
      #     fail-on-alert: true
      #     comment-on-alert: true
      #     comment-always: true
      #     github-token: ${{ secrets.GITHUB_TOKEN }}

      # - name: Store ${{ matrix.bench.title }} result (CacheGrind)
      #   uses: Roblox/rhysd-github-action-benchmark@v-luau
      #   if: ${{ matrix.os }} == ubuntu-latest
      #   with:
      #     name: ${{ matrix.bench.title }} (CacheGrind)
      #     tool: "roblox"
      #     output-file-path: ./${{ matrix.bench.script }}-output.txt
      #     external-data-json-path: ./gh-pages/dev/bench/data.json
      #     alert-threshold: 150%
      #     fail-threshold: 1000%
      #     fail-on-alert: false
      #     comment-on-alert: true
      #     github-token: ${{ secrets.GITHUB_TOKEN }}

      # - name: Push benchmark results
      #   if: github.event_name == 'push'
      #   run: |
      #     echo "Pushing benchmark results..."
      #     cd gh-pages
      #     git config user.name github-actions
      #     git config user.email github@users.noreply.github.com
      #     git add ./dev/bench/data.json
      #     git commit -m "Add benchmarks results for ${{ github.sha }}"
      #     git push
      #     cd ..