diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d704b462..42e67748 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -12,94 +12,94 @@ on: - "prototyping/**" jobs: - windows: - name: Run ${{ matrix.bench.title }} (Windows ${{matrix.arch}}) - strategy: - fail-fast: false - matrix: - os: [windows-latest] - arch: [Win32, x64] - engine: - - { channel: stable, version: latest } - bench: - - { - script: "run-benchmarks", - timeout: 12, - title: "Luau Benchmarks", - cachegrindTitle: "Performance", - cachegrindIterCount: 20, - } - benchResultsRepo: - - { name: "AllanJeremy/luau-benchmark-results", branch: "main" } + # windows: + # name: Run ${{ matrix.bench.title }} (Windows ${{matrix.arch}}) + # strategy: + # fail-fast: false + # matrix: + # os: [windows-latest] + # arch: [Win32, x64] + # engine: + # - { channel: stable, version: latest } + # bench: + # - { + # script: "run-benchmarks", + # timeout: 12, + # title: "Luau Benchmarks", + # cachegrindTitle: "Performance", + # cachegrindIterCount: 20, + # } + # benchResultsRepo: + # - { name: "AllanJeremy/luau-benchmark-results", branch: "main" } - runs-on: ${{ matrix.os }} - steps: - - name: Checkout Luau repository - uses: actions/checkout@v3 + # runs-on: ${{ matrix.os }} + # steps: + # - name: Checkout Luau repository + # uses: actions/checkout@v3 - - name: Build Luau - shell: bash # necessary for fail-fast - run: | - mkdir build && cd build - cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo - cmake --build . --target Luau.Repl.CLI --config RelWithDebInfo - cmake --build . --target Luau.Analyze.CLI --config RelWithDebInfo + # - name: Build Luau + # shell: bash # necessary for fail-fast + # run: | + # mkdir build && cd build + # cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo + # cmake --build . --target Luau.Repl.CLI --config RelWithDebInfo + # cmake --build . --target Luau.Analyze.CLI --config RelWithDebInfo - - name: Move build files to root - run: | - move build/RelWithDebInfo/* . + # - name: Move build files to root + # run: | + # move build/RelWithDebInfo/* . - - name: Check dir structure - run: | - ls build/RelWithDebInfo - ls - - uses: actions/setup-python@v3 - with: - python-version: "3.9" - architecture: "x64" + # - name: Check dir structure + # run: | + # ls build/RelWithDebInfo + # ls + # - uses: actions/setup-python@v3 + # with: + # python-version: "3.9" + # architecture: "x64" - - name: Install python dependencies - run: | - python -m pip install requests - python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose + # - name: Install python dependencies + # run: | + # python -m pip install requests + # python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose - - name: Run benchmark - run: | - python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt + # - name: Run benchmark + # run: | + # python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt - - name: Checkout Benchmark Results repository - uses: actions/checkout@v3 - with: - repository: ${{ matrix.benchResultsRepo.name }} - ref: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" + # - name: Checkout Benchmark Results repository + # uses: actions/checkout@v3 + # with: + # repository: ${{ matrix.benchResultsRepo.name }} + # ref: ${{ matrix.benchResultsRepo.branch }} + # token: ${{ secrets.BENCH_GITHUB_TOKEN }} + # path: "./gh-pages" - - name: Store ${{ matrix.bench.title }} result - uses: Roblox/rhysd-github-action-benchmark@v-luau - with: - name: ${{ matrix.bench.title }} - tool: "benchmarkluau" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data.json - alert-threshold: 150% - fail-threshold: 200% - fail-on-alert: true - comment-on-alert: true - comment-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} + # - name: Store ${{ matrix.bench.title }} result + # uses: Roblox/rhysd-github-action-benchmark@v-luau + # with: + # name: ${{ matrix.bench.title }} + # tool: "benchmarkluau" + # output-file-path: ./${{ matrix.bench.script }}-output.txt + # external-data-json-path: ./gh-pages/dev/bench/data.json + # alert-threshold: 150% + # fail-threshold: 200% + # fail-on-alert: true + # comment-on-alert: true + # comment-always: true + # github-token: ${{ secrets.GITHUB_TOKEN }} - # - name: Push benchmark results - # if: github.event_name == 'push' - # run: | - # echo "Pushing benchmark results..." - # cd gh-pages - # git config user.name github-actions - # git config user.email github@users.noreply.github.com - # git add ./dev/bench/data.json - # git commit -m "Add benchmarks results for ${{ github.sha }}" - # git push - # cd .. + # - name: Push benchmark results + # if: github.event_name == 'push' + # run: | + # echo "Pushing benchmark results..." + # cd gh-pages + # git config user.name github-actions + # git config user.email github@users.noreply.github.com + # git add ./dev/bench/data.json + # git commit -m "Add benchmarks results for ${{ github.sha }}" + # git push + # cd .. unix: name: Run ${{ matrix.bench.title }} (${{ matrix.os}}) @@ -123,71 +123,75 @@ jobs: - name: Checkout Luau repository uses: actions/checkout@v3 - - name: Build Luau - run: make config=release luau luau-analyze - - - uses: actions/setup-python@v3 - with: - python-version: "3.9" - architecture: "x64" - - - name: Install python dependencies + - name: Something that should run on ubuntu-latest only + if: ${{matrix.os}} == 'ubuntu-latest' run: | - python -m pip install requests - python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose + echo "Running on ubuntu-latest" + # - name: Build Luau + # run: make config=release luau luau-analyze - - name: Run benchmark - run: | - python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt + # - uses: actions/setup-python@v3 + # with: + # python-version: "3.9" + # architecture: "x64" - - name: Install valgrind - if: ${{ matrix.os }} == ubuntu-latest - run: | - sudo apt-get install valgrind + # - name: Install python dependencies + # run: | + # python -m pip install requests + # python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose - - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) - if: ${{ matrix.os }} == ubuntu-latest - run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 | tee -a ${{ matrix.bench.script }}-output.txt + # - name: Run benchmark + # run: | + # python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt - - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) - if: ${{ matrix.os }} == ubuntu-latest - run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt + # - name: Install valgrind + # if: ${{ matrix.os }} == ubuntu-latest + # run: | + # sudo apt-get install valgrind - - name: Checkout Benchmark Results repository - uses: actions/checkout@v3 - with: - repository: ${{ matrix.benchResultsRepo.name }} - ref: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" + # - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) + # if: ${{ matrix.os }} == ubuntu-latest + # run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 | tee -a ${{ matrix.bench.script }}-output.txt - - name: Store ${{ matrix.bench.title }} result - uses: Roblox/rhysd-github-action-benchmark@v-luau - with: - name: ${{ matrix.bench.title }} - tool: "benchmarkluau" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data.json - alert-threshold: 150% - fail-threshold: 200% - fail-on-alert: true - comment-on-alert: true - comment-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} + # - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) + # if: ${{ matrix.os }} == ubuntu-latest + # run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt - - name: Store ${{ matrix.bench.title }} result (CacheGrind) - uses: Roblox/rhysd-github-action-benchmark@v-luau - if: ${{ matrix.os }} == ubuntu-latest - with: - name: ${{ matrix.bench.title }} (CacheGrind) - tool: "roblox" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data.json - alert-threshold: 150% - fail-threshold: 1000% - fail-on-alert: false - comment-on-alert: true - github-token: ${{ secrets.GITHUB_TOKEN }} + # - name: Checkout Benchmark Results repository + # uses: actions/checkout@v3 + # with: + # repository: ${{ matrix.benchResultsRepo.name }} + # ref: ${{ matrix.benchResultsRepo.branch }} + # token: ${{ secrets.BENCH_GITHUB_TOKEN }} + # path: "./gh-pages" + + # - name: Store ${{ matrix.bench.title }} result + # uses: Roblox/rhysd-github-action-benchmark@v-luau + # with: + # name: ${{ matrix.bench.title }} + # tool: "benchmarkluau" + # output-file-path: ./${{ matrix.bench.script }}-output.txt + # external-data-json-path: ./gh-pages/dev/bench/data.json + # alert-threshold: 150% + # fail-threshold: 200% + # fail-on-alert: true + # comment-on-alert: true + # comment-always: true + # github-token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Store ${{ matrix.bench.title }} result (CacheGrind) + # uses: Roblox/rhysd-github-action-benchmark@v-luau + # if: ${{ matrix.os }} == ubuntu-latest + # with: + # name: ${{ matrix.bench.title }} (CacheGrind) + # tool: "roblox" + # output-file-path: ./${{ matrix.bench.script }}-output.txt + # external-data-json-path: ./gh-pages/dev/bench/data.json + # alert-threshold: 150% + # fail-threshold: 1000% + # fail-on-alert: false + # comment-on-alert: true + # github-token: ${{ secrets.GITHUB_TOKEN }} # - name: Push benchmark results # if: github.event_name == 'push'