diff --git a/.github/workflows/benchmark-dev.yml b/.github/workflows/benchmark-dev.yml new file mode 100644 index 00000000..cadbb706 --- /dev/null +++ b/.github/workflows/benchmark-dev.yml @@ -0,0 +1,341 @@ +name: benchmark-dev + +on: + push: + branches: + - master + paths-ignore: + - "docs/**" + - "papers/**" + - "rfcs/**" + - "*.md" + - "prototyping/**" + +jobs: + windows: + name: windows-${{matrix.arch}} + strategy: + fail-fast: false + matrix: + os: [windows-latest] + arch: [Win32, x64] + bench: + - { + script: "run-benchmarks", + timeout: 12, + title: "Luau Benchmarks", + cachegrindTitle: "Performance", + cachegrindIterCount: 20, + } + benchResultsRepo: + - { name: "luau-lang/benchmark-data", branch: "main" } + + runs-on: ${{ matrix.os }} + steps: + - name: Checkout Luau repository + uses: actions/checkout@v3 + + - name: Build Luau + shell: bash # necessary for fail-fast + run: | + mkdir build && cd build + cmake .. -DCMAKE_BUILD_TYPE=Release + cmake --build . --target Luau.Repl.CLI --config Release + cmake --build . --target Luau.Analyze.CLI --config Release + + - name: Move build files to root + run: | + move build/Release/* . + + - uses: actions/setup-python@v3 + with: + python-version: "3.9" + architecture: "x64" + + - name: Install python dependencies + run: | + python -m pip install requests + python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose + + - name: Run benchmark + run: | + python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt + + - name: Checkout Benchmark Results repository + uses: actions/checkout@v3 + with: + repository: ${{ matrix.benchResultsRepo.name }} + ref: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + + - name: Store ${{ matrix.bench.title }} result + uses: Roblox/rhysd-github-action-benchmark@v-luau + with: + name: ${{ matrix.bench.title }} (Windows ${{matrix.arch}}) + tool: "benchmarkluau" + output-file-path: ./${{ matrix.bench.script }}-output.txt + external-data-json-path: ./gh-pages/dev/bench/data.json + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Push benchmark results + if: github.event_name == 'push' + run: | + echo "Pushing benchmark results..." + cd gh-pages + git config user.name github-actions + git config user.email github@users.noreply.github.com + git add ./dev/bench/data.json + git commit -m "Add benchmarks results for ${{ github.sha }}" + git push + cd .. + + unix: + name: ${{matrix.os}} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + bench: + - { + script: "run-benchmarks", + timeout: 12, + title: "Luau Benchmarks", + cachegrindTitle: "Performance", + cachegrindIterCount: 20, + } + benchResultsRepo: + - { name: "luau-lang/benchmark-data", branch: "main" } + + runs-on: ${{ matrix.os }} + steps: + - name: Checkout Luau repository + uses: actions/checkout@v3 + + - name: Build Luau + run: make config=release luau luau-analyze + + - uses: actions/setup-python@v3 + with: + python-version: "3.9" + architecture: "x64" + + - name: Install python dependencies + run: | + python -m pip install requests + python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose + + - name: Run benchmark + run: | + python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt + + - name: Install valgrind + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get install valgrind + + - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) + if: matrix.os == 'ubuntu-latest' + run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 | tee -a ${{ matrix.bench.script }}-output.txt + + - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) + if: matrix.os == 'ubuntu-latest' + run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt + + - name: Checkout Benchmark Results repository + uses: actions/checkout@v3 + with: + repository: ${{ matrix.benchResultsRepo.name }} + ref: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + + - name: Store ${{ matrix.bench.title }} result + uses: Roblox/rhysd-github-action-benchmark@v-luau + with: + name: ${{ matrix.bench.title }} + tool: "benchmarkluau" + output-file-path: ./${{ matrix.bench.script }}-output.txt + external-data-json-path: ./gh-pages/dev/bench/data.json + github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} + + - name: Store ${{ matrix.bench.title }} result (CacheGrind) + if: matrix.os == 'ubuntu-latest' + uses: Roblox/rhysd-github-action-benchmark@v-luau + with: + name: ${{ matrix.bench.title }} (CacheGrind) + tool: "roblox" + output-file-path: ./${{ matrix.bench.script }}-output.txt + external-data-json-path: ./gh-pages/dev/bench/data.json + github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} + + - name: Push benchmark results + if: github.event_name == 'push' + run: | + echo "Pushing benchmark results..." + cd gh-pages + git config user.name github-actions + git config user.email github@users.noreply.github.com + git add ./dev/bench/data.json + git commit -m "Add benchmarks results for ${{ github.sha }}" + git push + cd .. + + callgrind: + name: callgrind + strategy: + fail-fast: false + matrix: + bench: + - { + script: "run-benchmarks", + timeout: 12, + title: "Luau Benchmarks (Callgrind)", + cachegrindTitle: "Performance", + cachegrindIterCount: 20, + } + benchResultsRepo: + - { name: "luau-lang/benchmark-data", branch: "main" } + + runs-on: ubuntu-latest + steps: + - name: Checkout Luau repository + uses: actions/checkout@v3 + + - name: Install valgrind + run: | + sudo apt-get install valgrind + + - uses: actions/setup-python@v3 + with: + python-version: "3.9" + architecture: "x64" + + - name: Install python dependencies + run: | + python -m pip install requests + python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose + + - name: Build Luau + run: make config=release CALLGRIND=1 luau + + - name: Run benchmark + run: | + python bench/bench.py --callgrind | tee ${{ matrix.bench.script }}-output.txt + + - name: Checkout Benchmark Results repository + uses: actions/checkout@v3 + with: + repository: ${{ matrix.benchResultsRepo.name }} + ref: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + + - name: Store ${{ matrix.bench.title }} result + uses: Roblox/rhysd-github-action-benchmark@v-luau + with: + name: ${{ matrix.bench.title }} + tool: "benchmarkluau" + output-file-path: ./${{ matrix.bench.script }}-output.txt + external-data-json-path: ./gh-pages/dev/bench/data.json + github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} + + - name: Push benchmark results + if: github.event_name == 'push' + run: | + echo "Pushing benchmark results..." + cd gh-pages + git config user.name github-actions + git config user.email github@users.noreply.github.com + git add ./dev/bench/data.json + git commit -m "Add benchmarks results for ${{ github.sha }}" + git push + cd .. + + static-analysis: + name: luau-analyze + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + bench: + - { + script: "run-analyze", + timeout: 12, + title: "Luau Analyze", + cachegrindTitle: "Performance", + cachegrindIterCount: 20, + } + benchResultsRepo: + - { name: "luau-lang/benchmark-data", branch: "main" } + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + with: + token: "${{ secrets.BENCH_GITHUB_TOKEN }}" + + - name: Build Luau + run: make config=release luau luau-analyze + + - uses: actions/setup-python@v4 + with: + python-version: "3.9" + architecture: "x64" + + - name: Install python dependencies + run: | + sudo pip install requests numpy scipy matplotlib ipython jupyter pandas sympy nose + + - name: Install valgrind + run: | + sudo apt-get install valgrind + + - name: Run Luau Analyze on static file + run: sudo python ./bench/measure_time.py ./build/release/luau-analyze bench/static_analysis/LuauPolyfillMap.lua | tee ${{ matrix.bench.script }}-output.txt + + - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) + run: sudo ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 ./build/release/luau-analyze bench/static_analysis/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt + + - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) + run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}" 1 ./build/release/luau-analyze bench/static_analysis/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt + + - name: Checkout Benchmark Results repository + uses: actions/checkout@v3 + with: + repository: ${{ matrix.benchResultsRepo.name }} + ref: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + + - name: Store ${{ matrix.bench.title }} result + uses: Roblox/rhysd-github-action-benchmark@v-luau + with: + name: ${{ matrix.bench.title }} + tool: "benchmarkluau" + + gh-pages-branch: "main" + output-file-path: ./${{ matrix.bench.script }}-output.txt + external-data-json-path: ./gh-pages/dev/bench/data.json + github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} + + - name: Store ${{ matrix.bench.title }} result (CacheGrind) + uses: Roblox/rhysd-github-action-benchmark@v-luau + with: + name: ${{ matrix.bench.title }} + tool: "roblox" + gh-pages-branch: "main" + output-file-path: ./${{ matrix.bench.script }}-output.txt + external-data-json-path: ./gh-pages/dev/bench/data.json + github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} + + - name: Push benchmark results + if: github.event_name == 'push' + run: | + echo "Pushing benchmark results..." + cd gh-pages + git config user.name github-actions + git config user.email github@users.noreply.github.com + git add ./dev/bench/data.json + git commit -m "Add benchmarks results for ${{ github.sha }}" + git push + cd .. diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 26ad5e46..e0ba8e86 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -13,175 +13,6 @@ on: - "prototyping/**" jobs: - windows: - name: windows-${{matrix.arch}} - strategy: - fail-fast: false - matrix: - os: [windows-latest] - arch: [Win32, x64] - bench: - - { - script: "run-benchmarks", - timeout: 12, - title: "Luau Benchmarks", - cachegrindTitle: "Performance", - cachegrindIterCount: 20, - } - benchResultsRepo: - - { name: "luau-lang/benchmark-data", branch: "main" } - - runs-on: ${{ matrix.os }} - steps: - - name: Checkout Luau repository - uses: actions/checkout@v3 - - - name: Build Luau - shell: bash # necessary for fail-fast - run: | - mkdir build && cd build - cmake .. -DCMAKE_BUILD_TYPE=Release - cmake --build . --target Luau.Repl.CLI --config Release - cmake --build . --target Luau.Analyze.CLI --config Release - - - name: Move build files to root - run: | - move build/Release/* . - - - uses: actions/setup-python@v3 - with: - python-version: "3.9" - architecture: "x64" - - - name: Install python dependencies - run: | - python -m pip install requests - python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose - - - name: Run benchmark - run: | - python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt - - - name: Checkout Benchmark Results repository - uses: actions/checkout@v3 - with: - repository: ${{ matrix.benchResultsRepo.name }} - ref: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - - - name: Store ${{ matrix.bench.title }} result - uses: Roblox/rhysd-github-action-benchmark@v-luau - with: - name: ${{ matrix.bench.title }} (Windows ${{matrix.arch}}) - tool: "benchmarkluau" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data.json - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Push benchmark results - if: github.event_name == 'push' - run: | - echo "Pushing benchmark results..." - cd gh-pages - git config user.name github-actions - git config user.email github@users.noreply.github.com - git add ./dev/bench/data.json - git commit -m "Add benchmarks results for ${{ github.sha }}" - git push - cd .. - - unix: - name: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - bench: - - { - script: "run-benchmarks", - timeout: 12, - title: "Luau Benchmarks", - cachegrindTitle: "Performance", - cachegrindIterCount: 20, - } - benchResultsRepo: - - { name: "luau-lang/benchmark-data", branch: "main" } - - runs-on: ${{ matrix.os }} - steps: - - name: Checkout Luau repository - uses: actions/checkout@v3 - - - name: Build Luau - run: make config=release luau luau-analyze - - - uses: actions/setup-python@v3 - with: - python-version: "3.9" - architecture: "x64" - - - name: Install python dependencies - run: | - python -m pip install requests - python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose - - - name: Run benchmark - run: | - python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt - - - name: Install valgrind - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt-get install valgrind - - - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) - if: matrix.os == 'ubuntu-latest' - run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 | tee -a ${{ matrix.bench.script }}-output.txt - - - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) - if: matrix.os == 'ubuntu-latest' - run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt - - - name: Checkout Benchmark Results repository - uses: actions/checkout@v3 - with: - repository: ${{ matrix.benchResultsRepo.name }} - ref: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - - - name: Store ${{ matrix.bench.title }} result - uses: Roblox/rhysd-github-action-benchmark@v-luau - with: - name: ${{ matrix.bench.title }} - tool: "benchmarkluau" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data.json - github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} - - - name: Store ${{ matrix.bench.title }} result (CacheGrind) - if: matrix.os == 'ubuntu-latest' - uses: Roblox/rhysd-github-action-benchmark@v-luau - with: - name: ${{ matrix.bench.title }} (CacheGrind) - tool: "roblox" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data.json - github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} - - - name: Push benchmark results - if: github.event_name == 'push' - run: | - echo "Pushing benchmark results..." - cd gh-pages - git config user.name github-actions - git config user.email github@users.noreply.github.com - git add ./dev/bench/data.json - git commit -m "Add benchmarks results for ${{ github.sha }}" - git push - cd .. - callgrind: name: callgrind strategy: @@ -207,16 +38,6 @@ jobs: run: | sudo apt-get install valgrind - - uses: actions/setup-python@v3 - with: - python-version: "3.9" - architecture: "x64" - - - name: Install python dependencies - run: | - python -m pip install requests - python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose - - name: Build Luau run: make config=release CALLGRIND=1 luau @@ -238,7 +59,7 @@ jobs: name: ${{ matrix.bench.title }} tool: "benchmarkluau" output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data.json + external-data-json-path: ./gh-pages/bench/data.json github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} - name: Push benchmark results @@ -248,95 +69,7 @@ jobs: cd gh-pages git config user.name github-actions git config user.email github@users.noreply.github.com - git add ./dev/bench/data.json - git commit -m "Add benchmarks results for ${{ github.sha }}" - git push - cd .. - - static-analysis: - name: luau-analyze - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - bench: - - { - script: "run-analyze", - timeout: 12, - title: "Luau Analyze", - cachegrindTitle: "Performance", - cachegrindIterCount: 20, - } - benchResultsRepo: - - { name: "luau-lang/benchmark-data", branch: "main" } - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - with: - token: "${{ secrets.BENCH_GITHUB_TOKEN }}" - - - name: Build Luau - run: make config=release luau luau-analyze - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - architecture: "x64" - - - name: Install python dependencies - run: | - sudo pip install requests numpy scipy matplotlib ipython jupyter pandas sympy nose - - - name: Install valgrind - run: | - sudo apt-get install valgrind - - - name: Run Luau Analyze on static file - run: sudo python ./bench/measure_time.py ./build/release/luau-analyze bench/static_analysis/LuauPolyfillMap.lua | tee ${{ matrix.bench.script }}-output.txt - - - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) - run: sudo ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 ./build/release/luau-analyze bench/static_analysis/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt - - - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) - run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}" 1 ./build/release/luau-analyze bench/static_analysis/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt - - - name: Checkout Benchmark Results repository - uses: actions/checkout@v3 - with: - repository: ${{ matrix.benchResultsRepo.name }} - ref: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - - - name: Store ${{ matrix.bench.title }} result - uses: Roblox/rhysd-github-action-benchmark@v-luau - with: - name: ${{ matrix.bench.title }} - tool: "benchmarkluau" - - gh-pages-branch: "main" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data.json - github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} - - - name: Store ${{ matrix.bench.title }} result (CacheGrind) - uses: Roblox/rhysd-github-action-benchmark@v-luau - with: - name: ${{ matrix.bench.title }} - tool: "roblox" - gh-pages-branch: "main" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data.json - github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} - - - name: Push benchmark results - if: github.event_name == 'push' - run: | - echo "Pushing benchmark results..." - cd gh-pages - git config user.name github-actions - git config user.email github@users.noreply.github.com - git add ./dev/bench/data.json + git add ./bench/data.json git commit -m "Add benchmarks results for ${{ github.sha }}" git push cd ..