mirror of
https://github.com/luau-lang/luau.git
synced 2024-12-13 21:40:43 +00:00
2460e38998
This change adds another file for benchmarking luau-analyze and sets up benchmarks for both non-strict/strict modes for analysis and all three optimization levels for compilation performance. To avoid issues with race conditions on repository update we do all this in the same job in benchmark.yml. To be able to benchmark both modes from a single file, luau-analyze gains --mode argument which allows to override the default typechecking mode. Not sure if we'll want this to be a hard override on top of the module-specified mode in the future, but this works for now.
105 lines
4.2 KiB
YAML
105 lines
4.2 KiB
YAML
name: benchmark
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- master
|
|
paths-ignore:
|
|
- "docs/**"
|
|
- "papers/**"
|
|
- "rfcs/**"
|
|
- "*.md"
|
|
- "prototyping/**"
|
|
|
|
jobs:
|
|
callgrind:
|
|
name: callgrind ${{ matrix.compiler }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
os: [ubuntu-22.04]
|
|
compiler: [g++]
|
|
benchResultsRepo:
|
|
- { name: "luau-lang/benchmark-data", branch: "main" }
|
|
|
|
runs-on: ${{ matrix.os }}
|
|
steps:
|
|
- name: Checkout Luau repository
|
|
uses: actions/checkout@v3
|
|
|
|
- name: Install valgrind
|
|
run: |
|
|
sudo apt-get install valgrind
|
|
|
|
- name: Build Luau
|
|
run: CXX=${{ matrix.compiler }} make config=release CALLGRIND=1 luau luau-analyze
|
|
|
|
- name: Run benchmark (bench)
|
|
run: |
|
|
python bench/bench.py --callgrind --vm "./luau -O2" | tee -a bench-output.txt
|
|
|
|
- name: Run benchmark (analyze)
|
|
run: |
|
|
filter() {
|
|
awk '/.*I\s+refs:\s+[0-9,]+/ {gsub(",", "", $4); X=$4} END {print "SUCCESS: '$1' : " X/1e7 "ms +/- 0% on luau-analyze"}'
|
|
}
|
|
valgrind --tool=callgrind ./luau-analyze --mode=nonstrict bench/other/LuauPolyfillMap.lua 2>&1 | filter map-nonstrict | tee -a analyze-output.txt
|
|
valgrind --tool=callgrind ./luau-analyze --mode=strict bench/other/LuauPolyfillMap.lua 2>&1 | filter map-strict | tee -a analyze-output.txt
|
|
valgrind --tool=callgrind ./luau-analyze --mode=nonstrict bench/other/regex.lua 2>&1 | filter regex-nonstrict | tee -a analyze-output.txt
|
|
valgrind --tool=callgrind ./luau-analyze --mode=strict bench/other/regex.lua 2>&1 | filter regex-strict | tee -a analyze-output.txt
|
|
|
|
- name: Run benchmark (compile)
|
|
run: |
|
|
filter() {
|
|
awk '/.*I\s+refs:\s+[0-9,]+/ {gsub(",", "", $4); X=$4} END {print "SUCCESS: '$1' : " X/1e7 "ms +/- 0% on luau --compile"}'
|
|
}
|
|
valgrind --tool=callgrind ./luau --compile=null -O0 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O0 | tee -a compile-output.txt
|
|
valgrind --tool=callgrind ./luau --compile=null -O1 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O1 | tee -a compile-output.txt
|
|
valgrind --tool=callgrind ./luau --compile=null -O2 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O2 | tee -a compile-output.txt
|
|
valgrind --tool=callgrind ./luau --compile=null -O0 bench/other/regex.lua 2>&1 | filter regex-O0 | tee -a compile-output.txt
|
|
valgrind --tool=callgrind ./luau --compile=null -O1 bench/other/regex.lua 2>&1 | filter regex-O1 | tee -a compile-output.txt
|
|
valgrind --tool=callgrind ./luau --compile=null -O2 bench/other/regex.lua 2>&1 | filter regex-O2 | tee -a compile-output.txt
|
|
|
|
- name: Checkout benchmark results
|
|
uses: actions/checkout@v3
|
|
with:
|
|
repository: ${{ matrix.benchResultsRepo.name }}
|
|
ref: ${{ matrix.benchResultsRepo.branch }}
|
|
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
|
|
path: "./gh-pages"
|
|
|
|
- name: Store results (bench)
|
|
uses: Roblox/rhysd-github-action-benchmark@v-luau
|
|
with:
|
|
name: callgrind ${{ matrix.compiler }}
|
|
tool: "benchmarkluau"
|
|
output-file-path: ./bench-output.txt
|
|
external-data-json-path: ./gh-pages/bench.json
|
|
|
|
- name: Store results (analyze)
|
|
uses: Roblox/rhysd-github-action-benchmark@v-luau
|
|
with:
|
|
name: luau-analyze
|
|
tool: "benchmarkluau"
|
|
output-file-path: ./analyze-output.txt
|
|
external-data-json-path: ./gh-pages/analyze.json
|
|
|
|
- name: Store results (compile)
|
|
uses: Roblox/rhysd-github-action-benchmark@v-luau
|
|
with:
|
|
name: luau --compile
|
|
tool: "benchmarkluau"
|
|
output-file-path: ./compile-output.txt
|
|
external-data-json-path: ./gh-pages/compile.json
|
|
|
|
- name: Push benchmark results
|
|
if: github.event_name == 'push'
|
|
run: |
|
|
echo "Pushing benchmark results..."
|
|
cd gh-pages
|
|
git config user.name github-actions
|
|
git config user.email github@users.noreply.github.com
|
|
git add *.json
|
|
git commit -m "Add benchmarks results for ${{ github.sha }}"
|
|
git push
|
|
cd ..
|