From 5953634fb481e47e523a7a91ec2eadacacf1bc35 Mon Sep 17 00:00:00 2001 From: Allan Jeremy Date: Wed, 17 Aug 2022 18:36:29 +0300 Subject: [PATCH] build: added base benchmark base --- .github/workflows/benchmark-dev.yml | 339 +++++++++++----------- .github/workflows/push-results/action.yml | 64 ++++ 2 files changed, 240 insertions(+), 163 deletions(-) create mode 100644 .github/workflows/push-results/action.yml diff --git a/.github/workflows/benchmark-dev.yml b/.github/workflows/benchmark-dev.yml index 2c6eae4b..aed50fc4 100644 --- a/.github/workflows/benchmark-dev.yml +++ b/.github/workflows/benchmark-dev.yml @@ -90,181 +90,194 @@ jobs: git push cd .. - unix: - name: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - bench: - - { - script: "run-benchmarks", - timeout: 12, - title: "Luau Benchmarks", - cachegrindTitle: "Performance", - cachegrindIterCount: 20, - } - benchResultsRepo: - - { name: "luau-lang/benchmark-data", branch: "main" } - - runs-on: ${{ matrix.os }} - steps: - - name: Checkout Luau repository - uses: actions/checkout@v3 - - - name: Build Luau - run: make config=release luau luau-analyze - - - uses: actions/setup-python@v3 - with: - python-version: "3.9" - architecture: "x64" - - - name: Install python dependencies - run: | - python -m pip install requests - python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose - - - name: Run benchmark - run: | - python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt - - - name: Install valgrind - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt-get install valgrind - - - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) - if: matrix.os == 'ubuntu-latest' - run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 | tee -a ${{ matrix.bench.script }}-output.txt - - - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) - if: matrix.os == 'ubuntu-latest' - run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt - - - name: Checkout Benchmark Results repository - uses: actions/checkout@v3 - with: - repository: ${{ matrix.benchResultsRepo.name }} - ref: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - - - name: Store ${{ matrix.bench.title }} result - uses: Roblox/rhysd-github-action-benchmark@v-luau - with: - name: ${{ matrix.bench.title }} - tool: "benchmarkluau" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json - github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} - - - name: Store ${{ matrix.bench.title }} result (CacheGrind) - if: matrix.os == 'ubuntu-latest' - uses: Roblox/rhysd-github-action-benchmark@v-luau - with: - name: ${{ matrix.bench.title }} (CacheGrind) - tool: "roblox" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json - github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} - - name: Push benchmark results if: github.event_name == 'push' - run: | - echo "Pushing benchmark results..." - cd gh-pages - git config user.name github-actions - git config user.email github@users.noreply.github.com - git add ./dev/bench/data-${{ matrix.os }}.json - git commit -m "Add benchmarks results for ${{ github.sha }}" - git push - cd .. + uses: ./.github/workflows/push-results + with: + repository: ${{ matrix.benchResultsRepo.name }} + branch: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.GITHUB_TOKEN }} + path: "./gh-pages" + bench_name: "${{ matrix.bench.title }} (Windows ${{matrix.arch}})" + bench_tool: "benchmarkluau" + bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" + bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - static-analysis: - name: luau-analyze - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - bench: - - { - script: "run-analyze", - timeout: 12, - title: "Luau Analyze", - cachegrindTitle: "Performance", - cachegrindIterCount: 20, - } - benchResultsRepo: - - { name: "luau-lang/benchmark-data", branch: "main" } - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - with: - token: "${{ secrets.BENCH_GITHUB_TOKEN }}" + # unix: + # name: ${{matrix.os}} + # strategy: + # fail-fast: false + # matrix: + # os: [ubuntu-latest, macos-latest] + # bench: + # - { + # script: "run-benchmarks", + # timeout: 12, + # title: "Luau Benchmarks", + # cachegrindTitle: "Performance", + # cachegrindIterCount: 20, + # } + # benchResultsRepo: + # - { name: "luau-lang/benchmark-data", branch: "main" } - - name: Build Luau - run: make config=release luau luau-analyze + # runs-on: ${{ matrix.os }} + # steps: + # - name: Checkout Luau repository + # uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - architecture: "x64" + # - name: Build Luau + # run: make config=release luau luau-analyze - - name: Install python dependencies - run: | - sudo pip install requests numpy scipy matplotlib ipython jupyter pandas sympy nose + # - uses: actions/setup-python@v3 + # with: + # python-version: "3.9" + # architecture: "x64" - - name: Install valgrind - run: | - sudo apt-get install valgrind + # - name: Install python dependencies + # run: | + # python -m pip install requests + # python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose - - name: Run Luau Analyze on static file - run: sudo python ./bench/measure_time.py ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee ${{ matrix.bench.script }}-output.txt + # - name: Run benchmark + # run: | + # python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt - - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) - run: sudo ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt + # - name: Install valgrind + # if: matrix.os == 'ubuntu-latest' + # run: | + # sudo apt-get install valgrind - - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) - run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}" 1 ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt + # - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) + # if: matrix.os == 'ubuntu-latest' + # run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 | tee -a ${{ matrix.bench.script }}-output.txt - - name: Checkout Benchmark Results repository - uses: actions/checkout@v3 - with: - repository: ${{ matrix.benchResultsRepo.name }} - ref: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" + # - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) + # if: matrix.os == 'ubuntu-latest' + # run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt - - name: Store ${{ matrix.bench.title }} result - uses: Roblox/rhysd-github-action-benchmark@v-luau - with: - name: ${{ matrix.bench.title }} - tool: "benchmarkluau" + # - name: Checkout Benchmark Results repository + # uses: actions/checkout@v3 + # with: + # repository: ${{ matrix.benchResultsRepo.name }} + # ref: ${{ matrix.benchResultsRepo.branch }} + # token: ${{ secrets.BENCH_GITHUB_TOKEN }} + # path: "./gh-pages" - gh-pages-branch: "main" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json - github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} + # - name: Store ${{ matrix.bench.title }} result + # uses: Roblox/rhysd-github-action-benchmark@v-luau + # with: + # name: ${{ matrix.bench.title }} + # tool: "benchmarkluau" + # output-file-path: ./${{ matrix.bench.script }}-output.txt + # external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json + # github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} - - name: Store ${{ matrix.bench.title }} result (CacheGrind) - uses: Roblox/rhysd-github-action-benchmark@v-luau - with: - name: ${{ matrix.bench.title }} - tool: "roblox" - gh-pages-branch: "main" - output-file-path: ./${{ matrix.bench.script }}-output.txt - external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json - github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} + # - name: Store ${{ matrix.bench.title }} result (CacheGrind) + # if: matrix.os == 'ubuntu-latest' + # uses: Roblox/rhysd-github-action-benchmark@v-luau + # with: + # name: ${{ matrix.bench.title }} (CacheGrind) + # tool: "roblox" + # output-file-path: ./${{ matrix.bench.script }}-output.txt + # external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json + # github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} - - name: Push benchmark results - if: github.event_name == 'push' - run: | - echo "Pushing benchmark results..." - cd gh-pages - git config user.name github-actions - git config user.email github@users.noreply.github.com - git add ./dev/bench/data-${{ matrix.os }}.json - git commit -m "Add benchmarks results for ${{ github.sha }}" - git push - cd .. + # - name: Push benchmark results + # if: github.event_name == 'push' + # run: | + # echo "Pushing benchmark results..." + # cd gh-pages + # git config user.name github-actions + # git config user.email github@users.noreply.github.com + # git add ./dev/bench/data-${{ matrix.os }}.json + # git commit -m "Add benchmarks results for ${{ github.sha }}" + # git push + # cd .. + + # static-analysis: + # name: luau-analyze + # strategy: + # fail-fast: false + # matrix: + # os: [ubuntu-latest] + # bench: + # - { + # script: "run-analyze", + # timeout: 12, + # title: "Luau Analyze", + # cachegrindTitle: "Performance", + # cachegrindIterCount: 20, + # } + # benchResultsRepo: + # - { name: "luau-lang/benchmark-data", branch: "main" } + # runs-on: ${{ matrix.os }} + # steps: + # - uses: actions/checkout@v3 + # with: + # token: "${{ secrets.BENCH_GITHUB_TOKEN }}" + + # - name: Build Luau + # run: make config=release luau luau-analyze + + # - uses: actions/setup-python@v4 + # with: + # python-version: "3.9" + # architecture: "x64" + + # - name: Install python dependencies + # run: | + # sudo pip install requests numpy scipy matplotlib ipython jupyter pandas sympy nose + + # - name: Install valgrind + # run: | + # sudo apt-get install valgrind + + # - name: Run Luau Analyze on static file + # run: sudo python ./bench/measure_time.py ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee ${{ matrix.bench.script }}-output.txt + + # - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) + # run: sudo ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt + + # - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) + # run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}" 1 ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt + + # - name: Checkout Benchmark Results repository + # uses: actions/checkout@v3 + # with: + # repository: ${{ matrix.benchResultsRepo.name }} + # ref: ${{ matrix.benchResultsRepo.branch }} + # token: ${{ secrets.BENCH_GITHUB_TOKEN }} + # path: "./gh-pages" + + # - name: Store ${{ matrix.bench.title }} result + # uses: Roblox/rhysd-github-action-benchmark@v-luau + # with: + # name: ${{ matrix.bench.title }} + # tool: "benchmarkluau" + + # gh-pages-branch: "main" + # output-file-path: ./${{ matrix.bench.script }}-output.txt + # external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json + # github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} + + # - name: Store ${{ matrix.bench.title }} result (CacheGrind) + # uses: Roblox/rhysd-github-action-benchmark@v-luau + # with: + # name: ${{ matrix.bench.title }} + # tool: "roblox" + # gh-pages-branch: "main" + # output-file-path: ./${{ matrix.bench.script }}-output.txt + # external-data-json-path: ./gh-pages/dev/bench/data-${{ matrix.os }}.json + # github-token: ${{ secrets.BENCH_GITHUB_TOKEN }} + + # - name: Push benchmark results + # if: github.event_name == 'push' + # run: | + # echo "Pushing benchmark results..." + # cd gh-pages + # git config user.name github-actions + # git config user.email github@users.noreply.github.com + # git add ./dev/bench/data-${{ matrix.os }}.json + # git commit -m "Add benchmarks results for ${{ github.sha }}" + # git push + # cd .. diff --git a/.github/workflows/push-results/action.yml b/.github/workflows/push-results/action.yml new file mode 100644 index 00000000..49630323 --- /dev/null +++ b/.github/workflows/push-results/action.yml @@ -0,0 +1,64 @@ +name: Checkout & push results +description: Checkout a given repo and push results to GitHub +inputs: + repository: + required: true + # type: string + description: The benchmark results repository to check out + branch: + required: true + # type: string + description: The benchmark results repository's branch to check out + token: + required: true + # type: string + description: The GitHub token to use for pushing results + path: + required: true + # type: string + description: The path to check out the results repository to + bench_name: + required: true + # type: string + bench_tool: + required: true + # type: string + bench_output_file_path: + required: true + # type: string + bench_external_data_json_path: + required: true + # type: string + +runs: + using: "composite" + steps: + - name: Checkout repository + uses: actions/checkout@v3 + shell: bash + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.branch }} + token: ${{ inputs.token }} + path: "./gh-pages" + + - name: Store results + uses: Roblox/rhysd-github-action-benchmark@v-luau + shell: bash + with: + name: ${{ inputs.bench_name }} + tool: ${{ inputs.bench_tool }} + output-file-path: ${{ inputs.bench_output_file_path }} + external-data-json-path: ${{ inputs.bench_external_data_json_path }} + + - name: Push benchmark results + shell: bash + run: | + echo "Pushing benchmark results..." + cd gh-pages + git config user.name github-actions + git config user.email github@users.noreply.github.com + git add *.json + git commit -m "Add benchmarks results for ${{ github.sha }}" + git push + cd ..