From da01056022ce7e85f402932488bdb7f4ef495a65 Mon Sep 17 00:00:00 2001 From: Allan N Jeremy Date: Tue, 14 Jun 2022 18:48:07 +0300 Subject: [PATCH] Added Luau Benchmark Workflows (#530) --- .github/workflows/benchmark.yml | 109 ++++++++++++++++++++++++++++++++ scripts/run-with-cachegrind.sh | 102 ++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 scripts/run-with-cachegrind.sh diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..68f63006 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,109 @@ +name: Luau Benchmarks + +on: + push: + branches: + - master + + paths-ignore: + - "docs/**" + - "papers/**" + - "rfcs/**" + - "*.md" + - "prototyping/**" + +jobs: + benchmarks-run: + name: Run ${{ matrix.bench.title }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + bench: + - { + script: "run-benchmarks", + timeout: 12, + title: "Luau Benchmarks", + cachegrindTitle: "Performance", + cachegrindIterCount: 20, + } + benchResultsRepo: + - { name: "luau-lang/benchmark-data", branch: "main" } + + runs-on: ${{ matrix.os }} + steps: + - name: Checkout Luau + uses: actions/checkout@v3 + + - name: Build Luau + run: make config=release luau luau-analyze + + - uses: actions/setup-python@v3 + with: + python-version: "3.9" + architecture: "x64" + + - name: Install python dependencies + run: | + python -m pip install requests + python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose + + - name: Install valgrind + run: | + sudo apt-get install valgrind + + - name: Run benchmark + run: | + python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt + + - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) + run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 | tee -a ${{ matrix.bench.script }}-output.txt + + - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) + run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt + + - name: Checkout Benchmark Results repository + uses: actions/checkout@v3 + with: + repository: ${{ matrix.benchResultsRepo.name }} + ref: ${{ matrix.benchResultsRepo.branch }} + token: ${{ secrets.BENCH_GITHUB_TOKEN }} + path: "./gh-pages" + + - name: Store ${{ matrix.bench.title }} result + uses: Roblox/rhysd-github-action-benchmark@v-luau + with: + name: ${{ matrix.bench.title }} + tool: "benchmarkluau" + output-file-path: ./${{ matrix.bench.script }}-output.txt + external-data-json-path: ./gh-pages/dev/bench/data.json + alert-threshold: 150% + fail-threshold: 1000% + fail-on-alert: false + comment-on-alert: true + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Store ${{ matrix.bench.title }} result + uses: Roblox/rhysd-github-action-benchmark@v-luau + with: + name: ${{ matrix.bench.title }} (CacheGrind) + tool: "roblox" + output-file-path: ./${{ matrix.bench.script }}-output.txt + external-data-json-path: ./gh-pages/dev/bench/data.json + alert-threshold: 150% + fail-threshold: 1000% + fail-on-alert: false + comment-on-alert: true + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Push benchmark results + + run: | + echo "Pushing benchmark results..." + cd gh-pages + git config user.name github-actions + git config user.email github@users.noreply.github.com + git add ./dev/bench/data.json + git commit -m "Add benchmarks results for ${{ github.sha }}" + git push + cd .. diff --git a/scripts/run-with-cachegrind.sh b/scripts/run-with-cachegrind.sh new file mode 100644 index 00000000..eb4a8c3f --- /dev/null +++ b/scripts/run-with-cachegrind.sh @@ -0,0 +1,102 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +declare -A event_map +event_map[Ir]="TotalInstructionsExecuted,executions\n" +event_map[I1mr]="L1_InstrReadCacheMisses,misses/op\n" +event_map[ILmr]="LL_InstrReadCacheMisses,misses/op\n" +event_map[Dr]="TotalMemoryReads,reads\n" +event_map[D1mr]="L1_DataReadCacheMisses,misses/op\n" +event_map[DLmr]="LL_DataReadCacheMisses,misses/op\n" +event_map[Dw]="TotalMemoryWrites,writes\n" +event_map[D1mw]="L1_DataWriteCacheMisses,misses/op\n" +event_map[DLmw]="LL_DataWriteCacheMisses,misses/op\n" +event_map[Bc]="ConditionalBranchesExecuted,executions\n" +event_map[Bcm]="ConditionalBranchMispredictions,mispredictions/op\n" +event_map[Bi]="IndirectBranchesExecuted,executions\n" +event_map[Bim]="IndirectBranchMispredictions,mispredictions/op\n" + +now_ms() { + echo -n $(date +%s%N | cut -b1-13) +} + +# Run cachegrind on a given benchmark and echo the results. +ITERATION_COUNT=$4 +START_TIME=$(now_ms) + +valgrind \ + --quiet \ + --tool=cachegrind \ + "$1" "$2" >/dev/null + +TIME_ELAPSED=$(bc <<< "$(now_ms) - ${START_TIME}") + +# Generate report using cg_annotate and extract the header and totals of the +# recorded events valgrind was configured to record. +CG_RESULTS=$(cg_annotate $(ls -t cachegrind.out.* | head -1)) +CG_HEADERS=$(grep -B2 'PROGRAM TOTALS$' <<< "$CG_RESULTS" | head -1 | sed -E 's/\s+/\n/g' | sed '/^$/d') +CG_TOTALS=$(grep 'PROGRAM TOTALS$' <<< "$CG_RESULTS" | head -1 | grep -Po '[0-9,]+\s' | tr -d ', ') + +TOTALS_ARRAY=($CG_TOTALS) +HEADERS_ARRAY=($CG_HEADERS) + +declare -A header_map +for i in "${!TOTALS_ARRAY[@]}"; do + header_map[${HEADERS_ARRAY[$i]}]=$i +done + +# Map the results to the format that the benchmark script expects. +for i in "${!TOTALS_ARRAY[@]}"; do + TOTAL=${TOTALS_ARRAY[$i]} + + # Labels and unit descriptions are packed together in the map. + EVENT_TUPLE=${event_map[${HEADERS_ARRAY[$i]}]} + IFS=$',' read -d '\n' -ra EVENT_VALUES < <(printf "%s" "$EVENT_TUPLE") + EVENT_NAME="${EVENT_VALUES[0]}" + UNIT="${EVENT_VALUES[1]}" + + case ${HEADERS_ARRAY[$i]} in + I1mr | ILmr) + REF=${TOTALS_ARRAY[header_map["Ir"]]} + OPS_PER_SEC=$(bc -l <<< "$TOTAL / $REF") + ;; + + D1mr | DLmr) + REF=${TOTALS_ARRAY[header_map["Dr"]]} + OPS_PER_SEC=$(bc -l <<< "$TOTAL / $REF") + ;; + + D1mw | DLmw) + REF=${TOTALS_ARRAY[header_map["Dw"]]} + OPS_PER_SEC=$(bc -l <<< "$TOTAL / $REF") + ;; + + Bcm) + REF=${TOTALS_ARRAY[header_map["Bc"]]} + OPS_PER_SEC=$(bc -l <<< "$TOTAL / $REF") + ;; + + Bim) + REF=${TOTALS_ARRAY[header_map["Bi"]]} + OPS_PER_SEC=$(bc -l <<< "$TOTAL / $REF") + ;; + + *) + OPS_PER_SEC=$(bc -l <<< "$TOTAL") + ;; + esac + + STD_DEV="0%" + RUNS="1" + + if [[ $OPS_PER_SEC =~ ^[+-]?[0-9]*$ ]] + then # $OPS_PER_SEC is integer + printf "%s#%s x %.0f %s ±%s (%d runs sampled)\n" \ + "$3" "$EVENT_NAME" "$OPS_PER_SEC" "$UNIT" "$STD_DEV" "$RUNS" + else # $OPS_PER_SEC is float + printf "%s#%s x %.10f %s ±%s (%d runs sampled)\n" \ + "$3" "$EVENT_NAME" "$OPS_PER_SEC" "$UNIT" "$STD_DEV" "$RUNS" + fi + +done