bench: Implement luau-analyze and luau --compile benchmarks (#575)

This change adds another file for benchmarking luau-analyze and sets up
benchmarks for both non-strict/strict modes for analysis and all three
optimization levels for compilation performance.

To avoid issues with race conditions on repository update we do all this
in the same job in benchmark.yml.

To be able to benchmark both modes from a single file, luau-analyze
gains --mode argument which allows to override the default typechecking
mode. Not sure if we'll want this to be a hard override on top of the
module-specified mode in the future, but this works for now.

.github/workflows/benchmark-dev.yml

@@ -220,13 +220,13 @@ jobs:
           sudo apt-get install valgrind
 
       - name: Run Luau Analyze on static file
-        run: sudo python ./bench/measure_time.py ./build/release/luau-analyze bench/static_analysis/LuauPolyfillMap.lua | tee ${{ matrix.bench.script }}-output.txt
+        run: sudo python ./bench/measure_time.py ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee ${{ matrix.bench.script }}-output.txt
 
       - name: Run ${{ matrix.bench.title }} (Cold Cachegrind)
-        run: sudo ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 ./build/release/luau-analyze bench/static_analysis/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt
+        run: sudo ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt
 
       - name: Run ${{ matrix.bench.title }} (Warm Cachegrind)
-        run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}" 1  ./build/release/luau-analyze bench/static_analysis/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt
+        run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}" 1  ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt
 
       - name: Checkout Benchmark Results repository
         uses: actions/checkout@v3

.github/workflows/benchmark.yml

@@ -32,11 +32,33 @@ jobs:
           sudo apt-get install valgrind
 
       - name: Build Luau
-        run: CXX=${{ matrix.compiler }} make config=release CALLGRIND=1 luau
+        run: CXX=${{ matrix.compiler }} make config=release CALLGRIND=1 luau luau-analyze
 
-      - name: Run benchmark
+      - name: Run benchmark (bench)
         run: |
-          python bench/bench.py --callgrind --vm "./luau -O2" | tee output.txt
+          python bench/bench.py --callgrind --vm "./luau -O2" | tee -a bench-output.txt
+
+      - name: Run benchmark (analyze)
+        run: |
+          filter() {
+            awk '/.*I\s+refs:\s+[0-9,]+/ {gsub(",", "", $4); X=$4} END {print "SUCCESS: '$1' : " X/1e7 "ms +/- 0% on luau-analyze"}'
+          }
+          valgrind --tool=callgrind ./luau-analyze --mode=nonstrict bench/other/LuauPolyfillMap.lua 2>&1 | filter map-nonstrict | tee -a analyze-output.txt
+          valgrind --tool=callgrind ./luau-analyze --mode=strict bench/other/LuauPolyfillMap.lua 2>&1 | filter map-strict | tee -a analyze-output.txt
+          valgrind --tool=callgrind ./luau-analyze --mode=nonstrict bench/other/regex.lua 2>&1 | filter regex-nonstrict | tee -a analyze-output.txt
+          valgrind --tool=callgrind ./luau-analyze --mode=strict bench/other/regex.lua 2>&1 | filter regex-strict | tee -a analyze-output.txt
+
+      - name: Run benchmark (compile)
+        run: |
+          filter() {
+            awk '/.*I\s+refs:\s+[0-9,]+/ {gsub(",", "", $4); X=$4} END {print "SUCCESS: '$1' : " X/1e7 "ms +/- 0% on luau --compile"}'
+          }
+          valgrind --tool=callgrind ./luau --compile=null -O0 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O0 | tee -a compile-output.txt
+          valgrind --tool=callgrind ./luau --compile=null -O1 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O1 | tee -a compile-output.txt
+          valgrind --tool=callgrind ./luau --compile=null -O2 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O2 | tee -a compile-output.txt
+          valgrind --tool=callgrind ./luau --compile=null -O0 bench/other/regex.lua 2>&1 | filter regex-O0 | tee -a compile-output.txt
+          valgrind --tool=callgrind ./luau --compile=null -O1 bench/other/regex.lua 2>&1 | filter regex-O1 | tee -a compile-output.txt
+          valgrind --tool=callgrind ./luau --compile=null -O2 bench/other/regex.lua 2>&1 | filter regex-O2 | tee -a compile-output.txt
 
       - name: Checkout benchmark results
         uses: actions/checkout@v3
@@ -46,13 +68,29 @@ jobs:
           token: ${{ secrets.BENCH_GITHUB_TOKEN }}
           path: "./gh-pages"
 
-      - name: Store results
+      - name: Store results (bench)
         uses: Roblox/rhysd-github-action-benchmark@v-luau
         with:
           name: callgrind ${{ matrix.compiler }}
           tool: "benchmarkluau"
-          output-file-path: ./output.txt
-          external-data-json-path: ./gh-pages/bench/data.json
+          output-file-path: ./bench-output.txt
+          external-data-json-path: ./gh-pages/bench.json
+
+      - name: Store results (analyze)
+        uses: Roblox/rhysd-github-action-benchmark@v-luau
+        with:
+          name: luau-analyze
+          tool: "benchmarkluau"
+          output-file-path: ./analyze-output.txt
+          external-data-json-path: ./gh-pages/analyze.json
+
+      - name: Store results (compile)
+        uses: Roblox/rhysd-github-action-benchmark@v-luau
+        with:
+          name: luau --compile
+          tool: "benchmarkluau"
+          output-file-path: ./compile-output.txt
+          external-data-json-path: ./gh-pages/compile.json
 
       - name: Push benchmark results
         if: github.event_name == 'push'
@@ -61,7 +99,7 @@ jobs:
           cd gh-pages
           git config user.name github-actions
           git config user.email github@users.noreply.github.com
-          git add ./bench/data.json
+          git add *.json
           git commit -m "Add benchmarks results for ${{ github.sha }}"
           git push
           cd ..

CLI/Analyze.cpp

@@ -8,6 +8,10 @@
 
 #include "FileUtils.h"
 
+#ifdef CALLGRIND
+#include <valgrind/callgrind.h>
+#endif
+
 LUAU_FASTFLAG(DebugLuauTimeTracing)
 LUAU_FASTFLAG(LuauTypeMismatchModuleNameResolution)
 
@@ -112,6 +116,7 @@ static void displayHelp(const char* argv0)
     printf("Available options:\n");
     printf("  --formatter=plain: report analysis errors in Luacheck-compatible format\n");
     printf("  --formatter=gnu: report analysis errors in GNU-compatible format\n");
+    printf("  --mode=strict: default to strict mode when typechecking\n");
     printf("  --timetrace: record compiler time tracing information into trace.json\n");
 }
 
@@ -178,9 +183,9 @@ struct CliConfigResolver : Luau::ConfigResolver
     mutable std::unordered_map<std::string, Luau::Config> configCache;
     mutable std::vector<std::pair<std::string, std::string>> configErrors;
 
-    CliConfigResolver()
+    CliConfigResolver(Luau::Mode mode)
     {
-        defaultConfig.mode = Luau::Mode::Nonstrict;
+        defaultConfig.mode = mode;
     }
 
     const Luau::Config& getConfig(const Luau::ModuleName& name) const override
@@ -229,6 +234,7 @@ int main(int argc, char** argv)
     }
 
     ReportFormat format = ReportFormat::Default;
+    Luau::Mode mode = Luau::Mode::Nonstrict;
     bool annotate = false;
 
     for (int i = 1; i < argc; ++i)
@@ -240,6 +246,8 @@ int main(int argc, char** argv)
             format = ReportFormat::Luacheck;
         else if (strcmp(argv[i], "--formatter=gnu") == 0)
             format = ReportFormat::Gnu;
+        else if (strcmp(argv[i], "--mode=strict") == 0)
+            mode = Luau::Mode::Strict;
         else if (strcmp(argv[i], "--annotate") == 0)
             annotate = true;
         else if (strcmp(argv[i], "--timetrace") == 0)
@@ -258,12 +266,16 @@ int main(int argc, char** argv)
     frontendOptions.retainFullTypeGraphs = annotate;
 
     CliFileResolver fileResolver;
-    CliConfigResolver configResolver;
+    CliConfigResolver configResolver(mode);
     Luau::Frontend frontend(&fileResolver, &configResolver, frontendOptions);
 
     Luau::registerBuiltinTypes(frontend.typeChecker);
     Luau::freeze(frontend.typeChecker.globalTypes);
 
+#ifdef CALLGRIND
+    CALLGRIND_ZERO_STATS;
+#endif
+
     std::vector<std::string> files = getSourceFiles(argc, argv);
 
     int failed = 0;

bench/other/LuauPolyfillMap.lua

@@ -1,5 +1,4 @@
 -- This file is part of the Roblox luau-polyfill repository and is licensed under MIT License; see LICENSE.txt for details
---!nonstrict
 -- #region Array
 -- Array related
 local Array = {}