test: add performance benchmarks using tachometer (#159)

* test: add performance benchmarks using tachometer

* test: fix benchmark names

* test: ensure emoji fonts are installed

* test: fix error check

* test: ignore scripts during yarn install

* test: optimize CI script some more

* test: improve setup scripts

.github/workflows/benchmarks.yml

@@ -0,0 +1,76 @@
+# Largely borrowed from https://github.com/lit/lit/blob/56e8efd/.github/workflows/benchmarks.yml
+name: Benchmarks
+
+on: [pull_request]
+
+jobs:
+  benchmarks:
+    name: benchmarks
+
+    # We can't currently run benchmarks on PRs from forked repos, because the
+    # tachometer action reports results by posting a comment, and we can't post
+    # comments without a github token.
+    if: github.event.pull_request == null || github.event.pull_request.head.repo.full_name == github.repository
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install system dependencies
+        run: sudo apt-get install -y fonts-noto-color-emoji
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 14
+
+      # via https://github.com/actions/cache/blob/0638051/examples.md#node---yarn
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: yarn install
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          yarn --immutable --ignore-scripts
+          PERF=1 yarn build:rollup
+          ./bin/setup-benchmark.sh
+
+      # first-load
+      - name: Benchmark first-load
+        run: |
+          cd test/benchmark
+          ../../node_modules/.bin/tach \
+            --config ./first-load.tachometer.json \
+            --json-file ./first-load.results.json
+
+      - name: Report first-load
+        uses: andrewiggins/tachometer-reporter-action@v2
+        with:
+          report-id: emoji-picker-element-first-load
+          path: test/benchmark/first-load.results.json
+          pr-bench-name: this-change
+          base-bench-name: tip-of-tree
+
+      # second-load
+      - name: Benchmark second-load
+        run: |
+          cd test/benchmark
+          ../../node_modules/.bin/tach \
+            --config ./second-load.tachometer.json \
+            --json-file ./second-load.results.json
+
+      - name: Report second-load
+        uses: andrewiggins/tachometer-reporter-action@v2
+        with:
+          report-id: emoji-picker-element-second-load
+          path: test/benchmark/second-load.results.json
+          pr-bench-name: this-change
+          base-bench-name: tip-of-tree

.gitignore

@@ -133,3 +133,5 @@ dist
 /docs-tmp
 /ts-tmp
 /bundle.js
+/test/benchmark/node_modules
+/test/benchmark/data.json

CONTRIBUTING.md

@@ -31,6 +31,10 @@ Check code coverage:
 
 ## Other
 
+Benchmark runtime performance:
+
+    yarn benchmark:runtime
+
 Benchmark memory usage:
 
     yarn benchmark:memory

bin/run-benchmark.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+./bin/setup-benchmark.sh
+
+cd ./test/benchmark
+
+../../node_modules/.bin/tach --config ./first-load.tachometer.json
+../../node_modules/.bin/tach --config ./second-load.tachometer.json
+

bin/setup-benchmark.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd ./test/benchmark
+
+# fix Tachometer not finding the local dependency using a symlink hack
+rm -fr ./node_modules
+mkdir ./node_modules
+ln -s ../../.. ./node_modules/emoji-picker-element
+
+# have Tachometer host the data.json right here
+rm -f ./data.json
+ln -s ../../node_modules/emoji-picker-element-data/en/emojibase/data.json ./data.json

package.json

@@ -25,6 +25,7 @@
     "build:typedoc": "typedoc --target ES5 --out docs-tmp --theme markdown --excludePrivate --excludeNotExported --hideSources --hideBreadcrumbs ./src/types && node ./bin/generateTypeDocs && shx rm -fr docs-tmp",
     "build:types": "tsc --target ES5 -d --outDir ./ts-tmp --project ./src/types && shx mv ./ts-tmp/*.d.ts ./ && shx rm -fr ts-tmp",
     "build:toc": "node ./bin/generateTOC",
+    "benchmark:runtime": "cross-env PERF=1 run-s build:rollup && ./bin/run-benchmark.sh",
     "benchmark:bundlesize": "run-s build:rollup benchmark:bundle benchmark:run-bundlesize",
     "benchmark:bundle": "rollup -c ./test/bundlesize/rollup.config.js",
     "benchmark:memory": "run-s build:rollup benchmark:bundle && run-p --race benchmark:memory:server benchmark:memory:test",
@@ -124,6 +125,7 @@
     "svelte-jester": "nolanlawson/svelte-jester#auto-preprocess",
     "svelte-preprocess": "^4.7.3",
     "svgo": "^2.3.1",
+    "tachometer": "^0.5.9",
     "typedoc": "^0.19.2",
     "typedoc-plugin-markdown": "^2.4.2",
     "typescript": "~4.0.0"
@@ -152,6 +154,7 @@
       "getComputedStyle",
       "indexedDB",
       "IDBKeyRange",
+      "Headers",
       "matchMedia",
       "performance",
       "ResizeObserver",

test/benchmark/benchmark.js

@@ -0,0 +1,31 @@
+// hijack the performance.mark/measure API so we can add our own marks/measures just for the benchmark
+const { mark, measure } = performance
+
+performance.mark = function (name) {
+  if (name === 'initialLoad') {
+    mark.call(performance, 'benchmark-start')
+  }
+}
+
+performance.measure = function (name, start) {
+  if (name === 'initialLoad' && start === 'initialLoad') {
+    // test to make sure the picker loaded with no errors
+    const hasErrors = !!document.querySelector('emoji-picker')
+      .shadowRoot.querySelector('.message:not(.gone)')
+    if (hasErrors) {
+      console.error('picker is showing an error message')
+    } else {
+      measure.call(performance, 'benchmark-total', 'benchmark-start')
+    }
+  }
+}
+
+// Fake an eTag on the headers for the emoji-picker data so that we actually reuse the cache.
+// Tachometer doesn't serve an eTag by default
+const nativeGet = Headers.prototype.get
+Headers.prototype.get = function (name) {
+  if (name.toLowerCase() === 'etag') {
+    return 'W/fakeEtag'
+  }
+  return nativeGet.call(this, name)
+}

test/benchmark/first-load.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+</head>
+<body>
+  <script type="module" src="./benchmark.js"></script>
+  <script type="module">
+    import { Picker } from 'emoji-picker-element'
+
+    const dataSource = '/data.json'
+
+    document.body.appendChild(new Picker({ dataSource }))
+  </script>
+</body>
+</html>

test/benchmark/first-load.tachometer.json

@@ -0,0 +1,43 @@
+{
+  "$schema": "https://raw.githubusercontent.com/Polymer/tachometer/master/config.schema.json",
+  "sampleSize": 50,
+  "timeout": 5,
+  "horizons": ["10%"],
+  "benchmarks": [
+    {
+      "url": "./first-load.html",
+      "browser": {
+        "name": "chrome",
+        "headless": true
+      },
+      "measurement": [
+        {
+          "mode": "performance",
+          "entryName": "benchmark-total"
+        }
+      ],
+      "expand": [
+        {
+          "name": "this-change"
+        },
+        {
+          "name": "tip-of-tree",
+          "packageVersions": {
+            "label": "tip-of-tree",
+            "dependencies": {
+              "emoji-picker-element": {
+                "kind": "git",
+                "repo": "https://github.com/nolanlawson/emoji-picker-element.git",
+                "ref": "master",
+                "setupCommands": [
+                  "yarn --immutable --ignore-scripts",
+                  "PERF=1 yarn build:rollup"
+                ]
+              }
+            }
+          }
+        }
+      ]
+    }
+  ]
+}

test/benchmark/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "emoji-picker-element-benchmark",
+  "private": true,
+  "version": "0.0.1",
+  "dependencies": {
+    "emoji-picker-element": "*"
+  }
+}

test/benchmark/second-load.html

@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+</head>
+<body>
+<script type="module" src="./benchmark.js"></script>
+  <script type="module">
+    import Database from 'emoji-picker-element/database'
+
+    (async () => {
+      const dataSource = '/data.json'
+
+      // populate IndexedDB so the Picker is just reading from the local store
+      const db = new Database({ dataSource })
+      await db.ready()
+      await db.close()
+
+      // lazy-load the picker so that its logic to determine emoji support runs during the perf measure
+      const { default: Picker } = await import('emoji-picker-element/picker.js')
+
+      document.body.appendChild(new Picker({ dataSource }))
+    })()
+  </script>
+</body>
+</html>

test/benchmark/second-load.tachometer.json

@@ -0,0 +1,43 @@
+{
+  "$schema": "https://raw.githubusercontent.com/Polymer/tachometer/master/config.schema.json",
+  "sampleSize": 50,
+  "timeout": 5,
+  "horizons": ["10%"],
+  "benchmarks": [
+    {
+      "url": "./second-load.html",
+      "browser": {
+        "name": "chrome",
+        "headless": true
+      },
+      "measurement": [
+        {
+          "mode": "performance",
+          "entryName": "benchmark-total"
+        }
+      ],
+      "expand": [
+        {
+          "name": "this-change"
+        },
+        {
+          "name": "tip-of-tree",
+          "packageVersions": {
+            "label": "tip-of-tree",
+            "dependencies": {
+              "emoji-picker-element": {
+                "kind": "git",
+                "repo": "https://github.com/nolanlawson/emoji-picker-element.git",
+                "ref": "master",
+                "setupCommands": [
+                  "yarn --immutable --ignore-scripts",
+                  "PERF=1 yarn build:rollup"
+                ]
+              }
+            }
+          }
+        }
+      ]
+    }
+  ]
+}