feat: implement custom emoji

README.md

@@ -1,7 +1,7 @@
 emoji-picker-element
 ====
 
-A lightweight emoji picker, distributed as a custom element.
+A lightweight emoji picker, built as a custom element.
 
 It's built on top of IndexedDB, so it consumes [far less memory](#benchmarks) than other emoji pickers.
 It also uses [Svelte](https://svelte.dev), so it has a minimal runtime footprint.
@@ -9,9 +9,10 @@ It also uses [Svelte](https://svelte.dev), so it has a minimal runtime footprint
 Design goals:
 
 - Store emoji data in IndexedDB
-- Render native emoji
+- Render native emoji, no spritesheets
 - Accessible
 - Drop-in as a vanilla web component
+- Support custom emoji
 
 ## Install
 
@@ -505,6 +506,50 @@ picker.addEventListener('skin-tone-change', event => {
 
 Note that skin tones are an integer from 0 (default) to 1 (light) through 5 (dark).
 
+### Custom emoji
+
+Both the Picker and the Database support custom emoji. Unlike regular emoji, custom emoji
+are kept in-memory. (It's assumed that they're small, and they might frequently change, so
+there's not much point in storing them in IndexedDB.)
+
+Custom emoji should follow the format:
+
+```js
+[
+  {
+    shortcode: 'foo',
+    url: 'http://example.com/foo.png'
+  },
+  {
+    shortcode: 'bar',
+    url: 'http://example.com/bar.png'
+  }  
+]
+```
+
+To pass custom emoji into the `Picker`:
+
+```js
+const picker = new Picker({
+  customEmoji: [/* ... */]
+})
+```
+
+Or the `Database`:
+
+```js
+const database = new Database({
+  customEmoji: [/* ... */]
+})
+```
+
+It can also be set at runtime:
+
+```js
+picker.customEmoji = [/* ... */ ]
+database.customEmoji = [/* ... */ ]
+```
+
 ### Tree-shaking
 
 If you want to import the `Database` without the `Picker`, or you want to code-split them separately, then do:

src/database/Database.js

@@ -16,6 +16,7 @@ import {
 } from './idbInterface'
 import { log } from '../shared/log'
 import { getETag, getETagAndData } from './utils/ajax'
+import { customEmojiIndex } from './customEmojiIndex'
 
 async function checkForUpdates (db, dataSource) {
   // just do a simple HEAD request first to see if the eTags match
@@ -53,12 +54,14 @@ async function loadDataForFirstTime (db, dataSource) {
 }
 
 export default class Database {
-  constructor ({ dataSource = DEFAULT_DATA_SOURCE, locale = DEFAULT_LOCALE } = {}) {
+  constructor ({ dataSource = DEFAULT_DATA_SOURCE, locale = DEFAULT_LOCALE, customEmoji = [] } = {}) {
     this._dataSource = dataSource
     this._locale = locale
     this._dbName = `emoji-picker-element-${this._locale}`
     this._db = undefined
     this._lazyUpdate = undefined
+    this._custom = customEmojiIndex(customEmoji)
+
     this._ready = this._init()
   }
 
@@ -91,13 +94,21 @@ export default class Database {
   async getEmojiBySearchQuery (query) {
     assertNonEmptyString(query)
     await this.ready()
-    const emojis = await getEmojiBySearchQuery(this._db, query)
-    return uniqEmoji(emojis)
+    const customs = this._custom.search(query)
+    const natives = uniqEmoji(await getEmojiBySearchQuery(this._db, query))
+    return [
+      ...customs,
+      ...natives
+    ]
   }
 
   async getEmojiByShortcode (shortcode) {
     assertNonEmptyString(shortcode)
     await this.ready()
+    const custom = this._custom.byShortcode(shortcode)
+    if (custom) {
+      return custom
+    }
     return getEmojiByShortcode(this._db, shortcode)
   }
 
@@ -118,16 +129,24 @@ export default class Database {
     return set(this._db, STORE_KEYVALUE, KEY_PREFERRED_SKINTONE, skinTone)
   }
 
-  async incrementFavoriteEmojiCount (unicode) {
-    assertNonEmptyString(unicode)
+  async incrementFavoriteEmojiCount (unicodeOrShortcode) {
+    assertNonEmptyString(unicodeOrShortcode)
     await this.ready()
-    return incrementFavoriteEmojiCount(this._db, unicode)
+    return incrementFavoriteEmojiCount(this._db, unicodeOrShortcode)
   }
 
-  async getTopFavoriteEmoji (n) {
-    assertNumber(n)
+  async getTopFavoriteEmoji (limit) {
+    assertNumber(limit)
     await this.ready()
-    return getTopFavoriteEmoji(this._db, n)
+    return getTopFavoriteEmoji(this._db, this._custom, limit)
+  }
+
+  set customEmoji (customEmojis) {
+    this._custom = customEmojiIndex(customEmojis)
+  }
+
+  get customEmoji () {
+    return this._custom.all
   }
 
   async _shutdown () {

src/database/customEmojiIndex.js

@@ -0,0 +1,42 @@
+import { trie } from './utils/trie'
+import { extractTokens } from './utils/extractTokens'
+import { assertCustomEmojis } from './utils/assertCustomEmojis'
+
+export function customEmojiIndex (customEmojis) {
+  assertCustomEmojis(customEmojis)
+  // sort custom emojis by shortcode
+  const all = customEmojis.sort((a, b) => a.shortcode.toLowerCase() < b.shortcode.toLowerCase() ? -1 : 1)
+
+  // search query to custom emoji. Similar to how we do this in IDB, the last token
+  // is treated as a prefix search, but every other one is treated as an exact match
+  // Then we AND the results together
+  const { byPrefix, byExactMatch } = trie(customEmojis, emoji => extractTokens(emoji.shortcode))
+  const search = query => {
+    const searchTokens = extractTokens(query)
+    const intermediateResults = [
+      ...searchTokens.slice(0, -1).map(byExactMatch),
+      byPrefix(searchTokens[searchTokens.length - 1])
+    ]
+    const shortestArray = intermediateResults.sort((a, b) => (a.length < b.length ? -1 : 1))[0]
+    const results = []
+    for (const item of shortestArray) {
+      // if this item is included in every array in the intermediate results, add it to the final results
+      if (!intermediateResults.some(array => array.findIndex(_ => _.shortcode === item.shortcode) === -1)) {
+        results.push(item)
+      }
+    }
+    return results.sort((a, b) => a.shortcode < b.shortcode ? -1 : 1)
+  }
+
+  // shortcodes to custom emoji
+  const shortcodeToEmoji = new Map()
+  for (const customEmoji of customEmojis) {
+    shortcodeToEmoji.set(customEmoji.shortcode.toLowerCase(), customEmoji)
+  }
+  const byShortcode = shortcode => shortcodeToEmoji.get(shortcode.toLowerCase())
+  return {
+    all,
+    search,
+    byShortcode
+  }
+}

src/database/idbInterface.js

@@ -152,26 +152,38 @@ export function incrementFavoriteEmojiCount (db, unicode) {
   })
 }
 
-export function getTopFavoriteEmoji (db, n) {
-  if (n === 0) {
+export function getTopFavoriteEmoji (db, customEmojiIndex, limit) {
+  if (limit === 0) {
     return []
   }
   return dbPromise(db, [STORE_FAVORITES, STORE_EMOJI], MODE_READONLY, ([favoritesStore, emojiStore], cb) => {
     const results = []
     favoritesStore.index(INDEX_COUNT).openCursor(undefined, 'prev').onsuccess = e => {
       const cursor = e.target.result
-      if (!cursor) {
+      if (!cursor) { // no more results
         return cb(results)
       }
-      // TODO: this could be optimized by doing the get and the cursor.continue() in parallel
-      getIDB(emojiStore, cursor.primaryKey, emoji => {
-        if (emoji) {
-          results.push(emoji)
-          if (results.length === n) {
-            return cb(results)
-          }
+
+      function addResult (result) {
+        results.push(result)
+        if (results.length === limit) {
+          return cb(results) // done, reached the limit
         }
         cursor.continue()
+      }
+
+      const unicodeOrShortcode = cursor.primaryKey
+      const custom = customEmojiIndex.byShortcode(unicodeOrShortcode)
+      if (custom) {
+        return addResult(custom)
+      }
+      // TODO: this could be optimized by doing the get and the cursor.continue() in parallel
+      getIDB(emojiStore, unicodeOrShortcode, emoji => {
+        if (emoji) {
+          return addResult(emoji)
+        }
+        // emoji not found somehow, ignore (may happen if custom emoji change)
+        cursor.continue()
       })
     }
   })

src/database/utils/assertCustomEmojis.js

@@ -0,0 +1,11 @@
+const requiredKeys = [
+  'shortcode',
+  'url'
+]
+
+export function assertCustomEmojis (customEmojis) {
+  if (!customEmojis || !Array.isArray(customEmojis ||
+    (customEmojis[0] && requiredKeys.some(key => !(key in customEmojis[0]))))) {
+    throw new Error('Expected custom emojis to be in correct format')
+  }
+}

src/database/utils/trie.js

@@ -0,0 +1,73 @@
+// trie data structure for prefix searches
+// loosely based on https://github.com/nolanlawson/substring-trie
+
+const CODA_MARKER = '' // marks the end of the string
+
+export function trie (arr, itemToTokens) {
+  const map = new Map()
+  for (const item of arr) {
+    const tokens = itemToTokens(item)
+    for (const token of tokens) {
+      let currentMap = map
+      for (let i = 0, len = token.length; i < len; i++) {
+        const char = token.charAt(i)
+        let nextMap = currentMap.get(char)
+        if (!nextMap) {
+          nextMap = new Map()
+          currentMap.set(char, nextMap)
+        }
+        currentMap = nextMap
+      }
+      let valuesAtCoda = currentMap.get(CODA_MARKER)
+      if (!valuesAtCoda) {
+        valuesAtCoda = []
+        currentMap.set(CODA_MARKER, valuesAtCoda)
+      }
+      valuesAtCoda.push(item)
+    }
+  }
+
+  const search = (query, exact) => {
+    let i = -1
+    const len = query.length
+    let currentMap = map
+    while (++i < len) {
+      const char = query.charAt(i)
+      const nextMap = currentMap.get(char)
+      if (nextMap) {
+        currentMap = nextMap
+      } else {
+        return []
+      }
+    }
+
+    if (exact) {
+      const results = currentMap.get(CODA_MARKER)
+      return results || []
+    }
+
+    const results = []
+    // traverse
+    const queue = currentMap ? [currentMap] : []
+    while (queue.length) {
+      const currentMap = queue.shift()
+      const entriesSortedByKey = [...currentMap.entries()].sort((a, b) => a[0] < b[0] ? -1 : 1)
+      for (const [key, value] of entriesSortedByKey) {
+        if (key === CODA_MARKER) { // CODA_MARKER always comes first; it's the empty string
+          results.push(...value)
+        } else {
+          queue.push(value)
+        }
+      }
+    }
+    return results
+  }
+
+  const byPrefix = query => search(query, false)
+  const byExactMatch = query => search(query, true)
+
+  return {
+    byPrefix,
+    byExactMatch
+  }
+}

src/types/database.ts

@@ -1,4 +1,4 @@
-import {Emoji, DatabaseConstructorOptions, SkinTone} from "./shared";
+import {Emoji, DatabaseConstructorOptions, SkinTone, CustomEmoji} from "./shared";
 
 export default class Database {
 
@@ -10,10 +10,12 @@ export default class Database {
    *
    * @param dataSource - URL to fetch the emojibase data from
    * @param locale - Locale string
+   * @param customEmoji - Array of custom emoji
    */
   constructor({
                 dataSource = 'https://cdn.jsdelivr.net/npm/emojibase-data@5/en/data.json',
-                locale = 'en'
+                locale = 'en',
+                customEmoji = []
               }: DatabaseConstructorOptions = {}) {
   }
 
@@ -89,22 +91,41 @@ export default class Database {
 
   /**
    * Increment the favorite count for an emoji by one. The unicode string must be non-empty. It should
-   * correspond to the base (non-skin-tone) unicode string from the emoji object.
+   * correspond to the base (non-skin-tone) unicode string from the emoji object, or in the case of
+   * custom emoji, it should be the shortcode.
    *
-   * @param unicode - unicode of the emoji to increment
+   * @param unicodeOrShortcode - unicode of the native emoji, or shortcode of a custom emoji
    */
-  incrementFavoriteEmojiCount (unicode: string): Promise<void> {
+  incrementFavoriteEmojiCount (unicodeOrShortcode: string): Promise<void> {
     return Promise.resolve()
   }
 
   /**
    * Get the top favorite emoji in descending order. If there are no favorite emoji yet, returns an empty array.
-   * @param n - maximum number of results to return
+   * @param limit - maximum number of results to return
    */
-  getTopFavoriteEmoji (n: number): Promise<Emoji[]> {
+  getTopFavoriteEmoji (limit: number): Promise<Emoji[]> {
     return Promise.resolve([])
   }
 
+  /**
+   * Set the custom emoji for this database. Throws an error if custom emoji are not in the correct format.
+   *
+   * Note that custom emoji are kept in-memory, not in IndexedDB. So they are not shared against separate
+   * Database instances.
+   *
+   * @param customEmoji
+   */
+  set customEmoji(customEmoji: CustomEmoji[]) {
+  }
+
+  /**
+   * Return the custom emoji associated with this Database, or the empty array if none.
+   */
+  get customEmoji(): CustomEmoji[] {
+    return []
+  }
+
   /**
    * Closes the underlying IndexedDB connection. The Database is not usable after that (or any other Databases
    * with the same locale).

src/types/shared.ts

@@ -29,6 +29,7 @@ export interface EmojiSkin {
 export interface DatabaseConstructorOptions {
   dataSource?: string
   locale?: string
+  customEmoji?: CustomEmoji[]
 }
 
 export interface PickerConstructorOptions {
@@ -91,4 +92,9 @@ export type SkinToneChangeEvent = Modify<UIEvent, {
 export interface EmojiPickerEventMap {
   "emoji-click": EmojiClickEvent;
   "skin-tone-change": SkinToneChangeEvent;
+}
+
+export interface CustomEmoji {
+  shortcode: string,
+  url: string
 }

test/spec/database/custom.test.js

@@ -0,0 +1,113 @@
+import { ALL_EMOJI, basicAfterEach, basicBeforeEach, truncatedEmoji } from '../shared'
+import Database from '../../../src/database/Database'
+
+const customEmojis = [
+  {
+    shortcode: 'CapitalLettersLikeThis',
+    url: 'caps.png'
+  },
+  {
+    shortcode: 'underscores_like_this',
+    url: 'underscores.png'
+  },
+  {
+    shortcode: 'a',
+    url: 'a.png'
+  },
+  {
+    shortcode: 'z',
+    url: 'z.png'
+  },
+  {
+    shortcode: 'monkey', // conflicts with native emoji
+    url: 'monkey.png'
+  }
+]
+
+const summarize = ({ unicode, shortcode, url }) => {
+  const res = { shortcode, url }
+  if (unicode) {
+    res.unicode = unicode
+  }
+  return res
+}
+
+const summaryByUnicode = unicode => {
+  const emoji = truncatedEmoji.find(_ => _.emoji === unicode)
+  return summarize({
+    unicode: emoji.emoji,
+    shortcode: emoji.shortcode
+  })
+}
+
+describe('custom emoji', () => {
+  let db
+
+  beforeEach(async () => {
+    basicBeforeEach()
+    db = new Database({ dataSource: ALL_EMOJI })
+    await db.ready()
+  })
+  afterEach(async () => {
+    basicAfterEach()
+    await db.delete()
+  })
+
+  test('errors', () => {
+    db.customEmoji = [] // empty arrays are fine
+    expect(() => { db.customEmoji = null }).toThrow()
+    expect(() => { db.customEmoji = 'foo' }).toThrow()
+    expect(() => { db.customEmoji = [{}] }).toThrow()
+  })
+
+  test('getEmojiByShortcode', async () => {
+    db.customEmoji = customEmojis
+    expect(db.customEmoji).toStrictEqual(customEmojis)
+    expect(await db.getEmojiByShortcode('capitalletterslikethis')).toStrictEqual(
+      { shortcode: 'CapitalLettersLikeThis', url: 'caps.png' }
+    )
+    expect(await db.getEmojiByShortcode('underscores_like_this')).toStrictEqual(
+      { shortcode: 'underscores_like_this', url: 'underscores.png' }
+    )
+    expect(await db.getEmojiByShortcode('Underscores_Like_This')).toStrictEqual(
+      { shortcode: 'underscores_like_this', url: 'underscores.png' }
+    )
+    // custom emoji take precedence over native in case of conflict
+    expect(await db.getEmojiByShortcode('monkey')).toStrictEqual(
+      { shortcode: 'monkey', url: 'monkey.png' }
+    )
+  })
+
+  test('getEmojiBySearchQuery', async () => {
+    db.customEmoji = customEmojis
+
+    expect((await db.getEmojiBySearchQuery('monkey')).map(summarize)).toStrictEqual([
+      { shortcode: 'monkey', url: 'monkey.png' },
+      summaryByUnicode('🐵'),
+      summaryByUnicode('🐒')
+    ])
+
+    expect((await db.getEmojiBySearchQuery('undersc'))).toStrictEqual([
+      { shortcode: 'underscores_like_this', url: 'underscores.png' }
+    ])
+
+    expect((await db.getEmojiBySearchQuery('underscores lik'))).toStrictEqual([
+      { shortcode: 'underscores_like_this', url: 'underscores.png' }
+    ])
+
+    expect((await db.getEmojiBySearchQuery('undersc like'))).toStrictEqual([
+    ])
+    expect((await db.getEmojiBySearchQuery('undersc lik'))).toStrictEqual([
+    ])
+
+    expect((await db.getEmojiBySearchQuery('underscores like'))).toStrictEqual([
+      { shortcode: 'underscores_like_this', url: 'underscores.png' }
+    ])
+    expect((await db.getEmojiBySearchQuery('underscores like th'))).toStrictEqual([
+      { shortcode: 'underscores_like_this', url: 'underscores.png' }
+    ])
+    expect((await db.getEmojiBySearchQuery('underscores like this'))).toStrictEqual([
+      { shortcode: 'underscores_like_this', url: 'underscores.png' }
+    ])
+  })
+})

test/spec/database/trie.test.js

@@ -0,0 +1,38 @@
+import { trie } from '../../../src/database/utils/trie'
+
+describe('trie tests', () => {
+  test('basic trie test', () => {
+    const mapper = _ => ([_])
+    const items = ['ban', 'bananas', 'banana', 'tomato', 'bandana', 'bandanas']
+
+    const { byPrefix, byExactMatch } = trie(items, mapper)
+
+    expect(byPrefix('banan')).toStrictEqual(['banana', 'bananas'])
+    expect(byPrefix('ban')).toStrictEqual(['ban', 'banana', 'bananas', 'bandana', 'bandanas'])
+    expect(byPrefix('bananaphone')).toStrictEqual([])
+    expect(byPrefix('band')).toStrictEqual(['bandana', 'bandanas'])
+    expect(byPrefix('banana')).toStrictEqual(['banana', 'bananas'])
+    expect(byPrefix('bananas')).toStrictEqual(['bananas'])
+
+    expect(byExactMatch('banan')).toStrictEqual([])
+    expect(byExactMatch('ban')).toStrictEqual(['ban'])
+    expect(byExactMatch('bananaphone')).toStrictEqual([])
+    expect(byExactMatch('band')).toStrictEqual([])
+    expect(byExactMatch('banana')).toStrictEqual(['banana'])
+    expect(byExactMatch('bananas')).toStrictEqual(['bananas'])
+  })
+
+  test('multiple results for same token', () => {
+    const mapper = _ => _.split(/\s+/)
+    const items = ['banana phone', 'banana split', 'gone bananas', 'bunch of bananas']
+
+    const { byPrefix, byExactMatch } = trie(items, mapper)
+
+    expect(byPrefix('ban').sort()).toStrictEqual(items.sort())
+    expect(byPrefix('banana').sort()).toStrictEqual(items.sort())
+
+    expect(byExactMatch('ban')).toStrictEqual([])
+    expect(byExactMatch('banana').sort()).toStrictEqual(['banana phone', 'banana split'].sort())
+    expect(byExactMatch('bananas').sort()).toStrictEqual(['gone bananas', 'bunch of bananas'].sort())
+  })
+})