There could be a lot more done if we target luajit only. Or if we port this to C. But for now let's keep it this way. It's 2-3x faster than before, and the issue is mostly only on very big inputs, which are not that common in practice. Previously computing scores allocated two matrices of lua tables per call. With hundreds of thousands of items this made fuzzy matching in console.lua GC-bound. When only the score is needed, keep two reused rolling matrix rows, making repeated scoring allocation-free. positions() still keeps all rows since tracing back the matched positions requires them. fzy.filter() of 300000 items drops from 1013 ms to 356 ms (on my setup) for a worst-case single-character needle, and a standalone score() call no longer allocates at all.
339 lines
9.8 KiB
Lua
339 lines
9.8 KiB
Lua
--[[ The MIT License (MIT)
|
|
|
|
Copyright (c) 2020 Seth Warn
|
|
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
of this software and associated documentation files (the "Software"), to deal
|
|
in the Software without restriction, including without limitation the rights
|
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
copies of the Software, and to permit persons to whom the Software is
|
|
furnished to do so, subject to the following conditions:
|
|
|
|
The above copyright notice and this permission notice shall be included in
|
|
all copies or substantial portions of the Software.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
THE SOFTWARE. ]]
|
|
|
|
-- The lua implementation of the fzy string matching algorithm
|
|
|
|
local SCORE_GAP_LEADING = -0.005
|
|
local SCORE_GAP_TRAILING = -0.005
|
|
local SCORE_GAP_INNER = -0.01
|
|
local SCORE_MATCH_CONSECUTIVE = 1.0
|
|
local SCORE_MATCH_SLASH = 0.9
|
|
local SCORE_MATCH_WORD = 0.8
|
|
local SCORE_MATCH_CAPITAL = 0.7
|
|
local SCORE_MATCH_DOT = 0.6
|
|
local SCORE_MAX = math.huge
|
|
local SCORE_MIN = -math.huge
|
|
local MATCH_MAX_LENGTH = 1024
|
|
|
|
local fzy = {}
|
|
|
|
local byte = string.byte
|
|
|
|
local CHAR_SLASH = byte("/")
|
|
local CHAR_BACKSLASH = byte("\\")
|
|
local CHAR_DASH = byte("-")
|
|
local CHAR_UNDERSCORE = byte("_")
|
|
local CHAR_SPACE = byte(" ")
|
|
local CHAR_DOT = byte(".")
|
|
local CHAR_A = byte("A")
|
|
local CHAR_Z = byte("Z")
|
|
local CHAR_a = byte("a")
|
|
local CHAR_z = byte("z")
|
|
|
|
-- Reusable memory arrays for repeating filtering calls. Profiling shown that
|
|
-- with huge payloads we spend a lot of time allocating and later dealociting,
|
|
-- making poor GC work overtime. Keep them allocated. Free them with fzy.gc()
|
|
-- after use.
|
|
local match_bonus, row_D1, row_M1, row_D2, row_M2 = {}, {}, {}, {}, {}
|
|
|
|
function fzy.gc()
|
|
match_bonus, row_D1, row_M1, row_D2, row_M2 = {}, {}, {}, {}, {}
|
|
end
|
|
|
|
-- Check if `needle` is a subsequence of the `haystack`.
|
|
--
|
|
-- Usually called before `score` or `positions`.
|
|
--
|
|
-- Args:
|
|
-- needle (string)
|
|
-- haystack (string)
|
|
-- case_sensitive (bool, optional): defaults to false
|
|
--
|
|
-- Returns:
|
|
-- bool
|
|
function fzy.has_match(needle, haystack, case_sensitive)
|
|
if not case_sensitive then
|
|
needle = string.lower(needle)
|
|
haystack = string.lower(haystack)
|
|
end
|
|
|
|
local j = 1
|
|
for i = 1, string.len(needle) do
|
|
j = string.find(haystack, needle:sub(i, i), j, true)
|
|
if not j then
|
|
return false
|
|
else
|
|
j = j + 1
|
|
end
|
|
end
|
|
|
|
return true
|
|
end
|
|
|
|
local function precompute_bonus(haystack)
|
|
local last_char = CHAR_SLASH
|
|
for i = 1, #haystack do
|
|
local this_char = byte(haystack, i)
|
|
local bonus = 0
|
|
if last_char == CHAR_SLASH or last_char == CHAR_BACKSLASH then
|
|
bonus = SCORE_MATCH_SLASH
|
|
elseif last_char == CHAR_DASH or last_char == CHAR_UNDERSCORE or last_char == CHAR_SPACE then
|
|
bonus = SCORE_MATCH_WORD
|
|
elseif last_char == CHAR_DOT then
|
|
bonus = SCORE_MATCH_DOT
|
|
elseif last_char >= CHAR_a and last_char <= CHAR_z and
|
|
this_char >= CHAR_A and this_char <= CHAR_Z then
|
|
bonus = SCORE_MATCH_CAPITAL
|
|
end
|
|
|
|
match_bonus[i] = bonus
|
|
last_char = this_char
|
|
end
|
|
|
|
return match_bonus
|
|
end
|
|
|
|
|
|
|
|
-- Run the dynamic programming loop over the score matrices D (score of the
|
|
-- best match ending at this position) and M (best score up to this position).
|
|
--
|
|
-- `needle` and `haystack` must already be case-folded as desired and both
|
|
-- non-empty, with #needle < #haystack <= MATCH_MAX_LENGTH. `haystack_cased`
|
|
-- is the original-case haystack, used for the match bonuses.
|
|
--
|
|
-- Returns the score, and, if `full` is true, also the D and M matrices as
|
|
-- freshly allocated arrays of rows, which are needed to trace back the
|
|
-- matched positions. Otherwise only two reused rows are kept, making
|
|
-- repeated scoring allocation-free.
|
|
local function compute(needle, haystack_cased, haystack, full)
|
|
local n = #needle
|
|
local m = #haystack
|
|
|
|
|
|
-- Note that the match bonuses must be computed before the arguments are
|
|
-- converted to lowercase, since there are bonuses for camelCase.
|
|
local match_bonus = precompute_bonus(haystack_cased)
|
|
|
|
local D, M
|
|
if full then
|
|
D, M = {}, {}
|
|
end
|
|
|
|
local prev_D, prev_M, cur_D, cur_M = row_D2, row_M2, row_D1, row_M1
|
|
|
|
for i = 1, n do
|
|
if full then
|
|
cur_D, cur_M = {}, {}
|
|
D[i], M[i] = cur_D, cur_M
|
|
end
|
|
|
|
local prev_score = SCORE_MIN
|
|
local gap_score = i == n and SCORE_GAP_TRAILING or SCORE_GAP_INNER
|
|
local needle_char = byte(needle, i)
|
|
|
|
for j = 1, m do
|
|
if needle_char == byte(haystack, j) then
|
|
local score = SCORE_MIN
|
|
if i == 1 then
|
|
score = (j - 1) * SCORE_GAP_LEADING + match_bonus[j]
|
|
elseif j > 1 then
|
|
local match = prev_M[j - 1] + match_bonus[j]
|
|
local consecutive = prev_D[j - 1] + SCORE_MATCH_CONSECUTIVE
|
|
score = match > consecutive and match or consecutive
|
|
end
|
|
cur_D[j] = score
|
|
prev_score = prev_score + gap_score
|
|
if score > prev_score then
|
|
prev_score = score
|
|
end
|
|
cur_M[j] = prev_score
|
|
else
|
|
cur_D[j] = SCORE_MIN
|
|
prev_score = prev_score + gap_score
|
|
cur_M[j] = prev_score
|
|
end
|
|
end
|
|
|
|
prev_D, prev_M, cur_D, cur_M = cur_D, cur_M, prev_D, prev_M
|
|
end
|
|
|
|
return prev_M[m], D, M
|
|
end
|
|
|
|
-- Compute a matching score.
|
|
--
|
|
-- Args:
|
|
-- needle (string): must be a subsequence of `haystack`, or the result is
|
|
-- undefined.
|
|
-- haystack (string)
|
|
-- case_sensitive (bool, optional): defaults to false
|
|
--
|
|
-- Returns:
|
|
-- number: higher scores indicate better matches. See also `get_score_min`
|
|
-- and `get_score_max`.
|
|
function fzy.score(needle, haystack, case_sensitive)
|
|
local n = #needle
|
|
local m = #haystack
|
|
|
|
if n == 0 or m == 0 or m > MATCH_MAX_LENGTH or n > m then
|
|
return SCORE_MIN
|
|
elseif n == m then
|
|
return SCORE_MAX
|
|
end
|
|
|
|
local haystack_cased = haystack
|
|
if not case_sensitive then
|
|
needle = string.lower(needle)
|
|
haystack = string.lower(haystack)
|
|
end
|
|
|
|
local score = compute(needle, haystack_cased, haystack, false)
|
|
return score
|
|
end
|
|
|
|
-- Compute the locations where fzy matches a string.
|
|
--
|
|
-- Determine where each character of the `needle` is matched to the `haystack`
|
|
-- in the optimal match.
|
|
--
|
|
-- Args:
|
|
-- needle (string): must be a subsequence of `haystack`, or the result is
|
|
-- undefined.
|
|
-- haystack (string)
|
|
-- case_sensitive (bool, optional): defaults to false
|
|
--
|
|
-- Returns:
|
|
-- {int,...}: indices, where `indices[n]` is the location of the `n`th
|
|
-- character of `needle` in `haystack`.
|
|
-- number: the same matching score returned by `score`
|
|
function fzy.positions(needle, haystack, case_sensitive)
|
|
local n = #needle
|
|
local m = #haystack
|
|
|
|
if n == 0 or m == 0 or m > MATCH_MAX_LENGTH or n > m then
|
|
return {}, SCORE_MIN
|
|
elseif n == m then
|
|
local consecutive = {}
|
|
for i = 1, n do
|
|
consecutive[i] = i
|
|
end
|
|
return consecutive, SCORE_MAX
|
|
end
|
|
|
|
local haystack_cased = haystack
|
|
if not case_sensitive then
|
|
needle = string.lower(needle)
|
|
haystack = string.lower(haystack)
|
|
end
|
|
|
|
local score, D, M = compute(needle, haystack_cased, haystack, true)
|
|
|
|
local positions = {}
|
|
local match_required = false
|
|
local j = m
|
|
for i = n, 1, -1 do
|
|
while j >= 1 do
|
|
if D[i][j] ~= SCORE_MIN and (match_required or D[i][j] == M[i][j]) then
|
|
match_required = (i ~= 1) and (j ~= 1) and (
|
|
M[i][j] == D[i - 1][j - 1] + SCORE_MATCH_CONSECUTIVE)
|
|
positions[i] = j
|
|
j = j - 1
|
|
break
|
|
else
|
|
j = j - 1
|
|
end
|
|
end
|
|
end
|
|
|
|
return positions, score
|
|
end
|
|
|
|
-- Apply `has_match` and `positions` to an array of haystacks.
|
|
--
|
|
-- Args:
|
|
-- needle (string)
|
|
-- haystack ({string, ...})
|
|
-- case_sensitive (bool, optional): defaults to false
|
|
--
|
|
-- Returns:
|
|
-- {{idx, positions, score}, ...}: an array with one entry per matching line
|
|
-- in `haystacks`, each entry giving the index of the line in `haystacks`
|
|
-- as well as the equivalent to the return value of `positions` for that
|
|
-- line.
|
|
function fzy.filter(needle, haystacks, case_sensitive)
|
|
local result = {}
|
|
|
|
for i, line in ipairs(haystacks) do
|
|
if fzy.has_match(needle, line, case_sensitive) then
|
|
local p, s = fzy.positions(needle, line, case_sensitive)
|
|
table.insert(result, {i, p, s})
|
|
end
|
|
end
|
|
|
|
return result
|
|
end
|
|
|
|
-- The lowest value returned by `score`.
|
|
--
|
|
-- In two special cases:
|
|
-- - an empty `needle`, or
|
|
-- - a `needle` or `haystack` larger than than `get_max_length`,
|
|
-- the `score` function will return this exact value, which can be used as a
|
|
-- sentinel. This is the lowest possible score.
|
|
function fzy.get_score_min()
|
|
return SCORE_MIN
|
|
end
|
|
|
|
-- The score returned for exact matches. This is the highest possible score.
|
|
function fzy.get_score_max()
|
|
return SCORE_MAX
|
|
end
|
|
|
|
-- The maximum size for which `fzy` will evaluate scores.
|
|
function fzy.get_max_length()
|
|
return MATCH_MAX_LENGTH
|
|
end
|
|
|
|
-- The minimum score returned for normal matches.
|
|
--
|
|
-- For matches that don't return `get_score_min`, their score will be greater
|
|
-- than than this value.
|
|
function fzy.get_score_floor()
|
|
return MATCH_MAX_LENGTH * SCORE_GAP_INNER
|
|
end
|
|
|
|
-- The maximum score for non-exact matches.
|
|
--
|
|
-- For matches that don't return `get_score_max`, their score will be less than
|
|
-- this value.
|
|
function fzy.get_score_ceiling()
|
|
return MATCH_MAX_LENGTH * SCORE_MATCH_CONSECUTIVE
|
|
end
|
|
|
|
-- The name of the currently-running implementation, "lua" or "native".
|
|
function fzy.get_implementation_name()
|
|
return "lua"
|
|
end
|
|
|
|
return fzy
|