mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2024-08-22 12:38:06 +02:00
* change to new code location * vendor * tagged version v0.2.0 * gitea-vet v0.2.1 Co-authored-by: techknowlogick <techknowlogick@gitea.io>
398 lines
9.9 KiB
Go
Vendored
398 lines
9.9 KiB
Go
Vendored
// Copyright 2019 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// Package fuzzy implements a fuzzy matching algorithm.
|
|
package fuzzy
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
)
|
|
|
|
const (
|
|
// MaxInputSize is the maximum size of the input scored against the fuzzy matcher. Longer inputs
|
|
// will be truncated to this size.
|
|
MaxInputSize = 127
|
|
// MaxPatternSize is the maximum size of the pattern used to construct the fuzzy matcher. Longer
|
|
// inputs are truncated to this size.
|
|
MaxPatternSize = 63
|
|
)
|
|
|
|
type scoreVal int
|
|
|
|
func (s scoreVal) val() int {
|
|
return int(s) >> 1
|
|
}
|
|
|
|
func (s scoreVal) prevK() int {
|
|
return int(s) & 1
|
|
}
|
|
|
|
func score(val int, prevK int /*0 or 1*/) scoreVal {
|
|
return scoreVal(val<<1 + prevK)
|
|
}
|
|
|
|
// Matcher implements a fuzzy matching algorithm for scoring candidates against a pattern.
|
|
// The matcher does not support parallel usage.
|
|
type Matcher struct {
|
|
pattern string
|
|
patternLower []byte // lower-case version of the pattern
|
|
patternShort []byte // first characters of the pattern
|
|
caseSensitive bool // set if the pattern is mix-cased
|
|
|
|
patternRoles []RuneRole // the role of each character in the pattern
|
|
roles []RuneRole // the role of each character in the tested string
|
|
|
|
scores [MaxInputSize + 1][MaxPatternSize + 1][2]scoreVal
|
|
|
|
scoreScale float32
|
|
|
|
lastCandidateLen int // in bytes
|
|
lastCandidateMatched bool
|
|
|
|
// Here we save the last candidate in lower-case. This is basically a byte slice we reuse for
|
|
// performance reasons, so the slice is not reallocated for every candidate.
|
|
lowerBuf [MaxInputSize]byte
|
|
rolesBuf [MaxInputSize]RuneRole
|
|
}
|
|
|
|
func (m *Matcher) bestK(i, j int) int {
|
|
if m.scores[i][j][0].val() < m.scores[i][j][1].val() {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// NewMatcher returns a new fuzzy matcher for scoring candidates against the provided pattern.
|
|
func NewMatcher(pattern string) *Matcher {
|
|
if len(pattern) > MaxPatternSize {
|
|
pattern = pattern[:MaxPatternSize]
|
|
}
|
|
|
|
m := &Matcher{
|
|
pattern: pattern,
|
|
patternLower: ToLower(pattern, nil),
|
|
}
|
|
|
|
for i, c := range m.patternLower {
|
|
if pattern[i] != c {
|
|
m.caseSensitive = true
|
|
break
|
|
}
|
|
}
|
|
|
|
if len(pattern) > 3 {
|
|
m.patternShort = m.patternLower[:3]
|
|
} else {
|
|
m.patternShort = m.patternLower
|
|
}
|
|
|
|
m.patternRoles = RuneRoles(pattern, nil)
|
|
|
|
if len(pattern) > 0 {
|
|
maxCharScore := 4
|
|
m.scoreScale = 1 / float32(maxCharScore*len(pattern))
|
|
}
|
|
|
|
return m
|
|
}
|
|
|
|
// Score returns the score returned by matching the candidate to the pattern.
|
|
// This is not designed for parallel use. Multiple candidates must be scored sequentially.
|
|
// Returns a score between 0 and 1 (0 - no match, 1 - perfect match).
|
|
func (m *Matcher) Score(candidate string) float32 {
|
|
if len(candidate) > MaxInputSize {
|
|
candidate = candidate[:MaxInputSize]
|
|
}
|
|
lower := ToLower(candidate, m.lowerBuf[:])
|
|
m.lastCandidateLen = len(candidate)
|
|
|
|
if len(m.pattern) == 0 {
|
|
// Empty patterns perfectly match candidates.
|
|
return 1
|
|
}
|
|
|
|
if m.match(candidate, lower) {
|
|
sc := m.computeScore(candidate, lower)
|
|
if sc > minScore/2 && !m.poorMatch() {
|
|
m.lastCandidateMatched = true
|
|
if len(m.pattern) == len(candidate) {
|
|
// Perfect match.
|
|
return 1
|
|
}
|
|
|
|
if sc < 0 {
|
|
sc = 0
|
|
}
|
|
normalizedScore := float32(sc) * m.scoreScale
|
|
if normalizedScore > 1 {
|
|
normalizedScore = 1
|
|
}
|
|
|
|
return normalizedScore
|
|
}
|
|
}
|
|
|
|
m.lastCandidateMatched = false
|
|
return -1
|
|
}
|
|
|
|
const minScore = -10000
|
|
|
|
// MatchedRanges returns matches ranges for the last scored string as a flattened array of
|
|
// [begin, end) byte offset pairs.
|
|
func (m *Matcher) MatchedRanges() []int {
|
|
if len(m.pattern) == 0 || !m.lastCandidateMatched {
|
|
return nil
|
|
}
|
|
i, j := m.lastCandidateLen, len(m.pattern)
|
|
if m.scores[i][j][0].val() < minScore/2 && m.scores[i][j][1].val() < minScore/2 {
|
|
return nil
|
|
}
|
|
|
|
var ret []int
|
|
k := m.bestK(i, j)
|
|
for i > 0 {
|
|
take := (k == 1)
|
|
k = m.scores[i][j][k].prevK()
|
|
if take {
|
|
if len(ret) == 0 || ret[len(ret)-1] != i {
|
|
ret = append(ret, i)
|
|
ret = append(ret, i-1)
|
|
} else {
|
|
ret[len(ret)-1] = i - 1
|
|
}
|
|
j--
|
|
}
|
|
i--
|
|
}
|
|
// Reverse slice.
|
|
for i := 0; i < len(ret)/2; i++ {
|
|
ret[i], ret[len(ret)-1-i] = ret[len(ret)-1-i], ret[i]
|
|
}
|
|
return ret
|
|
}
|
|
|
|
func (m *Matcher) match(candidate string, candidateLower []byte) bool {
|
|
i, j := 0, 0
|
|
for ; i < len(candidateLower) && j < len(m.patternLower); i++ {
|
|
if candidateLower[i] == m.patternLower[j] {
|
|
j++
|
|
}
|
|
}
|
|
if j != len(m.patternLower) {
|
|
return false
|
|
}
|
|
|
|
// The input passes the simple test against pattern, so it is time to classify its characters.
|
|
// Character roles are used below to find the last segment.
|
|
m.roles = RuneRoles(candidate, m.rolesBuf[:])
|
|
|
|
return true
|
|
}
|
|
|
|
func (m *Matcher) computeScore(candidate string, candidateLower []byte) int {
|
|
pattLen, candLen := len(m.pattern), len(candidate)
|
|
|
|
for j := 0; j <= len(m.pattern); j++ {
|
|
m.scores[0][j][0] = minScore << 1
|
|
m.scores[0][j][1] = minScore << 1
|
|
}
|
|
m.scores[0][0][0] = score(0, 0) // Start with 0.
|
|
|
|
segmentsLeft, lastSegStart := 1, 0
|
|
for i := 0; i < candLen; i++ {
|
|
if m.roles[i] == RSep {
|
|
segmentsLeft++
|
|
lastSegStart = i + 1
|
|
}
|
|
}
|
|
|
|
// A per-character bonus for a consecutive match.
|
|
consecutiveBonus := 2
|
|
wordIdx := 0 // Word count within segment.
|
|
for i := 1; i <= candLen; i++ {
|
|
|
|
role := m.roles[i-1]
|
|
isHead := role == RHead
|
|
|
|
if isHead {
|
|
wordIdx++
|
|
} else if role == RSep && segmentsLeft > 1 {
|
|
wordIdx = 0
|
|
segmentsLeft--
|
|
}
|
|
|
|
var skipPenalty int
|
|
if i == 1 || (i-1) == lastSegStart {
|
|
// Skipping the start of first or last segment.
|
|
skipPenalty++
|
|
}
|
|
|
|
for j := 0; j <= pattLen; j++ {
|
|
// By default, we don't have a match. Fill in the skip data.
|
|
m.scores[i][j][1] = minScore << 1
|
|
|
|
// Compute the skip score.
|
|
k := 0
|
|
if m.scores[i-1][j][0].val() < m.scores[i-1][j][1].val() {
|
|
k = 1
|
|
}
|
|
|
|
skipScore := m.scores[i-1][j][k].val()
|
|
// Do not penalize missing characters after the last matched segment.
|
|
if j != pattLen {
|
|
skipScore -= skipPenalty
|
|
}
|
|
m.scores[i][j][0] = score(skipScore, k)
|
|
|
|
if j == 0 || candidateLower[i-1] != m.patternLower[j-1] {
|
|
// Not a match.
|
|
continue
|
|
}
|
|
pRole := m.patternRoles[j-1]
|
|
|
|
if role == RTail && pRole == RHead {
|
|
if j > 1 {
|
|
// Not a match: a head in the pattern matches a tail character in the candidate.
|
|
continue
|
|
}
|
|
// Special treatment for the first character of the pattern. We allow
|
|
// matches in the middle of a word if they are long enough, at least
|
|
// min(3, pattern.length) characters.
|
|
if !bytes.HasPrefix(candidateLower[i-1:], m.patternShort) {
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Compute the char score.
|
|
var charScore int
|
|
// Bonus 1: the char is in the candidate's last segment.
|
|
if segmentsLeft <= 1 {
|
|
charScore++
|
|
}
|
|
// Bonus 2: Case match or a Head in the pattern aligns with one in the word.
|
|
// Single-case patterns lack segmentation signals and we assume any character
|
|
// can be a head of a segment.
|
|
if candidate[i-1] == m.pattern[j-1] || role == RHead && (!m.caseSensitive || pRole == RHead) {
|
|
charScore++
|
|
}
|
|
|
|
// Penalty 1: pattern char is Head, candidate char is Tail.
|
|
if role == RTail && pRole == RHead {
|
|
charScore--
|
|
}
|
|
// Penalty 2: first pattern character matched in the middle of a word.
|
|
if j == 1 && role == RTail {
|
|
charScore -= 4
|
|
}
|
|
|
|
// Third dimension encodes whether there is a gap between the previous match and the current
|
|
// one.
|
|
for k := 0; k < 2; k++ {
|
|
sc := m.scores[i-1][j-1][k].val() + charScore
|
|
|
|
isConsecutive := k == 1 || i-1 == 0 || i-1 == lastSegStart
|
|
if isConsecutive {
|
|
// Bonus 3: a consecutive match. First character match also gets a bonus to
|
|
// ensure prefix final match score normalizes to 1.0.
|
|
// Logically, this is a part of charScore, but we have to compute it here because it
|
|
// only applies for consecutive matches (k == 1).
|
|
sc += consecutiveBonus
|
|
}
|
|
if k == 0 {
|
|
// Penalty 3: Matching inside a segment (and previous char wasn't matched). Penalize for the lack
|
|
// of alignment.
|
|
if role == RTail || role == RUCTail {
|
|
sc -= 3
|
|
}
|
|
}
|
|
|
|
if sc > m.scores[i][j][1].val() {
|
|
m.scores[i][j][1] = score(sc, k)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
result := m.scores[len(candidate)][len(m.pattern)][m.bestK(len(candidate), len(m.pattern))].val()
|
|
|
|
return result
|
|
}
|
|
|
|
// ScoreTable returns the score table computed for the provided candidate. Used only for debugging.
|
|
func (m *Matcher) ScoreTable(candidate string) string {
|
|
var buf bytes.Buffer
|
|
|
|
var line1, line2, separator bytes.Buffer
|
|
line1.WriteString("\t")
|
|
line2.WriteString("\t")
|
|
for j := 0; j < len(m.pattern); j++ {
|
|
line1.WriteString(fmt.Sprintf("%c\t\t", m.pattern[j]))
|
|
separator.WriteString("----------------")
|
|
}
|
|
|
|
buf.WriteString(line1.String())
|
|
buf.WriteString("\n")
|
|
buf.WriteString(separator.String())
|
|
buf.WriteString("\n")
|
|
|
|
for i := 1; i <= len(candidate); i++ {
|
|
line1.Reset()
|
|
line2.Reset()
|
|
|
|
line1.WriteString(fmt.Sprintf("%c\t", candidate[i-1]))
|
|
line2.WriteString("\t")
|
|
|
|
for j := 1; j <= len(m.pattern); j++ {
|
|
line1.WriteString(fmt.Sprintf("M%6d(%c)\t", m.scores[i][j][0].val(), dir(m.scores[i][j][0].prevK())))
|
|
line2.WriteString(fmt.Sprintf("H%6d(%c)\t", m.scores[i][j][1].val(), dir(m.scores[i][j][1].prevK())))
|
|
}
|
|
buf.WriteString(line1.String())
|
|
buf.WriteString("\n")
|
|
buf.WriteString(line2.String())
|
|
buf.WriteString("\n")
|
|
buf.WriteString(separator.String())
|
|
buf.WriteString("\n")
|
|
}
|
|
|
|
return buf.String()
|
|
}
|
|
|
|
func dir(prevK int) rune {
|
|
if prevK == 0 {
|
|
return 'M'
|
|
}
|
|
return 'H'
|
|
}
|
|
|
|
func (m *Matcher) poorMatch() bool {
|
|
if len(m.pattern) < 2 {
|
|
return false
|
|
}
|
|
|
|
i, j := m.lastCandidateLen, len(m.pattern)
|
|
k := m.bestK(i, j)
|
|
|
|
var counter, len int
|
|
for i > 0 {
|
|
take := (k == 1)
|
|
k = m.scores[i][j][k].prevK()
|
|
if take {
|
|
len++
|
|
if k == 0 && len < 3 && m.roles[i-1] == RTail {
|
|
// Short match in the middle of a word
|
|
counter++
|
|
if counter > 1 {
|
|
return true
|
|
}
|
|
}
|
|
j--
|
|
} else {
|
|
len = 0
|
|
}
|
|
i--
|
|
}
|
|
return false
|
|
}
|