Merge branch 'WIP_fuzzyMatchUpdates' into fuzzyMatchUpdates

This commit is contained in:
Jeremy Wu 2020-01-07 22:31:21 +11:00
commit fe65e7a0c4
5 changed files with 207 additions and 178 deletions

View file

@ -12,7 +12,7 @@ namespace Wox.Infrastructure
{
public static MatchOption DefaultMatchOption = new MatchOption();
public static int UserSettingSearchPrecision { get; set; }
public static SearchPrecisionScore UserSettingSearchPrecision { get; set; }
public static bool ShouldUsePinyin { get; set; }
@ -41,7 +41,15 @@ namespace Wox.Infrastructure
}
/// <summary>
/// refer to https://github.com/mattyork/fuzzy
/// Current method:
/// Character matching + substring matching;
/// 1. Query search string is split into substrings, separator is whitespace.
/// 2. Check each query substring's characters against full compare string,
/// 3. if a character in the substring is matched, loop back to verify the previous character.
/// 4. If previous character also matches, and is the start of the substring, update list.
/// 5. Once the previous character is verified, move on to the next character in the query substring.
/// 6. Move onto the next substring's characters until all substrings are checked.
/// 7. Consider success and move onto scoring if every char or substring without whitespaces matched
/// </summary>
public static MatchResult FuzzySearch(string query, string stringToCompare, MatchOption opt)
{
@ -52,107 +60,93 @@ namespace Wox.Infrastructure
var fullStringToCompareWithoutCase = opt.IgnoreCase ? stringToCompare.ToLower() : stringToCompare;
var queryWithoutCase = opt.IgnoreCase ? query.ToLower() : query;
var querySubstrings = queryWithoutCase.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
int currentQuerySubstringIndex = 0;
var currentQuerySubstring = querySubstrings[currentQuerySubstringIndex];
var currentQuerySubstringCharacterIndex = 0;
int currentQueryToCompareIndex = 0;
var queryToCompareSeparated = queryWithoutCase.Split(' ');
var currentQueryToCompare = queryToCompareSeparated[currentQueryToCompareIndex];
var patternIndex = 0;
var firstMatchIndex = -1;
var firstMatchIndexInWord = -1;
var lastMatchIndex = 0;
bool allMatched = false;
bool isFullWordMatched = false;
bool allWordsFullyMatched = true;
bool allQuerySubstringsMatched = false;
bool matchFoundInPreviousLoop = false;
bool allSubstringsContainedInCompareString = true;
var indexList = new List<int>();
for (var index = 0; index < fullStringToCompareWithoutCase.Length; index++)
for (var compareStringIndex = 0; compareStringIndex < fullStringToCompareWithoutCase.Length; compareStringIndex++)
{
var ch = stringToCompare[index];
if (fullStringToCompareWithoutCase[index] == currentQueryToCompare[patternIndex])
if (fullStringToCompareWithoutCase[compareStringIndex] != currentQuerySubstring[currentQuerySubstringCharacterIndex])
{
if (firstMatchIndex < 0)
{ // first matched char will become the start of the compared string
firstMatchIndex = index;
}
matchFoundInPreviousLoop = false;
continue;
}
if (patternIndex == 0)
{ // first letter of current word
isFullWordMatched = true;
firstMatchIndexInWord = index;
}
else if (!isFullWordMatched)
{ // we want to verify that there is not a better match if this is not a full word
// in order to do so we need to verify all previous chars are part of the pattern
int startIndexToVerify = index - patternIndex;
bool allMatch = true;
for (int indexToCheck = 0; indexToCheck < patternIndex; indexToCheck++)
{
if (fullStringToCompareWithoutCase[startIndexToVerify + indexToCheck] !=
currentQueryToCompare[indexToCheck])
{
allMatch = false;
}
}
if (firstMatchIndex < 0)
{
// first matched char will become the start of the compared string
firstMatchIndex = compareStringIndex;
}
if (allMatch)
{ // update to this as a full word
isFullWordMatched = true;
if (currentQueryToCompareIndex == 0)
{ // first word so we need to update start index
firstMatchIndex = startIndexToVerify;
}
if (currentQuerySubstringCharacterIndex == 0)
{
// first letter of current word
matchFoundInPreviousLoop = true;
firstMatchIndexInWord = compareStringIndex;
}
else if (!matchFoundInPreviousLoop)
{
// we want to verify that there is not a better match if this is not a full word
// in order to do so we need to verify all previous chars are part of the pattern
var startIndexToVerify = compareStringIndex - currentQuerySubstringCharacterIndex;
indexList.RemoveAll(x => x >= firstMatchIndexInWord);
for (int indexToCheck = 0; indexToCheck < patternIndex; indexToCheck++)
{ // update the index list
indexList.Add(startIndexToVerify + indexToCheck);
}
}
}
lastMatchIndex = index + 1;
indexList.Add(index);
// increase the pattern matched index and check if everything was matched
if (++patternIndex == currentQueryToCompare.Length)
if (AllPreviousCharsMatched(startIndexToVerify, currentQuerySubstringCharacterIndex, fullStringToCompareWithoutCase, currentQuerySubstring))
{
if (++currentQueryToCompareIndex >= queryToCompareSeparated.Length)
{ // moved over all the words
allMatched = true;
break;
}
matchFoundInPreviousLoop = true;
// otherwise move to the next word
currentQueryToCompare = queryToCompareSeparated[currentQueryToCompareIndex];
patternIndex = 0;
if (!isFullWordMatched)
{ // if any of the words was not fully matched all are not fully matched
allWordsFullyMatched = false;
}
// if it's the begining character of the first query substring that is matched then we need to update start index
firstMatchIndex = currentQuerySubstringIndex == 0 ? startIndexToVerify : firstMatchIndex;
indexList = GetUpdatedIndexList(startIndexToVerify, currentQuerySubstringCharacterIndex, firstMatchIndexInWord, indexList);
}
}
else
lastMatchIndex = compareStringIndex + 1;
indexList.Add(compareStringIndex);
currentQuerySubstringCharacterIndex++;
// if finished looping through every character in the current substring
if (currentQuerySubstringCharacterIndex == currentQuerySubstring.Length)
{
isFullWordMatched = false;
// if any of the substrings was not matched then consider as all are not matched
allSubstringsContainedInCompareString = !matchFoundInPreviousLoop ? false : allSubstringsContainedInCompareString;
currentQuerySubstringIndex++;
allQuerySubstringsMatched = AllQuerySubstringsMatched(currentQuerySubstringIndex, querySubstrings.Length);
if (allQuerySubstringsMatched)
break;
// otherwise move to the next query substring
currentQuerySubstring = querySubstrings[currentQuerySubstringIndex];
currentQuerySubstringCharacterIndex = 0;
}
}
// return rendered string if we have a match for every char or all substring without whitespaces matched
if (allMatched)
// proceed to calculate score if every char or substring without whitespaces matched
if (allQuerySubstringsMatched)
{
// check if all query string was contained in string to compare
bool containedFully = lastMatchIndex - firstMatchIndex == queryWithoutCase.Length;
var score = CalculateSearchScore(query, stringToCompare, firstMatchIndex, lastMatchIndex - firstMatchIndex, containedFully, allWordsFullyMatched);
var score = CalculateSearchScore(query, stringToCompare, firstMatchIndex, lastMatchIndex - firstMatchIndex, allSubstringsContainedInCompareString);
var pinyinScore = ScoreForPinyin(stringToCompare, query);
var result = new MatchResult
{
Success = true,
MatchData = indexList,
RawScore = Math.Max(score, pinyinScore)
RawScore = Math.Max(score, pinyinScore),
AllSubstringsContainedInCompareString = allSubstringsContainedInCompareString
};
return result;
@ -161,8 +155,44 @@ namespace Wox.Infrastructure
return new MatchResult { Success = false };
}
private static int CalculateSearchScore(string query, string stringToCompare, int firstIndex, int matchLen,
bool isFullyContained, bool allWordsFullyMatched)
private static bool AllPreviousCharsMatched(int startIndexToVerify, int currentQuerySubstringCharacterIndex,
string fullStringToCompareWithoutCase, string currentQuerySubstring)
{
var allMatch = true;
for (int indexToCheck = 0; indexToCheck < currentQuerySubstringCharacterIndex; indexToCheck++)
{
if (fullStringToCompareWithoutCase[startIndexToVerify + indexToCheck] !=
currentQuerySubstring[indexToCheck])
{
allMatch = false;
}
}
return allMatch;
}
private static List<int> GetUpdatedIndexList(int startIndexToVerify, int currentQuerySubstringCharacterIndex, int firstMatchIndexInWord, List<int> indexList)
{
var updatedList = new List<int>();
indexList.RemoveAll(x => x >= firstMatchIndexInWord);
updatedList.AddRange(indexList);
for (int indexToCheck = 0; indexToCheck < currentQuerySubstringCharacterIndex; indexToCheck++)
{
updatedList.Add(startIndexToVerify + indexToCheck);
}
return updatedList;
}
private static bool AllQuerySubstringsMatched(int currentQuerySubstringIndex, int querySubstringsLength)
{
return currentQuerySubstringIndex >= querySubstringsLength;
}
private static int CalculateSearchScore(string query, string stringToCompare, int firstIndex, int matchLen, bool allSubstringsContainedInCompareString)
{
// A match found near the beginning of a string is scored more than a match found near the end
// A match is scored more if the characters in the patterns are closer to each other,
@ -179,15 +209,8 @@ namespace Wox.Infrastructure
score += 10;
}
if (isFullyContained)
{
score += 20; // honestly I'm not sure what would be a good number here or should it factor the size of the pattern
}
if (allWordsFullyMatched)
{
score += 20;
}
if (allSubstringsContainedInCompareString)
score += 10 * string.Concat(query.Where(c => !char.IsWhiteSpace(c))).Count();
return score;
}
@ -256,6 +279,11 @@ namespace Wox.Infrastructure
}
}
/// <summary>
/// Indicates if all query's substrings are contained in the string to compare
/// </summary>
public bool AllSubstringsContainedInCompareString { get; set; }
/// <summary>
/// Matched data to highlight.
/// </summary>
@ -268,7 +296,7 @@ namespace Wox.Infrastructure
private bool IsSearchPrecisionScoreMet(int score)
{
return score >= UserSettingSearchPrecision;
return score >= (int)UserSettingSearchPrecision;
}
private int ApplySearchPrecisionFilter(int score)

View file

@ -45,16 +45,19 @@ namespace Wox.Infrastructure.UserSettings
{
try
{
var precisionScore = (StringMatcher.SearchPrecisionScore)Enum.Parse(
typeof(StringMatcher.SearchPrecisionScore),
value);
var precisionScore = (StringMatcher.SearchPrecisionScore)Enum
.Parse(typeof(StringMatcher.SearchPrecisionScore), value);
QuerySearchPrecision = precisionScore;
StringMatcher.UserSettingSearchPrecision = (int)precisionScore;
StringMatcher.UserSettingSearchPrecision = precisionScore;
}
catch (System.Exception e)
catch (ArgumentException e)
{
// what do we do here?!
Logger.Log.Exception(nameof(Settings), "Fail to set QuerySearchPrecision", e);
Logger.Log.Exception(nameof(Settings), "Failed to load QuerySearchPrecisionString value from Settings file", e);
QuerySearchPrecision = StringMatcher.SearchPrecisionScore.Regular;
StringMatcher.UserSettingSearchPrecision = StringMatcher.SearchPrecisionScore.Regular;
throw;
}
}

View file

@ -4,7 +4,6 @@ using System.Diagnostics;
using System.Linq;
using NUnit.Framework;
using Wox.Infrastructure;
using Wox.Infrastructure.UserSettings;
using Wox.Plugin;
namespace Wox.Test
@ -122,115 +121,114 @@ namespace Wox.Test
}
}
[TestCase]
public void WhenGivenStringsForCalScoreMethodThenShouldReturnCurrentScoring()
[TestCase(Chrome, Chrome, 167)]
[TestCase(Chrome, LastIsChrome, 113)]
[TestCase(Chrome, HelpCureHopeRaiseOnMindEntityChrome, 21)]
[TestCase(Chrome, UninstallOrChangeProgramsOnYourComputer, 15)]
[TestCase(Chrome, CandyCrushSagaFromKing, 0)]
[TestCase("sql", MicrosoftSqlServerManagementStudio, 56)]
[TestCase("sql manag", MicrosoftSqlServerManagementStudio, 119)]//double spacing intended
public void WhenGivenQueryStringThenShouldReturnCurrentScoring(string queryString, string compareString, int expectedScore)
{
// Arrange
string searchTerm = "chrome"; // since this looks for specific results it will always be one case
var searchStrings = new List<string>
{
Chrome,//SCORE: 107
LastIsChrome,//SCORE: 53
HelpCureHopeRaiseOnMindEntityChrome,//SCORE: 21
UninstallOrChangeProgramsOnYourComputer, //SCORE: 15
CandyCrushSagaFromKing//SCORE: 0
}
.OrderByDescending(x => x)
.ToList();
// When, Given
var rawScore = StringMatcher.FuzzySearch(queryString, compareString).RawScore;
// Act
var results = new List<Result>();
foreach (var str in searchStrings)
{
results.Add(new Result
{
Title = str,
Score = StringMatcher.FuzzySearch(searchTerm, str).RawScore
});
}
// Assert
VerifyResult(147, Chrome);
VerifyResult(93, LastIsChrome);
VerifyResult(41, HelpCureHopeRaiseOnMindEntityChrome);
VerifyResult(35, UninstallOrChangeProgramsOnYourComputer);
VerifyResult(0, CandyCrushSagaFromKing);
void VerifyResult(int expectedScore, string expectedTitle)
{
var result = results.FirstOrDefault(x => x.Title == expectedTitle);
if (result == null)
{
Assert.Fail($"Fail to find result: {expectedTitle} in result list");
}
Assert.AreEqual(expectedScore, result.Score, $"Expected score for {expectedTitle}: {expectedScore}, Actual: {result.Score}");
}
// Should
Assert.AreEqual(expectedScore, rawScore, $"Expected score for compare string '{compareString}': {expectedScore}, Actual: {rawScore}");
}
[TestCase("goo", "Google Chrome", (int)StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("chr", "Google Chrome", (int)StringMatcher.SearchPrecisionScore.Low, true)]
[TestCase("chr", "Chrome", (int)StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("chr", "Help cure hope raise on mind entity Chrome", (int)StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("chr", "Help cure hope raise on mind entity Chrome", (int)StringMatcher.SearchPrecisionScore.Low, true)]
[TestCase("chr", "Candy Crush Saga from King", (int)StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("chr", "Candy Crush Saga from King", (int)StringMatcher.SearchPrecisionScore.None, true)]
[TestCase("ccs", "Candy Crush Saga from King", (int)StringMatcher.SearchPrecisionScore.Low, true)]
[TestCase("cand", "Candy Crush Saga from King", (int)StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("cand", "Help cure hope raise on mind entity Chrome", (int)StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("goo", "Google Chrome", StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("chr", "Google Chrome", StringMatcher.SearchPrecisionScore.Low, true)]
[TestCase("chr", "Chrome", StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("chr", "Help cure hope raise on mind entity Chrome", StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("chr", "Help cure hope raise on mind entity Chrome", StringMatcher.SearchPrecisionScore.Low, true)]
[TestCase("chr", "Candy Crush Saga from King", StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("chr", "Candy Crush Saga from King", StringMatcher.SearchPrecisionScore.None, true)]
[TestCase("ccs", "Candy Crush Saga from King", StringMatcher.SearchPrecisionScore.Low, true)]
[TestCase("cand", "Candy Crush Saga from King",StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("cand", "Help cure hope raise on mind entity Chrome", StringMatcher.SearchPrecisionScore.Regular, false)]
public void WhenGivenDesiredPrecisionThenShouldReturnAllResultsGreaterOrEqual(
string queryString,
string compareString,
int expectedPrecisionScore,
StringMatcher.SearchPrecisionScore expectedPrecisionScore,
bool expectedPrecisionResult)
{
// Arrange
var expectedPrecisionString = (StringMatcher.SearchPrecisionScore)expectedPrecisionScore;
StringMatcher.UserSettingSearchPrecision = expectedPrecisionScore; // this is why static state is evil...
// When
StringMatcher.UserSettingSearchPrecision = expectedPrecisionScore;
// Act
// Given
var matchResult = StringMatcher.FuzzySearch(queryString, compareString);
// Assert
Debug.WriteLine("");
Debug.WriteLine("###############################################");
Debug.WriteLine($"QueryString: {queryString} CompareString: {compareString}");
Debug.WriteLine($"RAW SCORE: {matchResult.RawScore.ToString()}, PrecisionLevelSetAt: {expectedPrecisionScore} ({(int)expectedPrecisionScore})");
Debug.WriteLine("###############################################");
Debug.WriteLine("");
// Should
Assert.AreEqual(expectedPrecisionResult, matchResult.IsSearchPrecisionScoreMet(),
$"Query:{queryString}{Environment.NewLine} " +
$"Compare:{compareString}{Environment.NewLine}" +
$"Raw Score: {matchResult.RawScore}{Environment.NewLine}" +
$"Precision Level: {expectedPrecisionString}={expectedPrecisionScore}");
$"Precision Score: {(int)expectedPrecisionScore}");
}
[TestCase("exce", "OverLeaf-Latex: An online LaTeX editor", (int)StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("term", "Windows Terminal (Preview)", (int)StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("sql s managa", MicrosoftSqlServerManagementStudio, (int)StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("sql' s manag", MicrosoftSqlServerManagementStudio, (int)StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("sql s manag", MicrosoftSqlServerManagementStudio, (int)StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("sql manag", MicrosoftSqlServerManagementStudio, (int)StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("sql", MicrosoftSqlServerManagementStudio, (int)StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("sql serv", MicrosoftSqlServerManagementStudio, (int)StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("mic", MicrosoftSqlServerManagementStudio, (int)StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("chr", "Shutdown", (int)StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("chr", "Change settings for text-to-speech and for speech recognition (if installed).", (int)StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("a test", "This is a test", (int)StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("test", "This is a test", (int)StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("exce", "OverLeaf-Latex: An online LaTeX editor", StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("term", "Windows Terminal (Preview)", StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("sql s managa", MicrosoftSqlServerManagementStudio, StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("sql' s manag", MicrosoftSqlServerManagementStudio, StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("sql s manag", MicrosoftSqlServerManagementStudio, StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("sql manag", MicrosoftSqlServerManagementStudio, StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("sql", MicrosoftSqlServerManagementStudio, StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("sql serv", MicrosoftSqlServerManagementStudio, StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("sql studio", MicrosoftSqlServerManagementStudio, StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("mic", MicrosoftSqlServerManagementStudio, StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("chr", "Shutdown", StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("chr", "Change settings for text-to-speech and for speech recognition (if installed).", StringMatcher.SearchPrecisionScore.Regular, false)]
[TestCase("a test", "This is a test", StringMatcher.SearchPrecisionScore.Regular, true)]
[TestCase("test", "This is a test", StringMatcher.SearchPrecisionScore.Regular, true)]
public void WhenGivenQueryShouldReturnResultsContainingAllQuerySubstrings(
string queryString,
string compareString,
int expectedPrecisionScore,
StringMatcher.SearchPrecisionScore expectedPrecisionScore,
bool expectedPrecisionResult)
{
// Arrange
var expectedPrecisionString = (StringMatcher.SearchPrecisionScore)expectedPrecisionScore;
StringMatcher.UserSettingSearchPrecision = expectedPrecisionScore; // this is why static state is evil...
// When
StringMatcher.UserSettingSearchPrecision = expectedPrecisionScore;
// Act
// Given
var matchResult = StringMatcher.FuzzySearch(queryString, compareString);
// Assert
Debug.WriteLine("");
Debug.WriteLine("###############################################");
Debug.WriteLine($"QueryString: {queryString} CompareString: {compareString}");
Debug.WriteLine($"RAW SCORE: {matchResult.RawScore.ToString()}, PrecisionLevelSetAt: {expectedPrecisionScore} ({(int)expectedPrecisionScore})");
Debug.WriteLine("###############################################");
Debug.WriteLine("");
// Should
Assert.AreEqual(expectedPrecisionResult, matchResult.IsSearchPrecisionScoreMet(),
$"Query:{queryString}{Environment.NewLine} " +
$"Compare:{compareString}{Environment.NewLine}" +
$"Raw Score: {matchResult.RawScore}{Environment.NewLine}" +
$"Precision Level: {expectedPrecisionString}={expectedPrecisionScore}");
$"Precision Score: {(int)expectedPrecisionScore}");
}
[TestCase("sql servman", MicrosoftSqlServerManagementStudio, false)]
[TestCase("sql serv man", MicrosoftSqlServerManagementStudio, true)]
[TestCase("sql", MicrosoftSqlServerManagementStudio, true)]
[TestCase("sqlserv", MicrosoftSqlServerManagementStudio, false)]
[TestCase("mssms", MicrosoftSqlServerManagementStudio, false)]
[TestCase("chr", "Change settings for text-to-speech and for speech recognition (if installed).", false)]
[TestCase("ch r", "Change settings for text-to-speech and for speech recognition (if installed).", true)]
public void WhenGivenQueryShouldEvaluateTrueFalseIfCompareStringContainsAllSubstrings(string queryString, string compareString, bool expectedResult)
{
// When, Given
var matchResult = StringMatcher.FuzzySearch(queryString, compareString).AllSubstringsContainedInCompareString;
// Should
Assert.AreEqual(matchResult, expectedResult);
}
}
}

View file

@ -55,7 +55,7 @@ namespace Wox
Alphabet.Initialize(_settings);
StringMatcher.UserSettingSearchPrecision = (int)_settings.QuerySearchPrecision;
StringMatcher.UserSettingSearchPrecision = _settings.QuerySearchPrecision;
StringMatcher.ShouldUsePinyin = _settings.ShouldUsePinyin;
PluginManager.LoadPlugins(_settings.PluginSettings);

View file

@ -73,7 +73,7 @@ namespace Wox.ViewModel
public List<string> QuerySearchPrecisionStrings
{
get
{
{
var precisionStrings = new List<string>();
var enumList = Enum.GetValues(typeof(StringMatcher.SearchPrecisionScore)).Cast<StringMatcher.SearchPrecisionScore>().ToList();