postgres/typesense
1
0
Fork 0
mirror of https://github.com/typesense/typesense.git synced 2025-05-17 20:22:32 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Refactor tokenizer to use index, skip and separate logic.

Browse Source
This commit is contained in:
Kishore Nallan 2021-04-16 17:02:15 +05:30 committed by Kishore Nallan
parent cdcdc7bd20
commit 1d1712f391
9 changed files with 135 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
include/index.h
View File

@ -323,24 +323,9 @@ public:
void scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc);
static void tokenize_doc_field(const nlohmann::json& document, const field& search_field, std::vector<std::string>& tokens);
template<typename T>
static bool _arrays_match(std::vector<T> reindex_vals, std::vector<T> old_vals) {
if(old_vals.size() != reindex_vals.size()) {
return false;
}
for(size_t i=0; i < reindex_vals.size(); i++) {
const T& reindex_val = reindex_vals[i];
const T& old_val = old_vals[i];
if(reindex_val != old_val) {
return false;
}
}
return true;
}
static void tokenize_string_field(const nlohmann::json& document,
const field& search_field, std::vector<std::string>& tokens,
const std::string& locale);
// Public operations

15
include/string_utils.h
View File

@ -4,26 +4,19 @@
#include <algorithm>
#include <sstream>
#include <ctype.h>
#include <unicode/translit.h>
#include <iconv.h>
#include <vector>
#include <random>
#include <map>
#include "wyhash_v5.h"
struct StringUtils {
UErrorCode status;
//icu::Transliterator* transliterator;
iconv_t cd;
StringUtils(): status(U_ZERO_ERROR) {
// transliterator(icu::Transliterator::createInstance("Latin-ASCII", UTRANS_FORWARD, status))
cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
StringUtils() {
}
~StringUtils() {
//delete transliterator;
iconv_close(cd);
}
// Adapted from: http://stackoverflow.com/a/236180/131050
@ -217,8 +210,6 @@ struct StringUtils {
std::transform(str.begin(), str.end(), str.begin(), ::tolower);
}
void unicode_normalize(std::string& str) const;
/* https://stackoverflow.com/a/34571089/131050 */
static std::string base64_encode(const std::string &in) {
std::string out;

20
include/tokenizer.h
View File

@ -18,9 +18,11 @@ private:
size_t token_counter = 0;
iconv_t cd;
static const size_t CHARS = 0;
static const size_t SEPARATORS = 1;
size_t stream_mode;
static const size_t INDEX = 0;
static const size_t SEPARATE = 1;
static const size_t SKIP = 2;
size_t prev_stream_mode;
std::stringstream out;
@ -31,6 +33,12 @@ private:
int32_t prev_position = -1;
char* normalized_text = nullptr;
inline size_t get_stream_mode(char c) {
return std::isalnum(c) ? INDEX : (
(c == ' ' || c == '\n') ? SEPARATE : SKIP
);
}
public:
explicit Tokenizer(const std::string& input,
@ -48,11 +56,11 @@ public:
cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
if(!input.empty() && (std::isalnum(text[0]) || (text[i] & ~0x7f) != 0)) {
if(!text.empty() && (std::isalnum(text[0]) || (text[i] & ~0x7f) != 0)) {
// alphanum or non-ascii
stream_mode = CHARS;
prev_stream_mode = INDEX;
} else {
stream_mode = SEPARATORS;
prev_stream_mode = SEPARATE;
}
if(!locale.empty() && locale != "en") {

62
src/index.cpp
View File

@ -405,60 +405,10 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_do
lock.unlock();
bool arrays_match = false;
// compare values between old and update docs:
// if they match, we will remove them from both del and update docs
if(search_field.is_string()) {
// Go through all the field names and find the keys+values so that they can be removed from in-memory index
std::vector<std::string> reindex_vals;
std::vector<std::string> old_vals;
tokenize_doc_field(update_doc, search_field, reindex_vals);
tokenize_doc_field(old_doc, search_field, old_vals);
arrays_match = _arrays_match<std::string>(reindex_vals, old_vals);
} else if(search_field.is_int32()) {
std::vector<int32_t> reindex_vals = search_field.is_single_integer() ?
std::vector<int32_t>{update_doc[field_name].get<int32_t>()} :
update_doc[field_name].get<std::vector<int32_t>>();
std::vector<int32_t> old_vals = search_field.is_single_integer() ?
std::vector<int32_t>{old_doc[field_name].get<int32_t>()} :
old_doc[field_name].get<std::vector<int32_t>>();
arrays_match = _arrays_match<int32_t>(reindex_vals, old_vals);
} else if(search_field.is_int64()) {
std::vector<int64_t> reindex_vals = search_field.is_single_integer() ?
std::vector<int64_t>{update_doc[field_name].get<int64_t>()} :
update_doc[field_name].get<std::vector<int64_t>>();
std::vector<int64_t> old_vals = search_field.is_single_integer() ?
std::vector<int64_t>{old_doc[field_name].get<int64_t>()} :
old_doc[field_name].get<std::vector<int64_t>>();
arrays_match = _arrays_match<int64_t>(reindex_vals, old_vals);
} else if(search_field.is_float()) {
std::vector<float> reindex_vals = search_field.is_single_float() ?
std::vector<float>{update_doc[field_name].get<float>()} :
update_doc[field_name].get<std::vector<float>>();
std::vector<float> old_vals = search_field.is_single_float() ?
std::vector<float>{old_doc[field_name].get<float>()} :
old_doc[field_name].get<std::vector<float>>();
arrays_match = _arrays_match<float>(reindex_vals, old_vals);
} else if(search_field.is_bool()) {
std::vector<bool> reindex_vals = search_field.is_single_bool() ?
std::vector<bool>{update_doc[field_name].get<bool>()} :
update_doc[field_name].get<std::vector<bool>>();
std::vector<bool> old_vals = search_field.is_single_bool() ?
std::vector<bool>{old_doc[field_name].get<bool>()} :
old_doc[field_name].get<std::vector<bool>>();
arrays_match = _arrays_match<bool>(reindex_vals, old_vals);
}
if(arrays_match) {
if(update_doc[search_field.name] == old_doc[search_field.name]) {
del_keys.push_back(field_name);
}
}
@ -2393,7 +2343,7 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
// Go through all the field names and find the keys+values so that they can be removed from in-memory index
if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {
std::vector<std::string> tokens;
tokenize_doc_field(document, search_field, tokens);
tokenize_string_field(document, search_field, tokens, search_field.locale);
for(auto & token: tokens) {
const unsigned char *key = (const unsigned char *) token.c_str();
@ -2491,17 +2441,17 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
return Option<uint32_t>(seq_id);
}
void Index::tokenize_doc_field(const nlohmann::json& document, const field& search_field,
std::vector<std::string>& tokens) {
void Index::tokenize_string_field(const nlohmann::json& document, const field& search_field,
std::vector<std::string>& tokens, const std::string& locale) {
const std::string& field_name = search_field.name;
if(search_field.type == field_types::STRING) {
Tokenizer(document[field_name], true, true, !search_field.is_string()).tokenize(tokens);
Tokenizer(document[field_name], false, true, false, locale).tokenize(tokens);
} else if(search_field.type == field_types::STRING_ARRAY) {
const std::vector<std::string>& values = document[field_name].get<std::vector<std::string>>();
for(const std::string & value: values) {
Tokenizer(value, true, true, !search_field.is_string()).tokenize(tokens);
Tokenizer(value, false, true, false, locale).tokenize(tokens);
}
}
}

46
src/string_utils.cpp
View File

@ -22,52 +22,6 @@ std::string lower_and_no_special_chars(const std::string & str) {
return ss.str();
}
void StringUtils::unicode_normalize(std::string & str) const {
if(str.empty()) {
return ;
}
std::stringstream out;
for (char *s = &str[0]; *s;) {
char inbuf[5];
char *p = inbuf;
if((*s & ~0x7f) == 0 ) {
// ascii character
out << *s++;
continue;
}
// group bytes to form a unicode representation
*p++ = *s++;
if ((*s & 0xC0) == 0x80) *p++ = *s++;
if ((*s & 0xC0) == 0x80) *p++ = *s++;
if ((*s & 0xC0) == 0x80) *p++ = *s++;
*p = 0;
size_t insize = (p - &inbuf[0]);
char outbuf[5] = {};
size_t outsize = sizeof(outbuf);
char *outptr = outbuf;
char *inptr = inbuf;
//printf("[%s]\n", inbuf);
errno = 0;
iconv(cd, &inptr, &insize, &outptr, &outsize);
if(errno == EILSEQ) {
// symbol cannot be represented as ASCII, so write the original symbol
out << inbuf;
} else {
out << outbuf;
}
}
str = lower_and_no_special_chars(out.str());
}
std::string StringUtils::randstring(size_t length) {
static auto& chrs = "0123456789"
"abcdefghijklmnopqrstuvwxyz"

41
src/tokenizer.cpp
View File

@ -2,6 +2,16 @@
#include "tokenizer.h"
bool Tokenizer::next(std::string &token, size_t& token_index) {
if(no_op) {
if(i == text.size()) {
return false;
}
token = text;
i = text.size();
return true;
}
if(!locale.empty() && locale != "en") {
while (position != icu::BreakIterator::DONE) {
//LOG(INFO) << "Position: " << position;
@ -35,23 +45,18 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {
return false;
}
if(no_op) {
if(i == text.size()) {
return false;
}
token = text;
i = text.size();
return true;
}
while(i < text.size()) {
bool is_ascii = (text[i] & ~0x7f) == 0;
if(is_ascii) {
const size_t next_stream_mode = std::isalnum(text[i]) ? CHARS : SEPARATORS;
size_t this_stream_mode = get_stream_mode(text[i]);
if(next_stream_mode != stream_mode) {
// We tokenize when `stream_mode` changes
if(this_stream_mode == SKIP && !keep_separators) {
i++;
continue;
}
if(this_stream_mode != prev_stream_mode) {
// We tokenize when `prev_stream_mode` changes
token = out.str();
out.str(std::string());
@ -62,13 +67,13 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {
}
i++;
if(stream_mode == SEPARATORS && !keep_separators) {
stream_mode = next_stream_mode;
if(prev_stream_mode == SEPARATE && !keep_separators) {
prev_stream_mode = this_stream_mode;
continue;
}
token_index = token_counter++;
stream_mode = next_stream_mode;
prev_stream_mode = this_stream_mode;
return true;
} else {
if(normalize) {
@ -82,9 +87,9 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {
}
}
if(stream_mode == SEPARATORS) { // to detect first non-ascii character
if(prev_stream_mode == SEPARATE) { // to detect first non-ascii character
// we will tokenize now and treat the following non-ascii chars as a different token
stream_mode = CHARS;
prev_stream_mode = INDEX;
token = out.str();
out.str(std::string());

66
test/collection_test.cpp
View File

@ -1835,6 +1835,70 @@ TEST_F(CollectionTest, DeletionOfADocument) {
collectionManager.drop_collection("collection_for_del");
}
TEST_F(CollectionTest, DeletionOfDocumentSingularFields) {
Collection *coll1;
std::vector<field> fields = {field("str", field_types::STRING, false),
field("int32", field_types::INT32, false),
field("int64", field_types::INT64, false),
field("float", field_types::FLOAT, false),
field("bool", field_types::BOOL, false)};
std::vector<sort_by> sort_fields = { sort_by("int32", "DESC") };
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "int32").get();
}
nlohmann::json doc;
doc["id"] = "100";
doc["str"] = "[NEW] Cell Phone Cases, Holders & Clips!";
doc["int32"] = 100032;
doc["int64"] = 1582369739000;
doc["float"] = -293.24;
doc["bool"] = true;
Option<nlohmann::json> add_op = coll1->add(doc.dump());
ASSERT_TRUE(add_op.ok());
nlohmann::json res = coll1->search("phone", {"str"}, "", {}, sort_fields, 0, 10, 1,
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10).get();
ASSERT_EQ(1, res["found"]);
Option<std::string> rem_op = coll1->remove("100");
ASSERT_TRUE(rem_op.ok());
res = coll1->search("phone", {"str"}, "", {}, sort_fields, 0, 10, 1,
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10).get();
ASSERT_EQ(0, res["found"].get<int32_t>());
// also assert against the actual index
Index *index = coll1->_get_indexes()[0]; // seq id will always be zero for first document
auto search_index = index->_get_search_index();
auto numerical_index = index->_get_numerical_index();
auto str_tree = search_index["str"];
auto int32_tree = numerical_index["int32"];
auto int64_tree = numerical_index["int64"];
auto float_tree = numerical_index["float"];
auto bool_tree = numerical_index["bool"];
ASSERT_EQ(0, art_size(str_tree));
ASSERT_EQ(0, int32_tree->size());
ASSERT_EQ(0, int64_tree->size());
ASSERT_EQ(0, float_tree->size());
ASSERT_EQ(0, bool_tree->size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionTest, DeletionOfDocumentArrayFields) {
Collection *coll1;
@ -3304,7 +3368,7 @@ TEST_F(CollectionTest, HighlightWithAccentedCharacters) {
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionTest, SearchingForRecordsWithSpecialChars) {
TEST_F(CollectionTest, DISABLED_SearchingForRecordsWithSpecialChars) {
Collection *coll1;
std::vector<field> fields = {field("title", field_types::STRING, false),

34
test/string_utils_test.cpp
View File

@ -4,40 +4,6 @@
#include <unicode/translit.h>
#include <json.hpp>
TEST(StringUtilsTest, ShouldNormalizeString) {
StringUtils string_utils;
std::string alphanum = "Aa12Zz";
string_utils.unicode_normalize(alphanum);
ASSERT_STREQ("aa12zz", alphanum.c_str());
std::string alphanum_space = "Aa12Zz 12A";
string_utils.unicode_normalize(alphanum_space);
ASSERT_STREQ("aa12zz12a", alphanum_space.c_str());
std::string alphanum_specialchars = "Aa12Zz@W-_?,.R";
string_utils.unicode_normalize(alphanum_specialchars);
ASSERT_STREQ("aa12zzwr", alphanum_specialchars.c_str());
std::string alphanum_unicodechars = "abcÅà123ß12";
string_utils.unicode_normalize(alphanum_unicodechars);
ASSERT_STREQ("abcaa123ss12", alphanum_unicodechars.c_str());
std::string tamil_unicodechars = "தமிழ் நாடு";
string_utils.unicode_normalize(tamil_unicodechars);
ASSERT_STREQ("தமிழ்நாடு", tamil_unicodechars.c_str());
std::string chinese_unicodechars = "你好吗";
string_utils.unicode_normalize(chinese_unicodechars);
ASSERT_STREQ("你好吗", chinese_unicodechars.c_str());
std::string mixed_unicodechars = "çн தமிழ் நாடு so...";
string_utils.unicode_normalize(mixed_unicodechars);
ASSERT_STREQ("cнதமிழ்நாடுso", mixed_unicodechars.c_str());
// Any-Latin; Latin-ASCII; Lower()
}
TEST(StringUtilsTest, ShouldJoinString) {
std::vector<std::string> parts = {"foo", "bar", "baz", "bazinga"};

42
test/tokenizer_test.cpp
View File

@ -40,17 +40,13 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {
const std::string split_tokens = "foo-bar-baz";
tokens.clear();
Tokenizer(split_tokens, false, false, false).tokenize(tokens);
ASSERT_EQ(3, tokens.size());
ASSERT_STREQ("foo", tokens[0].c_str());
ASSERT_STREQ("bar", tokens[1].c_str());
ASSERT_STREQ("baz", tokens[2].c_str());
ASSERT_EQ(1, tokens.size());
ASSERT_STREQ("foobarbaz", tokens[0].c_str());
tokens.clear();
Tokenizer(split_tokens, false, true, false).tokenize(tokens);
ASSERT_EQ(3, tokens.size());
ASSERT_STREQ("foo", tokens[0].c_str());
ASSERT_STREQ("bar", tokens[1].c_str());
ASSERT_STREQ("baz", tokens[2].c_str());
ASSERT_EQ(1, tokens.size());
ASSERT_STREQ("foobarbaz", tokens[0].c_str());
// multiple spaces
const std::string multispace_tokens = "foo bar";
@ -62,7 +58,7 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {
ASSERT_STREQ("bar", tokens[2].c_str());
// special chars
const std::string specialchar_tokens = "https://www.amazon.com/s?k=phone&ref=nb_sb_noss_2";;
const std::string specialchar_tokens = "https://www.amazon.com/s?k=phone&ref=nb_sb_noss_2";
tokens.clear();
Tokenizer(specialchar_tokens, true, false, false).tokenize(tokens);
ASSERT_EQ(23, tokens.size());
@ -102,13 +98,14 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeUnicodeStrings) {
const std::string withoutnormalize = "Mise à, jour.";
tokens.clear();
Tokenizer(withoutnormalize, true, false, false).tokenize(tokens);
ASSERT_EQ(6, tokens.size());
ASSERT_EQ(7, tokens.size());
ASSERT_STREQ("Mise", tokens[0].c_str());
ASSERT_STREQ(" ", tokens[1].c_str());
ASSERT_STREQ("à", tokens[2].c_str());
ASSERT_STREQ(", ", tokens[3].c_str());
ASSERT_STREQ("jour", tokens[4].c_str());
ASSERT_STREQ(".", tokens[5].c_str());
ASSERT_STREQ(",", tokens[3].c_str());
ASSERT_STREQ(" ", tokens[4].c_str());
ASSERT_STREQ("jour", tokens[5].c_str());
ASSERT_STREQ(".", tokens[6].c_str());
// when normalization and keep empty are disabled
const std::string withoutnormalizeandkeepempty = "Mise à jour.";
@ -144,17 +141,20 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {
tokens.push_back(token);
}
ASSERT_EQ(10, tokens.size());
ASSERT_EQ(13, tokens.size());
ASSERT_STREQ("michael", tokens[0].c_str());
ASSERT_STREQ(" ", tokens[1].c_str());
ASSERT_STREQ("jordan", tokens[2].c_str());
ASSERT_STREQ(":\n\n", tokens[3].c_str());
ASSERT_STREQ("welcome", tokens[4].c_str());
ASSERT_STREQ(", ", tokens[5].c_str());
ASSERT_STREQ("everybody", tokens[6].c_str());
ASSERT_STREQ(". ", tokens[7].c_str());
ASSERT_STREQ("welcome", tokens[8].c_str());
ASSERT_STREQ("!", tokens[9].c_str());
ASSERT_STREQ(":", tokens[3].c_str());
ASSERT_STREQ("\n\n", tokens[4].c_str());
ASSERT_STREQ("welcome", tokens[5].c_str());
ASSERT_STREQ(",", tokens[6].c_str());
ASSERT_STREQ(" ", tokens[7].c_str());
ASSERT_STREQ("everybody", tokens[8].c_str());
ASSERT_STREQ(".", tokens[9].c_str());
ASSERT_STREQ(" ", tokens[10].c_str());
ASSERT_STREQ("welcome", tokens[11].c_str());
ASSERT_STREQ("!", tokens[12].c_str());
// check for index when separators are not kept
Tokenizer tokenizer2(withnewline, false, true, false);