Merge pull request #1144 from krunal1313/gzip_indexing

Tests for gzip indexing
This commit is contained in:
Kishore Nallan 2023-08-22 20:15:57 +05:30 committed by GitHub
commit 55bc819c7c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 60 additions and 2 deletions

1
BUILD
View File

@ -137,6 +137,7 @@ filegroup(
"test/**/*.txt",
"test/**/*.ini",
"test/**/*.jsonl",
"test/**/*.gz",
]),
)

View File

@ -142,8 +142,6 @@ private:
butil::EndPoint peering_endpoint;
Option<bool> handle_gzip(const std::shared_ptr<http_req>& request);
public:
static constexpr const char* log_dir_name = "log";
@ -241,6 +239,8 @@ public:
std::string get_leader_url() const;
static Option<bool> handle_gzip(const std::shared_ptr<http_req>& request);
private:
friend class ReplicationClosure;

View File

@ -4,6 +4,7 @@
#include <collection_manager.h>
#include <core_api.h>
#include "core_api_utils.h"
#include "raft_server.h"
class CoreAPIUtilsTest : public ::testing::Test {
protected:
@ -621,6 +622,7 @@ TEST_F(CoreAPIUtilsTest, PresetSingleSearch) {
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
Collection* coll1 = op.get();
auto preset_value = R"(
{"collection":"preset_coll", "per_page": "12"}
@ -1157,4 +1159,59 @@ TEST_F(CoreAPIUtilsTest, TestProxyTimeout) {
ASSERT_EQ(408, resp->status_code);
ASSERT_EQ("Server error on remote server. Please try again later.", nlohmann::json::parse(resp->body)["message"]);
}
TEST_F(CoreAPIUtilsTest, SampleGzipIndexTest) {
Collection *coll_hnstories;
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
coll_hnstories = collectionManager.get_collection("coll_hnstories").get();
if(coll_hnstories == nullptr) {
coll_hnstories = collectionManager.create_collection("coll_hnstories", 4, fields, "title").get();
}
auto req = std::make_shared<http_req>();
std::ifstream infile(std::string(ROOT_DIR)+"test/resources/hnstories.jsonl.gz");
std::stringstream outbuffer;
infile.seekg (0, infile.end);
int length = infile.tellg();
infile.seekg (0, infile.beg);
req->body.resize(length);
infile.read(&req->body[0], length);
auto res = ReplicationState::handle_gzip(req);
if (!res.error().empty()) {
LOG(ERROR) << res.error();
FAIL();
} else {
outbuffer << req->body;
}
std::vector<std::string> doc_lines;
std::string line;
while(std::getline(outbuffer, line)) {
doc_lines.push_back(line);
}
ASSERT_EQ(14, doc_lines.size());
ASSERT_EQ("{\"points\":1,\"title\":\"DuckDuckGo Settings\"}", doc_lines[0]);
ASSERT_EQ("{\"points\":1,\"title\":\"Making Twitter Easier to Use\"}", doc_lines[1]);
ASSERT_EQ("{\"points\":2,\"title\":\"London refers Uber app row to High Court\"}", doc_lines[2]);
ASSERT_EQ("{\"points\":1,\"title\":\"Young Global Leaders, who should be nominated? (World Economic Forum)\"}", doc_lines[3]);
ASSERT_EQ("{\"points\":1,\"title\":\"Blooki.st goes BETA in a few hours\"}", doc_lines[4]);
ASSERT_EQ("{\"points\":1,\"title\":\"Unicode Security Data: Beta Review\"}", doc_lines[5]);
ASSERT_EQ("{\"points\":2,\"title\":\"FileMap: MapReduce on the CLI\"}", doc_lines[6]);
ASSERT_EQ("{\"points\":1,\"title\":\"[Full Video] NBC News Interview with Edward Snowden\"}", doc_lines[7]);
ASSERT_EQ("{\"points\":1,\"title\":\"Hybrid App Monetization Example with Mobile Ads and In-App Purchases\"}", doc_lines[8]);
ASSERT_EQ("{\"points\":1,\"title\":\"We need oppinion from Android Developers\"}", doc_lines[9]);
ASSERT_EQ("{\"points\":1,\"title\":\"\\\\t Why Mobile Developers Should Care About Deep Linking\"}", doc_lines[10]);
ASSERT_EQ("{\"points\":2,\"title\":\"Are we getting too Sassy? Weighing up micro-optimisation vs. maintainability\"}", doc_lines[11]);
ASSERT_EQ("{\"points\":2,\"title\":\"Google's XSS game\"}", doc_lines[12]);
ASSERT_EQ("{\"points\":1,\"title\":\"Telemba Turns Your Old Roomba and Tablet Into a Telepresence Robot\"}", doc_lines[13]);
infile.close();
}

Binary file not shown.