From 1f8c0d73dd35f1f626bc25502c3b1298f8e89eac Mon Sep 17 00:00:00 2001 From: krunal Date: Mon, 7 Aug 2023 16:23:43 +0530 Subject: [PATCH 1/3] adding tests --- include/raft_server.h | 4 ++-- test/core_api_utils_test.cpp | 27 +++++++++++++++++++++++++++ test/resources/hnstories.jsonl.gz | Bin 0 -> 519 bytes 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 test/resources/hnstories.jsonl.gz diff --git a/include/raft_server.h b/include/raft_server.h index dddaccb9..d5fcfec7 100644 --- a/include/raft_server.h +++ b/include/raft_server.h @@ -142,8 +142,6 @@ private: butil::EndPoint peering_endpoint; - Option handle_gzip(const std::shared_ptr& request); - public: static constexpr const char* log_dir_name = "log"; @@ -241,6 +239,8 @@ public: std::string get_leader_url() const; + static Option handle_gzip(const std::shared_ptr& request); + private: friend class ReplicationClosure; diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp index d24b44bf..080152d3 100644 --- a/test/core_api_utils_test.cpp +++ b/test/core_api_utils_test.cpp @@ -4,6 +4,7 @@ #include #include #include "core_api_utils.h" +#include "raft_server.h" class CoreAPIUtilsTest : public ::testing::Test { protected: @@ -1137,4 +1138,30 @@ TEST_F(CoreAPIUtilsTest, TestProxyInvalid) { ASSERT_EQ(400, resp->status_code); ASSERT_EQ("Headers must be a JSON object.", nlohmann::json::parse(resp->body)["message"]); +} + +TEST_F(CoreAPIUtilsTest, SampleGzipIndexTest) { + Collection *coll_hnstories; + + std::vector fields = {field("title", field_types::STRING, false), + field("points", field_types::INT32, false),}; + + coll_hnstories = collectionManager.get_collection("coll_hnstories").get(); + if(coll_hnstories == nullptr) { + coll_hnstories = collectionManager.create_collection("coll_hnstories", 4, fields, "title").get(); + } + + auto req = std::make_shared(); + char buf[196605]; //chunk size limit + std::ifstream infile(std::string(ROOT_DIR)+"test/resources/hnstories.jsonl.gz"); + + while(infile >> buf) { + req->body = buf; + + auto res = ReplicationState::handle_gzip(req); + if(!res.error().empty()) { + LOG(ERROR) << res.error(); + FAIL(); + } + } } \ No newline at end of file diff --git a/test/resources/hnstories.jsonl.gz b/test/resources/hnstories.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..ce37418939440daa7e52bf1c4c95b714c318c261 GIT binary patch literal 519 zcmV+i0{HzOiwFo>vCw1y188n@bZ>HLWpgo0E^2dcZfpQ`Q_G6mKoGp=D+*nb!0tkF z^kwmT7wk>&BTmTXFw(T8!J}z04=c`+e@_o4EP?1C33|G#da6nep@6wyRW z#i*F2>Xe@)V@oj_SBK*Fo0tE!uGLA7vFlBY`m>cvBBQ%_>MoDno8mA)Zd%Qcam*TwQ zT&%EQpW63B;1kPyGIt@OW(t_hcVid_g(LM4ct0q`-kFLuiDXssCSEX8QH?O6sIZ~4 zp?*k^g|W0&-3ap9kWLjr^ob2&x_TU<`@Trl;)~J5Kkzh};)(izM<+I9E{G!y=DJs2 zW9zurTXi$}d zctNDWU1wYdJ$i02cG|O0&ge{*I~rE?`8ndyT>jF(4q$sVd#b!_RdR}QNF7$j4@5H9yPkR&hHQU}S2=UeWgX-&9_aSnHAmEg&vWEXLGh-1lITY4Mt zb-P8aS{l0Cjx4pM!Y+9iAi4JVBv;xn=W3RDr%FpP+scx`+~0W$ Date: Mon, 7 Aug 2023 20:53:10 +0530 Subject: [PATCH 2/3] updating test --- test/core_api_utils_test.cpp | 49 ++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp index 82236699..d2ae2599 100644 --- a/test/core_api_utils_test.cpp +++ b/test/core_api_utils_test.cpp @@ -933,7 +933,7 @@ TEST_F(CoreAPIUtilsTest, ExportIncludeExcludeFields) { // exclude fields - delete dynamic_cast(req->data); + delete dynamic_cast(req->data); req->data = nullptr; res->body.clear(); req->params.erase("include_fields"); @@ -994,7 +994,7 @@ TEST_F(CoreAPIUtilsTest, ExportIncludeExcludeFieldsWithFilter) { // exclude fields - delete dynamic_cast(req->data); + delete dynamic_cast(req->data); req->data = nullptr; res->body.clear(); req->params.erase("include_fields"); @@ -1173,16 +1173,45 @@ TEST_F(CoreAPIUtilsTest, SampleGzipIndexTest) { } auto req = std::make_shared(); - char buf[196605]; //chunk size limit std::ifstream infile(std::string(ROOT_DIR)+"test/resources/hnstories.jsonl.gz"); + std::stringstream outbuffer; - while(infile >> buf) { - req->body = buf; + infile.seekg (0, infile.end); + int length = infile.tellg(); + infile.seekg (0, infile.beg); - auto res = ReplicationState::handle_gzip(req); - if(!res.error().empty()) { - LOG(ERROR) << res.error(); - FAIL(); - } + req->body.resize(length); + infile.read(&req->body[0], length); + + auto res = ReplicationState::handle_gzip(req); + if (!res.error().empty()) { + LOG(ERROR) << res.error(); + FAIL(); + } else { + outbuffer << req->body; } + + std::vector doc_lines; + std::string line; + while(std::getline(outbuffer, line)) { + doc_lines.push_back(line); + } + + ASSERT_EQ(14, doc_lines.size()); + ASSERT_EQ("{\"points\":1,\"title\":\"DuckDuckGo Settings\"}", doc_lines[0]); + ASSERT_EQ("{\"points\":1,\"title\":\"Making Twitter Easier to Use\"}", doc_lines[1]); + ASSERT_EQ("{\"points\":2,\"title\":\"London refers Uber app row to High Court\"}", doc_lines[2]); + ASSERT_EQ("{\"points\":1,\"title\":\"Young Global Leaders, who should be nominated? (World Economic Forum)\"}", doc_lines[3]); + ASSERT_EQ("{\"points\":1,\"title\":\"Blooki.st goes BETA in a few hours\"}", doc_lines[4]); + ASSERT_EQ("{\"points\":1,\"title\":\"Unicode Security Data: Beta Review\"}", doc_lines[5]); + ASSERT_EQ("{\"points\":2,\"title\":\"FileMap: MapReduce on the CLI\"}", doc_lines[6]); + ASSERT_EQ("{\"points\":1,\"title\":\"[Full Video] NBC News Interview with Edward Snowden\"}", doc_lines[7]); + ASSERT_EQ("{\"points\":1,\"title\":\"Hybrid App Monetization Example with Mobile Ads and In-App Purchases\"}", doc_lines[8]); + ASSERT_EQ("{\"points\":1,\"title\":\"We need oppinion from Android Developers\"}", doc_lines[9]); + ASSERT_EQ("{\"points\":1,\"title\":\"\\\\t Why Mobile Developers Should Care About Deep Linking\"}", doc_lines[10]); + ASSERT_EQ("{\"points\":2,\"title\":\"Are we getting too Sassy? Weighing up micro-optimisation vs. maintainability\"}", doc_lines[11]); + ASSERT_EQ("{\"points\":2,\"title\":\"Google's XSS game\"}", doc_lines[12]); + ASSERT_EQ("{\"points\":1,\"title\":\"Telemba Turns Your Old Roomba and Tablet Into a Telepresence Robot\"}", doc_lines[13]); + + infile.close(); } \ No newline at end of file From e697af56e04fa2bcc80c9a6cabe031b39c964326 Mon Sep 17 00:00:00 2001 From: krunal Date: Tue, 22 Aug 2023 14:46:05 +0530 Subject: [PATCH 3/3] adding gzip file type in test files --- BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/BUILD b/BUILD index 75df2b3e..df4270f9 100644 --- a/BUILD +++ b/BUILD @@ -137,6 +137,7 @@ filegroup( "test/**/*.txt", "test/**/*.ini", "test/**/*.jsonl", + "test/**/*.gz", ]), )