Merge pull request #1144 from krunal1313/gzip_indexing

Tests for gzip indexing

BUILD

@@ -137,6 +137,7 @@ filegroup(
         "test/**/*.txt",
         "test/**/*.ini",
         "test/**/*.jsonl",
+        "test/**/*.gz",
     ]),
 )
 

include/raft_server.h

@@ -142,8 +142,6 @@ private:
 
     butil::EndPoint peering_endpoint;
 
-    Option<bool> handle_gzip(const std::shared_ptr<http_req>& request);
-
 public:
 
     static constexpr const char* log_dir_name = "log";
@@ -241,6 +239,8 @@ public:
 
     std::string get_leader_url() const;
 
+    static Option<bool> handle_gzip(const std::shared_ptr<http_req>& request);
+
 private:
 
     friend class ReplicationClosure;

test/core_api_utils_test.cpp

@@ -4,6 +4,7 @@
 #include <collection_manager.h>
 #include <core_api.h>
 #include "core_api_utils.h"
+#include "raft_server.h"
 
 class CoreAPIUtilsTest : public ::testing::Test {
 protected:
@@ -621,6 +622,7 @@ TEST_F(CoreAPIUtilsTest, PresetSingleSearch) {
 
     auto op = collectionManager.create_collection(schema);
     ASSERT_TRUE(op.ok());
+    Collection* coll1 = op.get();
 
     auto preset_value = R"(
         {"collection":"preset_coll", "per_page": "12"}
@@ -1157,4 +1159,59 @@ TEST_F(CoreAPIUtilsTest, TestProxyTimeout) {
 
     ASSERT_EQ(408, resp->status_code);
     ASSERT_EQ("Server error on remote server. Please try again later.", nlohmann::json::parse(resp->body)["message"]);
+}
+
+TEST_F(CoreAPIUtilsTest, SampleGzipIndexTest) {
+    Collection *coll_hnstories;
+
+    std::vector<field> fields = {field("title", field_types::STRING, false),
+                                 field("points", field_types::INT32, false),};
+
+    coll_hnstories = collectionManager.get_collection("coll_hnstories").get();
+    if(coll_hnstories == nullptr) {
+        coll_hnstories = collectionManager.create_collection("coll_hnstories", 4, fields, "title").get();
+    }
+
+    auto req = std::make_shared<http_req>();
+    std::ifstream infile(std::string(ROOT_DIR)+"test/resources/hnstories.jsonl.gz");
+    std::stringstream outbuffer;
+
+    infile.seekg (0, infile.end);
+    int length = infile.tellg();
+    infile.seekg (0, infile.beg);
+
+    req->body.resize(length);
+    infile.read(&req->body[0], length);
+
+    auto res = ReplicationState::handle_gzip(req);
+    if (!res.error().empty()) {
+        LOG(ERROR) << res.error();
+        FAIL();
+    } else {
+        outbuffer << req->body;
+    }
+
+    std::vector<std::string> doc_lines;
+    std::string line;
+    while(std::getline(outbuffer, line)) {
+        doc_lines.push_back(line);
+    }
+
+    ASSERT_EQ(14, doc_lines.size());
+    ASSERT_EQ("{\"points\":1,\"title\":\"DuckDuckGo Settings\"}", doc_lines[0]);
+    ASSERT_EQ("{\"points\":1,\"title\":\"Making Twitter Easier to Use\"}", doc_lines[1]);
+    ASSERT_EQ("{\"points\":2,\"title\":\"London refers Uber app row to High Court\"}", doc_lines[2]);
+    ASSERT_EQ("{\"points\":1,\"title\":\"Young Global Leaders, who should be nominated? (World Economic Forum)\"}", doc_lines[3]);
+    ASSERT_EQ("{\"points\":1,\"title\":\"Blooki.st goes BETA in a few hours\"}", doc_lines[4]);
+    ASSERT_EQ("{\"points\":1,\"title\":\"Unicode Security Data: Beta Review\"}", doc_lines[5]);
+    ASSERT_EQ("{\"points\":2,\"title\":\"FileMap: MapReduce on the CLI\"}", doc_lines[6]);
+    ASSERT_EQ("{\"points\":1,\"title\":\"[Full Video] NBC News Interview with Edward Snowden\"}", doc_lines[7]);
+    ASSERT_EQ("{\"points\":1,\"title\":\"Hybrid App Monetization Example with Mobile Ads and In-App Purchases\"}", doc_lines[8]);
+    ASSERT_EQ("{\"points\":1,\"title\":\"We need oppinion from Android Developers\"}", doc_lines[9]);
+    ASSERT_EQ("{\"points\":1,\"title\":\"\\\\t Why Mobile Developers Should Care About Deep Linking\"}", doc_lines[10]);
+    ASSERT_EQ("{\"points\":2,\"title\":\"Are we getting too Sassy? Weighing up micro-optimisation vs. maintainability\"}", doc_lines[11]);
+    ASSERT_EQ("{\"points\":2,\"title\":\"Google's XSS game\"}", doc_lines[12]);
+    ASSERT_EQ("{\"points\":1,\"title\":\"Telemba Turns Your Old Roomba and Tablet Into a Telepresence Robot\"}", doc_lines[13]);
+
+    infile.close();
 }