diff --git a/src/collection.cpp b/src/collection.cpp index 32024266..c2c35572 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -211,6 +211,9 @@ nlohmann::json Collection::add_many(std::vector& json_lines) { index_res["error"] = doc_seq_id_op.error(); index_res["success"] = false; + // FIXME: + LOG(INFO) << "Document parsing error, bad json_line is: " << json_line; + // NOTE: we overwrite the input json_lines with result to avoid memory pressure json_lines[i] = index_res.dump(); continue; @@ -733,6 +736,9 @@ Option Collection::search(const std::string & query, const std:: size_t total_found = 0; spp::sparse_hash_set groups_processed; // used to calculate total_found for grouped query + // FIXME: + LOG(INFO) << "Num indices used for querying: " << indices.size(); + // send data to individual index threads size_t index_id = 0; for(Index* index: indices) { diff --git a/src/core_api.cpp b/src/core_api.cpp index 9b6567bb..b35803fc 100644 --- a/src/core_api.cpp +++ b/src/core_api.cpp @@ -589,6 +589,9 @@ bool post_import_documents(http_req& req, http_res& res) { if(req.body_index == 0) { // will log for every major chunk of request body LOG(INFO) << "Import, req.body.size=" << req.body.size() << ", batch_size=" << IMPORT_BATCH_SIZE; + // FIXME: + int nminusten_pos = std::max(0, int(req.body.size())-10); + LOG(INFO) << "Last 10 chars: " << req.body.substr(nminusten_pos); } CollectionManager & collectionManager = CollectionManager::get_instance(); @@ -601,12 +604,16 @@ bool post_import_documents(http_req& req, http_res& res) { return false; } - //LOG(INFO) << "Import, " << "req.body_index=" << req.body_index << ", req.body.size: " << req.body.size(); + // FIXME: + LOG(INFO) << "Import, " << "req.body_index=" << req.body_index << ", req.body.size: " << req.body.size(); //LOG(INFO) << "req body %: " << (float(req.body_index)/req.body.size())*100; std::vector json_lines; req.body_index = StringUtils::split(req.body, json_lines, "\n", false, req.body_index, IMPORT_BATCH_SIZE); + // FIXME: + LOG(INFO) << "json_lines.size before: " << json_lines.size(); + bool stream_proceed = false; // default state if(req.body_index == req.body.size()) { @@ -615,6 +622,8 @@ bool post_import_documents(http_req& req, http_res& res) { stream_proceed = true; if(req.last_chunk_aggregate) { + // FIXME: + LOG(INFO) << "req.last_chunk_aggregate is true"; req.body = ""; } else { if(!json_lines.empty()) { @@ -639,30 +648,35 @@ bool post_import_documents(http_req& req, http_res& res) { } } + // FIXME: + LOG(INFO) << "json_lines.size after: " << json_lines.size(); //LOG(INFO) << "json_lines.size: " << json_lines.size() << ", req.stream_state: " << req.stream_state; // When only one partial record arrives as a chunk, an empty body is pushed to response stream bool single_partial_record_body = (json_lines.empty() && !req.body.empty()); - std::stringstream ss; + std::stringstream response_stream; + + // FIXME: + LOG(INFO) << "single_partial_record_body: " << single_partial_record_body; if(!single_partial_record_body) { nlohmann::json json_res = collection->add_many(json_lines); //const std::string& import_summary_json = json_res.dump(); - //ss << import_summary_json << "\n"; + //response_stream << import_summary_json << "\n"; for (size_t i = 0; i < json_lines.size(); i++) { if(i == json_lines.size()-1 && req.body_index == req.body.size() && req.last_chunk_aggregate) { // indicates last record of last batch - ss << json_lines[i]; + response_stream << json_lines[i]; } else { - ss << json_lines[i] << "\n"; + response_stream << json_lines[i] << "\n"; } } } res.content_type_header = "text/plain; charset=utf8"; res.status_code = 200; - res.body += ss.str(); + res.body += response_stream.str(); if(stream_proceed) { HttpServer::stream_response(req, res);