From f52b2597a1c6c0b7c519bb742fc00b5e07ecaf48 Mon Sep 17 00:00:00 2001
From: kishorenc <kishorenc@gmail.com>
Date: Thu, 19 Nov 2020 23:36:57 +0530
Subject: [PATCH] Fixed a bug in array exclusion affecting pinned hits.

---
 src/array_utils.cpp       | 11 +++--------
 src/index.cpp             |  1 -
 test/array_utils_test.cpp | 15 +++++++++++++++
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/array_utils.cpp b/src/array_utils.cpp
index 5a88dffd..9f0c7f4a 100644
--- a/src/array_utils.cpp
+++ b/src/array_utils.cpp
@@ -129,16 +129,11 @@ size_t ArrayUtils::exclude_scalar(const uint32_t *A, const size_t lenA,
       results[res_index] = A[indexA];
       res_index++;
       indexA++;
+    } else if (A[indexA] == B[indexB]) {
+        indexA++;
+        indexB++;
     } else {
-      if (A[indexA] == B[indexB]) {
-        indexA++;
         indexB++;
-      } else {
-        results[res_index] = A[indexA];
-        res_index++;
-        indexA++;
-        indexB++;
-      }
     }
   }
 
diff --git a/src/index.cpp b/src/index.cpp
index 57e9326d..3a5258a4 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1574,7 +1574,6 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co
                         token_to_costs.erase(token_to_costs.begin()+token_index);
                         tokens.erase(tokens.begin()+token_index);
                         costs.erase(costs.begin()+token_index);
-                        token_index--;
                     }
                 }
 
diff --git a/test/array_utils_test.cpp b/test/array_utils_test.cpp
index ccaf3dd0..0fa4622a 100644
--- a/test/array_utils_test.cpp
+++ b/test/array_utils_test.cpp
@@ -1,5 +1,6 @@
 #include <gtest/gtest.h>
 #include "array_utils.h"
+#include "logger.h"
 
 TEST(SortedArrayTest, AndScalar) {
     const size_t size1 = 9;
@@ -154,6 +155,20 @@ TEST(SortedArrayTest, FilterArray) {
     results_size = ArrayUtils::exclude_scalar(arr1, size1, arr2, vec2.size(), &results);
     ASSERT_EQ(0, results_size);
 
+    // on a larger array
+    results = nullptr;
+    
+    std::vector<uint32_t> vec3 = {58, 118, 185, 260, 322, 334, 353};
+    std::vector<uint32_t> filter_ids = {58, 103, 116, 117, 137, 154, 191, 210, 211, 284, 299, 302, 306, 309, 332, 334, 360};
+    std::vector<uint32_t> expected_res = {118, 185, 260, 322, 353};
+
+    results_size = ArrayUtils::exclude_scalar(&vec3[0], vec3.size(), &filter_ids[0], filter_ids.size(), &results);
+    ASSERT_EQ(expected_res.size(), results_size);
+    
+    for(size_t i=0; i<expected_res.size(); i++) {
+        ASSERT_EQ(expected_res[i], results[i]);
+    }
+
     delete[] arr2;
     delete[] arr1;
     delete[] results;