improve index robustness

keyvi/include/keyvi/index/internal/index_reader_worker.h

@@ -30,6 +30,7 @@
 #include <atomic>
 #include <chrono>  //NOLINT
 #include <ctime>
+#include <exception>
 #include <memory>
 #include <mutex>  //NOLINT
 #include <string>
@@ -152,6 +153,11 @@ class IndexReaderWorker final {
     index_toc.ParseStream(isw);
     TRACE("index_toc loaded");
 
+    if (index_toc.HasParseError() || !index_toc.IsObject() || !index_toc.HasMember("files") ||
+        !index_toc["files"].IsArray()) {
+      throw std::invalid_argument("invalid toc file");
+    }
+
     TRACE("reading segments");
 
     read_only_segments_t new_segments = std::make_shared<read_only_segment_vec_t>();
@@ -189,12 +195,22 @@ class IndexReaderWorker final {
   }
 
   void UpdateWatcher() {
+    int retries_left = 3;
     while (!stop_update_thread_) {
       TRACE("UpdateWatcher: Check for new segments");
-      // reload
-      ReloadIndex();
-      ReloadDeletedKeys();
-      // sleep for next refresh
+      try {
+        ReloadIndex();
+        ReloadDeletedKeys();
+        retries_left = 0;
+      } catch (const std::exception& ex) {
+        TRACE("UpdateWatcher: reload failed: %s, retries left: %d", ex.what(), retries_left);
+        last_modification_time_ = 0;
+        if (retries_left > 0) {
+          --retries_left;
+          continue;
+        }
+        retries_left = 3;
+      }
       std::this_thread::sleep_for(refresh_interval_);
     }
   }

keyvi/tests/keyvi/index/read_only_index_test.cpp

@@ -23,7 +23,9 @@
  *      Author: hendrik
  */
 #include <chrono>  //NOLINT
+#include <string>
 #include <thread>  //NOLINT
+#include <vector>
 
 #include <boost/filesystem.hpp>
 #include <boost/test/unit_test.hpp>
@@ -298,6 +300,50 @@ BOOST_AUTO_TEST_CASE(nearMatching) {
                    {"\"pizzeria in Munich 4\"", "\"pizzeria in Munich 1\""});
 }
 
+BOOST_AUTO_TEST_CASE(reloadRecoveryAfterMissingSegment) {
+  testing::IndexMock index;
+
+  std::vector<std::pair<std::string, std::string>> test_data = {
+      {"abc", "{a:1}"},
+      {"def", "{b:2}"},
+  };
+  index.AddSegment(&test_data);
+
+  ReadOnlyIndex reader(index.GetIndexFolder(), {{"refresh_interval", "100"}});
+  BOOST_CHECK(reader.Contains("abc"));
+  BOOST_CHECK(reader.Contains("def"));
+
+  // simulate the race condition: write a TOC referencing a non-existent segment
+  std::this_thread::sleep_for(std::chrono::seconds(1));
+  {
+    boost::filesystem::path toc_file(index.GetIndexFolder());
+    toc_file /= "index.toc";
+    std::ofstream toc(toc_file.string());
+    toc << R"({"files": ["kv-0.kv", "kv-nonexistent.kv"]})";
+  }
+
+  // wait for a few reload cycles — the reader should not crash
+  std::this_thread::sleep_for(std::chrono::milliseconds(500));
+
+  // the reader should still serve the old segments
+  BOOST_CHECK(reader.Contains("abc"));
+  BOOST_CHECK(reader.Contains("def"));
+
+  // now fix the TOC by adding a real new segment
+  std::this_thread::sleep_for(std::chrono::seconds(1));
+  std::vector<std::pair<std::string, std::string>> test_data_2 = {
+      {"ghi", "{c:3}"},
+  };
+  index.AddSegment(&test_data_2);
+
+  // wait for reload to pick up the fixed TOC
+  std::this_thread::sleep_for(std::chrono::milliseconds(500));
+
+  BOOST_CHECK(reader.Contains("abc"));
+  BOOST_CHECK(reader.Contains("def"));
+  BOOST_CHECK(reader.Contains("ghi"));
+}
+
 BOOST_AUTO_TEST_SUITE_END()
 
 } /* namespace index */