divide idx to sub directories for preventing from scanning all files.

diff --git a/src/thread_idx.cxx b/src/thread_idx.cxx
index efc9c9e..d9f09c8 100644 (file)
--- a/src/thread_idx.cxx
+++ b/src/thread_idx.cxx
@@ -37,7 +37,11 @@ namespace dialektos {
 ThreadIdx::ThreadIdx() : title_(), line_count_(0), last_modified_(), etag_()
 {}
 
-void ThreadIdx::to_xml(const boost::filesystem::path& xml) {
+void ThreadIdx::to_xml(const boost::filesystem::path& _xml) {
+  const std::string prefix = _xml.filename().substr(0, 3);
+  const boost::filesystem::path xml =
+    _xml.parent_path() / prefix / _xml.filename();
+
   if (!misc::create_directories(xml.parent_path())) return;
   std::ofstream ofs(xml.file_string().c_str());
   try {
@@ -49,9 +53,13 @@ void ThreadIdx::to_xml(const boost::filesystem::path& xml) {
   ofs.close();
 }
 
-ThreadIdx ThreadIdx::from_xml(const boost::filesystem::path& xml) {
+ThreadIdx ThreadIdx::from_xml(const boost::filesystem::path& _xml) {
   ThreadIdx idx;
 
+  const std::string prefix = _xml.filename().substr(0, 3);
+  const boost::filesystem::path xml =
+    _xml.parent_path() / prefix / _xml.filename();
+
   if (!boost::filesystem::exists(xml) ||
       !boost::filesystem::is_regular_file(xml))
     return idx;

diff --git a/src/thread_idx_cache.cxx b/src/thread_idx_cache.cxx
index 02c1e61..c5c9276 100644 (file)
--- a/src/thread_idx_cache.cxx
+++ b/src/thread_idx_cache.cxx
@@ -180,15 +180,86 @@ void ThreadIdxCache::merge_idx(const boost::filesystem::path& idx_dir,
 void ThreadIdxCache::idx_dir_scan(const boost::filesystem::path& idx_dir,
     boost::unordered_map<ThreadID, ThreadIdxCache>& caches) {
 
-  const boost::unordered_set<ThreadID> exist_ids = get_exist_ids(idx_dir);
+  std::vector<DirectoryTimeStamp> _directories =
+    directory_timestamp_from_xml(idx_dir / "dirs.xml");
+  boost::unordered_map<std::string, std::time_t> directories;
+  BOOST_FOREACH(const DirectoryTimeStamp& dir, _directories) {
+    directories[dir.filename] = dir.last_modified;
+  }
+
+  try {
+    const boost::filesystem::directory_iterator it_end;
+    for (boost::filesystem::directory_iterator it(idx_dir);
+        it != it_end; ++it) {
+
+      const boost::filesystem::path sub_dir = it->path();
+      if (!boost::filesystem::is_directory(sub_dir)) continue;
+
+      const std::string filename = sub_dir.filename();
+      const std::time_t last_modified =
+        boost::filesystem::last_write_time(sub_dir);
+      if (directories.find(filename) != directories.end() &&
+          directories[filename] == last_modified) {
+        continue;
+      }
+
+      directories[filename] = last_modified;
+
+      const boost::unordered_set<ThreadID> exist_ids = get_exist_ids(sub_dir);
+      // TODO remove deleted ids.
+//      const std::vector<ThreadID> deleted_ids =
+//        get_deleted_ids(caches, exist_ids);
+//      remove_deleted_ids(caches, deleted_ids);
+      merge_idx(sub_dir, exist_ids, caches);
+
+    }
+  } catch(const boost::filesystem::filesystem_error& e) {
+    std::cerr << e.what() << std::endl;
+  }
+
+  _directories.clear();
+  typedef boost::unordered_map<std::string, std::time_t>::value_type PairType;
+  BOOST_FOREACH(const PairType& pair, directories) {
+    DirectoryTimeStamp dir;
+    dir.filename = pair.first;
+    dir.last_modified = pair.second;
+    _directories.push_back(dir);
+  }
+  directory_timestamp_to_xml(idx_dir / "dirs.xml", _directories);
+}
 
-  const std::vector<ThreadID> deleted_ids = get_deleted_ids(caches, exist_ids);
+std::vector<DirectoryTimeStamp> ThreadIdxCache::directory_timestamp_from_xml(
+    const boost::filesystem::path& xml) {
+  std::vector<DirectoryTimeStamp> cache_vector;
 
-  remove_deleted_ids(caches, deleted_ids);
+  if (boost::filesystem::exists(xml) && boost::filesystem::is_regular_file(xml)) {
+    std::ifstream ifs(xml.file_string().c_str());
+    try {
+      boost::archive::xml_iarchive ia(ifs);
+      ia >> boost::serialization::make_nvp("DirectoryTimeStamp", cache_vector);
+    } catch (const boost::archive::archive_exception& e) {
+      std::cerr << "directory_timestamp_from_xml(): " << e.what() << std::endl;
+    }
+    ifs.close();
+  }
 
-  merge_idx(idx_dir, exist_ids, caches);
+  return cache_vector;
 }
 
+void ThreadIdxCache::directory_timestamp_to_xml(
+    const boost::filesystem::path& xml,
+    const std::vector<DirectoryTimeStamp>& cache) {
+  if (cache.empty()) return;
+
+  if (!misc::create_directories(xml.parent_path())) return;
+  std::ofstream ofs(xml.file_string().c_str());
+  try {
+    boost::archive::xml_oarchive oa(ofs);
+    oa << boost::serialization::make_nvp("DirectoryTimeStamp", cache);
+  } catch (const boost::archive::archive_exception& e) {
+    std::cerr << "directory_timestamp_to_xml(): " << e.what() << std::endl;
+  }
+}
 
 
 } // namespace dialektos

diff --git a/src/thread_idx_cache.hxx b/src/thread_idx_cache.hxx
index c5ad980..1f5340f 100644 (file)
--- a/src/thread_idx_cache.hxx
+++ b/src/thread_idx_cache.hxx
@@ -31,6 +31,20 @@
 
 namespace dialektos {
 
+
+struct DirectoryTimeStamp {
+  std::string filename;
+  std::time_t last_modified;
+private:
+  friend class boost::serialization::access;
+  template <typename ArchiveType>
+  void serialize(ArchiveType& ar, const unsigned int version) {
+    ar & boost::serialization::make_nvp("Directory", filename);
+    ar & boost::serialization::make_nvp("LastModified", last_modified);
+  }
+};
+
+
 struct ThreadIdxCache {
 
   typedef std::string ThreadID;
@@ -61,6 +75,11 @@ private:
       const boost::unordered_set<ThreadID>& exist_ids);
   static boost::unordered_set<ThreadID> get_exist_ids(
         const boost::filesystem::path&);
+  static std::vector<DirectoryTimeStamp> directory_timestamp_from_xml(
+      const boost::filesystem::path&);
+  static void directory_timestamp_to_xml(
+      const boost::filesystem::path&, const std::vector<DirectoryTimeStamp>&);
+
 
   friend class boost::serialization::access;
   template <typename ArchiveType>