IndexManager.h

#ifndef LIBTPC_TPCINDEXREADER_H
#define LIBTPC_TPCINDEXREADER_H

#include <vector>
#include <string>
#include <lucene++/LuceneHeaders.h>
#include <cfloat>
#include "CASManager.h"
#include "DataStructures.h"

namespace tpc {

    namespace index {

        static const std::string INDEX_ROOT_LOCATION("/usr/local/textpresso/luceneindex/");
        static const std::string CORPUS_COUNTER_FILENAME("cc.cfg");
        static const std::string DOCUMENT_INDEXNAME("fulltext");
        static const std::string SENTENCE_INDEXNAME("sentence");
        static const std::string DOCUMENT_INDEXNAME_CS("fulltext_cs");
        static const std::string SENTENCE_INDEXNAME_CS("sentence_cs");

        static const int MAX_HITS(1000000);
        static const int FIELD_CACHE_MIN_HITS(30000);

        static const int MAX_NUM_SENTENCES_IN_QUERY(200);
        static const int MAX_NUM_DOCIDS_IN_QUERY(200);

        static const std::set<std::string> INDEX_TYPES{DOCUMENT_INDEXNAME, SENTENCE_INDEXNAME, DOCUMENT_INDEXNAME_CS,
                                                       SENTENCE_INDEXNAME_CS};
        static const std::string SUBINDEX_NAME = "subindex";
        static const std::set<std::string> DOCUMENTS_FIELDS_DETAILED{"accession_compressed", "title_compressed",
                                                                    "author_compressed", "journal_compressed", "year",
                                                                    "abstract_compressed", "filepath",
                                                                    "corpus", "doc_id",
                                                                    "fulltext_compressed", "type_compressed",
                                                                     "fulltext_cat_compressed"};
        static const std::set<std::string> SENTENCE_FIELDS_DETAILED{"sentence_id", "begin", "end",
                                                                    "sentence_compressed", "sentence_cat_compressed"};

        struct TmpConf {
            std::string new_index_flag;
            std::string index_descriptor;
            std::string tmp_dir;
        };

        class tpc_exception : public std::runtime_error {
        public:
            explicit tpc_exception(char const* const message) throw(): std::runtime_error(message) { }
            virtual char const* what() const throw() { return std::exception::what(); }
        };

        class IndexManager {
        public:

            IndexManager() = default;

            explicit IndexManager(const std::string& index_path, bool read_only = true, bool external = false):
                    index_dir(index_path),
                    readonly(read_only),
                    external(external),
                    readers_map(),
                    corpus_doc_counter(),
                    externalIndexManager() { };
            ~IndexManager() {
                close();
            };
            IndexManager(const IndexManager& other) {
                readers_map = other.readers_map;
                index_dir = other.index_dir;
                readonly = other.readonly;
                external = other.external;
                corpus_doc_counter = other.corpus_doc_counter;
                externalIndexManager = other.externalIndexManager;
            };
            IndexManager& operator=(const IndexManager& other) {
                readers_map = other.readers_map;
                index_dir = other.index_dir;
                readonly = other.readonly;
                external = other.external;
                corpus_doc_counter = other.corpus_doc_counter;
                externalIndexManager = other.externalIndexManager;
            };
            IndexManager(IndexManager&& other) noexcept :
                    readers_map(std::move(other.readers_map)),
                    readonly(other.readonly),
                    external(other.external),
                    index_dir(std::move(other.index_dir)),
                    corpus_doc_counter(std::move(other.corpus_doc_counter)),
                    externalIndexManager(std::move(other.externalIndexManager)) {}
            IndexManager& operator=(IndexManager&& other) noexcept {
                readers_map = std::move(other.readers_map);
                index_dir = std::move(other.index_dir);
                readonly = other.readonly;
                external = other.external;
                corpus_doc_counter = std::move(other.corpus_doc_counter);
                externalIndexManager = std::move(other.externalIndexManager);
            };

            void close() {
                for (auto &it : readers_map) {
                    it.second->close();
                }
            }

            static std::vector<std::string> get_available_corpora();

            std::vector<std::string> get_additional_corpora();

            int get_num_articles_in_corpus(const std::string& corpus, bool external = false);

            SearchResults search_documents(const Query &query, bool matches_only = false,
                                           const std::set<std::string> &doc_ids = {},
                                           const SearchResults& partialResults = SearchResults());

            DocumentDetails get_document_details(const DocumentSummary &doc_summary,
                                                 bool include_sentences_details = true,
                                                 std::set<std::string> include_doc_fields = DOCUMENTS_FIELDS_DETAILED,
                                                 std::set<std::string> include_match_sentences_fields = SENTENCE_FIELDS_DETAILED,
                                                 const std::set<std::string> &exclude_doc_fields = {},
                                                 const std::set<std::string> &exclude_match_sentences_fields = {});

            std::vector<DocumentDetails> get_documents_details(const std::vector<DocumentSummary> &doc_summaries,
                                                               bool sort_by_year,
                                                               bool include_sentences_details = true,
                                                               std::set<std::string> include_doc_fields = DOCUMENTS_FIELDS_DETAILED,
                                                               std::set<std::string> include_match_sentences_fields = SENTENCE_FIELDS_DETAILED,
                                                               const std::set<std::string> &exclude_doc_fields = {},
                                                               const std::set<std::string> &exclude_match_sentences_fields = {});

            // comparators for reverse sorting of documents and sentence objects
            static bool document_score_gt(const Document &a, const Document &b) { return a.score > b.score; }
            static bool document_year_score_gt(const Document &a, const Document &b) {
                if (a.year != b.year) return a.year > b.year;
                return a.score > b.score;
            }

            static bool sentence_greater_than(const SentenceSummary &a, const SentenceSummary &b) {
                return a.score > b.score;
            }

            void create_index_from_existing_cas_dir(const std::string &input_cas_dir,
                                                    const std::set<std::string>& file_list = {},
                                                    int max_num_papers_per_subindex = 50000);

            void add_file_to_index(const std::string& file_path, int max_num_papers_per_subindex = 50000);

            void remove_file_from_index(const std::string& identifier);

            void calculate_and_save_corpus_counter();

            void save_all_doc_ids_for_sentences_to_db();

            void save_all_years_for_documents_to_db();

            bool has_external_index();

            void set_external_index(std::string external_idx_path);

            void remove_external_index();

            std::vector<std::string> get_external_corpora();

        private:

            Lucene::Collection<Lucene::IndexReaderPtr> get_subreaders(QueryType type, bool case_sensitive = false);

            SearchResults read_documents_summaries(const Lucene::Collection<Lucene::ScoreDocPtr> &matches_collection,
                                                   bool sort_by_year = false);

            SearchResults read_sentences_summaries(const Lucene::Collection<Lucene::ScoreDocPtr> &matches_collection,
                                                   bool sort_by_year = false);

            void update_sentences_details_for_document(const DocumentSummary &doc_summary,
                                                       DocumentDetails &doc_details,
                                                       Lucene::QueryParserPtr sent_parser,
                                                       Lucene::SearcherPtr searcher,
                                                       Lucene::FieldSelectorPtr fsel,
                                                       const std::set<Lucene::String> &fields,
                                                       bool use_lucene_internal_ids,
                                                       Lucene::MultiReaderPtr sent_reader);

            static std::set<Lucene::String> compose_field_set(const std::set<std::string> &include_fields,
                                                              const std::set<std::string> &exclude_fields,
                                                              const std::set<std::string> &required_fields = {});

            void update_document_details(DocumentDetails &doc_details, Lucene::String field,
                                         Lucene::DocumentPtr doc_ptr);

            std::vector<DocumentDetails> read_documents_details(const std::vector<DocumentSummary> &doc_summaries,
                                                                Lucene::QueryParserPtr doc_parser,
                                                                Lucene::SearcherPtr searcher,
                                                                Lucene::FieldSelectorPtr fsel,
                                                                const std::set<Lucene::String> &fields,
                                                                bool use_lucene_internal_ids,
                                                                Lucene::MultiReaderPtr doc_reader);

            static TmpConf write_tmp_conf_files(const std::string &index_path);

            static void create_subindex_dir_structure(const std::string &index_path);

            int add_cas_file_to_index(const char *file_path, std::string index_descriptor, std::string tempDir,
                                      bool update_db);

            bool process_single_file(const std::string &filepath, bool &first_paper, const TmpConf &tmp_conf,
                                     bool update_db = false);

            std::string remove_document_from_index(std::string identifier, bool case_sensitive);
            void remove_sentences_for_document(const std::string& doc_id, bool case_sensitive);

            void add_doc_and_sentences_to_bdb(std::string identifier);

            void save_corpus_counter();

            void update_corpus_counter();

            void load_corpus_counter();
            int get_num_docs_in_corpus_from_index(const std::string& corpus);

            std::map<std::string, Lucene::IndexReaderPtr> readers_map;
            std::string index_dir;
            bool readonly;
            bool external;
            std::map<std::string, int> corpus_doc_counter;
            std::shared_ptr<IndexManager> externalIndexManager;
        };
    }
}


#endif //LIBTPC_TPCINDEXREADER_H