搜索引擎源码及流程

 1 #ifndef _PAGELIB_H_
 2 #define _PAGELIB_H_
 3 #include <iostream>
 4 #include <string>
 5 #include <vector>
 6 #include <fstream>
 7 #include <unistd.h>
 8 #include <dirent.h>
 9 #include <sys/types.h>
10 #include <sys/stat.h>
11 #include <stdio.h>
12 #include <stdlib.h>
13 #include <string.h>
14 #include <time.h>
15 #include <pwd.h>
16 #include <grp.h>
17 
18 class pagelib
19 {
20     public:
21         pagelib(const std::string &dir, 
22                 const std::string &libname)
23             :dir_(dir),
24             libname_(libname)
25     {}
26         void readdirname()
27         {
28             readfile(dir_);
29         }
30         void store_to_lib(const std::string&);
31     private:
32         void readfile(const std::string &);
33         void store_to_vector(const std::string&);
34         std::string dir_;
35         std::string libname_;
36         static int i;
37         std::vector<std::string> vec;
38 };
39 
40 #endif /*PAGELIB_H*/ 

 1 #include "pagelib.h"
 2 using namespace std;
 3 int pagelib::i = 1;
 4 
 5 void pagelib::readfile(const string &s)
 6 {
 7     DIR *dir;
 8     struct dirent * mydir;
 9     struct stat mystat;
10     char str[256];
11     dir=opendir(s.c_str());
12     if(!dir)
13     {
14         cout << s << endl;
15         exit(-1);
16     }
17 
18     while((mydir=readdir(dir))!=NULL)
19     {
20         sprintf(str, "%s/%s", s.c_str(), mydir->d_name);
21         stat(str, &mystat);
22         if(!strcmp(mydir->d_name, ".") || !strcmp(mydir->d_name, ".."))
23             continue;
24         if( mystat.st_mode & 0040000 ) 
25         {
26             string s1 = str;
27             readfile(s1);
28         }
29         else
30         {
31             cout << str << endl;
32             store_to_vector(str);
33         }
34     }
35 }
36 
37 void pagelib::store_to_vector(const string &str)
38 {
39     string content, line;
40     ifstream is(str.c_str());
41 
42     string title;
43     getline(is, title);
44     {
45         for (size_t i = 0; i < title.size(); i++) {
46             if(title[i] == ‘\r‘)
47                 title[i] = ‘ ‘;
48         }
49     }
50 
51     while(getline(is, line) > 0)
52     {
53         for (size_t i = 0; i < line.size(); i++) {
54             if(line[i] == ‘\r‘)
55                 line[i] = ‘\n‘;
56         }
57         content += line;
58     }
59     char s1[5];
60     sprintf(s1, "%d", i++);
61     string doc = string("<doc>\n  <docid>") + s1 + "</docid>\n   <url>" + string(str) 
62         + "</url>\n   <title>" + title + "</title>\n     <content>\n" 
63         + content + "     </content>\n</doc>\n\n";
64     vec.push_back(doc);
65     is.close();
66 }
67 
68 void pagelib::store_to_lib(const string& index)
69 {
70     ofstream os_lib(libname_.c_str());
71     ofstream os_index(index.c_str());
72     vector<string>::iterator it = vec.begin();
73     int j = 1;
74     os_lib << "<pagelib>\n" << endl;
75     for(; it != vec.end(); it++)
76     {
77         os_index << j++ << " " << os_lib.tellp() ;
78         os_lib << *it;
79         os_index << " " << (*it).size() << endl;
80     }
81     os_lib << "</pagelib>\n" << endl;
82     os_index.close();
83     os_lib.close();
84 }

 1 #ifndef _SPLIT_H_
 2 #define _SPLIT_H_
 3 #include <utility>
 4 #include "MixSegment.hpp"
 5 #include <map>
 6 #include <string>
 7 #include <fstream>
 8 #include <queue>
 9 #include <sstream>
10 #include <set>
11 #include <algorithm>
12 
13 typedef struct qelem
14 {
15     std::string word;
16     int frequence;
17     bool operator<(const qelem &other) const
18     {
19         return frequence < other.frequence;
20     }
21 }qelem;
22 
23 
24 class splitpagelib
25 {
26     public:
27         splitpagelib(const std::string &libname,
28                 const std::string &indexname,
29                 const std::string &stopfilename,
30                 const std::string &splitfilename
31                 )
32                 :libname_(libname),
33             indexname_(indexname),
34             stopfilename_(stopfilename),
35             splitfilename_(splitfilename)
36     {}
37         void startsplit(const std::string& dic_path, 
38                         const std::string& model_path);
39         void deletesame();
40         void newindexlib(const std::string&,
41                          const std::string&);
42         void make_reverse_index(const std::string& ,
43                                 const std::string&);
44     private:
45         void compute_power();
46         bool ifsame(const std::map<std::string, int> &,
47                 const std::map<std::string, int> &);
48         void topten(int, 
49                     std::ofstream &, 
50                     const std::map<std::string, int> &);
51         void weightogether();
52 
53         std::string libname_;//从lib文件读内容
54         std::string indexname_;//从index读每篇文档相应的offset，读出相应文章
55         std::string stopfilename_;//停用词文件
56         std::string splitfilename_;//分词后的文件 
57 
58         std::set<std::string> set_stop;//读入停用词
59         std::map<int, std::string> map_doc;//根据索引把每篇doc读入map
60         std::map<int, std::map<std::string, int> > map_lib;
61         std::map<std::string, int> words;//将去去重后的所有单词及频数写入map
62         std::map<int, std::map<std::string, double> >doc_word_wei;
63         std::map<std::string, std::map<int, double> >rev_index;//倒排索引
64         std::map<std::string, std::map<int, int> > map_word_fre;
65 };  
66 
67 #endif /*SPLIT_H*/ 

  1 #include "split.h"
  2 #include <stdio.h>
  3 using namespace std;
  4 using namespace CppJieba;
  5 
  6 void splitpagelib::startsplit(const string &dic_path, 
  7         const string &model_path)    
  8 {
  9     MixSegment  segment_(dic_path, model_path);
 10     //读入停用词到set_stop中去
 11     ifstream is_stop(stopfilename_.c_str());
 12     string stopword;
 13     while(getline(is_stop, stopword) > 0)
 14     {
 15         set_stop.insert(stopword);
 16     }
 17     //通过index来将lib中文档读出并分词，统计出现最高的10个 放入map中
 18     vector<string> words;//用来存放切割后的单词
 19     ifstream is_lib(libname_.c_str());
 20     ifstream is_index(indexname_.c_str());
 21     ofstream os_split(splitfilename_.c_str());
 22     string lib_line, index_line;
 23     cout << "begin to cut word :" << indexname_ 
 24          <<" " << libname_ << endl;
 25     while(getline(is_index, index_line) > 0)
 26     {
 27         map<string, int> map_;
 28         int docid, offset, size;
 29         istringstream ss(index_line);
 30         ss >> docid >> offset >> size;
 31         is_lib.seekg(offset);
 32         string every_doc = "";
 33         while(is_lib.tellg() < (offset + size))
 34         {
 35             getline(is_lib, lib_line);
 36             every_doc += lib_line + "\n";
 37             for(size_t i = 0; i < lib_line.size(); i++)
 38             {
 39                 if(!(lib_line[i] & (1 << 7)))
 40                     lib_line[i] = ‘ ‘;
 41             }
 42 
 43             segment_.cut(lib_line.c_str(), words);
 44             for(size_t i = 0; i < words.size(); i++)
 45             {
 46                 set<string>::iterator it = set_stop.find(words[i]);
 47                 if(it == set_stop.end() && words[i] != " ")//不去掉的话把空格当单词
 48                     ++map_[words[i]];
 49             }
 50             words.clear();
 51         }
 52         os_split << docid << endl;
 53         topten(docid, os_split, map_);
 54         os_split << endl;
 55         map_doc[docid] = every_doc;//将每篇文章读入map_doc
 56     }
 57     os_split.close();
 58     is_lib.close();
 59     is_index.close();
 60     cout << "cut word over :write to"  << splitfilename_ << endl;
 61 }
 62 
 63 void splitpagelib::topten(int docid,
 64         ofstream &os,
 65         const map<string, int> &map_)
 66 {
 67     priority_queue<qelem> prique_;
 68     qelem ql;
 69     for(map<string, int>::const_iterator it = map_.begin(); 
 70             it != map_.end(); 
 71             ++it)
 72     {
 73         ql.word = it->first;
 74         ql.frequence = it->second;
 75         prique_.push(ql);
 76     }
 77 
 78     int i = 0;
 79     while(i < 10 && !prique_.empty())
 80     {
 81         ql = prique_.top() ;
 82         map_lib[docid][ql.word] = ql.frequence;
 83         prique_.pop();
 84         i++;
 85         os << ql.word << " " << ql.frequence << " ";
 86     }
 87 }
 88 
 89 void splitpagelib::deletesame()
 90 {    //去重
 91     cout << "begin to delete the same" << endl;
 92     int *arr = new int [map_lib.size() + 1];
 93     for (size_t i = 1; i < map_lib.size() + 1; i++) 
 94     {
 95         arr[i] = 1;
 96     }
 97     size_t ix1, ix2;
 98     for(ix1 = 1; ix1 < map_lib.size() + 1; ++ix1)
 99     {
100         if(arr[ix1] == 0)
101             continue; 
102         for (ix2 = ix1 +1; ix2 < map_lib.size() + 1; ix2++) 
103         {
104             if(arr[ix2] == 0)
105                 continue;
106             if(ifsame(map_lib[ix1], map_lib[ix2]))
107             {
108                 arr[ix2] = 0;
109                 map_lib.erase(ix2);
110             }
111         }        
112     }
113     delete [] arr;
114     cout << "delsanme over" << endl;
115 }
116 
117 bool splitpagelib::ifsame(const map<string, int>& map1, 
118         const map<string, int>& map2)
119 {
120     int i = 0;
121     for(map<string, int>::const_iterator it = map2.begin(); it != map2.end(); ++it)
122     {
123         if(map1.count(it->first) > 0)
124             i++;
125     }
126     if(i > 6)
127         return true;
128     else
129         return false;
130 }
131 
132 void splitpagelib::newindexlib(const string& index,
133                         const string& lib)
134 {
135     ofstream os_index(index.c_str());
136     ofstream os_lib(lib.c_str());
137 
138     map<int, map<string, double> >::iterator it;
139     size_t i = 1;
140     cout << "begin write to index lib" << index 
141          << " " << lib <<  endl;
142     for(it = doc_word_wei.begin(); it != doc_word_wei.end(); it++)
143     {
144         os_index << i << " " << os_lib.tellp() << " " ;
145         
146         int begin = map_doc[it->first].find("<docid>");
147         int end = map_doc[it->first].find("</docid>");
148         char s[10] = {0};
149         sprintf(s, "%d", i);
150         map_doc[it->first].replace(begin + 7, end - begin - 7, s);
151         os_lib << map_doc[it->first] << endl;
152 
153         os_index << map_doc[it->first].size() << endl;//写index    
154         i++;
155     }
156     cout << "write over" << endl;
157     os_index.close();
158     os_lib.close();
159 }
160 
161 void splitpagelib::compute_power()
162 {
163     map<int, map<string, int> >::iterator it1;    
164     map<string, int>::iterator it2 ;
165     for(it1 = map_lib.begin(); it1 != map_lib.end(); it1++)
166     {
167         for(it2 = it1->second.begin(); it2 != it1->second.end(); ++it2)
168         {
169             map_word_fre[it2->first][it1->first] = it2->second;
170         }
171     }
172 
173     for(it1 = map_lib.begin(); it1 != map_lib.end(); it1++)
174     {
175         for(it2 = it1->second.begin(); it2 != it1->second.end(); ++it2)
176         {
177             double d = (it2->second) * log( (double)(map_word_fre.size()) / (double)(map_word_fre[it2->first].size() +1) );
178             doc_word_wei[it1->first][it2->first] = d;
179         }
180     }
181 }
182 
183 void splitpagelib::make_reverse_index(const string &index,
184                                         const string &power)
185 {
186     //计算每篇中的权重
187     compute_power();
188     //对每篇文档的词作归一化
189    weightogether();
190    //建立倒排索引
191    int j = 1;
192    for(map<int, map<string, double> >::iterator it = doc_word_wei.begin();
193            it != doc_word_wei.end();
194            it++)
195    {
196        for(map<string, double>::iterator it1 = it->second.begin(); 
197                it1 != it->second.end(); 
198                ++it1)
199        {
200            rev_index[it1->first][j] = it1->second;  
201        }
202        j++;
203    }
204     cout << "write reverse index" << index <<endl;
205     ofstream os_index(index.c_str());
206     for(map<string, map<int, double> >::iterator it = rev_index.begin(); 
207             it != rev_index.end(); 
208             ++it)
209     {
210         os_index << it->first << endl;
211         for(map<int, double> ::iterator it1 = it->second.begin(); 
212                 it1 != it->second.end(); 
213                 ++it1)
214         {
215             os_index << it1->first << " " << it1->second << " ";
216         }
217         os_index << endl;
218     }
219     os_index.close();
220     cout << "write reverse index over" << endl;
221 
222     cout << "write doc_word_wei begin" << power << endl;
223     ofstream os_power(power.c_str());
224     map<int, map<string, double> >::iterator it = doc_word_wei.begin();
225     map<string, double>::iterator it1;
226     int i = 1;
227     for(; it != doc_word_wei.end(); ++it)
228     {
229         os_power << i << endl;
230         for(it1 = it->second.begin(); it1 != it->second.end(); ++it1)
231         {
232             os_power << it1->first << " " << it1->second << " " ;
233         }
234         os_power << endl;
235         i++; 
236     }
237     os_power.close();
238     cout << "write power end" << endl;
239 }
240 
241 void splitpagelib::weightogether()//归一化
242 {
243     map<int, map<string, double> >::iterator it;
244     map<string, double>::iterator it1;
245     double s;
246     for(it = doc_word_wei.begin(); it != doc_word_wei.end(); it++)
247     {
248         s = 0;
249         for(it1 = it->second.begin(); it1 != it->second.end(); it1++)
250         {
251             s += it1->second * it1->second;
252         }
253 
254         for(it1 = it->second.begin(); it1 != it->second.end(); it1++)
255         {
256             it1->second /= sqrt(s);
257         }
258       }
259 }

 1 #ifndef _QUERY_H_
 2 #define _QUERY_H_
 3 
 4 #include <map>
 5 #include "MixSegment.hpp"
 6 #include <string>
 7 #include <fstream>
 8 #include <sstream>
 9 #include <queue>
10 #include <algorithm>
11 
12 typedef struct simi
13 {
14     int docid;
15     double simidegre;
16     bool operator<( const simi &right)const
17     {
18         return simidegre < right.simidegre;
19     }
20 }simi;
21 
22 class query
23 {
24     public:
25         query(){}
26      void read_to_map(const std::string&,
27            const std::string&,
28            const std::string&,
29            const std::string&);
30      std::string search(const std::string&);
31     private:
32      static void similar(std::map<std::string, double>&,
33                          std::map<std::string, double>&,
34                          int,
35                          std::priority_queue<simi>&);
36      
37       std::string offsetfile_;
38       std::string reverseindexfile_;
39       std::string docfile_;
40       std::string doc_weifile_;
41      
42       std::map<std::string, std::map<int, double> > map_reindex;//存储倒排索引
43       std::map<int, std::map<std::string, double> > map_wei;
44       std::map<int, std::string> map_doc;
45 };
46 
47 #endif /*QUERY_H*/ 

  1 #include "query.h"
  2 
  3 using namespace std;
  4 using namespace CppJieba;
  5 
  6 void query::read_to_map(
  7          const string &offsetfile,
  8         const string &reverseindexfile,
  9         const string &docfile,
 10         const string &doc_weifile)
 11 {
 12   offsetfile_ = offsetfile;
 13   reverseindexfile_ = reverseindexfile;
 14   doc_weifile_ = doc_weifile;
 15   docfile_ = docfile;
 16 
 17     //读倒排索引文件
 18     ifstream is_revindex(reverseindexfile_.c_str());
 19     string line;
 20     while(getline(is_revindex, line) > 0)
 21     {
 22         string word;
 23         word = line;
 24         getline(is_revindex, line) ;
 25         istringstream sstream(line);
 26         int docid;
 27         double weight;
 28         while(sstream >> docid >> weight)
 29         {
 30             map_reindex[word][docid] = weight;
 31         }
 32     }
 33     is_revindex.close();
 34 
 35     //读偏移索引文件和doc文件，将每篇文档读入map
 36     ifstream is_offset(offsetfile_.c_str());
 37     ifstream is_doc(docfile_.c_str());
 38     string line_off;
 39     while(getline(is_offset, line_off) > 0)
 40     {
 41         istringstream sstream(line_off);
 42         int docid;
 43         int offset;
 44         int size;
 45         sstream >> docid >> offset >> size;
 46         is_doc.seekg(offset);
 47         string every_doc = "";
 48         while(is_doc.tellg() < (offset + size))
 49         {
 50             string line;
 51             getline(is_doc, line);
 52             every_doc += line + "\n";
 53         }
 54         map_doc[docid] = every_doc;
 55     }
 56     is_offset.close();
 57     is_doc.close();
 58     //读每篇doc归一化后的top10词文件
 59     ifstream is_map_wei(doc_weifile_.c_str());
 60     string line_wei;
 61     while(getline(is_map_wei, line_wei) > 0)
 62     {
 63         int docid;
 64         istringstream ss(line_wei);
 65         ss >> docid;
 66         getline(is_map_wei, line_wei);
 67         istringstream sstream(line_wei);
 68         string word;
 69         double weight;
 70         while(sstream >>  word >> weight)
 71         {
 72             map_wei[docid][word] = weight;
 73         }
 74     }
 75     is_map_wei.close();
 76 }
 77 
 78 string query::search(const std::string &word)
 79 {
 80     priority_queue<simi> result_doc;
 81     //切词,算出单词权重存储到一个map，相当于一片doc
 82     MixSegment segment("../dict/jieba.dict.utf8",
 83             "../dict/hmm_model.utf8");
 84     vector<string> words;
 85     map<string, double> map_que;
 86     segment.cut(word, words);
 87     double d = 0;
 88     for (size_t i = 0; i < words.size(); i++) 
 89     {
 90         double s = 1 * log( (double)map_reindex.size() / (double)(map_reindex[words[i]].size()+1) );
 91         cout << words[i] << s << endl;
 92         map_que[ words[i] ] = s;
 93         d += s*s;
 94     }
 95     for(size_t i = 0; i < words.size(); i++)
 96     {
 97         map_que[ words[i] ] /= sqrt(d);
 98     }
 99     //找出单词交集的docid 
100     map<int, double> com(map_reindex[ words[0] ]);
101 
102     for (size_t i = 1; i < words.size(); i++) 
103     {
104         if(map_reindex[ words[i] ].size() == 0)
105             continue;
106         for(map<int, double>::iterator it = com.begin(); it != com.end(); ++it)
107         {
108             if(map_reindex[ words[i] ].count(it->first) == 0)
109                 com.erase(it->first);
110         }
111     }
112     cout << com.size() << endl;
113     for(map<int, double>::iterator it = com.begin(); it != com.end(); ++it)
114     {
115         similar(map_que, map_wei[it->first], it->first, result_doc);
116     }
117 
118     simi si;
119     string result = "";
120     while(!result_doc.empty())
121     {
122         si = result_doc.top();
123         char s[20] = {0};
124         sprintf(s, "%d %f\n", si.docid, si.simidegre);
125         result += s + map_doc[si.docid];
126         result_doc.pop();
127     }
128     return result;
129 }
130 
131 void query::similar(map<string, double> &map_doc1,
132                     map<string, double> &map_doc2,
133                     int docid,
134                     priority_queue<simi> &result_doc)
135 {
136     double s = 0;
137     for(map<string, double>::iterator it = map_doc1.begin();
138                                     it != map_doc1.end();
139                                         ++it)
140     {
141         if(map_doc2.count(it->first) > 0)
142         {
143             s += it->second * map_doc2[it->first]; 
144         }
145     }
146     simi si;
147     si.docid = docid;
148     si.simidegre = s;
149     result_doc.push(si);
150 }