火力全开——仿造Baidu简单实现基于Lucene.net的全文检索的功能

private void CreateIndex()
        {
            //索引库存放在这个文件夹里
            string indexPath = ConfigurationManager.AppSettings["pathIndex"];
            //Directory表示索引文件保存的地方，是抽象类，两个子类FSDirectory表示文件中，RAMDirectory 表示存储在内存中
            FSDirectory directory = FSDirectory.Open(new DirectoryInfo(indexPath), new NativeFSLockFactory());
            //判断目录directory是否是一个索引目录。
            bool isUpdate = IndexReader.IndexExists(directory);
            logger.Debug("索引库存在状态:"+isUpdate);
            if (isUpdate)
            {
                if (IndexWriter.IsLocked(directory))
                {
                    IndexWriter.Unlock(directory);
                }
            }
            //第三个参数为是否创建索引文件夹,Bool Create,如果为True，则新创建的索引会覆盖掉原来的索引文件，反之，则不必创建,更新即可。
            IndexWriter write = new IndexWriter(directory, new PanGuAnalyzer(), !isUpdate, IndexWriter.MaxFieldLength.UNLIMITED);

            WebClient wc = new WebClient();
            //编码，防止乱码
            wc.Encoding = Encoding.UTF8;
            int maxID;
            try
            {
                //读取rss，获得第一个item中的链接的编号部分就是最大的帖子编号
                maxID = GetMaxID();
            }
            catch (WebException webEx)
            {
                logger.Error("获得最大帖子号出错",webEx);
                return;
                
            }
            for (int i = 1; i <= maxID; i++)
            {
                try
                {
                    string url = "http://localhost:8080/showtopic-" + i + ".aspx";
                    logger.Debug("开始下载:"+url);
                    string html = wc.DownloadString(url);
                    HTMLDocumentClass doc = new HTMLDocumentClass();

                    doc.designMode = "on";//不让解析引擎尝试去执行
                    doc.IHTMLDocument2_write(html);
                    doc.close();

                    string title = doc.title;
                    string body = doc.body.innerText;
                    //为避免重复索引，先输出number=i的记录，在重新添加
                    write.DeleteDocuments(new Term("number", i.ToString()));

                    Document document = new Document();
                    //Field为字段，只有对全文检索的字段才分词，Field.Store是否存储
                    document.Add(new Field("number", i.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
                    document.Add(new Field("title", title, Field.Store.YES, Field.Index.NOT_ANALYZED));
                    document.Add(new Field("body", body, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
                    write.AddDocument(document);
                    logger.Debug("索引" + i.ToString() + "完毕");
                }
                catch (WebException webEx)
                {

                    logger.Error("下载"+i.ToString()+"失败",webEx);
                }

            }
            write.Close();
            directory.Close();
            logger.Debug("全部索引完毕");
        }
        //取最大帖子号
        private int GetMaxID()
        {
            XDocument xdoc = XDocument.Load("Http://localhost:8080/tools/rss.aspx");
            XElement channel = xdoc.Root.Element("channel");
            XElement fitstItem = channel.Elements("item").First();
            XElement link = fitstItem.Element("link");
            Match match = Regex.Match(link.Value, @"http://localhost:8080/showtopic-(\d+)\.aspx");
            string id = match.Groups[1].Value;
            return Convert.ToInt32(id);
        }

   public class Global : System.Web.HttpApplication
    {

        private static ILog logger = LogManager.GetLogger(typeof(Global));
        private IScheduler sched;
        protected void Application_Start(object sender, EventArgs e)
        {
            //控制台就放在Main
            logger.Debug("Application_Start");
            log4net.Config.XmlConfigurator.Configure();
            //从配置中读取任务启动时间
            int indexStartHour = Convert.ToInt32(ConfigurationManager.AppSettings["IndexStartHour"]);
            int indexStartMin = Convert.ToInt32(ConfigurationManager.AppSettings["IndexStartMin"]);


            ISchedulerFactory sf = new StdSchedulerFactory();
            sched = sf.GetScheduler();
            JobDetail job = new JobDetail("job1", "group1", typeof(IndexJob));//IndexJob为实现了IJob接口的类
            Trigger trigger = TriggerUtils.MakeDailyTrigger("tigger1", indexStartHour, indexStartMin);//每天10点3分执行
            trigger.JobName = "job1";
            trigger.JobGroup = "group1";
            trigger.Group = "group1";

            sched.AddJob(job, true);
            sched.ScheduleJob(trigger);
            //IIS启动了就不会来了
            sched.Start();


        }

        protected void Session_Start(object sender, EventArgs e)
        {

        }

        protected void Application_BeginRequest(object sender, EventArgs e)
        {

        }

        protected void Application_AuthenticateRequest(object sender, EventArgs e)
        {

        }

        protected void Application_Error(object sender, EventArgs e)
        {
            logger.Debug("网络出现未处理异常:",HttpContext.Current.Server.GetLastError());
        }

        protected void Session_End(object sender, EventArgs e)
        {

        }

        protected void Application_End(object sender, EventArgs e)
        {
            logger.Debug("Application_End");
            sched.Shutdown(true);
        }
    }

       private string Preview(string body,string keyword)
        {
            PanGu.HighLight.SimpleHTMLFormatter simpleHTMLFormatter = new PanGu.HighLight.SimpleHTMLFormatter("<font color=\"Red\">","</font>");
            PanGu.HighLight.Highlighter highlighter = new PanGu.HighLight.Highlighter(simpleHTMLFormatter, new Segment());
            highlighter.FragmentSize = 100;
            string bodyPreview = highlighter.GetBestFragment(keyword, body);
            return bodyPreview;
        }

public IEnumerable<Model.SearchSum> GetHotWords()
        { 
            //缓存
            var data=HttpRuntime.Cache["hotwords"];
            if (data==null)
            {
                IEnumerable<Model.SearchSum> hotWords = DoSelect();
                HttpRuntime.Cache.Insert("hotwords",hotWords,null,DateTime.Now.AddMilliseconds(30),TimeSpan.Zero );
                return hotWords;
            }
            return (IEnumerable<Model.SearchSum>)data;
        }
        private IEnumerable<Model.SearchSum> DoSelect()
        {
            DataTable dt = SqlHelper.ExecuteDataTable(@"
select top 5 Keyword,count(*) as searchcount  from keywords 
where datediff(day,searchdatetime,getdate())<7
group by Keyword 
order by count(*) desc ");
            List<Model.SearchSum> list = new List<Model.SearchSum>();
            if (dt!=null&&dt.Rows!=null&&dt.Rows.Count>0)
            {
                foreach (DataRow row in dt.Rows)
                {
                    Model.SearchSum oneModel=new Model.SearchSum ();
                    oneModel.Keyword = Convert.ToString(row["keyword"]);
                    oneModel.SearchCount = Convert.ToInt32(row["SearchCount"]);
                    list.Add(oneModel);
                }
            }
            return list;
        }

 public IEnumerable<Model.SearchSum> GetSuggestion(string kw)
        {
            DataTable dt = SqlHelper.ExecuteDataTable(@"select top 5 Keyword,count(*) as searchcount  from keywords 
where datediff(day,searchdatetime,getdate())<7
and keyword like @keyword
group by Keyword 
order by count(*) desc",new SqlParameter("@keyword","%"+kw+"%"));
            List<Model.SearchSum> list = new List<Model.SearchSum>();
            if (dt != null && dt.Rows != null && dt.Rows.Count > 0)
            {
                foreach (DataRow row in dt.Rows)
                {
                    Model.SearchSum oneModel = new Model.SearchSum();
                    oneModel.Keyword = Convert.ToString(row["keyword"]);
                    oneModel.SearchCount = Convert.ToInt32(row["SearchCount"]);
                    list.Add(oneModel);
                }
            }
            return list;
        }

       protected void Page_Load(object sender, EventArgs e)
        {
            //加载热词

            hotwordsRepeater.DataSource = new Dao.KeywordDao().GetHotWords();
            hotwordsRepeater.DataBind();
            kw = Request["kw"];
            if (string.IsNullOrWhiteSpace(kw))
            {
                return;
            }
            //处理：将用户的搜索记录加入数据库，方便统计热词
            Model.SerachKeyword model = new Model.SerachKeyword();
            model.Keyword = kw;
            model.SearchDateTime = DateTime.Now;
            model.ClinetAddress = Request.UserHostAddress;

            new Dao.KeywordDao().Add(model);
            //分页控件
            MyPage pager = new MyPage();
            pager.TryParseCurrentPageIndex(Request["pagenum"]);
            //超链接href属性
            pager.UrlFormat = "CreateIndex.aspx?pagenum={n}&kw=" + Server.UrlEncode(kw);
            
            int startRowIndex = (pager.CurrentPageIndex - 1) * pager.PageSize;


            int totalCount = -1;
            List<SearchResult> list = DoSearch(startRowIndex,pager.PageSize,out totalCount);
            pager.TotalCount = totalCount;
            RenderToHTML = pager.RenderToHTML();
            dataRepeater.DataSource = list;
            dataRepeater.DataBind();
        }

        private List<SearchResult> DoSearch(int startRowIndex,int pageSize,out int totalCount)
        {
            string indexPath = "C:/Index";
            FSDirectory directory = FSDirectory.Open(new DirectoryInfo(indexPath), new NoLockFactory());
            IndexReader reader = IndexReader.Open(directory, true);
            //IndexSearcher是进行搜索的类
            IndexSearcher searcher = new IndexSearcher(reader);
            PhraseQuery query = new PhraseQuery();
            
            foreach (string word in CommonHelper.SplitWord(kw))
            {
                query.Add(new Term("body", word));
            }
            query.SetSlop(100);//相聚100以内才算是查询到
            TopScoreDocCollector collector = TopScoreDocCollector.create(1024, true);//最大1024条记录
            searcher.Search(query, null, collector);
            totalCount = collector.GetTotalHits();//返回总条数
            ScoreDoc[] docs = collector.TopDocs(startRowIndex, pageSize).scoreDocs;//分页,下标应该从0开始吧，0是第一条记录
            List<SearchResult> list = new List<SearchResult>();
            for (int i = 0; i < docs.Length; i++)
            {
                int docID = docs[i].doc;//取文档的编号，这个是主键，lucene.net分配
                //检索结果中只有文档的id，如果要取Document，则需要Doc再去取
                //降低内容占用
                Document doc = searcher.Doc(docID);
                string number = doc.Get("number");
                string title = doc.Get("title");
                string body = doc.Get("body");

                SearchResult searchResult = new SearchResult() { Number = number, Title = title, BodyPreview = Preview(body, kw) };
                list.Add(searchResult);

            }
            return list;
        }

   <script type="text/javascript">
        $(function () {
            $("#txtKeyword").autocomplete(
            {   source: "SearchSuggestion.ashx",
                select: function (event, ui) { $("#txtKeyword").val(ui.item.value); $("#form1").submit(); }
            });
        });
    </script>

 <div align="center">
        <input type="text" id="txtKeyword" name="kw" value='<%=kw %>'/>
        <%-- <asp:Button ID="createIndexButton" runat="server" onclick="searchButton_Click" 
            Text="创建索引库" />--%>
        <input type="submit" name="searchButton" value="搜索" style="width: 91px" /><br />
    </div>
    <br />
    <ul id="hotwordsUL">
          <asp:Repeater ID="hotwordsRepeater" runat="server">
            <ItemTemplate>
                <li><a href='CreateIndex.aspx?kw=<%#Eval("Keyword") %>'><%#Eval("Keyword") %></a></li>
            </ItemTemplate>
          </asp:Repeater>
    </ul>
    &nbsp;<br />
  
    <asp:Repeater ID="dataRepeater" runat="server" EnableViewState="true">
        <HeaderTemplate>
            <ul>
        </HeaderTemplate>
        <ItemTemplate>
            <li>
                <a href='http://localhost:8080/showtopic-<%#Eval("Number") %>.aspx'><%#Eval("Title") %></a>
                <br />
                <%#Eval("BodyPreview") %>
            
            </li>
        </ItemTemplate>
        <FooterTemplate>
        </ul>
        </FooterTemplate>
    </asp:Repeater>
    <br />
    <div class="pager"><%=RenderToHTML%></div>