一步一步学asp.net_Lucene.net站内搜索

大多数情况下,我们的搜索一般用的是sql的模糊搜索,但是这个模糊搜索,总是不够精确,而且总达不到我们的要求,于是乎,偶专门上网找了一些资料,研究了一下,现在比较流行的Lucene.net,感觉还蛮不错的,搜索效果也蛮好的,再配合盘古分词,感觉超酷.

嗯,我就动手尝试了一下类似百度的多模块搜索,感觉蛮好玩的.

网上一般都只做了一个搜索,借鉴他们的代码,我在这里专门设计怎么做多个模块搜索,做完可以考虑,怎么把这些内容整合搜索!

网上介绍lucene.net也蛮多了,这里就说一下,我做lucene.net的心得,lucene.net一般做出一个搜索比较容易,但是涉及到多个索引比较麻烦,这里我就专门看了一些资料写了一个多模块的搜索,正在看怎么把多种索引组合起来,能够综合性搜索,主要实现了2个模块,新闻和工艺知识,

进入正题:

做站内搜索主要涉及以下几个技术:

多线程技术,

Lucene.net,

盘古分词.

Lucene.net实际上就是把数据建立一个索引库保存起来,然后以后就像翻书一样搜索.

盘古分词就是专门为了把一段话分词,比如李明去吃饭.盘古分词就会拆分出关键词,李明 吃饭,这样,就可以到索引库去查找这两个词.

多线程为了保证让系统自动去索引我们写入的文章或者工艺知识等内容,每次做增删改系统就自动去更新索引库.

这里主要涉及以下几个dll,

其中还包括盘古分词的高亮显示和盘古分词的配置文件.

主要代码:

IndexJobItem类,

这个类定义了关于luncene.net的子任务信息类

using System;
using System.Data;
using System.Configuration;
using System.Linq;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.HtmlControls;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Xml.Linq;

/// <summary>
///任务类型
/// </summary>
public class IndexJobItem
{
    //任务类型
   public enum JobType
    {
        Delete, Add
    }

    public JobType ItemType { get; set; }
    public long ThreadId { get; set; }
    public int Id { get; set; }

    public override bool Equals(object obj)
    {
        IndexJobItem item = obj as IndexJobItem;
        if (item == null)
        {
            return false;
        }
        return this.ItemType == item.ItemType && this.ThreadId == item.ThreadId;
        //return base.Equals(obj);
    }
    public override int GetHashCode()
    {
        return base.GetHashCode();
    }
    public override string ToString()
    {
        return ItemType + ":" + ThreadId;
    }
    public IndexJobItem()
    {
        //
        //TODO: 在此处添加构造函数逻辑
        //
    }
}

IndexManager类

这个是专门线程进行索引操作

using System;
using System.Data;
using System.Configuration;
using System.Linq;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.HtmlControls;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Xml.Linq;
using log4net;
using System.Web.Hosting;
using Lucene.Net.Store;
using Lucene.Net.Index;
using System.IO;
using Lucene.Net.Analysis.PanGu;
using System.Net;
using czcraft.BLL;
using mshtml;
using czcraft;
using System.Text;
using Lucene.Net.Documents;
using System.Text.RegularExpressions;
using Quartz.Collection;
using System.Collections.Generic;
using System.Threading;

/// <summary>
///IndexManager 只能通过Instance实例化
/// </summary>
public class IndexManager
{
    //单例模式
    public readonly static IndexManager Instance = new IndexManager();
    /// <summary>
    /// 任务是否停止
    /// </summary>
    private bool IsStopped;
    /// <summary>
    /// 搜索类别枚举
    /// </summary>
    public enum JobSearchType
    {
        Product, News, Knowledge
    }
    /// <summary>
    /// 搜索类别
    /// </summary>
    public JobSearchType jobSearchType { get; set; }
    /// <summary>
    /// 索引任务集合列表
    /// </summary>
    private List<IndexJobItem> jobs = new List<IndexJobItem>();
    private static ILog log = LogManager.GetLogger(typeof(IndexManager));
    /// <summary>
    /// 私有构造函数所有的地方要对索引库进行修改都通过IndexManger,所以要单例,因为同时只能有一个在写索引库,
    /// 别的地方要写索引库要请求这个类来进行索引
    /// </summary>
    private IndexManager()
    {
        //
        //TODO: 在此处添加构造函数逻辑
        //
    }
    /// <summary>
    /// 启动任务
    /// </summary>
    public void Start()
    {
        IsStopped = false;
        Thread thread = new Thread(ScanThread);
        //背景线程
        thread.IsBackground = true;
        thread.Start();

    }
    /// <summary>
    /// 停止任务
    /// </summary>
    public void Stop()
    {
        IsStopped = true;

    }
    /// <summary>
    /// 扫描线程
    /// </summary>
    private void ScanThread()
    {
        //如果停止,则不在无限循环
        while (!IsStopped)
        {
            //休息5秒钟,尽可能多的积累任务
            Thread.Sleep(5000);
            if (jobs.Count <= 0)
            {
                //如果没有任务,线程等待
                log.Debug("没有任务,继续线程等待");
                Thread.Sleep(10 * 1000);
                continue;
            }
            //为什么每次循环都要打开,关闭索引库,因为关闭索引库以后才会把写入的数据提交到索引库中.也可以每次操作都"提交"(参考Lucene.net文档)
            //Enum.Parse(typeof(JobSearchType), jobSearchType).ToString()获取枚举名称
            string indexPath = System.IO.Path.Combine(HostingEnvironment.ApplicationPhysicalPath, ConfigurationManager.AppSettings["path"] + @"\" + Enum.Parse(typeof(JobSearchType), jobSearchType.ToString ()).ToString());
            FSDirectory directory = FSDirectory.Open(new DirectoryInfo(indexPath), new NativeFSLockFactory());

            bool isUpdate = IndexReader.IndexExists(directory);
            log.Debug("索引库是否存在:" + isUpdate);
            if (isUpdate)
            {
                //如果索引目录被锁定(比如索引过程中程序异常退出),则首先解锁
                if (IndexWriter.IsLocked(directory))
                {
                    log.Debug("开始解锁索引库");
                    IndexWriter.Unlock(directory);
                    log.Debug("解锁库完成");
                }
            }
            //索引
            IndexWriter writer = new IndexWriter(directory, new PanGuAnalyzer(), !isUpdate, Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED);

            //开始建立索引
            ProcessJob(writer);

            writer.Close();
            //不要忘了close
            directory.Close();
            log.Debug("全部索引完毕");
        }
    }
    /// <summary>
    /// 索引任务
    /// </summary>
    /// <param name="writer"></param>
    private void ProcessJob(IndexWriter writer)
    {
        foreach (var job in jobs.ToArray())
        {
            //删除任务
            jobs.Remove(job);
            //因为自己的网站,直接读取数据库,不用WebClient
            //为避免重复索引,所以先删除number=i的记录,再重新添加
            writer.DeleteDocuments(new Term("number", job.Id.ToString()));
            //索引
            Document document = new Document();
            string TypeName = "";
            //如果"添加" 任务则再添加
            if (job.ItemType == IndexJobItem.JobType.Add)
            {
                switch (jobSearchType)
                {
                    //索引工艺知识
                    case JobSearchType.Knowledge:
                        document = AddDocumentBycraftknowledge(job);
                        TypeName = "工艺知识";
                        break;
                    case JobSearchType.News:
                        document = AddDocumentByNews(job);
                        TypeName = "新闻";
                        break;
                    case JobSearchType.Product:
                        AddDocumentByProduct(job);
                        TypeName = "商品";
                        break;
                    default:
                        log.Debug("未设置JobSearchType属性,无法索引");
                        return;



                }

                writer.AddDocument(document);
                log.Debug("索引" + TypeName + ":" + job.Id + "完成!");



            }
        }
    }
    /// <summary>
    /// 给商品添加索引
    /// </summary>
    /// <param name="job"></param>
    /// <returns></returns>
    public Document AddDocumentByProduct(IndexJobItem job)
    {
        return null;

    }
    /// <summary>
    /// 给新闻添加索引
    /// </summary>
    /// <param name="job"></param>
    /// <returns></returns>
    public Document AddDocumentByNews(IndexJobItem job)
    {
        newsBLL bll = new newsBLL();
        //有可能刚添加就被删除了
        if (bll == null)
        {
            return null;

        }
        var craftknowledge = bll.Get(job.Id);
        string title = craftknowledge.Title;
        //这里要去除标签
        string body = Common.Tools.HtmlToTxt(craftknowledge.Content);
        Document document = new Document();
        document.Add(new Field("number", job.Id.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
        document.Add(new Field("ArticleHtmlUrl", craftknowledge.ArticleHtmlUrl, Field.Store.YES, Field.Index.NOT_ANALYZED));
        //以下内容要索引
        document.Add(new Field("title", title, Field.Store.YES, Field.Index.ANALYZED));
        document.Add(new Field("body", body, Field.Store.YES, Field.Index.ANALYZED, Lucene.Net.Documents.Field.TermVector.WITH_POSITIONS_OFFSETS));
        return document;
    }
    /// <summary>
    /// 给工艺知识添加索引
    /// </summary>
    /// <returns></returns>
    public Document AddDocumentBycraftknowledge(IndexJobItem job)
    {
        craftknowledgeBLL bll = new craftknowledgeBLL();
        //有可能刚添加就被删除了
        if (bll == null)
        {
            return null;

        }
        var craftknowledge = bll.Get(job.Id);
        string title = craftknowledge.Title;
        //这里要去除标签
        string body = Common.Tools.HtmlToTxt(craftknowledge.Content);
        Document document = new Document();
        document.Add(new Field("number", job.Id.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
        document.Add(new Field("ArticleHtmlUrl", craftknowledge.ArticleHtmlUrl, Field.Store.YES, Field.Index.NOT_ANALYZED));
        //以下内容要索引
        document.Add(new Field("title", title, Field.Store.YES, Field.Index.ANALYZED));
        document.Add(new Field("body", body, Field.Store.YES, Field.Index.ANALYZED, Lucene.Net.Documents.Field.TermVector.WITH_POSITIONS_OFFSETS));
        return document;
    }
    /// <summary>
    /// 添加任务
    /// </summary>
    /// <param name="Id">根据id</param>
    public void AddJob(int Id)
    {
        IndexJobItem job = new IndexJobItem();
        job.Id = Id;
        job.ItemType = IndexJobItem.JobType.Add;
        log.Debug(Id + "加入到任务列表中");
        //把任务加入任务列表
        jobs.Add(job);

    }
    /// <summary>
    /// 删除任务
    /// </summary>
    /// <param name="Id">根据Id</param>
    public void RemoveJob(int Id)
    {
        IndexJobItem job = new IndexJobItem();
        job.Id = Id;
        job.ItemType = IndexJobItem.JobType.Delete;
        log.Debug(Id + "加入删除任务列表");
        jobs.Add(job);
    }
    /// <summary>
    /// 实例化
    /// </summary>
    /// <returns></returns>
    public static IndexManager GetInstance(JobSearchType jobType)
    {
        //设置job的类别为
        Instance.jobSearchType = jobType;
         return Instance;
    }



}

接下来在全局配置文件中开启线程

在webconfig配置索引目录

在这里一个很重要的问题就是,我的商品,工艺知识,和新闻是根据一个枚举来判断到底是给哪个进行索引,

索引的目录也是动态的,根据枚举判断的

索引的目录

搜索BLL

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using Lucene.Net.Store;
using System.IO;
using Lucene.Net.Index;
using Lucene.Net.Analysis.PanGu;
using System.Net;
using Lucene.Net.Documents;
using log4net;
using Lucene.Net.Search;
using System.Text;
using mshtml;
using PanGu;
using System.Xml.Linq;
using System.Text.RegularExpressions;
using czcraft.BLL;
using czcraft.Model;
using System.Collections;
using System.Web.Hosting;
using System.Configuration;

namespace czcraft.BLL
{
    public partial class SearchBLL
    {
        private ILog logger = LogManager.GetLogger(typeof(SearchBLL));

        /// <summary>
        /// 搜索
        /// </summary>
        /// <param name="kw">关键词</param>
        /// <param name="startIndex">开始页码</param>
        /// <param name="pageSize">每页显示个数</param>
        /// <param name="totalCount">总个数</param>
        /// <returns></returns>
        public IEnumerable<SearchResult> Search(string kw, int startIndex, int pageSize, out int totalCount,SearchSum.searchType Type)
        {
            string indexPath = System.IO.Path.Combine(HostingEnvironment.ApplicationPhysicalPath, ConfigurationManager.AppSettings["path"] + @"\" + Type.ToString ());
            FSDirectory directory = FSDirectory.Open(new DirectoryInfo(indexPath), new NoLockFactory());
            IndexReader reader = IndexReader.Open(directory, true);
            IndexSearcher searcher = new IndexSearcher(reader);
            PhraseQuery query = new PhraseQuery();

            //todo:把用户输入的关键词进行拆词

            foreach (string word in CommonHelper.SplitWord(kw))//先用空格，让用户去分词，空格分隔的就是词“计算机 专业”
            {
                query.Add(new Term("body", word));
            }

            query.SetSlop(50);
            TopScoreDocCollector collector = TopScoreDocCollector.create(1000, true);
            searcher.Search(query, null, collector);
            totalCount = collector.GetTotalHits();//返回总条数
            ScoreDoc[] docs = collector.TopDocs(startIndex, pageSize).scoreDocs;
            List<SearchResult> listResult = new List<SearchResult>();
            for (int i = 0; i < docs.Length; i++)
            {
                int docId = docs[i].doc;//取到文档的编号（主键，这个是Lucene .net分配的）
                //检索结果中只有文档的id，如果要取Document，则需要Doc再去取
                //降低内容占用
                Document doc = searcher.Doc(docId);//根据id找Document
                string number = doc.Get("number");
                string title = doc.Get("title");
                string body = doc.Get("body");
                string ArticleHtmlUrl = doc.Get("ArticleHtmlUrl");
                SearchResult result = new SearchResult();
                result.Number = number;
                result.Title = title;


                result.BodyPreview = Preview(body, kw);
                result.ArticleHtmlUrl = ArticleHtmlUrl;
                listResult.Add(result);
            }
            return listResult;
        }
        /// <summary>
        /// 设置高亮显示
        /// </summary>
        /// <param name="body">文章主体</param>
        /// <param name="keyword">关键词</param>
        /// <returns></returns>
        private static string Preview(string body, string keyword)
        {
            //创建HTMLFormatter,参数为高亮单词的前后缀
            PanGu.HighLight.SimpleHTMLFormatter simpleHTMLFormatter =
                   new PanGu.HighLight.SimpleHTMLFormatter("<font color=\"red\">", "</font>");
            //创建 Highlighter ，输入HTMLFormatter 和 盘古分词对象Semgent
            PanGu.HighLight.Highlighter highlighter =
                            new PanGu.HighLight.Highlighter(simpleHTMLFormatter,
                            new Segment());
            //设置每个摘要段的字符数
            highlighter.FragmentSize = 100;
            //获取最匹配的摘要段
            String bodyPreview = highlighter.GetBestFragment(keyword, body);
            return bodyPreview;
        }
    }
}

搜索也根据搜索类别枚举动态判断搜索类别!

前端页面设计:

<%@ Page Language="C#" MasterPageFile="~/Top_Down.master" AutoEventWireup="true"
    CodeFile="SearchKnowledge.aspx.cs" Inherits="Search_SearchKnowledge" Title="找找看-工艺知识" %>

<asp:Content ID="Content1" ContentPlaceHolderID="head" runat="Server">
    <link href="../css/baidu.css" rel="stylesheet" type="text/css" />
    <link href="../css/other.css" rel="stylesheet" type="text/css" />
    <link href="../css/ui-lightness/jquery-ui-1.8.2.custom.css" rel="stylesheet" type="text/css" />
    <link href="../css/Pager.css" rel="stylesheet" type="text/css" />
    <link href="../css/Search.css" rel="stylesheet" type="text/css" />
    <script src="../Admin/scripts/jquery-1.7.1.min.js" type="text/javascript"></script>

    <script src="../js/jquery-ui-1.8.2.custom.min.js" type="text/javascript"></script>

    <script type="text/javascript">

      $(function () {
            $("#kw").autocomplete(
            { source: "Data/SearchSuggestion.ashx",
                select: function (event, ui) { $("#kw").val(ui.item.value); $("#form1").submit(); }
            });
        });
    </script>


</asp:Content>
<asp:Content ID="Content2" ContentPlaceHolderID="ContentPlaceHolder1" runat="Server">
    <div class="content">
        <div class="left_side">
            <div class="logo_bottom">
            </div>
        </div>
        <div class="gjss_load">
            <h4>
                找找看</h4>
            <span>当前位置：<a href="#">首页</a> &gt; <a href="#">找找看</a></span>
        </div>
        <div class="gjss">
            <div class="gjss_top">
            </div>
            <div class="gjss_c">
                <table width="804">
                    <tr>
                        <td colspan="7" align="center">
                            <label id="lbNews"  style="margin-left:260px" class="tab"><a href="SearchNews.aspx">新闻</a></label>      <label id="lbKnowledge"  style="margin-left:50px" class="tab"><a href="SearchKnowledge.aspx">工艺知识</a></label><label id="lbProduct" style="margin-left:50px" class="tab"><a href="#">商品</a></label>

                            </td>
                    </tr>
                    <tr>
                        <td class="style1">
                            <div id="m" align="center">
                                <div id="fm">
                                    <form name="form1">
                                    <span class="s_ipt_wr" style="float: left">
                                        <input id="kw" class="s_ipt" name="kw" maxlength="100" value='<%=Request["kw"] %>' />
                                    </span><span class="s_btn_wr">
                                        <input id="su" class="s_btn" onmouseout="this.className='s_btn'" onmousedown="this.className='s_btn s_btn_h'"
                                            value="找找看" type="submit" /></span></form>
                                </div>
                            </div>
                        </td>
                    </tr>
                    <tr>
                        <td colspan="7" align="center" class="style1">
                            <div style="text-align: center">
                                <ul id="hotwordsUL" class="hotWords">
                                    <asp:Repeater ID="repeaterHotWords" runat="server">
                                        <ItemTemplate>
                                            <li><a href='SearchKnowledge.aspx?kw=<%#Eval("KeyWord") %>'>
                                                <%#Eval("KeyWord") %>
                                            </a></li>
                                        </ItemTemplate>
                                    </asp:Repeater>
                                </ul>
                            </div>
                        </td>
                    </tr>
                    <tr>
                        <td colspan="7" align="center">
                            <br />
                            <ul id="ulResult" class="hotWords">
                                <asp:Repeater EnableViewState="false" ID="repeaterResult" runat="server">
                                    <ItemTemplate>
                                        <li><span><%--<a  href='../CraftKnowledge/ViewCraftKnowledge.aspx?KnowledgeId=<%#Eval("Number") %>'>--%>
                                        <a  href='<%#Eval("ArticleHtmlUrl") %>'>
                                            <%#Eval("Title") %></a></span>
                                           <br />
                                           <span> <%#Eval("BodyPreview")%></span>
                                        </li>
                                    </ItemTemplate>
                                </asp:Repeater>
                            </ul>
                            <br />
                            <div class="pager">
                                <%=PageHtml%>
                            </div>

                        </td>
                    </tr>
                </table>
            </div>
        </div>
    </div>
</asp:Content>

后台代码:

using System;
using System.Collections;
using System.Configuration;
using System.Data;
using System.Linq;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.HtmlControls;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Xml.Linq;
using czcraft;
using czcraft.BLL;
using Common;
using czcraft.Model;
using System.Collections.Generic;

public partial class Search_SearchKnowledge : System.Web.UI.Page
{
    //分页控件
    public string PageHtml { get;private set; }
    protected void Page_Load(object sender, EventArgs e)
    {
        //加载热词
        repeaterHotWords.DataSource = new SearchInfoBLL().GetHotWords(SearchSum.searchType.Knowledge);
        repeaterHotWords.DataBind();

        //如果kw为空，则是第一次进入界面
        string kw = Request["kw"];

        if (!Tools.IsValidInput(ref kw,true)||string.IsNullOrEmpty(kw))
        {
            return;
        }
        //把搜索记录加入数据库
        SearchInfo kwLog = new SearchInfo();
        kwLog.KeyWord = kw;
        kwLog.DateTime = DateTime.Now;
        kwLog.Ip = Request.UserHostAddress;
        kwLog.SearchType = SearchSum.searchType.Knowledge.GetHashCode().ToString ();
        new SearchInfoBLL().AddNew(kwLog);

        var pager =new Common.RupengPager();
        pager.UrlFormat = "SearchKnowledge.aspx?pagenum={n}&kw=" + Server.UrlEncode(kw);
        pager.PageSize = 10;
        //解析当前页面
        pager.TryParseCurrentPageIndex(Request["pagenum"]);
        int startRowIndex = (pager.CurrentPageIndex - 1) * pager.PageSize;

        int totalCount;
        IEnumerable<SearchResult> result = new SearchBLL().Search(kw, startRowIndex, 10, out totalCount, SearchSum.searchType.Knowledge);
        pager.TotalCount = totalCount;
        PageHtml = pager.Render();//渲染页码条HTML

        repeaterResult.DataSource = result;
        repeaterResult.DataBind();

    }
}

我们还可以再做做当前热点,这里详细粘贴代码了

效果图:

这里复制了好多重复数据,不是程序问题………………

哈哈,一个站内搜索给网站增添了不少亮点,

这个搜索框当然是copy的百度的,哈哈