网络采集软件核心技术剖析系列（1）---如何使用C#语言获取博客园某个博主的全部随笔链接及标题

 protected void GatherInitCnblogsFirstUrls()
        {
            string strPagePre = "http://www.cnblogs.com/";
            string strPagePost = "/default.html?page={0}&OnlyTitle=1";
            string strPage = strPagePre + this.txtBoxCnblogsBlogID.Text + strPagePost;


            for (int i = 500; i > 0; i--)
            {
                string strTemp = string.Format(strPage, i);
                m_wd.AddUrlQueue(strTemp);

            }
        }

     public string GetPageByHttpWebRequest(string url, Encoding encoding, string strRefer)
        {

            string result = null;
   
            WebResponse response = null;
            StreamReader reader = null;

            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
                request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)";
                request.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";

                if (!string.IsNullOrEmpty(strRefer))
                {
                    Uri u = new Uri(strRefer);
                    request.Referer = u.Host;
                }
                else
                {
                    request.Referer = strRefer;
                }
                request.Method = "GET";
                response = request.GetResponse();
                reader = new StreamReader(response.GetResponseStream(), encoding);
                result = reader.ReadToEnd();
                
            }
            catch (Exception ex)
            {
                result = "";
            }
            finally
            {
                if (reader != null)
                    reader.Close();
                if (response != null)
                    response.Close();
                
            }
            return result;
        }

    private void GetReferences()
        {
            HtmlNodeCollection hrefs = m_Doc.DocumentNode.SelectNodes("//a[@href]");

            if (Equals(hrefs, null))
            {
                References = new string[0];
                return;
            }

            References = hrefs.
                Select(href => href.Attributes["href"].Value).
                Distinct().
                ToArray();
        }

private void GetReferencesText()
        {
            try
            {
                m_dicLink2Text.Clear();
                HtmlNodeCollection hrefs = m_Doc.DocumentNode.SelectNodes("//a[@href]");

                if (Equals(hrefs, null))
                {
                    return;
                }

                foreach (HtmlNode node in hrefs)
                {
                    if (!m_dicLink2Text.Keys.Contains(node.Attributes["href"].Value.ToString()))
                        if(!HttpUtility.HtmlDecode(node.InnerHtml).Contains("img src")
                            && !HttpUtility.HtmlDecode(node.InnerHtml).Contains("img ")
                            && !HttpUtility.HtmlDecode(node.InnerHtml).Contains(" src"))
                         m_dicLink2Text.Add(node.Attributes["href"].Value.ToString(), HttpUtility.HtmlDecode(node.InnerHtml));
                }
                int a = 0;
            }
            catch (System.Exception e)
            {
                System.Console.WriteLine(e.ToString());
            }

        }

正则有两种模式，一种为贪婪模式（默认），另外一种为懒惰模式，以下为例：
（abc）dfe（gh）
对上面这个字符串使用（.*）将会匹配整个字符串，因为正则默认是尽可能多的匹配。
虽然（abc）满足我们的表达式，但是（abc）dfe（gh）也同样满足，所以正则会匹配多的那个。
如果我们只想匹配（abc）和（gh）就需要用到以下的表达式
（.*?）
在重复元字符*或者+后面跟一个?，作用就是在满足的条件下尽可能少匹配。

   MatchCollection matchs = Regex.Matches(normalizedLink, m_strCnblogsUrlFilterRule, RegexOptions.Singleline);
                if (matchs.Count > 0)
                {
                    string strLinkText = "";

                    if (links.m_dicLink2Text.Keys.Contains(normalizedLink))
                        strLinkText = links.m_dicLink2Text[normalizedLink];

                    if (strLinkText == "")
                    {
                        if (links.m_dicLink2Text.Keys.Contains(link))
                            strLinkText = links.m_dicLink2Text[link].TrimEnd().TrimStart();
                    }

                    PrintLog(strLinkText + "\n");
                    PrintLog(normalizedLink + "\n");
                    

                    lstThisTimesUrls.Add(normalizedLink);
                }

  private bool CheckArticles(List<string> lstThisTimesUrls)
        {
            bool bRet = true;
            foreach (string strTemp in lstThisTimesUrls)
            {
                if (!m_lstUrls.Contains(strTemp))
                {
                    bRet = false;
                    break;
                }
            }
            foreach (string strTemp in lstThisTimesUrls)
            {
                if (!m_lstUrls.Contains(strTemp))
                    m_lstUrls.Add(strTemp);
            }
         
            return bRet;
        }

        TaskDelegate deles = new TaskDelegate(new ccTaskDelegate(RefreshTask));
        
        public void RefreshTask(DelegatePara dp)
        {
            //如果需要在安全的线程上下文中执行
            if (this.InvokeRequired)
            {
                this.Invoke(new ccTaskDelegate(RefreshTask), dp);
                return;
            }
          
            //转换参数
            string strLog = (string)dp.strLog;
            WriteLog(strLog);

        }
        protected void PrintLog(string strLog)
        {
            DelegatePara dp = new DelegatePara();

            dp.strLog = strLog;
            deles.Refresh(dp);
        }
        public void WriteLog(string strLog)
        {
            try
            {
                strLog = System.DateTime.Now.ToLongTimeString() + " : " + strLog;
           
                this.richTextBoxLog.AppendText(strLog);
                this.richTextBoxLog.SelectionStart = int.MaxValue;
                this.richTextBoxLog.ScrollToCaret();
            }
            catch
            {
            }
        }