HttpWebRequest多线程抓取17Track的物流信息

公司的一个系统需要去抓17Track的物流信息，贴上代码有需要的朋友可以参考一下↓

//17Track的抓取地址以及开启的线程数量
  <add key="url" value="http://www.17track.net/restapi/handlertrack.ashx"/>
  <add key="threadCount" value="3"/>

//使用HttpWebRequest去请求
private static JObject PostData(string url, string shipmentNumber)
        {
            HttpWebRequest request =
                   (HttpWebRequest)WebRequest.Create(url);
            request.Method = "Post";
            request.ContentType = "application/json; charset=UTF-8";
            request.Headers["Accept-Language"] = "zh-CN,zh;q=0.";
            request.Headers["Accept-Charset"] = "GBK,utf-8;q=0.7,*;q=0.3";

            JArray shipmentNumberArrary = new JArray();
            JObject shipmentJson = new JObject();
            shipmentJson.Add("num", shipmentNumber);

            shipmentNumberArrary.Add(shipmentJson);


            JObject requestParms = new JObject();
            requestParms.Add("guid", "");
            requestParms.Add("data", shipmentNumberArrary);

            Encoding encoding = Encoding.UTF8;
            byte[] postData = encoding.GetBytes(requestParms.ToString());
            request.ContentLength = postData.Length;

            Stream requestStream = request.GetRequestStream();
            requestStream.Write(postData, 0, postData.Length);
            HttpWebResponse response = (HttpWebResponse)request.GetResponse();
            Stream responseStream = response.GetResponseStream();

            StreamReader streamReader = new StreamReader(responseStream, encoding);
            string retString = streamReader.ReadToEnd();
            streamReader.Close();
            responseStream.Close();

            return JObject.Parse(retString);
        }

  
//开启多线程去抓取
try
            {
                _taskList = new List<Task>();


                DataTable shipmentList = GetDataForTrack();

                if (shipmentList.Rows.Count == 0)
                {
                    WriteLog.addTOLog(_logPath, "Get17Track", "No Shipment to Get Track.");
                    return;
                }
                else
                {
                    WriteLog.addTOLog(_logPath, "Get17Track", "【Total shipment to Get track :" + shipmentList.Rows.Count + "】");
                }

                //开启的线程数
                int threadCount = Convert.ToInt32(ConfigurationManager.AppSettings["threadCount"]);
                //每条线程执行的记录数
                int eachThreadRow = shipmentList.Rows.Count/threadCount;

                for (int i = 0; i < threadCount; i++)
                {
                    //如果总记录数小于线程数，则直接开启一个线程
                    if (eachThreadRow <=0 )
                    {
                        GetTrack(shipmentList, i * eachThreadRow, shipmentList.Rows.Count-1);
                        break;
                    }
                    //如果不能整除，则最后一条线程的执行记录数需要加上余数
                    if (i == threadCount-1 && shipmentList.Rows.Count%threadCount != 0)
                    {
                        GetTrack(shipmentList, i*eachThreadRow,(i + 1)*eachThreadRow - 1 + shipmentList.Rows.Count%threadCount);
                        continue;
                    }
                    //开启线程执行该线程需要执行的记录范围
                    GetTrack(shipmentList, i*eachThreadRow,(i+1)*eachThreadRow-1);
                }
                Task.WaitAll(_taskList.ToArray());
                WriteLog.addTOLog(_logPath, "Get17Track","Get Track End.");
            }
            catch (Exception ex)
            {
                WriteLog.addTOLog(_logPath, "Get17Track", ex.Message);
            }

  
//解析抓取到的数据，存入数据库
private  void GetTrack(DataTable dt,int strartRow,int endRow)
        {
          _taskList.Add(Task.Factory.StartNew(() =>
          {
              SqlHelper help2 = new SqlHelper(_sqlConnectionStr);
                for (int i = strartRow; i <= endRow; i++)
                {
                    string shipmentNumber = dt.Rows[i]["shipment_number"].ToString().Trim();
                    try
                    {
                        var responseJosn = PostData(_url, shipmentNumber);
                        if (responseJosn["ret"].ToString() != "1") continue;
                        var shipmentListJsonArray = JArray.Parse(responseJosn["dat"].ToString());
                        if (!shipmentListJsonArray.Any()) continue;
                        foreach (var trackListJsonArray in shipmentListJsonArray)
                        {
                            if (!trackListJsonArray["track"].Any()) continue;
                            var trakJson = JObject.Parse(trackListJsonArray["track"].ToString());
                            var trackDetailsJsonArray = JArray.Parse(trakJson["z2"].ToString());
                            if (!trackDetailsJsonArray.Any()) continue;

                            for (int j = trackDetailsJsonArray.Count-1; j >=0; j--)
                            {
                                string date = trackDetailsJsonArray[j]["a"].ToString();
                                string location = trackDetailsJsonArray[j]["c"].ToString();
                                string desc = trackDetailsJsonArray[j]["z"].ToString();

                                SqlParameter[] parms = new SqlParameter[]
                                {
                                    new SqlParameter("@datetime", date),
                                    new SqlParameter("@desc", desc),
                                    new SqlParameter("@location", location),
                                    new SqlParameter("@shipmentNumber", shipmentNumber),
                                    new SqlParameter("@type", "I")
                                };
                                help2.ExecuteTransactionWithSp("SP_AwbTracker_17Track", parms);
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        WriteLog.addTOLog(_logPath, "Get17Track",shipmentNumber +" Error: "+ ex.Message);
                    }
                }
            }));
        }