C#小爬虫，通过URL进行模拟发送接收数据

 1 public async Task<string> SendDataAsync(HttpMethod httpMethod, string requestUrl, HttpContent postContent = null, string cookies = "")
 2         {
 3             HttpClientHandler httpHandler = new HttpClientHandler()
 4             {
 5                 AllowAutoRedirect = true,
 6                 //CookieContainer = new CookieContainer(),
 7                 UseCookies = false,
 8                 AutomaticDecompression = DecompressionMethods.GZip
 9             };
10 
11             using (HttpClient httpClient = new HttpClient(httpHandler))
12             {
13                 httpClient.DefaultRequestHeaders.ExpectContinue = false;
14                 httpClient.DefaultRequestHeaders.Add("Accept", "*/*");
15                 httpClient.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate");
16                 httpClient.DefaultRequestHeaders.Add("Accept-Language", " zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
17                 httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0");
18                 //var response = await httpClient.GetAsync(sUrl);
19                 //return (await response.Content.ReadAsAsync<AvResult>());
20                 string sResult;
21                 try
22                 {
23                     HttpRequestMessage request = new HttpRequestMessage(httpMethod, requestUrl);
24                     bool isNullCookie = string.IsNullOrEmpty(cookies);
25                     if (!isNullCookie)
26                     {
27                         //httpHandler.CookieContainer.SetCookies(new Uri(requestUrl), cookies);
28                         request.Headers.Add("Cookie", cookies);
29                     }
30                     if (httpMethod.Method == "POST")
31                         request.Content = postContent;
32                     var response = await httpClient.SendAsync(request);
33                     sResult = await response.Content.ReadAsStringAsync();
34                 }
35                 catch (Exception ex)
36                 {
37                     sResult = ex.Message;
38                 }
39                 return sResult;
40             }
41         }