让libcurl支持多线程下载

  1 // MutilDownload.cpp : 定义控制台应用程序的入口点。
  2 //
  3 
  4 #include "stdafx.h"
  5 #include <stdio.h>
  6 #include <io.h>
  7 #include "curl/curl.h"
  8 #include <string>
  9 
 10 
 11 #ifdef _DEBUG
 12 #pragma comment(lib, "../lib/libcurld.lib")
 13 #pragma comment(lib, "../lib/libeay32d.lib")
 14 #pragma comment(lib, "../lib/ssleay32d.lib")
 15 #else
 16 #pragma comment(lib, "../lib/libcurl.lib")
 17 #pragma comment(lib, "../lib/libeay32.lib")
 18 #pragma comment(lib, "../lib/ssleay32.lib")
 19 #endif // _DEBUG
 20 
 21 #pragma comment(lib, "ws2_32.lib")
 22 #pragma comment(lib, "Wldap32.lib")
 23 
 24 
 25 using namespace std;
 26 
 27 struct tNode
 28 {
 29         FILE *fp;
 30         long startidx;
 31         long endidx;
 32         void *_curl;
 33         DWORD _tid;
 34         string _url;
 35         string _header;
 36         bool _started;
 37 };
 38 
 39 bool bError = false;
 40 int threadCnt = 0;
 41 long long downloaded_size = 0;
 42 long long total_size = 0;
 43 static CRITICAL_SECTION foo_mutex;
 44 
 45 static size_t downLoadPackage(void *ptr, size_t size, size_t nmemb, void *userdata)
 46 {
 47         tNode *node = (tNode*)userdata;
 48         size_t written = 0;
 49         EnterCriticalSection(&foo_mutex);
 50         if(node->_started) printf("%s\n", node->_header.c_str());
 51         node->_started = false;
 52         if (node->startidx <= node->endidx)
 53         {
 54                 int real_size = size * nmemb;
 55                 if (node->startidx + real_size > node->endidx)
 56                 {
 57                         real_size = node->endidx - node->startidx + 1;
 58                 }
 59 
 60                 if (fseek(node->fp, node->startidx, SEEK_SET) != 0)
 61                 {
 62                         perror("fseek");
 63                 }
 64                 else
 65                 {
 66                         written = fwrite(ptr, 1, real_size, node->fp);
 67                         node->startidx += real_size;
 68                 }
 69                 downloaded_size += real_size;
 70         }
 71         
 72         LeaveCriticalSection(&foo_mutex);
 73         return written;
 74 }
 75 
 76 static size_t RetriveHeaderFunction(void *ptr, size_t size, size_t nmemb, void *stream)
 77 {
 78         std::string* receive_header = (std::string*)stream;
 79         if (receive_header && ptr)
 80         {
 81                 receive_header->append(reinterpret_cast<const char*>(ptr), size * nmemb);
 82         }
 83 
 84         return nmemb * size;
 85 }
 86 
 87 int assetsManagerProgressFunc(void *ptr, double totalToDownload, double nowDownloaded, double totalToUpLoad, double nowUpLoaded)
 88 {
 89         long long tmp = ((long long)downloaded_size * 100)/ total_size;
 90 
 91         printf("\r下载进度%d", tmp);
 92 
 93         return 0;
 94 }
 95 
 96 /************************************************************************/
 97 /* 获取要下载的远程文件的大小                                             */
 98 /************************************************************************/
 99 long getDownloadFileLenth(const char *url){
100         double downloadFileLenth = 0;
101         CURL *handle = curl_easy_init();
102         curl_easy_setopt(handle, CURLOPT_URL, url);
103         curl_easy_setopt(handle, CURLOPT_HEADER, 1);    //只需要header头
104         curl_easy_setopt(handle, CURLOPT_NOBODY, 1);    //不需要body
105         if (curl_easy_perform(handle) == CURLE_OK)
106         {
107                 curl_easy_getinfo(handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &downloadFileLenth);
108         }
109         else
110         {
111                 downloadFileLenth = -1;
112         }
113         curl_easy_cleanup(handle);
114 
115         return static_cast<long>(downloadFileLenth);
116 }
117 
118 DWORD WINAPI workThread(void* pData)
119 {
120         tNode* pNode = (tNode*)pData;
121 
122         CURL* _curl = curl_easy_init();
123 
124         pNode->_curl = _curl;
125 
126         curl_easy_setopt(_curl, CURLOPT_URL, pNode->_url.c_str());
127 
128         std::string useragent = _T("Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0");
129         curl_easy_setopt(_curl, CURLOPT_USERAGENT, useragent.c_str());
130 
131         // 设置重定向的最大次数
132         curl_easy_setopt(_curl, CURLOPT_MAXREDIRS, 5);
133 
134         // 设置301、302跳转跟随location
135         curl_easy_setopt(_curl, CURLOPT_FOLLOWLOCATION, 1);
136 
137         curl_easy_setopt(_curl, CURLOPT_NOSIGNAL, 1L);
138         curl_easy_setopt(_curl, CURLOPT_POST, false);
139 
140         curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, downLoadPackage);
141         curl_easy_setopt(_curl, CURLOPT_WRITEDATA, pNode);
142 
143         curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, RetriveHeaderFunction);
144         curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &pNode->_header);
145 
146         curl_easy_setopt(_curl, CURLOPT_NOPROGRESS, false);
147         curl_easy_setopt(_curl, CURLOPT_PROGRESSFUNCTION, assetsManagerProgressFunc);
148 
149         curl_easy_setopt(_curl, CURLOPT_LOW_SPEED_LIMIT, 1L);
150         curl_easy_setopt(_curl, CURLOPT_LOW_SPEED_TIME, 5L);
151 
152         CString down_range;
153         down_range.Format(_T("%ld-%ld"), pNode->startidx, pNode->endidx);
154         curl_easy_setopt(_curl, CURLOPT_RANGE, (LPCTSTR)down_range);
155         pNode->_started = true;
156 
157         CURLcode res = curl_easy_perform(pNode->_curl);
158 
159         if (res != 0)
160         {
161                 const char* err_string = curl_easy_strerror(res);
162                 printf("download error: %s\n", err_string);
163                 bError = true;
164         }
165 
166         curl_easy_cleanup(pNode->_curl);
167 
168         EnterCriticalSection(&foo_mutex);
169         threadCnt--;
170         LeaveCriticalSection(&foo_mutex);
171 
172         delete pNode;
173 
174         return 0;
175 }
176 
177 bool downLoad(int threadNum, std::string _packageUrl, std::string _storagePath, std::string fileName)
178 {
179         long fileLength = getDownloadFileLenth(_packageUrl.c_str());
180 
181         if (fileLength <= 0)
182         {
183                 printf("get the file error...");
184                 return false;
185         }
186 
187 
188         // Create a file to save package.
189         string outFileName = _storagePath + fileName;
190         string srcFileName = outFileName;
191         outFileName += ".dl";
192 
193         FILE *fp = nullptr;
194         fopen_s(&fp, outFileName.c_str(), "wb");
195         if (!fp)
196         {
197                 return false;
198         }
199 
200         long gap = fileLength / threadNum;
201         total_size = fileLength;
202         downloaded_size = 0;
203 
204         for (int i = 0; i < threadNum; i++)
205         {
206                 tNode* pNode = new tNode();
207 
208                 if (i < threadNum - 1)
209                 {
210                         pNode->startidx = i * gap;
211                         pNode->endidx = pNode->startidx + gap - 1;
212                 }
213                 else
214                 {
215                         pNode->startidx = i * gap;
216                         pNode->endidx = fileLength - 1;
217                 }
218 
219                 printf("idx = %d, startidx = %d, endidx = %d\n", i, pNode->startidx, pNode->endidx);
220 
221                 pNode->fp = fp;
222                 pNode->_url = _packageUrl;
223 
224                 EnterCriticalSection(&foo_mutex);
225                 threadCnt++;
226                 LeaveCriticalSection(&foo_mutex);
227 
228                 HANDLE hThread = CreateThread(NULL, 0, workThread, pNode, 0, &(pNode->_tid));
229                 CloseHandle(hThread);
230         }
231 
232         while (threadCnt > 0)
233         {
234                 Sleep(1000);
235         }
236 
237         fclose(fp);
238 
239         if (bError)
240         {
241                 printf("\ndownload failed......\n");
242                 DeleteFile(outFileName.c_str());
243         }
244         else
245         {
246                 printf("\ndownload succed......\n");
247                 MoveFileEx(outFileName.c_str(), srcFileName.c_str(), MOVEFILE_REPLACE_EXISTING);
248         }
249         
250         return true;
251 }
252 
253 int _tmain(int argc, _TCHAR* argv[])
254 {
255         InitializeCriticalSection(&foo_mutex);
256 
257         int thread_number = 4;
258         if (argc > 1)
259         {
260                 char* str_thread_number = argv[1];
261                 int temp_number = _ttoi(str_thread_number);
262                 if (temp_number > 0)
263                 {
264                         thread_number = temp_number;
265                 }
266         }
267 
268         if (thread_number > 10)
269         {
270                 thread_number = 10;
271         }
272 
273         downLoad(thread_number, "http://dlsw.baidu.com/sw-search-sp/soft/71/10998/OfflineBaiduPlayer_151_V4.1.2.263.1432003947.exe", "./", "BaiduPlayer.exe");
274 
275         getchar();
276 
277         DeleteCriticalSection(&foo_mutex);
278 
279         return 0;
280 }