数据统计之数据多线程数据传输

标签：

说实话这段代码对于数据库的提升要看读写比例，数据库插入的数据的方式以及引起的数据库锁表示行锁还是页锁还是表锁有关系，线程数据量依据服务器的不同设定也不一样。提升效率比较难以计算。但是毕竟也是一种尝试

就贴出来共享了

           SqlHelper.CommandTimeout = 1800;
            long minAutoID = GetSourceMaxOrMinID(curstatdatevalue, DownLogSourceDBConnString, true);
            long maxAutoID = GetSourceMaxOrMinID(curstatdatevalue, DownLogSourceDBConnString, false);
            //数据分片，确定每个分片的数据量
            long everyCount = (maxAutoID - minAutoID) / PartsCount + 1;
            List<AutoIdArea> list = new List<AutoIdArea>();
            for (int i = 0; i < PartsCount; i++)
            {
                list.Add(new AutoIdArea(minAutoID + i * everyCount, minAutoID + (i + 1) * everyCount - 1, "D_Copy_ThreadAutoId_" + curstatdatevalue / 100 + "_" + i));
            }
            
            Utility.WriteLog("开始 将明细数据转换成原始统计数据保存到ResDownloadLog_Temp表中", LogDir);
            //设置信号灯
            Semaphore semaphores = new Semaphore(ThreadCount, ThreadCount);
            //定义方法
            Action<AutoIdArea, Semaphore, int> fun = (areaInfo, s, curstat) =>
                {
                    try
                    {
                        long minId =
                            Convert.ToInt64(Utility.GetStatAutoID(areaInfo.ThreadKey, areaInfo.BeginInt,
                                                                  ComputingDB_ConnString));
                        if (minId <= areaInfo.BeginInt)
                            minId = areaInfo.BeginInt;
                        //在别的线程没有出现异常的情况下继续跑
                        while (Import(areaInfo, ref minId, curstat, DownLogSourceDBConnString) > 0) ;
                    }
                    catch (Exception ex)
                    {
                        ThreadException = ex;
                        Utility.WriteLog(
                            string.Format("异步导入数据异常:beginId={0},endId={1},ex:{2}", areaInfo.BeginInt, areaInfo.EndInt,
                                          ex.ToString()), LogDir);
                        Utility.SendMail("异步导入数据异常",
                                         string.Format("beginId={0},endId={1},ex:{2}", areaInfo.BeginInt,
                                                       areaInfo.EndInt, ex.ToString()), false);
                    }
                    finally
                    {
                        s.Release();
                    }

                };
            List<IAsyncResult> listResult = new List<IAsyncResult>();

            for (int i = 0; i < list.Count; i++)
            {
                semaphores.WaitOne();
                if (ThreadException != null)
                {
                    //等待所有异步线程完成
                    for (int j = 0; j < listResult.Count; j++)
                    {
                        listResult[j].AsyncWaitHandle.WaitOne();
                    }
                    throw ThreadException;
                }
                listResult.Add(fun.BeginInvoke(list[i], semaphores, curstatdatevalue, null, null));
            }

            //等待所有异步线程完成
            for (int i = 0; i < listResult.Count; i++)
            {
                listResult[i].AsyncWaitHandle.WaitOne();
            }
            if (ThreadException != null)
            {
                throw ThreadException;
            }

数据统计之数据多线程数据传输

标签：

原文地址：http://my.oschina.net/osenlin/blog/523468