批量插入数据, 将DataTable里的数据批量写入数据库的方法

大量数据导入操作, 也就是直接将DataTable里的内容写入到数据库

通用方法: 拼接Insert语句, 好土鳖

1. MS Sql Server:   使用SqlBulkCopy

2. MySql:

adapter.update()批量更新

MySqlBulkLoader, 这个是从文件里边到的, 有个实现是先将DATATable编程CSV文件, 在用MySqlBulkLoader导入MySql

参考文章: http://theonetechnologies.com/outsourcing/post/mysql-bulk-data-import-using-net-connector-mysqlbulkloader-class.aspx

原文引用于Rocho.J的批量插入数据, 将DataTable里的数据批量写入数据库的方法

//参考代码

3. MS Access: 只能用批量更新了, adapter.update()

备注:   此处先标记个思路, 等我这实现完了, 贴个示例

==============

其实早就做完了, 都忘记这回事了... 今天看到这篇, 补一下代码

//枚举

//公共方法

//DALFactory.cs

//FileHelper.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

using System.IO;
using System.Data;

namespace ETLUtilityDAL.Common
{
    public class FileHelper
    {
        public static string ReadFileToString(string fileFullPath, Encoding codeType)
        {
            string result = "";
            if (string.IsNullOrEmpty(fileFullPath))
                throw new ArgumentNullException( "fileFullPath","File path can not be null or empty! ");
            using (FileStream fileStream = new FileStream(fileFullPath, FileMode.OpenOrCreate, FileAccess.Read))
            {
                if (!File.Exists(fileFullPath))
                    throw new FileNotFoundException("File not found! ");
            }

            using (StreamReader sReader = new StreamReader(fileFullPath, codeType))
            {
                try
                {
                    result = sReader.ReadToEnd();
                }
                catch (Exception ex)
                {
                    throw new IOException(ex.Message);
                }
            }
            return result;
        }

        public static string ReadFileToString(string fileFullPath)
        {
            return ReadFileToString(fileFullPath, Encoding.Default);
        }

        public static void WriteDataTableToCSVFile(DataTable dataTable, string fileFullPath, Encoding codeType)
        {
            using (Stream stream = new FileStream(fileFullPath,FileMode.Create,FileAccess.Write))
            using (StreamWriter swriter = new StreamWriter(stream, codeType))
            {
                try
                {
                    int icolcount = dataTable.Columns.Count;
                    foreach (DataRow drow in dataTable.Rows)
                    {
                        for (int i = 0; i < icolcount; i++)
                        {
                            if (!Convert.IsDBNull(drow[i]))
                            {
                                swriter.Write(drow[i].ToString());
                            }
                            if (i < icolcount - 1)
                            {
                                swriter.Write("|");
                            }
                        }
                        swriter.Write(swriter.NewLine);
                    }
                }
                catch (Exception ex)
                {
                    throw new IOException(ex.Message);
                }
            }
        }

        public static void WriteDataTableToCSVFile(DataTable dataTable, string fileFullPath)
        {
            WriteDataTableToCSVFile(dataTable, fileFullPath, Encoding.Default);
        }

        public static string[] GetFileFullPathList(string directoryPath, string fileType, bool IsRecursive)
        {
            return IsRecursive ? Directory.GetFiles(directoryPath, fileType, SearchOption.AllDirectories) : Directory.GetFiles(directoryPath, fileType, SearchOption.TopDirectoryOnly);
        }

        public static string[] GetSubDirectorys(string directoryPath, string containsName, bool IsRecursive)
        {
            return IsRecursive ? Directory.GetDirectories(directoryPath, containsName, SearchOption.AllDirectories) : Directory.GetDirectories(directoryPath, containsName, SearchOption.TopDirectoryOnly);
        }

        public static void WriteStringToFile(string fileFullPath, bool isAppend ,string fileContent)
        {
            WriteStringToFile(fileFullPath, isAppend, fileContent, Encoding.Default);
        }

        public static void WriteStringToFile(string fileFullPath, bool isAppend, string fileContent, Encoding codeType)
        {
            //using (FileStream fileStream = new FileStream(fileFullPath, FileMode.OpenOrCreate, FileAccess.Write))
            using (StreamWriter sWriter = new StreamWriter(fileFullPath,isAppend,codeType))
            {
                try
                {
                    if (!File.Exists(fileFullPath))
                        File.Create(fileFullPath);
                    sWriter.Write(fileContent);
                }
                catch (Exception ex)
                {
                    throw new IOException(ex.Message);
                }
            }
        }
    }
}

//XMLHelper.cs, 用List模拟堆栈实现XML结点的操作

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

using System.IO;
using System.Xml;
using System.Xml.XPath;
using System.Xml.Schema;
using ETLUtilityDAL.Enums;
using System.Collections.Specialized;
using ETLUtilityModel;
using ETLUtilityModel.Enums;

namespace ETLUtilityDAL.Common
{
    public class XMLHelper
    {
        #region XMLStream
        public static List<ETLXmlNode> GetAllNodesFromXMLFile(string xmlFileFullName)
        {
            List<ETLXmlNode> xmlNodeLst = new List<ETLXmlNode>();

            using (Stream stream = new FileStream(xmlFileFullName, FileMode.Open, FileAccess.Read))
            using (XmlTextReader xmlTxtReader = new XmlTextReader(stream))
            {
                int fathId = 0;  //root node: 0
                int elemCount = 1;  //Element Count
                XmlNodeType nodeType;
                ETLXmlNode xNode;
                ETLXmlNodeStack nodStack = ETLXmlNodeStack.CreateETLXmlNodeStack();
                while (xmlTxtReader.Read())
                {
                    xNode = null;
                    nodeType = xmlTxtReader.NodeType;
                    switch (nodeType)
                    {
                        case XmlNodeType.Element:
                            bool isEmpty = false;
                            if (isEmpty = xmlTxtReader.IsEmptyElement)
                                xNode = new ETLXmlNode(elemCount, xmlTxtReader.Name, ETLXmlNodeType.OddNode, fathId);
                            else
                                xNode = new ETLXmlNode(elemCount, xmlTxtReader.Name, ETLXmlNodeType.EvenNode, fathId);

                            fathId = elemCount;
                            elemCount++;

                            //Deal with the Attribute
                            if (xmlTxtReader.HasAttributes)
                            {
                                NameValueCollection nvc = xNode.NodeAttributes;
                                for (int i = 0; i < xmlTxtReader.AttributeCount; i++)
                                {
                                    xmlTxtReader.MoveToAttribute(i);
                                    nvc.Add(xmlTxtReader.Name, xmlTxtReader.Value);
                                }
                            }
                            if (isEmpty)
                            {
                                xmlNodeLst.Add(xNode);
                                fathId = xNode.FatherNodeId;
                            }
                            else
                                ETLXmlNodeStack.Push(xNode);
                            break;
                        case XmlNodeType.EndElement:
                            xNode = ETLXmlNodeStack.Pop();
                            xmlNodeLst.Add(xNode);
                            fathId = xNode.FatherNodeId;
                            break;
                        case XmlNodeType.Text:
                            xNode = ETLXmlNodeStack.Pop();
                            xNode.NodeText = xmlTxtReader.Value;
                            ETLXmlNodeStack.Push(xNode);
                            break;
                        default:
                            break;
                    }
                }
            }

            return xmlNodeLst;
        }
        #endregion


        #region XPath   --- Not Implement
        #endregion

        #region XPathExpress   --- Not Implement

        #endregion

        #region Common
        public static bool IsValidateXmlFile(string xmlSchemaFileFullName)
        {
            bool result = false;
            using (Stream stream = new FileStream(xmlSchemaFileFullName, FileMode.Open, FileAccess.Read))
            using (XmlTextReader xmlTxtReader = new XmlTextReader(stream))
            {
                XmlSchema schema = XmlSchema.Read(stream, new ValidationEventHandler(dealSchemaValidation));
                XmlReaderSettings settings = new XmlReaderSettings();
                settings.Schemas.Add(schema);
                settings.ValidationType = ValidationType.Schema;
                settings.ValidationEventHandler += new ValidationEventHandler(dealSchemaValidation);        
                
                //Execute Validate
                try
                {
                    while (xmlTxtReader.Read())
                    { }
                    result = true;
                }
                catch (XmlSchemaValidationException xsve)
                {
                    result = false;
                    throw xsve;
                }
            }
            return result;
        }

        private static void dealSchemaValidation(object sender, System.Xml.Schema.ValidationEventArgs e)
        {
            throw new XmlSchemaValidationException(string.Format("Validation Error, Error Level:{0}\r\n. Error Details:\r\n{1}", e.Severity, e.Message));
        }
        #endregion

        static void TestMethod()
        {
            #region 使用XML流处理, 每次只处理一个节点, 速度快, 但缺点是: 不支持结构化查询, 适合从头到尾一次性处理                     
            //使用xml流输出字符            
            using (System.Xml.XmlWriter xmlwriter = System.Xml.XmlWriter.Create("Output.xml"))            
            {               
                xmlwriter.WriteStartDocument();                
                xmlwriter.WriteStartElement("human");   //</humen>                
                xmlwriter.WriteStartElement("man"); //子元素                
                //写元素属性                
                xmlwriter.WriteAttributeString("name", "father");    //属性               
                xmlwriter.WriteString("Mike");  //文本区               
                xmlwriter.WriteEndElement();                
                xmlwriter.WriteElementString("women", "jean");   //<women>jean</women>                
                xmlwriter.WriteStartElement("children");               
                xmlwriter.WriteAttributeString("name", "kiddy");                
                xmlwriter.WriteString("nickey kiddy");    //文本区               
                xmlwriter.WriteEndElement();                
                xmlwriter.WriteEndElement();            
            }
            #endregion
            #region 使用优化的XPath--XPathDocument类, 速度快, 也支持结构化的查询方式. 缺点: 只能读不能写            
            //1.创建XPathDocument对象            
            System.Xml.XPath.XPathDocument xpdoc = new System.Xml.XPath.XPathDocument("XMLOperation.xml");            
            //2.通过导航器进行查找            
            System.Xml.XPath.XPathNavigator xpnav = xpdoc.CreateNavigator();                      
            //3.经过编译的XPath            
            string xpath = "/configuration/system.web/httpHandlers/cellphone";            
            System.Xml.XPath.XPathExpression xpe = System.Xml.XPath.XPathExpression.Compile(xpath);            
            //4.使用导航器的Select迭代器进行查找, 查找的结果还是导航器            
            System.Xml.XPath.XPathNavigator resultNav = xpnav.SelectSingleNode(xpe);            
            Console.WriteLine("----------XPathDocument的查询单个结果----------");            
            Console.WriteLine(resultNav.Value);            
            //查找多个结果            
            Console.WriteLine("----------XPathDocument的查询多个结果----------");            
            xpath = "/configuration/system.web/httpHandlers/add/@type"; //查找add元素的type属性内容            
            xpe = System.Xml.XPath.XPathExpression.Compile(xpath);            
            System.Xml.XPath.XPathNodeIterator xpniter = xpnav.Select(xpe);            
            foreach (System.Xml.XPath.XPathNavigator xpn in xpniter)            
            {                
                Console.WriteLine(xpn.Value);            
            }            
            #endregion
        }
    }

    /// <summary>
    /// 用List模拟堆栈操作, 用于读取XML中的结点
    /// </summary>
    public class ETLXmlNodeStack
    {
        private List<ETLXmlNode> _xmlStack;

        private ETLXmlNodeStack()
        {
            this._xmlStack = new List<ETLXmlNode>(100);
        }

        private static readonly ETLXmlNodeStack inner;
        static ETLXmlNodeStack()
        {
            inner = new ETLXmlNodeStack();
        }
        
        public static ETLXmlNodeStack ETLXmlNodeStackInfo
        {
            get
            {
                return inner;
            }
        }

        public static int Count
        {
            get
            {
                return inner._xmlStack.Count;
            }
        }

        public static ETLXmlNodeStack CreateETLXmlNodeStack()
        {
            return inner;
        }

        public static void Push(ETLXmlNode etlXmlNode)
        {
            inner._xmlStack.Add(etlXmlNode);
        }

        public static ETLXmlNode Pop()
        {
            ETLXmlNode result = null;
            if (inner._xmlStack != null && inner._xmlStack.Count > 0)
            {
                result = inner._xmlStack[inner._xmlStack.Count - 1];
                inner._xmlStack.RemoveAt(inner._xmlStack.Count - 1);
            }
            return result;
        }
    }
}

//接口

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

using System.Data;
using ETLUtilityDAL.Enums;

namespace ETLUtilityDAL.Interfaces
{
    /// <summary>
    /// Utility Of Data Access Layer
    /// </summary>
    public interface IDBHelper
    {
        #region BaseExecute
        int ExecNonQuery(string sqlText, CommandType cmdType, string[] paramNames, object[] paramValues);
        T ExecScalar<T>(string sqlText, CommandType cmdType, string[] paramNames, object[] paramValues);
        T ExecReader<T>(string sqlText, CommandType cmdType, string[] paramNames, object[] paramValues);
        DataTable ExecDataTable(string sqlText, CommandType cmdType, string[] paramNames, object[] paramValues);
        DataSet ExecDataSet(string sqlText, CommandType cmdType, string[] paramNames, object[] paramValues);
        #endregion

        #region TxtExecute
        int TxtExecuteNonQuery(string sqlText);
        int TxtExecuteNonQuery(string sqlText, string[] paramNames, object[] paramValues);
        T TxtExecuteScalar<T>(string sqlText, string[] paramNames, object[] paramValues);
        T TxtExecuteReader<T>(string sqlText, string[] paramNames, object[] paramValues);
        DataTable TxtExecuteDataTable(string sqlText, string[] paramNames, object[] paramValues);
        DataSet TxtExecuteDataSet(string sqlText, string[] paramNames, object[] paramValues);
        #endregion

        #region SpExecute
        int SpExecuteNonQuery(string sqlText);
        int SpExecuteNonQuery(string spName, string[] paramNames, object[] paramValues);
        T SpExecuteScalar<T>(string spName, string[] paramNames, object[] paramValues);
        T SpExecuteReader<T>(string spName, string[] paramNames, object[] paramValues);
        DataTable SpExecuteDataTable(string spName, string[] paramNames, object[] paramValues);
        DataSet SpExecuteDataSet(string spName, string[] paramNames, object[] paramValues);
        #endregion

        #region Common
        bool BulkInsert<T, T1>(T sqlBulkCopy, DataTable dataTable, T1 sqlTrasaction);
        bool BulkInsert(DataTable dataTable);
        bool BulkInsert(DataSet dataSet);

        string DBName { get; }
        T GetConnection<T>();
        #endregion
    }
}

//MsAccess实现

//MsSqlServer实现

//MySql实现