一个控制台程序，主要业务就是读取某目录的zip压缩包文件，解压后遍历每个xml文件，解析文件内容然后存储到oracle11g(互联网主机)上。
程序第一版：
1)实现方式：使用动软的代码生成器里常用的DbHelperOra类改成了使用ODP.NET的DbHelper。
DbHelper类是个静态类，每个方法都单独new一个OracleConnection，每次Open了再马上Close并释放掉OracleConnection。
ps:使用这种类ado.net的手工写法是为了使用OracleBulkCopy实现批量插入数据库。解析一个xml后需要批量插入数据库的记录多则上万，少则数千，所以使用ORM估计效率低下，故选用此方案。
2)效果：
解析一个文件需要2-3分钟。插入数据库时经常出错： ORA-00054、ORA-00604等。多次调优，发现没有在使用OracleBulkCopy时使用事务。
/// <summary> /// 执行OracleBulkCopy(批量数据插入) /// </summary> /// <param name="tableName">Oracle里的表名</param> /// <param name="dt">临时表（包含数据)表结构要和数据库表一致</param> public void ExecuteOracleBulkCopy(string tableName, DataTable dt) { if (dbConn.State != ConnectionState.Open) dbConn.Open(); using (OracleBulkCopy obc = new OracleBulkCopy(dbConn)) { try { obc.BatchSize = dt.Rows.Count; obc.BulkCopyTimeout = 1800; obc.DestinationTableName = tableName; obc.WriteToServer(dt); } catch (OracleException ex) { throw new Exception(ex.Message); } finally { obc.Close(); dbConn.Close(); } } }
程序第二版：
1)调优过程：
a、DbHelper类调优，一个应用程序只建立一个OracleConnection，实现IDisposable。
程序在使用多线程时出现较多的异常，所以考虑先写单线程业务实现了再调优。这里没有使用单例模式。
using System; using System.Collections; using System.Collections.Specialized; using System.Data; using System.Configuration; using Oracle.DataAccess; using Oracle.DataAccess.Client; namespace ConsoleFacDataTrans { /// <summary> /// Copyright (C) DataCool /// 数据访问基础类(基于Oracle ODP.NET) /// </summary> public class DbHelperOra : IDisposable { private OracleConnection dbConn = null; public DbHelperOra(string connStr) { dbConn = new OracleConnection(connStr); } #region 公用方法 public int GetMaxID(string FieldName, string TableName) { string strsql = "select max(" + FieldName + ")+1 from " + TableName; object obj = GetSingle(strsql); if (obj == null) { return 1; } else { return int.Parse(obj.ToString()); } } public bool Exists(string strSql) { object obj = GetSingle(strSql); int cmdresult; if ((Object.Equals(obj, null)) || (Object.Equals(obj, System.DBNull.Value))) { cmdresult = 0; } else { cmdresult = int.Parse(obj.ToString()); } if (cmdresult == 0) { return false; } else { return true; } } public bool Exists(string strSql, params OracleParameter[] cmdParms) { object obj = GetSingle(strSql, cmdParms); int cmdresult; if ((Object.Equals(obj, null)) || (Object.Equals(obj, System.DBNull.Value))) { cmdresult = 0; } else { cmdresult = int.Parse(obj.ToString()); } if (cmdresult == 0) { return false; } else { return true; } } #endregion #region 执行简单SQL语句 /// <summary> /// 执行SQL语句，返回影响的记录数 /// </summary> /// <param name="SQLString">SQL语句</param> /// <returns>影响的记录数</returns> public int ExecuteSql(string SQLString) { using (OracleCommand cmd = new OracleCommand(SQLString, dbConn)) { try { if (dbConn.State != ConnectionState.Open) dbConn.Open(); int rows = cmd.ExecuteNonQuery(); return rows; } catch (OracleException ex) { dbConn.Close(); throw new Exception(ex.Message); } } } /// <summary> /// 执行多条SQL语句，实现数据库事务。 /// </summary> /// <param name="SQLStringList">多条SQL语句</param> public void ExecuteSqlTran(ArrayList SQLStringList) { if (dbConn.State != ConnectionState.Open) dbConn.Open(); OracleCommand cmd = dbConn.CreateCommand(); OracleTransaction tx = dbConn.BeginTransaction(); cmd.Transaction = tx; try { for (int n = 0; n < SQLStringList.Count; n++) { string strsql = SQLStringList[n].ToString(); if (strsql.Trim().Length > 1) { cmd.CommandText = strsql; cmd.ExecuteNonQuery(); } } tx.Commit(); } catch (OracleException E) { tx.Rollback(); throw new Exception(E.Message); } } /// <summary> /// 执行一条计算查询结果语句，返回查询结果（object）。 /// </summary> /// <param name="SQLString">计算查询结果语句</param> /// <returns>查询结果（object）</returns> public object GetSingle(string SQLString) { using (OracleCommand cmd = new OracleCommand(SQLString, dbConn)) { try { if (dbConn.State != ConnectionState.Open) dbConn.Open(); object obj = cmd.ExecuteScalar(); if ((Object.Equals(obj, null)) || (Object.Equals(obj, System.DBNull.Value))) { return null; } else { return obj; } } catch (OracleException e) { dbConn.Close(); throw new Exception(e.Message); } } } /// <summary> /// 执行查询语句，返回OracleDataReader ( 注意：调用该方法后，一定要对SqlDataReader进行Close ) /// </summary> /// <param name="strSQL">查询语句</param> /// <returns>OracleDataReader</returns> public OracleDataReader ExecuteReader(string strSQL) { OracleCommand cmd = new OracleCommand(strSQL, dbConn); try { if (dbConn.State != ConnectionState.Open) dbConn.Open(); OracleDataReader myReader = cmd.ExecuteReader(CommandBehavior.CloseConnection); return myReader; } catch (OracleException e) { throw new Exception(e.Message); } } /// <summary> /// 执行查询语句，返回DataSet /// </summary> /// <param name="SQLString">查询语句</param> /// <returns>DataSet</returns> public DataSet Query(string SQLString) { DataSet ds = new DataSet(); try { if (dbConn.State != ConnectionState.Open) dbConn.Open(); OracleDataAdapter command = new OracleDataAdapter(SQLString, dbConn); command.Fill(ds, "ds"); } catch (OracleException ex) { throw new Exception(ex.Message); } return ds; } #endregion #region 执行带参数的SQL语句 /// <summary> /// 执行SQL语句，返回影响的记录数 /// </summary> /// <param name="SQLString">SQL语句</param> /// <returns>影响的记录数</returns> public int ExecuteSql(string SQLString, params OracleParameter[] cmdParms) { using (OracleCommand cmd = new OracleCommand()) { try { PrepareCommand(cmd, dbConn, null, SQLString, cmdParms); int rows = cmd.ExecuteNonQuery(); cmd.Parameters.Clear(); return rows; } catch (OracleException E) { throw new Exception(E.Message); } } } /// <summary> /// 执行多条SQL语句，实现数据库事务。 /// </summary> /// <param name="SQLStringList">SQL语句的哈希表（key为sql语句，value是该语句的OracleParameter[]）</param> public void ExecuteSqlTran(Hashtable SQLStringList) { if (dbConn.State != ConnectionState.Open) dbConn.Open(); using (OracleTransaction trans = dbConn.BeginTransaction()) { OracleCommand cmd = new OracleCommand(); try { //循环 foreach (DictionaryEntry myDE in SQLStringList) { string cmdText = myDE.Key.ToString(); OracleParameter[] cmdParms = (OracleParameter[])myDE.Value; PrepareCommand(cmd, dbConn, trans, cmdText, cmdParms); int val = cmd.ExecuteNonQuery(); cmd.Parameters.Clear(); trans.Commit(); } } catch { trans.Rollback(); throw; } } } /// <summary> /// 执行一条计算查询结果语句，返回查询结果（object）。 /// </summary> /// <param name="SQLString">计算查询结果语句</param> /// <returns>查询结果（object）</returns> public object GetSingle(string SQLString, params OracleParameter[] cmdParms) { using (OracleCommand cmd = new OracleCommand()) { try { PrepareCommand(cmd, dbConn, null, SQLString, cmdParms); object obj = cmd.ExecuteScalar(); cmd.Parameters.Clear(); if ((Object.Equals(obj, null)) || (Object.Equals(obj, System.DBNull.Value))) { return null; } else { return obj; } } catch (OracleException e) { throw new Exception(e.Message); } } } /// <summary> /// 执行查询语句，返回OracleDataReader ( 注意：调用该方法后，一定要对SqlDataReader进行Close ) /// </summary> /// <param name="strSQL">查询语句</param> /// <returns>OracleDataReader</returns> public OracleDataReader ExecuteReader(string SQLString, params OracleParameter[] cmdParms) { OracleCommand cmd = new OracleCommand(); try { PrepareCommand(cmd, dbConn, null, SQLString, cmdParms); OracleDataReader myReader = cmd.ExecuteReader(CommandBehavior.CloseConnection); cmd.Parameters.Clear(); return myReader; } catch (OracleException e) { throw new Exception(e.Message); } } /// <summary> /// 执行查询语句，返回DataSet /// </summary> /// <param name="SQLString">查询语句</param> /// <returns>DataSet</returns> public DataSet Query(string SQLString, params OracleParameter[] cmdParms) { OracleCommand cmd = new OracleCommand(); PrepareCommand(cmd, dbConn, null, SQLString, cmdParms); using (OracleDataAdapter da = new OracleDataAdapter(cmd)) { DataSet ds = new DataSet(); try { da.Fill(ds, "ds"); cmd.Parameters.Clear(); } catch (OracleException ex) { throw new Exception(ex.Message); } return ds; } } private void PrepareCommand(OracleCommand cmd, OracleConnection conn, OracleTransaction trans, string cmdText, OracleParameter[] cmdParms) { if (conn.State != ConnectionState.Open) conn.Open(); cmd.Connection = conn; cmd.CommandText = cmdText; if (trans != null) cmd.Transaction = trans; cmd.CommandType = CommandType.Text;//cmdType; if (cmdParms != null) { foreach (OracleParameter parm in cmdParms) cmd.Parameters.Add(parm); } } #endregion #region 存储过程操作 /// <summary> /// 执行存储过程 返回SqlDataReader ( 注意：调用该方法后，一定要对SqlDataReader进行Close ) /// </summary> /// <param name="storedProcName">存储过程名</param> /// <param name="parameters">存储过程参数</param> /// <returns>OracleDataReader</returns> public OracleDataReader RunProcedure(string storedProcName, IDataParameter[] parameters) { OracleDataReader returnReader; if (dbConn.State != ConnectionState.Open) dbConn.Open(); OracleCommand command = BuildQueryCommand(dbConn, storedProcName, parameters); command.CommandType = CommandType.StoredProcedure; returnReader = command.ExecuteReader(CommandBehavior.CloseConnection); return returnReader; } /// <summary> /// 执行存储过程 /// </summary> /// <param name="storedProcName">存储过程名</param> /// <param name="parameters">存储过程参数</param> /// <param name="tableName">DataSet结果中的表名</param> /// <returns>DataSet</returns> public DataSet RunProcedure(string storedProcName, IDataParameter[] parameters, string tableName) { DataSet dataSet = new DataSet(); if (dbConn.State != ConnectionState.Open) dbConn.Open(); OracleDataAdapter sqlDA = new OracleDataAdapter(); sqlDA.SelectCommand = BuildQueryCommand(dbConn, storedProcName, parameters); sqlDA.Fill(dataSet, tableName); dbConn.Close(); return dataSet; } /// <summary> /// 构建 OracleCommand 对象(用来返回一个结果集，而不是一个整数值) /// </summary> /// <param name="connection">数据库连接</param> /// <param name="storedProcName">存储过程名</param> /// <param name="parameters">存储过程参数</param> /// <returns>OracleCommand</returns> private OracleCommand BuildQueryCommand(OracleConnection connection, string storedProcName, IDataParameter[] parameters) { OracleCommand command = new OracleCommand(storedProcName, connection); command.CommandType = CommandType.StoredProcedure; foreach (OracleParameter parameter in parameters) { command.Parameters.Add(parameter); } return command; } /// <summary> /// 执行存储过程，返回影响的行数 /// </summary> /// <param name="storedProcName">存储过程名</param> /// <param name="parameters">存储过程参数</param> /// <param name="rowsAffected">影响的行数</param> /// <returns></returns> public int RunProcedure(string storedProcName, IDataParameter[] parameters, out int rowsAffected) { int result; if (dbConn.State != ConnectionState.Open) dbConn.Open(); OracleCommand command = BuildIntCommand(dbConn, storedProcName, parameters); rowsAffected = command.ExecuteNonQuery(); result = (int)command.Parameters["ReturnValue"].Value; return result; } /// <summary> /// 创建 OracleCommand 对象实例(用来返回一个整数值) /// </summary> /// <param name="storedProcName">存储过程名</param> /// <param name="parameters">存储过程参数</param> /// <returns>OracleCommand 对象实例</returns> private OracleCommand BuildIntCommand(OracleConnection connection, string storedProcName, IDataParameter[] parameters) { OracleCommand command = BuildQueryCommand(connection, storedProcName, parameters); command.Parameters.Add(new OracleParameter("ReturnValue", OracleDbType.Int32, 4, ParameterDirection.ReturnValue, false, 0, 0, string.Empty, DataRowVersion.Default, null)); return command; } #endregion /// <summary> /// 执行OracleBulkCopy(批量数据插入) /// </summary> /// <param name="tableName">Oracle里的表名</param> /// <param name="dt">临时表（包含数据)表结构要和数据库表一致</param> public void ExecuteOracleBulkCopy(string tableName, DataTable dt) { if (dbConn.State != ConnectionState.Open) dbConn.Open(); using (OracleBulkCopy obc = new OracleBulkCopy(dbConn)) { OracleTransaction transaction = dbConn.BeginTransaction(); try { obc.BatchSize = dt.Rows.Count; obc.BulkCopyTimeout = 1800; obc.DestinationTableName = tableName; obc.WriteToServer(dt); transaction.Commit(); } catch (OracleException ex) { transaction.Rollback(); throw new Exception(ex.Message); } finally { obc.Close(); transaction.Dispose(); dbConn.Close(); } } } /// <summary> /// 执行无参数的存储过程 /// </summary> /// <param name="spName"></param> public void ExecuteStoreProcedure(string spName) { using (OracleCommand cmd = new OracleCommand(spName, dbConn)) { cmd.CommandType = CommandType.StoredProcedure; try { if (dbConn.State != ConnectionState.Open) dbConn.Open(); cmd.ExecuteNonQuery(); } catch (OracleException ex) { dbConn.Close(); throw new Exception(ex.Message); } } } public void Dispose() { if (dbConn != null) { dbConn.Close(); dbConn.Dispose(); } } } }
主要原因是每次调用一个方法都重新创建一个OracleConnection对象，每次都要Open、Close等效果当然不好了。这不是B/S程序，没必要。
b)在使用OracleBulkCopy时，使用事务！这是最关键的优化！
/// <summary> /// 执行OracleBulkCopy(批量数据插入) /// </summary> /// <param name="tableName">Oracle里的表名</param> /// <param name="dt">临时表（包含数据)表结构要和数据库表一致</param> public void ExecuteOracleBulkCopy(string tableName, DataTable dt) { if (dbConn.State != ConnectionState.Open) dbConn.Open(); using (OracleBulkCopy obc = new OracleBulkCopy(dbConn)) { OracleTransaction transaction = dbConn.BeginTransaction(); try { obc.BatchSize = dt.Rows.Count; obc.BulkCopyTimeout = 1800; obc.DestinationTableName = tableName; obc.WriteToServer(dt); transaction.Commit(); } catch (OracleException ex) { transaction.Rollback(); throw new Exception(ex.Message); } finally { obc.Close(); transaction.Dispose(); dbConn.Close(); } } }
2)程序效果：
解析一个xml文件并成功导入到数据库耗时仅需要1-10秒！秒哦，100倍的效率提升！
搞定收工，记录下来。以后备忘！