CREATE TABLE `demo_blog_4` ( `jobid` BIGINT, `product` STRING, `email` STRING, `jobname` STRING, `jobtype` TINYINT, `stype` TINYINT, `dtype` TINYINT, `dir` STRING, `dburl` STRING, `tablename` STRING, `username` STRING, `password` STRING, `hdbname` STRING, `htable` STRING, `params` STRING, `cluster` STRING, `begin_time` BIGINT, `end_time` BIGINT, `duration` BIGINT, `status` TINYINT, `created_at` STRING, `updated_at` STRING) COMMENT 'Imported by sqoop on 2015/04/13 10:16:31' ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; LOAD DATA INPATH 'hdfs://hp1/user/intern/pf_sqoop' OVERWRITE INTO TABLE `demo_blog_4`;<span style="font-family: Tahoma; background-color: rgb(255, 255, 255);"> </span>
Beginning code generation Executing SQL statement: SELECT t.* FROM test AS t WHERE 1=0 Error executing statement: java.sql.SQLException: ORA-00933: SQL command not properly ended java.sql.SQLException: ORA-00933: SQL command not properly ended
jarFile = codeGenerator.generateORM(options, tableName);
1831 protected Map<String, Integer> getColumnTypes() throws IOException {
1832 if (options.getCall() == null) {
1833 return connManager.getColumnTypes(tableName, options.getSqlQuery());
1834 } else {
1835 return connManager.getColumnTypesForProcedure(options.getCall());
1836 }
1837 }
121 public ConnManager getManager(JobData data) throws IOException {
122 com.cloudera.sqoop.SqoopOptions options = data.getSqoopOptions();
123 String manualDriver = options.getDriverClassName();
124 String managerClassName = options.getConnManagerClassName();
125 <span style="color:#ff6666;"> //如果在命令中指定了driver但是没有指定connManager那么就使用通用的JDBC的connManager，也会输出warn日志</span>
131 if (manualDriver != null && managerClassName == null) {
132 LOG.warn("Parameter --driver is set to an explicit driver however"
133 + " appropriate connection manager is not being set (via"
134 + " --connection-manager). Sqoop is going to fall back to "
135 + GenericJdbcManager.class.getCanonicalName() + ". Please specify"
136 + " explicitly which connection manager should be used next time."
137 );
138 return new GenericJdbcManager(manualDriver, options);
139 }
140 <span style="color:#ff6666;"> //如果制定了connManager，直接创建该类对象，不存在则报错。</span>
142 if (managerClassName != null){
143 ConnManager connManager = null;
144
145 try {
146 Class<ConnManager> cls = (Class<ConnManager>)
147 Class.forName(managerClassName);
148
152 if (manualDriver == null) {
153 Constructor<ConnManager> constructor =
154 cls.getDeclaredConstructor(com.cloudera.sqoop.SqoopOptions.class);
155 connManager = constructor.newInstance(options);
156 } else {
157 Constructor<ConnManager> constructor =
158 cls.getDeclaredConstructor(String.class,
159 com.cloudera.sqoop.SqoopOptions.class);
160 connManager = constructor.newInstance(manualDriver, options);
161 }
162 } catch (ClassNotFoundException e) {
163 LOG.error("Sqoop could not found specified connection manager class "
164 + managerClassName + ". Please check that you've specified the "
165 + "class correctly.");
166 throw new IOException(e);
167 } catch (NoSuchMethodException e) {
168 LOG.error("Sqoop wasn't able to create connnection manager properly. "
169 + "Some of the connectors supports explicit --driver and some "
170 + "do not. Please try to either specify --driver or leave it out.");
171 throw new IOException(e);
172 } catch (Exception e) {
173 LOG.error("Problem with bootstrapping connector manager:"
174 + managerClassName);
175 LOG.error(e);
176 throw new IOException(e);
177 }
178 return connManager;
179 }
180<span style="white-space:pre"> </span><span style="color:#ff0000;">//最后，也就是说既没有指定driver也没有指定connManager，就使用系统预定的以connManager的factory创建一个connManager</span>
181 // Try all the available manager factories.
182 for (ManagerFactory factory : factories) {
183 LOG.debug("Trying ManagerFactory: " + factory.getClass().getName());
184 ConnManager mgr = factory.accept(data);
185 if (null != mgr) {
186 LOG.debug("Instantiated ConnManager " + mgr.toString());
187 return mgr;
188 }
189 }
190
191 throw new IOException("No manager for connect string: "
192 + data.getSqoopOptions().getConnectString());
193 }
74 public static final String[] DEFAULT_FACTORY_CLASS_NAMES_ARR =
75 {OraOopManagerFactory.class.getName(),
76 DefaultManagerFactory.class.getName(), };
38 public ConnManager More ...accept(JobData data) {
39 SqoopOptions options = data.getSqoopOptions();
40
41 String scheme = extractScheme(options);
42 if (null == scheme) {
43 // We don't know if this is a mysql://, hsql://, etc.
44 // Can't do anything with this.
45 LOG.warn("Null scheme associated with connect string.");
46 return null;
47 }
48
49 LOG.debug("Trying with scheme: " + scheme);
50
51 if (scheme.equals("jdbc:mysql:")) {
52 if (options.isDirect()) {
53 return new DirectMySQLManager(options);
54 } else {
55 return new MySQLManager(options);
56 }
57 } else if (scheme.equals("jdbc:postgresql:")) {
58 if (options.isDirect()) {
59 return new DirectPostgresqlManager(options);
60 } else {
61 return new PostgresqlManager(options);
62 }
63 } else if (scheme.startsWith("jdbc:hsqldb:")) {
64 return new HsqldbManager(options);
65 } else if (scheme.startsWith("jdbc:oracle:")) {
66 return new OracleManager(options);
67 } else if (scheme.startsWith("jdbc:sqlserver:")) {
68 return new SQLServerManager(options);
69 } else if (scheme.startsWith("jdbc:db2:")) {
70 return new Db2Manager(options);
71 } else if (scheme.startsWith("jdbc:netezza:")) {
72 if (options.isDirect()) {
73 return new DirectNetezzaManager(options);
74 } else {
75 return new NetezzaManager(options);
76 }
77 } else if (scheme.startsWith("jdbc:cubrid:")) {
78 return new CubridManager(options);
79 } else {
80 return null;
81 }
82 }
107 protected String getColNamesQuery(String tableName) {
108 // adding where clause to prevent loading a big table
109 return "SELECT t.* FROM " + escapeTableName(tableName) + " AS t WHERE 1=0";
110 }
261 protected String More ...getColNamesQuery(String tableName) {
262 // SqlManager uses "tableName AS t" which doesn't work in Oracle.
263 String query = "SELECT t.* FROM " + escapeTableName(tableName)
264 + " t WHERE 1=0";
265
266 LOG.debug("Using column names query: " + query);
267 return query;
268 }
Beginning code generation Time zone has been set to GMT Executing SQL statement: SELECT t.* FROM TEST t WHERE 1=0
MServer[new_lighttest_ddb457]监控线程启动完成
Error executing statement: java.sql.SQLException: Not supported function!
java.sql.SQLException: Not supported function!
at com.netease.backend.db.DBConnection.setTransactionIsolation(DBConnection.java:709)
at org.apache.sqoop.manager.SqlManager.makeConnection(SqlManager.java:883)
at org.apache.sqoop.manager.GenericJdbcManager.getConnection(GenericJdbcManager.java:52)
at org.apache.sqoop.manager.SqlManager.execute(SqlManager.java:736)
at org.apache.sqoop.manager.SqlManager.execute(SqlManager.java:759)
Job job_1428578888362_0207 running in uber mode : false map 0% reduce 0% Task Id : attempt_1428578888362_0207_m_000000_0, Status : FAILED Error: java.lang.ClassNotFoundException: com.netease.backend.db.common.exceptions.SQLExceptionWithCause at java.net.URLClassLoader$1.run(URLClassLoader.java:366) at java.net.URLClassLoader$1.run(URLClassLoader.java:355) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:354) at java.lang.ClassLoader.loadClass(ClassLoader.java:425) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308) at java.lang.ClassLoader.loadClass(ClassLoader.java:358) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:191) at org.apache.sqoop.mapreduce.db.DBConfiguration.getConnection(DBConfiguration.java:277) at org.apache.sqoop.mapreduce.db.DBInputFormat.getConnection(DBInputFormat.java:218) at org.apache.sqoop.mapreduce.db.DBInputFormat.setConf(DBInputFormat.java:165) at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:726) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:340) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:168) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1614) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:163)
251 Job job = createJob(conf);
252 try {
253 // Set the external jar to use for the job.
254 job.getConfiguration().set("mapred.jar", ormJarFile);
255 if (options.getMapreduceJobName() != null) {
256 job.setJobName(options.getMapreduceJobName());
257 }
258 <span style="color:#ff6666;">//配置job的一些参数</span>
259 propagateOptionsToJob(job);
260 configureInputFormat(job, tableName, tableClassName, splitByCol);
261 configureOutputFormat(job, tableName, tableClassName);
262 configureMapper(job, tableName, tableClassName);
263 configureNumTasks(job);//将一些使用的jar文件加入到配置中
264 cacheJars(job, getContext().getConnManager());
265
266 jobSetup(job);
267 setJob(job);
268 boolean success = runJob(job);
269 if (!success) {
270 throw new ImportException("Import job failed!");
271 }
272
273 completeImport(job);
145 Configuration conf = job.getConfiguration();
146 FileSystem fs = FileSystem.getLocal(conf);
147 Set<String> localUrls = new HashSet<String>();
148
149 addToCache(Jars.getSqoopJarPath(), fs, localUrls);
150 if (null != mgr) {
151 addToCache(Jars.getDriverClassJar(mgr), fs, localUrls);
152 addToCache(Jars.getJarPathForClass(mgr.getClass()), fs, localUrls);
153 }
199 if (localUrls.isEmpty()) {
200 return;
201 }
202
203 // Add these to the 'tmpjars' array, which the MR JobSubmitter
204 // will upload to HDFS and put in the DistributedCache libjars.
205 String tmpjars = conf.get("tmpjars");
206 StringBuilder sb = new StringBuilder();
207 if (null != tmpjars) {
208 sb.append(tmpjars);
209 sb.append(",");
210 }
211 sb.append(StringUtils.arrayToString(localUrls.toArray(new String[0])));
212 conf.set("tmpjars", sb.toString());
原文地址：http://blog.csdn.net/yu616568/article/details/45030739
