代码提交

src/main/java/com/jz/dmp/cmdexectool/common/constant/CommConstant.java

@@ -53,6 +53,7 @@ public class CommConstant {
 	public static final String WATERDROP_FTL_SOURCE_JDBC = "source_jdbc.ftl";
 	public static final String WATERDROP_FTL_SOURCE_ELASTICSEARCH = "source_elasticsearch.ftl";
 	public static final String WATERDROP_FTL_SOURCE_SFTP = "source_sftp.ftl";
+	public static final String WATERDROP_FTL_SOURCE_HIVE = "source_hive.ftl";
 	public static final String WATERDROP_FTL_TRANSFORM_SQL = "transform_sql.ftl";
 	public static final String WATERDROP_FTL_TRANSFORM_JSON2 = "transform_json2.ftl";
 	public static final String WATERDROP_FTL_SINK_CONSOLE = "sink_console.ftl";
@@ -67,4 +68,9 @@ public class CommConstant {
 	public static final String FTL_DOCTRANS = "doctrans.ftl";//文件转码
 	public static final String FTL_HDFS_UPLOAD = "hdfs_upload.ftl";//HDFS上传
 	
+	/***************************************************/
+	//执行引擎
+	public static final String EXECUTION_ENGINE_JDBC = "jdbc";//jdbc
+	public static final String EXECUTION_ENGINE_SPARK = "spark";//spark
+	
 }

src/main/java/com/jz/dmp/cmdexectool/scheduler/dao/datasource/BaseDataSource.java

@@ -67,8 +67,6 @@ public abstract class BaseDataSource {
    */
   private String principal;
   
-  private String dbType;
-
   public String getPrincipal() {
     return principal;
   }
@@ -227,14 +225,6 @@ public abstract class BaseDataSource {
     this.other = other;
   }
 
-	public String getDbType() {
-		return dbType;
-	}
-
-	public void setDbType(String dbType) {
-		this.dbType = dbType;
-	}
-
 	public String getJdbcUrlDirect() {
 		return jdbcUrlDirect;
 	}

src/main/java/com/jz/dmp/cmdexectool/scheduler/dao/utils/DatabaseUtils.java

@@ -36,6 +36,7 @@ import com.jz.dmp.cmdexectool.scheduler.common.utils.CommonUtils;
 import com.jz.dmp.cmdexectool.scheduler.common.utils.JSONUtils;
 import com.jz.dmp.cmdexectool.scheduler.common.utils.ParameterUtils;
 import com.jz.dmp.cmdexectool.scheduler.dao.datasource.BaseDataSource;
+import com.jz.dmp.cmdexectool.scheduler.dao.datasource.MyBaseDataSource;
 import com.jz.dmp.cmdexectool.scheduler.server.utils.ParamUtils;
 
 public class DatabaseUtils {
@@ -51,7 +52,7 @@ public class DatabaseUtils {
      * @param postStatementsBinds   post statements binds
      * @param createFuncs           create functions
      */
-    public static void executeUpdateSql(List<SqlBinds> statementsBinds, BaseDataSource baseDataSource){
+    public static void executeUpdateSql(List<SqlBinds> statementsBinds, MyBaseDataSource myBaseDataSource){
         Connection connection = null;
         PreparedStatement stmt = null;
         ResultSet resultSet = null;
@@ -59,7 +60,7 @@ public class DatabaseUtils {
             // if upload resource is HDFS and kerberos startup
             CommonUtils.loadKerberosConf();
             // create connection
-            connection = createConnection(baseDataSource);
+            connection = createConnection(myBaseDataSource);
             // create temp function
             /*
             if (CollectionUtils.isNotEmpty(createFuncs)) {
@@ -89,7 +90,7 @@ public class DatabaseUtils {
                                         List<SqlBinds> preStatementsBinds,
                                         List<SqlBinds> postStatementsBinds,
                                         List<String> createFuncs,
-                                        BaseDataSource baseDataSource){
+                                        MyBaseDataSource myBaseDataSource){
         Connection connection = null;
         PreparedStatement stmt = null;
         ResultSet resultSet = null;
@@ -97,7 +98,7 @@ public class DatabaseUtils {
             // if upload resource is HDFS and kerberos startup
             CommonUtils.loadKerberosConf();
             // create connection
-            connection = createConnection(baseDataSource);
+            connection = createConnection(myBaseDataSource);
             // create temp function
             /*
             if (CollectionUtils.isNotEmpty(createFuncs)) {
@@ -171,24 +172,24 @@ public class DatabaseUtils {
      * @return connection
      * @throws Exception Exception
      */
-    private static Connection createConnection(BaseDataSource baseDataSource) throws Exception{
+    private static Connection createConnection(MyBaseDataSource myBaseDataSource) throws Exception{
         // if hive , load connection params if exists
         Connection connection = null;
-        if (HIVE == DbType.valueOf(baseDataSource.getDbType())) {
+        if (HIVE == DbType.valueOf(myBaseDataSource.getMyDbType().getDbType().name())) {
             Properties paramProp = new Properties();
-            paramProp.setProperty(USER, baseDataSource.getUser());
-            paramProp.setProperty(PASSWORD, baseDataSource.getPassword());
+            paramProp.setProperty(USER, myBaseDataSource.getUser());
+            paramProp.setProperty(PASSWORD, myBaseDataSource.getPassword());
             Map<String, String> connParamMap = CollectionUtils.stringToMap("",
                     SEMICOLON,
                     HIVE_CONF);
             paramProp.putAll(connParamMap);
 
-            connection = DriverManager.getConnection(baseDataSource.getJdbcUrlDirect(),
+            connection = DriverManager.getConnection(myBaseDataSource.getJdbcUrlDirect(),
                     paramProp);
         }else{
-        	connection = DriverManager.getConnection(baseDataSource.getJdbcUrlDirect(),
-                    baseDataSource.getUser(),
-                    baseDataSource.getPassword());
+        	connection = DriverManager.getConnection(myBaseDataSource.getJdbcUrlDirect(),
+        			myBaseDataSource.getUser(),
+        			myBaseDataSource.getPassword());
         }
         return connection;
     }

src/main/java/com/jz/dmp/cmdexectool/scheduler/server/worker/task/sql/SqlTask.java

@@ -36,6 +36,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.springframework.util.CollectionUtils;
 
+import com.jz.dmp.cmdexectool.common.constant.CommConstant;
 import com.jz.dmp.cmdexectool.scheduler.common.Constants;
 import com.jz.dmp.cmdexectool.scheduler.common.enums.DbType;
 import com.jz.dmp.cmdexectool.scheduler.common.process.Property;
@@ -45,6 +46,7 @@ import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlParameters;
 import com.jz.dmp.cmdexectool.scheduler.common.utils.OSUtils;
 import com.jz.dmp.cmdexectool.scheduler.common.utils.ParameterUtils;
 import com.jz.dmp.cmdexectool.scheduler.dao.datasource.BaseDataSource;
+import com.jz.dmp.cmdexectool.scheduler.dao.datasource.MyBaseDataSource;
 import com.jz.dmp.cmdexectool.scheduler.dao.utils.DatabaseUtils;
 import com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext;
 import com.jz.dmp.cmdexectool.scheduler.server.utils.ParamUtils;
@@ -101,25 +103,7 @@ public class SqlTask extends AbstractTask {
 	public void handle() throws Exception {
 		try {
 			
-			BaseDataSource baseDataSource = new BaseDataSource() {
-				
-				@Override
-				public String driverClassSelector() {
-					// TODO Auto-generated method stub
-					return null;
-				}
-				
-				@Override
-				public DbType dbTypeSelector() {
-					// TODO Auto-generated method stub
-					return null;
-				}
-			};
-			
-			baseDataSource.setDbType(sqlParameters.getMyDbType().getDbType().name());
-			baseDataSource.setUser(sqlParameters.getUser());
-			baseDataSource.setPassword(sqlParameters.getPassword());
-			baseDataSource.setAddress(sqlParameters.getJdbcUrl());
+			MyBaseDataSource targetBaseDataSource = sqlParameters.getTargetBaseDataSource();
 			
 			List<SqlBinds> preStatementSqlBinds = Optional.ofNullable(sqlParameters.getPreStatements())
                     .orElse(new ArrayList<>())
@@ -136,20 +120,34 @@ public class SqlTask extends AbstractTask {
 			
 			//判断是否需要运行前置sql
             if (!CollectionUtils.isEmpty(preStatementSqlBinds)) {
-            	DatabaseUtils.executeUpdateSql(preStatementSqlBinds, baseDataSource);
+            	DatabaseUtils.executeUpdateSql(preStatementSqlBinds, targetBaseDataSource);
 			}
 			
+            if (sqlParameters.getExecutioEngine().equals(CommConstant.EXECUTION_ENGINE_JDBC)) {
+            	
+            	List<String> mainSqlScript = new ArrayList<String>();
+            	mainSqlScript.add(sqlParameters.getSqlScript());
+            	
+            	List<SqlBinds> mainStatementSqlBinds = Optional.ofNullable(mainSqlScript)
+                        .orElse(new ArrayList<>())
+                        .stream()
+                        .map(DatabaseUtils::getSqlAndSqlParamsMap)
+                        .collect(Collectors.toList());
+            	
+            	DatabaseUtils.executeUpdateSql(mainStatementSqlBinds, sqlParameters.getSourceBaseDataSource());
+			}else {
 				// construct process
 				CommandExecuteResult commandExecuteResult = waterdropCommandExecutor.run(buildCommand());
+				setExitStatusCode(commandExecuteResult.getExitStatusCode());
+				setAppIds(commandExecuteResult.getAppIds());
+				setProcessId(commandExecuteResult.getProcessId());
+			}
 			
 			//判断是否运行后置sql
 			if (!CollectionUtils.isEmpty(postStatementSqlBinds)) {
-            	DatabaseUtils.executeUpdateSql(postStatementSqlBinds, baseDataSource);
+            	DatabaseUtils.executeUpdateSql(postStatementSqlBinds, targetBaseDataSource);
 			}
 			
-			setExitStatusCode(commandExecuteResult.getExitStatusCode());
-			setAppIds(commandExecuteResult.getAppIds());
-			setProcessId(commandExecuteResult.getProcessId());
 		} catch (Exception e) {
 			logger.error("sql task error", e);
 			setExitStatusCode(Constants.EXIT_CODE_FAILURE);

src/test/java/com/jz/cmdexectool/test/task/shell/SQLCommandExecutorTest.java

@@ -23,6 +23,7 @@ import com.jz.dmp.cmdexectool.ApiApplication;
 import com.jz.dmp.cmdexectool.scheduler.common.Constants;
 import com.jz.dmp.cmdexectool.scheduler.common.enums.DbType;
 import com.jz.dmp.cmdexectool.scheduler.common.enums.ExecutionStatus;
+import com.jz.dmp.cmdexectool.scheduler.common.enums.MyDbType;
 import com.jz.dmp.cmdexectool.scheduler.common.process.Property;
 import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlBinds;
 import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlType;
@@ -32,6 +33,7 @@ import com.jz.dmp.cmdexectool.scheduler.common.utils.JSONUtils;
 import com.jz.dmp.cmdexectool.scheduler.common.utils.ParameterUtils;
 import com.jz.dmp.cmdexectool.scheduler.dao.datasource.BaseDataSource;
 import com.jz.dmp.cmdexectool.scheduler.dao.datasource.DataSourceFactory;
+import com.jz.dmp.cmdexectool.scheduler.dao.datasource.MyBaseDataSource;
 import com.jz.dmp.cmdexectool.scheduler.dao.datasource.MySQLDataSource;
 import com.jz.dmp.cmdexectool.scheduler.dao.utils.DatabaseUtils;
 import com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext;
@@ -134,21 +136,23 @@ public class SQLCommandExecutorTest {
             // load class
             DataSourceFactory.loadClass(DbType.valueOf("MYSQL"));
             
-            MySQLDataSource mySQLDataSource = new MySQLDataSource();
-            mySQLDataSource.setAddress("192.168.1.140:3307");
-            mySQLDataSource.setUser("dmp");
-            mySQLDataSource.setPassword("Ioubuy123");
+            MyBaseDataSource myBaseDataSource = new MyBaseDataSource();
+            myBaseDataSource.setJdbcUrlDirect("jdbc:mysql://192.168.1.140:3307/dmp_web_new");
+            myBaseDataSource.setUser("dmp");
+            myBaseDataSource.setPassword("Ioubuy123");
+            myBaseDataSource.setMyDbType(MyDbType.MySQL);
             
             //String json = JSONObject.toJSONString(mySQLDataSource);
 
             // get datasource
             // BaseDataSource baseDataSource = DataSourceFactory.getDatasource(DbType.valueOf("MYSQL"), json);
             
-            BaseDataSource baseDataSource = mySQLDataSource;
-            baseDataSource.setDbType(DbType.MYSQL.name());
-            baseDataSource.setJdbcUrlDirect("jdbc:mysql://192.168.1.140:3307/dmp_web_new");
-            baseDataSource.setUser("dmp");
-            baseDataSource.setPassword("Ioubuy123");
+			/*
+			 * BaseDataSource baseDataSource = mySQLDataSource;
+			 * baseDataSource.s(DbType.MYSQL.name());
+			 * baseDataSource.setJdbcUrlDirect("jdbc:mysql://192.168.1.140:3307/dmp_web_new"
+			 * ); baseDataSource.setUser("dmp"); baseDataSource.setPassword("Ioubuy123");
+			 */
             
             // ready to execute SQL and parameter entity Map
             SqlBinds mainSqlBinds = getSqlAndSqlParamsMap("insert into test(id, name) values(1, 'test')");
@@ -177,7 +181,7 @@ public class SQLCommandExecutorTest {
             // execute sql task
             //DatabaseUtils.executeFuncAndSql(mainSqlBinds, preStatementSqlBinds, postStatementSqlBinds, createFuncs);
             
-            DatabaseUtils.executeUpdateSql(preStatementSqlBinds, baseDataSource);
+            DatabaseUtils.executeUpdateSql(preStatementSqlBinds, myBaseDataSource);
 
             //setExitStatusCode(Constants.EXIT_CODE_SUCCESS);
         } catch (Exception e) {