Skip to content

J
jz-dmp-cmdexectool
Commit e967006e authored by sml's avatar sml
Browse files
Options

代码提交

parent 02dd0b7c
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 397 additions and 199 deletions
src/main/java/com/jz/dmp/cmdexectool/common/constant/CommConstant.java
View file @ e967006e
......@@ -53,6 +53,7 @@ public class CommConstant {
public static final String WATERDROP_FTL_SOURCE_JDBC = "source_jdbc.ftl";
public static final String WATERDROP_FTL_SOURCE_ELASTICSEARCH = "source_elasticsearch.ftl";
public static final String WATERDROP_FTL_SOURCE_SFTP = "source_sftp.ftl";
public static final String WATERDROP_FTL_SOURCE_HIVE = "source_hive.ftl";
public static final String WATERDROP_FTL_TRANSFORM_SQL = "transform_sql.ftl";
public static final String WATERDROP_FTL_TRANSFORM_JSON2 = "transform_json2.ftl";
public static final String WATERDROP_FTL_SINK_CONSOLE = "sink_console.ftl";
......@@ -66,5 +67,10 @@ public class CommConstant {
public static final String FTL_UNZIPFILE = "unzipfile.ftl";//文件解压
public static final String FTL_DOCTRANS = "doctrans.ftl";//文件转码
public static final String FTL_HDFS_UPLOAD = "hdfs_upload.ftl";//HDFS上传
/***************************************************/
//执行引擎
public static final String EXECUTION_ENGINE_JDBC = "jdbc";//jdbc
public static final String EXECUTION_ENGINE_SPARK = "spark";//spark
}
src/main/java/com/jz/dmp/cmdexectool/scheduler/dao/datasource/BaseDataSource.java
View file @ e967006e
......@@ -67,8 +67,6 @@ public abstract class BaseDataSource {
*/
private String principal;
private String dbType;
public String getPrincipal() {
return principal;
}
......@@ -227,14 +225,6 @@ public abstract class BaseDataSource {
this.other = other;
}
public String getDbType() {
return dbType;
}
public void setDbType(String dbType) {
this.dbType = dbType;
}
public String getJdbcUrlDirect() {
return jdbcUrlDirect;
}
......
src/main/java/com/jz/dmp/cmdexectool/scheduler/dao/utils/DatabaseUtils.java
View file @ e967006e
......@@ -36,6 +36,7 @@ import com.jz.dmp.cmdexectool.scheduler.common.utils.CommonUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.JSONUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.ParameterUtils;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.BaseDataSource;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.MyBaseDataSource;
import com.jz.dmp.cmdexectool.scheduler.server.utils.ParamUtils;
public class DatabaseUtils {
......@@ -51,7 +52,7 @@ public class DatabaseUtils {
* @param postStatementsBinds post statements binds
* @param createFuncs create functions
*/
public static void executeUpdateSql(List<SqlBinds> statementsBinds, BaseDataSource baseDataSource){
public static void executeUpdateSql(List<SqlBinds> statementsBinds, MyBaseDataSource myBaseDataSource){
Connection connection = null;
PreparedStatement stmt = null;
ResultSet resultSet = null;
......@@ -59,7 +60,7 @@ public class DatabaseUtils {
// if upload resource is HDFS and kerberos startup
CommonUtils.loadKerberosConf();
// create connection
connection = createConnection(baseDataSource);
connection = createConnection(myBaseDataSource);
// create temp function
/*
if (CollectionUtils.isNotEmpty(createFuncs)) {
......@@ -89,7 +90,7 @@ public class DatabaseUtils {
List<SqlBinds> preStatementsBinds,
List<SqlBinds> postStatementsBinds,
List<String> createFuncs,
BaseDataSource baseDataSource){
MyBaseDataSource myBaseDataSource){
Connection connection = null;
PreparedStatement stmt = null;
ResultSet resultSet = null;
......@@ -97,7 +98,7 @@ public class DatabaseUtils {
// if upload resource is HDFS and kerberos startup
CommonUtils.loadKerberosConf();
// create connection
connection = createConnection(baseDataSource);
connection = createConnection(myBaseDataSource);
// create temp function
/*
if (CollectionUtils.isNotEmpty(createFuncs)) {
......@@ -171,24 +172,24 @@ public class DatabaseUtils {
* @return connection
* @throws Exception Exception
*/
private static Connection createConnection(BaseDataSource baseDataSource) throws Exception{
private static Connection createConnection(MyBaseDataSource myBaseDataSource) throws Exception{
// if hive , load connection params if exists
Connection connection = null;
if (HIVE == DbType.valueOf(baseDataSource.getDbType())) {
if (HIVE == DbType.valueOf(myBaseDataSource.getMyDbType().getDbType().name())) {
Properties paramProp = new Properties();
paramProp.setProperty(USER, baseDataSource.getUser());
paramProp.setProperty(PASSWORD, baseDataSource.getPassword());
paramProp.setProperty(USER, myBaseDataSource.getUser());
paramProp.setProperty(PASSWORD, myBaseDataSource.getPassword());
Map<String, String> connParamMap = CollectionUtils.stringToMap("",
SEMICOLON,
HIVE_CONF);
paramProp.putAll(connParamMap);
connection = DriverManager.getConnection(baseDataSource.getJdbcUrlDirect(),
connection = DriverManager.getConnection(myBaseDataSource.getJdbcUrlDirect(),
paramProp);
}else{
connection = DriverManager.getConnection(baseDataSource.getJdbcUrlDirect(),
baseDataSource.getUser(),
baseDataSource.getPassword());
connection = DriverManager.getConnection(myBaseDataSource.getJdbcUrlDirect(),
myBaseDataSource.getUser(),
myBaseDataSource.getPassword());
}
return connection;
}
......
src/main/java/com/jz/dmp/cmdexectool/scheduler/server/worker/task/sql/SqlTask.java
View file @ e967006e
......@@ -36,6 +36,7 @@ import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.springframework.util.CollectionUtils;
import com.jz.dmp.cmdexectool.common.constant.CommConstant;
import com.jz.dmp.cmdexectool.scheduler.common.Constants;
import com.jz.dmp.cmdexectool.scheduler.common.enums.DbType;
import com.jz.dmp.cmdexectool.scheduler.common.process.Property;
......@@ -45,6 +46,7 @@ import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlParameters;
import com.jz.dmp.cmdexectool.scheduler.common.utils.OSUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.ParameterUtils;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.BaseDataSource;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.MyBaseDataSource;
import com.jz.dmp.cmdexectool.scheduler.dao.utils.DatabaseUtils;
import com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext;
import com.jz.dmp.cmdexectool.scheduler.server.utils.ParamUtils;
......@@ -101,25 +103,7 @@ public class SqlTask extends AbstractTask {
public void handle() throws Exception {
try {
BaseDataSource baseDataSource = new BaseDataSource() {
@Override
public String driverClassSelector() {
// TODO Auto-generated method stub
return null;
}
@Override
public DbType dbTypeSelector() {
// TODO Auto-generated method stub
return null;
}
};
baseDataSource.setDbType(sqlParameters.getMyDbType().getDbType().name());
baseDataSource.setUser(sqlParameters.getUser());
baseDataSource.setPassword(sqlParameters.getPassword());
baseDataSource.setAddress(sqlParameters.getJdbcUrl());
MyBaseDataSource targetBaseDataSource = sqlParameters.getTargetBaseDataSource();
List<SqlBinds> preStatementSqlBinds = Optional.ofNullable(sqlParameters.getPreStatements())
.orElse(new ArrayList<>())
......@@ -136,20 +120,34 @@ public class SqlTask extends AbstractTask {
//判断是否需要运行前置sql
if (!CollectionUtils.isEmpty(preStatementSqlBinds)) {
DatabaseUtils.executeUpdateSql(preStatementSqlBinds, baseDataSource);
DatabaseUtils.executeUpdateSql(preStatementSqlBinds, targetBaseDataSource);
}
// construct process
CommandExecuteResult commandExecuteResult = waterdropCommandExecutor.run(buildCommand());
if (sqlParameters.getExecutioEngine().equals(CommConstant.EXECUTION_ENGINE_JDBC)) {
List<String> mainSqlScript = new ArrayList<String>();
mainSqlScript.add(sqlParameters.getSqlScript());
List<SqlBinds> mainStatementSqlBinds = Optional.ofNullable(mainSqlScript)
.orElse(new ArrayList<>())
.stream()
.map(DatabaseUtils::getSqlAndSqlParamsMap)
.collect(Collectors.toList());
DatabaseUtils.executeUpdateSql(mainStatementSqlBinds, sqlParameters.getSourceBaseDataSource());
}else {
// construct process
CommandExecuteResult commandExecuteResult = waterdropCommandExecutor.run(buildCommand());
setExitStatusCode(commandExecuteResult.getExitStatusCode());
setAppIds(commandExecuteResult.getAppIds());
setProcessId(commandExecuteResult.getProcessId());
}
//判断是否运行后置sql
if (!CollectionUtils.isEmpty(postStatementSqlBinds)) {
DatabaseUtils.executeUpdateSql(postStatementSqlBinds, baseDataSource);
DatabaseUtils.executeUpdateSql(postStatementSqlBinds, targetBaseDataSource);
}
setExitStatusCode(commandExecuteResult.getExitStatusCode());
setAppIds(commandExecuteResult.getAppIds());
setProcessId(commandExecuteResult.getProcessId());
} catch (Exception e) {
logger.error("sql task error", e);
setExitStatusCode(Constants.EXIT_CODE_FAILURE);
......
src/test/java/com/jz/cmdexectool/test/task/shell/SQLCommandExecutorTest.java
View file @ e967006e
......@@ -23,6 +23,7 @@ import com.jz.dmp.cmdexectool.ApiApplication;
import com.jz.dmp.cmdexectool.scheduler.common.Constants;
import com.jz.dmp.cmdexectool.scheduler.common.enums.DbType;
import com.jz.dmp.cmdexectool.scheduler.common.enums.ExecutionStatus;
import com.jz.dmp.cmdexectool.scheduler.common.enums.MyDbType;
import com.jz.dmp.cmdexectool.scheduler.common.process.Property;
import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlBinds;
import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlType;
......@@ -32,6 +33,7 @@ import com.jz.dmp.cmdexectool.scheduler.common.utils.JSONUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.ParameterUtils;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.BaseDataSource;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.DataSourceFactory;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.MyBaseDataSource;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.MySQLDataSource;
import com.jz.dmp.cmdexectool.scheduler.dao.utils.DatabaseUtils;
import com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext;
......@@ -134,21 +136,23 @@ public class SQLCommandExecutorTest {
// load class
DataSourceFactory.loadClass(DbType.valueOf("MYSQL"));
MySQLDataSource mySQLDataSource = new MySQLDataSource();
mySQLDataSource.setAddress("192.168.1.140:3307");
mySQLDataSource.setUser("dmp");
mySQLDataSource.setPassword("Ioubuy123");
MyBaseDataSource myBaseDataSource = new MyBaseDataSource();
myBaseDataSource.setJdbcUrlDirect("jdbc:mysql://192.168.1.140:3307/dmp_web_new");
myBaseDataSource.setUser("dmp");
myBaseDataSource.setPassword("Ioubuy123");
myBaseDataSource.setMyDbType(MyDbType.MySQL);
//String json = JSONObject.toJSONString(mySQLDataSource);
// get datasource
// BaseDataSource baseDataSource = DataSourceFactory.getDatasource(DbType.valueOf("MYSQL"), json);
BaseDataSource baseDataSource = mySQLDataSource;
baseDataSource.setDbType(DbType.MYSQL.name());
baseDataSource.setJdbcUrlDirect("jdbc:mysql://192.168.1.140:3307/dmp_web_new");
baseDataSource.setUser("dmp");
baseDataSource.setPassword("Ioubuy123");
/*
* BaseDataSource baseDataSource = mySQLDataSource;
* baseDataSource.s(DbType.MYSQL.name());
* baseDataSource.setJdbcUrlDirect("jdbc:mysql://192.168.1.140:3307/dmp_web_new"
* ); baseDataSource.setUser("dmp"); baseDataSource.setPassword("Ioubuy123");
*/
// ready to execute SQL and parameter entity Map
SqlBinds mainSqlBinds = getSqlAndSqlParamsMap("insert into test(id, name) values(1, 'test')");
......@@ -177,7 +181,7 @@ public class SQLCommandExecutorTest {
// execute sql task
//DatabaseUtils.executeFuncAndSql(mainSqlBinds, preStatementSqlBinds, postStatementSqlBinds, createFuncs);
DatabaseUtils.executeUpdateSql(preStatementSqlBinds, baseDataSource);
DatabaseUtils.executeUpdateSql(preStatementSqlBinds, myBaseDataSource);
//setExitStatusCode(Constants.EXIT_CODE_SUCCESS);
} catch (Exception e) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment