Skip to content

J
jz-dmp-cmdexectool
Commit 8e1fa3fc authored by mcb's avatar mcb
Browse files
Options

冲突修改

parents 6237d0f6 c460e8db
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 170 additions and 108 deletions
src/main/java/com/jz/dmp/cmdexectool/ApiApplication.java
View file @ 8e1fa3fc
......@@ -2,13 +2,18 @@ package com.jz.dmp.cmdexectool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.Banner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.WebApplicationType;
import org.springframework.boot.actuate.health.Health;
import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean;
import com.jz.dmp.cmdexectool.common.utils.ApplicationContextUtil;
import com.jz.dmp.cmdexectool.scheduler.service.process.ProcessService;
......@@ -21,8 +26,8 @@ public class ApiApplication implements HealthIndicator {
public static void main(String[] args) {
long start = System.currentTimeMillis();
SpringApplication springApplication = new SpringApplication(ApiApplication.class);
ConfigurableApplicationContext context = springApplication.run(args);
ConfigurableApplicationContext context = new SpringApplicationBuilder(ApiApplication.class).web(WebApplicationType.NONE).bannerMode(Banner.Mode.OFF).run(args);
Integer taskId = Integer.parseInt(args[0]);
if (taskId==null) {
......@@ -41,8 +46,6 @@ public class ApiApplication implements HealthIndicator {
long cost = System.currentTimeMillis() - start;
logger.info(" started status: {}, cost: {}", "SUCCESS!", cost);
springApplication.exit(context);
}
@Override
......
src/main/java/com/jz/dmp/cmdexectool/common/utils/FreeMarkerUtils.java
View file @ 8e1fa3fc
......@@ -4,9 +4,11 @@ import java.io.IOException;
import java.io.StringWriter;
import java.util.Map;
import org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean;
import org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer;
import freemarker.core.ParseException;
import freemarker.template.Configuration;
import freemarker.template.MalformedTemplateNameException;
import freemarker.template.Template;
import freemarker.template.TemplateException;
......@@ -22,7 +24,7 @@ public class FreeMarkerUtils {
* @return
* @author Bellamy
*/
public static String freemakerJson(String ftlName, Map<String, String> dataModel, FreeMarkerConfigurer freeMarkerConfig) {
public static String freemakerJson2(String ftlName, Map<String, String> dataModel, FreeMarkerConfigurer freeMarkerConfig) {
StringWriter stringWriter = new StringWriter();
try {
......@@ -45,5 +47,39 @@ public class FreeMarkerUtils {
}
return stringWriter.toString();
}
/**
* 使用freemaker模板生成 kafka connector 请求参数
*
* @param type 模板类型
* @param dataModel 模板里定义的变量数据对象
* @return
* @author Bellamy
*/
public static String freemakerNoneWebJson(String ftlName, Map<String, String> dataModel, FreeMarkerConfigurationFactoryBean freeMarkerConfigurationFactoryBean) {
StringWriter stringWriter = new StringWriter();
try {
Configuration configuration = freeMarkerConfigurationFactoryBean.createConfiguration();
Template template = configuration.getTemplate(ftlName);
if (template != null) {
try {
template.process(dataModel, stringWriter);
} catch (TemplateException e) {
e.printStackTrace();
}
}
} catch (TemplateNotFoundException e) {
e.printStackTrace();
} catch (MalformedTemplateNameException e) {
e.printStackTrace();
} catch (ParseException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (TemplateException e) {
e.printStackTrace();
}
return stringWriter.toString();
}
}
src/main/java/com/jz/dmp/cmdexectool/config/FreemarkerConfig.java 0 → 100644
View file @ 8e1fa3fc
package com.jz.dmp.cmdexectool.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean;
@Configuration
public class FreemarkerConfig {
@Bean(value = "freemarkerConfiguration")
public FreeMarkerConfigurationFactoryBean getFreeMarkerConfigurationFactoryBean(){
FreeMarkerConfigurationFactoryBean freeMarkerConfigurationFactoryBean = new FreeMarkerConfigurationFactoryBean();
freeMarkerConfigurationFactoryBean.setTemplateLoaderPath("classpath:templates");
return freeMarkerConfigurationFactoryBean;
}
}
src/main/java/com/jz/dmp/cmdexectool/scheduler/common/task/docTrans/DoctransParameters.java
View file @ 8e1fa3fc
......@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean;
import org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer;
/**
......@@ -54,7 +55,7 @@ public class DoctransParameters extends AbstractParameters {
}
public DoctransParameters(String script, FreeMarkerConfigurer freeMarkerConfig) {
public DoctransParameters(String script, FreeMarkerConfigurationFactoryBean freeMarkerConfigurationFactoryBean) {
this.script = script;
JSONObject scriptObj = JSONObject.parseObject(script);
......@@ -74,7 +75,7 @@ public class DoctransParameters extends AbstractParameters {
doctransModel.put("source_convert", sourceConvert);
doctransModel.put("sink_convert", sinkConvert);
doctransModel.put("file_suffix", fileSuffix);
this.cmdScript = FreeMarkerUtils.freemakerJson(CommConstant.FTL_DOCTRANS, doctransModel, freeMarkerConfig);
this.cmdScript = FreeMarkerUtils.freemakerNoneWebJson(CommConstant.FTL_DOCTRANS, doctransModel, freeMarkerConfigurationFactoryBean);
}
......
src/main/java/com/jz/dmp/cmdexectool/scheduler/common/task/ftp/FtpParameters.java
View file @ 8e1fa3fc
......@@ -22,6 +22,7 @@ import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean;
import org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer;
import com.alibaba.fastjson.JSONObject;
......@@ -58,7 +59,7 @@ public class FtpParameters extends AbstractParameters {
}
public FtpParameters(String script, FreeMarkerConfigurer freeMarkerConfig) {
public FtpParameters(String script, FreeMarkerConfigurationFactoryBean freeMarkerConfigurationFactoryBean) {
this.script = script;
JSONObject scriptObj = JSONObject.parseObject(script);
......@@ -94,7 +95,7 @@ public class FtpParameters extends AbstractParameters {
logger.info("调用ftp command模板参数【{}】", JSONObject.toJSONString(ftpModel));
this.cmdScript = FreeMarkerUtils.freemakerJson(CommConstant.FTL_SFTP_DOWNLOAD, ftpModel, freeMarkerConfig);
this.cmdScript = FreeMarkerUtils.freemakerNoneWebJson(CommConstant.FTL_SFTP_DOWNLOAD, ftpModel, freeMarkerConfigurationFactoryBean);
}
......
src/main/java/com/jz/dmp/cmdexectool/scheduler/common/task/hdfs/HdfsParameters.java
View file @ 8e1fa3fc
......@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean;
import org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer;
/**
......@@ -54,7 +55,7 @@ public class HdfsParameters extends AbstractParameters {
}
public HdfsParameters(String script, FreeMarkerConfigurer freeMarkerConfig) {
public HdfsParameters(String script, FreeMarkerConfigurationFactoryBean freeMarkerConfigurationFactoryBean) {
this.script = script;
JSONObject scriptObj = JSONObject.parseObject(script);
......@@ -68,7 +69,7 @@ public class HdfsParameters extends AbstractParameters {
HdfsModel.put("src_dir", srcDir);
HdfsModel.put("file_suffix", fileSuffix);
HdfsModel.put("des_dir", desDir);
this.cmdScript = FreeMarkerUtils.freemakerJson(CommConstant.FTL_HDFS_UPLOAD, HdfsModel, freeMarkerConfig);
this.cmdScript = FreeMarkerUtils.freemakerNoneWebJson(CommConstant.FTL_HDFS_UPLOAD, HdfsModel, freeMarkerConfigurationFactoryBean);
}
......
src/main/java/com/jz/dmp/cmdexectool/scheduler/common/task/unzipfile/UnzipfileParameters.java
View file @ 8e1fa3fc
......@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean;
import org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer;
/**
......@@ -54,7 +55,7 @@ public class UnzipfileParameters extends AbstractParameters {
}
public UnzipfileParameters(String script, FreeMarkerConfigurer freeMarkerConfig) {
public UnzipfileParameters(String script, FreeMarkerConfigurationFactoryBean freeMarkerConfigurationFactoryBean) {
this.script = script;
JSONObject scriptObj = JSONObject.parseObject(script);
......@@ -69,7 +70,7 @@ public class UnzipfileParameters extends AbstractParameters {
unzipfileModel.put("src_dir", srcDir);
unzipfileModel.put("des_dir", desDir);
unzipfileModel.put("type", type);
this.cmdScript = FreeMarkerUtils.freemakerJson(CommConstant.FTL_UNZIPFILE, unzipfileModel, freeMarkerConfig);
this.cmdScript = FreeMarkerUtils.freemakerNoneWebJson(CommConstant.FTL_UNZIPFILE, unzipfileModel, freeMarkerConfigurationFactoryBean);
}
......
src/main/java/com/jz/dmp/cmdexectool/scheduler/service/process/ProcessService.java
View file @ 8e1fa3fc
......@@ -21,13 +21,14 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.jz.dmp.cmdexectool.scheduler.common.task.sync.SyncParameters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean;
import org.springframework.util.CollectionUtils;
//import org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer;
import org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer;
import com.alibaba.fastjson.JSONArray;
......@@ -49,6 +50,7 @@ import com.jz.dmp.cmdexectool.scheduler.common.task.ftp.FtpParameters;
import com.jz.dmp.cmdexectool.scheduler.common.task.hdfs.HdfsParameters;
import com.jz.dmp.cmdexectool.scheduler.common.task.shell.ShellParameters;
import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlParameters;
import com.jz.dmp.cmdexectool.scheduler.common.task.sync.SyncParameters;
import com.jz.dmp.cmdexectool.scheduler.common.task.unzipfile.UnzipfileParameters;
import com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.AbstractTask;
......@@ -73,8 +75,10 @@ public class ProcessService {
private DmpProjectConfigInfoMapper dmpProjectConfigInfoMapper;
@Autowired
private DmpSyncingDatasourceDao dmpSyncingDatasourceDao;
//@Autowired
//private FreeMarkerConfigurer freeMarkerConfigurer;
@Autowired
private FreeMarkerConfigurer freeMarkerConfigurer;
private FreeMarkerConfigurationFactoryBean freeMarkerConfigurationFactoryBean;
/**
* @Title: taskStart @Description: TODO(启动task) @param 参数 @return void
......@@ -212,37 +216,37 @@ public class ProcessService {
break;
case sql:
SqlParameters sqlParameters = new SqlParameters(script, projectConfigInfoDto, dmpSyncingDatasourceDao, freeMarkerConfigurer, publicKey);
SqlParameters sqlParameters = new SqlParameters(script, projectConfigInfoDto, dmpSyncingDatasourceDao, freeMarkerConfigurationFactoryBean, publicKey);
sqlParameters.setTaskAppId(taskAppId);
taskExecutionContext = new TaskExecutionContext(sqlParameters, projectConfigInfoDto);
break;
case sync:
SyncParameters sync = new SyncParameters(script, projectConfigInfoDto, dmpSyncingDatasourceDao, freeMarkerConfigurer, publicKey);
SyncParameters sync = new SyncParameters(script, projectConfigInfoDto, dmpSyncingDatasourceDao, freeMarkerConfigurationFactoryBean, publicKey);
sync.setTaskAppId(taskAppId);
taskExecutionContext = new TaskExecutionContext(sync, projectConfigInfoDto);
break;
case subprocess:
break;
case ftp:
FtpParameters ftpParameters = new FtpParameters(script, freeMarkerConfigurer);
FtpParameters ftpParameters = new FtpParameters(script, freeMarkerConfigurationFactoryBean);
ftpParameters.setTaskAppId(taskAppId);
taskExecutionContext = new TaskExecutionContext(ftpParameters, projectConfigInfoDto);
break;
case unzipFile:
UnzipfileParameters unzipfileParameters = new UnzipfileParameters(script, freeMarkerConfigurer);
UnzipfileParameters unzipfileParameters = new UnzipfileParameters(script, freeMarkerConfigurationFactoryBean);
unzipfileParameters.setScript(script);
unzipfileParameters.setTaskAppId(taskAppId);
taskExecutionContext = new TaskExecutionContext(unzipfileParameters, projectConfigInfoDto);
break;
case docTrans:
DoctransParameters doctransParameters = new DoctransParameters(script, freeMarkerConfigurer);
DoctransParameters doctransParameters = new DoctransParameters(script, freeMarkerConfigurationFactoryBean);
doctransParameters.setScript(script);
doctransParameters.setTaskAppId(taskAppId);
taskExecutionContext = new TaskExecutionContext(doctransParameters, projectConfigInfoDto);
break;
case hdfs:
HdfsParameters hdfsParameters = new HdfsParameters(script, freeMarkerConfigurer);
HdfsParameters hdfsParameters = new HdfsParameters(script, freeMarkerConfigurationFactoryBean);
hdfsParameters.setScript(script);
hdfsParameters.setTaskAppId(taskAppId);
taskExecutionContext = new TaskExecutionContext(hdfsParameters, projectConfigInfoDto);
......
src/test/java/com/jz/cmdexectool/test/task/shell/SQLCommandExecutorTest.java
View file @ 8e1fa3fc
......@@ -16,35 +16,12 @@
*/
package com.jz.cmdexectool.test.task.shell;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.jz.dmp.cmdexectool.ApiApplication;
import com.jz.dmp.cmdexectool.scheduler.common.Constants;
import com.jz.dmp.cmdexectool.scheduler.common.enums.DbType;
import com.jz.dmp.cmdexectool.scheduler.common.enums.ExecutionStatus;
import com.jz.dmp.cmdexectool.scheduler.common.enums.MyDbType;
import com.jz.dmp.cmdexectool.scheduler.common.process.Property;
import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlBinds;
import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlType;
import com.jz.dmp.cmdexectool.scheduler.common.utils.CollectionUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.CommonUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.JSONUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.ParameterUtils;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.BaseDataSource;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.DataSourceFactory;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.MyBaseDataSource;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.MySQLDataSource;
import com.jz.dmp.cmdexectool.scheduler.dao.utils.DatabaseUtils;
import com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext;
import com.jz.dmp.cmdexectool.scheduler.server.utils.ParamUtils;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.AbstractTask;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.TaskManager;
import com.jz.dmp.cmdexectool.scheduler.service.process.ProcessService;
import static com.jz.dmp.cmdexectool.scheduler.common.Constants.HIVE_CONF;
import static com.jz.dmp.cmdexectool.scheduler.common.Constants.PASSWORD;
import static com.jz.dmp.cmdexectool.scheduler.common.Constants.SEMICOLON;
import static com.jz.dmp.cmdexectool.scheduler.common.Constants.USER;
import static com.jz.dmp.cmdexectool.scheduler.common.enums.DbType.HIVE;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
......@@ -72,8 +49,28 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import static com.jz.dmp.cmdexectool.scheduler.common.Constants.*;
import static com.jz.dmp.cmdexectool.scheduler.common.enums.DbType.HIVE;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.jz.dmp.cmdexectool.ApiApplication;
import com.jz.dmp.cmdexectool.scheduler.common.Constants;
import com.jz.dmp.cmdexectool.scheduler.common.enums.DbType;
import com.jz.dmp.cmdexectool.scheduler.common.enums.ExecutionStatus;
import com.jz.dmp.cmdexectool.scheduler.common.enums.MyDbType;
import com.jz.dmp.cmdexectool.scheduler.common.process.Property;
import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlBinds;
import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlType;
import com.jz.dmp.cmdexectool.scheduler.common.utils.CollectionUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.CommonUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.JSONUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.ParameterUtils;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.DataSourceFactory;
import com.jz.dmp.cmdexectool.scheduler.dao.datasource.MyBaseDataSource;
import com.jz.dmp.cmdexectool.scheduler.dao.utils.DatabaseUtils;
import com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext;
import com.jz.dmp.cmdexectool.scheduler.server.utils.ParamUtils;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.AbstractTask;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.TaskManager;
import com.jz.dmp.cmdexectool.scheduler.service.process.ProcessService;
/**
* python shell command executor test
......@@ -89,6 +86,11 @@ public class SQLCommandExecutorTest {
private static final int LIMIT = 10000;
@Test
public void test22() {
System.out.println("test");
}
//@Test
public void test() {
try {
......@@ -121,11 +123,6 @@ public class SQLCommandExecutorTest {
}
//@Test
public void test22() {
System.out.println("test");
}
@Test
public void testJdbcHandler() {
// set the name of the current thread
//String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, "test");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment