Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
J
jz-dmp-cmdexectool
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
姚本章
jz-dmp-cmdexectool
Commits
8e1fa3fc
Commit
8e1fa3fc
authored
Mar 11, 2021
by
mcb
Browse files
Options
Browse Files
Download
Plain Diff
冲突修改
parents
6237d0f6
c460e8db
Changes
11
Show whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
170 additions
and
108 deletions
+170
-108
ApiApplication.java
src/main/java/com/jz/dmp/cmdexectool/ApiApplication.java
+7
-4
FreeMarkerUtils.java
.../com/jz/dmp/cmdexectool/common/utils/FreeMarkerUtils.java
+37
-1
FreemarkerConfig.java
.../java/com/jz/dmp/cmdexectool/config/FreemarkerConfig.java
+16
-0
DoctransParameters.java
...ol/scheduler/common/task/docTrans/DoctransParameters.java
+3
-2
FtpParameters.java
.../cmdexectool/scheduler/common/task/ftp/FtpParameters.java
+3
-2
HdfsParameters.java
...mdexectool/scheduler/common/task/hdfs/HdfsParameters.java
+3
-2
SqlParameters.java
.../cmdexectool/scheduler/common/task/sql/SqlParameters.java
+31
-30
SyncParameters.java
...mdexectool/scheduler/common/task/sync/SyncParameters.java
+23
-22
UnzipfileParameters.java
.../scheduler/common/task/unzipfile/UnzipfileParameters.java
+3
-2
ProcessService.java
...cmdexectool/scheduler/service/process/ProcessService.java
+12
-8
SQLCommandExecutorTest.java
...z/cmdexectool/test/task/shell/SQLCommandExecutorTest.java
+32
-35
No files found.
src/main/java/com/jz/dmp/cmdexectool/ApiApplication.java
View file @
8e1fa3fc
...
@@ -2,13 +2,18 @@ package com.jz.dmp.cmdexectool;
...
@@ -2,13 +2,18 @@ package com.jz.dmp.cmdexectool;
import
org.slf4j.Logger
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.boot.Banner
;
import
org.springframework.boot.SpringApplication
;
import
org.springframework.boot.SpringApplication
;
import
org.springframework.boot.WebApplicationType
;
import
org.springframework.boot.actuate.health.Health
;
import
org.springframework.boot.actuate.health.Health
;
import
org.springframework.boot.actuate.health.HealthIndicator
;
import
org.springframework.boot.actuate.health.HealthIndicator
;
import
org.springframework.boot.autoconfigure.SpringBootApplication
;
import
org.springframework.boot.autoconfigure.SpringBootApplication
;
import
org.springframework.boot.builder.SpringApplicationBuilder
;
import
org.springframework.cache.annotation.EnableCaching
;
import
org.springframework.cache.annotation.EnableCaching
;
import
org.springframework.context.ConfigurableApplicationContext
;
import
org.springframework.context.ConfigurableApplicationContext
;
import
org.springframework.transaction.annotation.EnableTransactionManagement
;
import
org.springframework.transaction.annotation.EnableTransactionManagement
;
import
org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean
;
import
com.jz.dmp.cmdexectool.common.utils.ApplicationContextUtil
;
import
com.jz.dmp.cmdexectool.common.utils.ApplicationContextUtil
;
import
com.jz.dmp.cmdexectool.scheduler.service.process.ProcessService
;
import
com.jz.dmp.cmdexectool.scheduler.service.process.ProcessService
;
...
@@ -21,8 +26,8 @@ public class ApiApplication implements HealthIndicator {
...
@@ -21,8 +26,8 @@ public class ApiApplication implements HealthIndicator {
public
static
void
main
(
String
[]
args
)
{
public
static
void
main
(
String
[]
args
)
{
long
start
=
System
.
currentTimeMillis
();
long
start
=
System
.
currentTimeMillis
();
SpringApplication
springApplication
=
new
SpringApplication
(
ApiApplication
.
class
);
ConfigurableApplicationContext
context
=
springApplication
.
run
(
args
);
ConfigurableApplicationContext
context
=
new
SpringApplicationBuilder
(
ApiApplication
.
class
).
web
(
WebApplicationType
.
NONE
).
bannerMode
(
Banner
.
Mode
.
OFF
)
.
run
(
args
);
Integer
taskId
=
Integer
.
parseInt
(
args
[
0
]);
Integer
taskId
=
Integer
.
parseInt
(
args
[
0
]);
if
(
taskId
==
null
)
{
if
(
taskId
==
null
)
{
...
@@ -41,8 +46,6 @@ public class ApiApplication implements HealthIndicator {
...
@@ -41,8 +46,6 @@ public class ApiApplication implements HealthIndicator {
long
cost
=
System
.
currentTimeMillis
()
-
start
;
long
cost
=
System
.
currentTimeMillis
()
-
start
;
logger
.
info
(
" started status: {}, cost: {}"
,
"SUCCESS!"
,
cost
);
logger
.
info
(
" started status: {}, cost: {}"
,
"SUCCESS!"
,
cost
);
springApplication
.
exit
(
context
);
}
}
@Override
@Override
...
...
src/main/java/com/jz/dmp/cmdexectool/common/utils/FreeMarkerUtils.java
View file @
8e1fa3fc
...
@@ -4,9 +4,11 @@ import java.io.IOException;
...
@@ -4,9 +4,11 @@ import java.io.IOException;
import
java.io.StringWriter
;
import
java.io.StringWriter
;
import
java.util.Map
;
import
java.util.Map
;
import
org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
freemarker.core.ParseException
;
import
freemarker.core.ParseException
;
import
freemarker.template.Configuration
;
import
freemarker.template.MalformedTemplateNameException
;
import
freemarker.template.MalformedTemplateNameException
;
import
freemarker.template.Template
;
import
freemarker.template.Template
;
import
freemarker.template.TemplateException
;
import
freemarker.template.TemplateException
;
...
@@ -22,7 +24,7 @@ public class FreeMarkerUtils {
...
@@ -22,7 +24,7 @@ public class FreeMarkerUtils {
* @return
* @return
* @author Bellamy
* @author Bellamy
*/
*/
public
static
String
freemakerJson
(
String
ftlName
,
Map
<
String
,
String
>
dataModel
,
FreeMarkerConfigurer
freeMarkerConfig
)
{
public
static
String
freemakerJson
2
(
String
ftlName
,
Map
<
String
,
String
>
dataModel
,
FreeMarkerConfigurer
freeMarkerConfig
)
{
StringWriter
stringWriter
=
new
StringWriter
();
StringWriter
stringWriter
=
new
StringWriter
();
try
{
try
{
...
@@ -46,4 +48,38 @@ public class FreeMarkerUtils {
...
@@ -46,4 +48,38 @@ public class FreeMarkerUtils {
return
stringWriter
.
toString
();
return
stringWriter
.
toString
();
}
}
/**
* 使用freemaker模板生成 kafka connector 请求参数
*
* @param type 模板类型
* @param dataModel 模板里定义的变量数据对象
* @return
* @author Bellamy
*/
public
static
String
freemakerNoneWebJson
(
String
ftlName
,
Map
<
String
,
String
>
dataModel
,
FreeMarkerConfigurationFactoryBean
freeMarkerConfigurationFactoryBean
)
{
StringWriter
stringWriter
=
new
StringWriter
();
try
{
Configuration
configuration
=
freeMarkerConfigurationFactoryBean
.
createConfiguration
();
Template
template
=
configuration
.
getTemplate
(
ftlName
);
if
(
template
!=
null
)
{
try
{
template
.
process
(
dataModel
,
stringWriter
);
}
catch
(
TemplateException
e
)
{
e
.
printStackTrace
();
}
}
}
catch
(
TemplateNotFoundException
e
)
{
e
.
printStackTrace
();
}
catch
(
MalformedTemplateNameException
e
)
{
e
.
printStackTrace
();
}
catch
(
ParseException
e
)
{
e
.
printStackTrace
();
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
catch
(
TemplateException
e
)
{
e
.
printStackTrace
();
}
return
stringWriter
.
toString
();
}
}
}
src/main/java/com/jz/dmp/cmdexectool/config/FreemarkerConfig.java
0 → 100644
View file @
8e1fa3fc
package
com
.
jz
.
dmp
.
cmdexectool
.
config
;
import
org.springframework.context.annotation.Bean
;
import
org.springframework.context.annotation.Configuration
;
import
org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean
;
@Configuration
public
class
FreemarkerConfig
{
@Bean
(
value
=
"freemarkerConfiguration"
)
public
FreeMarkerConfigurationFactoryBean
getFreeMarkerConfigurationFactoryBean
(){
FreeMarkerConfigurationFactoryBean
freeMarkerConfigurationFactoryBean
=
new
FreeMarkerConfigurationFactoryBean
();
freeMarkerConfigurationFactoryBean
.
setTemplateLoaderPath
(
"classpath:templates"
);
return
freeMarkerConfigurationFactoryBean
;
}
}
src/main/java/com/jz/dmp/cmdexectool/scheduler/common/task/docTrans/DoctransParameters.java
View file @
8e1fa3fc
...
@@ -26,6 +26,7 @@ import java.util.HashMap;
...
@@ -26,6 +26,7 @@ import java.util.HashMap;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
import
org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
/**
/**
...
@@ -54,7 +55,7 @@ public class DoctransParameters extends AbstractParameters {
...
@@ -54,7 +55,7 @@ public class DoctransParameters extends AbstractParameters {
}
}
public
DoctransParameters
(
String
script
,
FreeMarkerConfigur
er
freeMarkerConfig
)
{
public
DoctransParameters
(
String
script
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
)
{
this
.
script
=
script
;
this
.
script
=
script
;
JSONObject
scriptObj
=
JSONObject
.
parseObject
(
script
);
JSONObject
scriptObj
=
JSONObject
.
parseObject
(
script
);
...
@@ -74,7 +75,7 @@ public class DoctransParameters extends AbstractParameters {
...
@@ -74,7 +75,7 @@ public class DoctransParameters extends AbstractParameters {
doctransModel
.
put
(
"source_convert"
,
sourceConvert
);
doctransModel
.
put
(
"source_convert"
,
sourceConvert
);
doctransModel
.
put
(
"sink_convert"
,
sinkConvert
);
doctransModel
.
put
(
"sink_convert"
,
sinkConvert
);
doctransModel
.
put
(
"file_suffix"
,
fileSuffix
);
doctransModel
.
put
(
"file_suffix"
,
fileSuffix
);
this
.
cmdScript
=
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
FTL_DOCTRANS
,
doctransModel
,
freeMarkerConfig
);
this
.
cmdScript
=
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
FTL_DOCTRANS
,
doctransModel
,
freeMarkerConfigurationFactoryBean
);
}
}
...
...
src/main/java/com/jz/dmp/cmdexectool/scheduler/common/task/ftp/FtpParameters.java
View file @
8e1fa3fc
...
@@ -22,6 +22,7 @@ import java.util.Map;
...
@@ -22,6 +22,7 @@ import java.util.Map;
import
org.slf4j.Logger
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
com.alibaba.fastjson.JSONObject
;
import
com.alibaba.fastjson.JSONObject
;
...
@@ -58,7 +59,7 @@ public class FtpParameters extends AbstractParameters {
...
@@ -58,7 +59,7 @@ public class FtpParameters extends AbstractParameters {
}
}
public
FtpParameters
(
String
script
,
FreeMarkerConfigur
er
freeMarkerConfig
)
{
public
FtpParameters
(
String
script
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
)
{
this
.
script
=
script
;
this
.
script
=
script
;
JSONObject
scriptObj
=
JSONObject
.
parseObject
(
script
);
JSONObject
scriptObj
=
JSONObject
.
parseObject
(
script
);
...
@@ -94,7 +95,7 @@ public class FtpParameters extends AbstractParameters {
...
@@ -94,7 +95,7 @@ public class FtpParameters extends AbstractParameters {
logger
.
info
(
"调用ftp command模板参数【{}】"
,
JSONObject
.
toJSONString
(
ftpModel
));
logger
.
info
(
"调用ftp command模板参数【{}】"
,
JSONObject
.
toJSONString
(
ftpModel
));
this
.
cmdScript
=
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
FTL_SFTP_DOWNLOAD
,
ftpModel
,
freeMarkerConfig
);
this
.
cmdScript
=
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
FTL_SFTP_DOWNLOAD
,
ftpModel
,
freeMarkerConfigurationFactoryBean
);
}
}
...
...
src/main/java/com/jz/dmp/cmdexectool/scheduler/common/task/hdfs/HdfsParameters.java
View file @
8e1fa3fc
...
@@ -26,6 +26,7 @@ import java.util.HashMap;
...
@@ -26,6 +26,7 @@ import java.util.HashMap;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
import
org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
/**
/**
...
@@ -54,7 +55,7 @@ public class HdfsParameters extends AbstractParameters {
...
@@ -54,7 +55,7 @@ public class HdfsParameters extends AbstractParameters {
}
}
public
HdfsParameters
(
String
script
,
FreeMarkerConfigur
er
freeMarkerConfig
)
{
public
HdfsParameters
(
String
script
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
)
{
this
.
script
=
script
;
this
.
script
=
script
;
JSONObject
scriptObj
=
JSONObject
.
parseObject
(
script
);
JSONObject
scriptObj
=
JSONObject
.
parseObject
(
script
);
...
@@ -68,7 +69,7 @@ public class HdfsParameters extends AbstractParameters {
...
@@ -68,7 +69,7 @@ public class HdfsParameters extends AbstractParameters {
HdfsModel
.
put
(
"src_dir"
,
srcDir
);
HdfsModel
.
put
(
"src_dir"
,
srcDir
);
HdfsModel
.
put
(
"file_suffix"
,
fileSuffix
);
HdfsModel
.
put
(
"file_suffix"
,
fileSuffix
);
HdfsModel
.
put
(
"des_dir"
,
desDir
);
HdfsModel
.
put
(
"des_dir"
,
desDir
);
this
.
cmdScript
=
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
FTL_HDFS_UPLOAD
,
HdfsModel
,
freeMarkerConfig
);
this
.
cmdScript
=
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
FTL_HDFS_UPLOAD
,
HdfsModel
,
freeMarkerConfigurationFactoryBean
);
}
}
...
...
src/main/java/com/jz/dmp/cmdexectool/scheduler/common/task/sql/SqlParameters.java
View file @
8e1fa3fc
...
@@ -24,6 +24,7 @@ import java.util.Map;
...
@@ -24,6 +24,7 @@ import java.util.Map;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
com.alibaba.fastjson.JSONObject
;
import
com.alibaba.fastjson.JSONObject
;
import
com.jz.dmp.cmdexectool.common.constant.CommConstant
;
import
com.jz.dmp.cmdexectool.common.constant.CommConstant
;
...
@@ -112,7 +113,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -112,7 +113,7 @@ public class SqlParameters extends AbstractParameters {
*/
*/
private
List
<
ResourceInfo
>
resourceList
;
private
List
<
ResourceInfo
>
resourceList
;
public
SqlParameters
(
String
script
,
DmpProjectConfigInfoDto
projectConfigInfoDto
,
DmpSyncingDatasourceDao
dmpSyncingDatasourceDao
,
FreeMarkerConfigur
er
freeMarkerConfig
,
String
publicKey
)
{
public
SqlParameters
(
String
script
,
DmpProjectConfigInfoDto
projectConfigInfoDto
,
DmpSyncingDatasourceDao
dmpSyncingDatasourceDao
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
,
String
publicKey
)
{
source
=
""
;
source
=
""
;
env
=
""
;
env
=
""
;
sink
=
""
;
sink
=
""
;
...
@@ -131,28 +132,28 @@ public class SqlParameters extends AbstractParameters {
...
@@ -131,28 +132,28 @@ public class SqlParameters extends AbstractParameters {
this
.
executioEngine
=
executioEngine
;
this
.
executioEngine
=
executioEngine
;
//产生evn模板
//产生evn模板
generatorEnvStr
(
freeMarkerConfig
);
generatorEnvStr
(
freeMarkerConfig
urationFactoryBean
);
//source
//source
generatorSourceStr
(
dmpSyncingDatasourceDao
,
freeMarkerConfig
,
publicKey
,
scriptObj
);
generatorSourceStr
(
dmpSyncingDatasourceDao
,
freeMarkerConfig
urationFactoryBean
,
publicKey
,
scriptObj
);
if
(
CommConstant
.
OUTPUT_TYPE_CONSOLE
.
equals
(
outputType
))
{
if
(
CommConstant
.
OUTPUT_TYPE_CONSOLE
.
equals
(
outputType
))
{
//产生console transform and sink
//产生console transform and sink
geneConsoleTransfAndSink
(
freeMarkerConfig
);
geneConsoleTransfAndSink
(
freeMarkerConfig
urationFactoryBean
);
}
else
if
(
CommConstant
.
OUTPUT_TYPE_HDFS
.
equals
(
outputType
))
{
}
else
if
(
CommConstant
.
OUTPUT_TYPE_HDFS
.
equals
(
outputType
))
{
//产生hdfs transform and sink
//产生hdfs transform and sink
geneHdfsTransfAndSink
(
freeMarkerConfig
,
scriptObj
);
geneHdfsTransfAndSink
(
freeMarkerConfig
urationFactoryBean
,
scriptObj
);
}
else
if
(
CommConstant
.
OUTPUT_TYPE_TABLE
.
equals
(
outputType
))
{
}
else
if
(
CommConstant
.
OUTPUT_TYPE_TABLE
.
equals
(
outputType
))
{
//产生table transform and sink
//产生table transform and sink
geneTableTransfAndSink
(
dmpSyncingDatasourceDao
,
freeMarkerConfig
,
scriptObj
,
publicKey
);
geneTableTransfAndSink
(
dmpSyncingDatasourceDao
,
freeMarkerConfig
urationFactoryBean
,
scriptObj
,
publicKey
);
}
else
if
(
CommConstant
.
OUTPUT_TYPE_TOPIC
.
equals
(
outputType
))
{
}
else
if
(
CommConstant
.
OUTPUT_TYPE_TOPIC
.
equals
(
outputType
))
{
//产生topic transform and sink
//产生topic transform and sink
geneTopicTransfAndSink
(
freeMarkerConfig
,
scriptObj
);
geneTopicTransfAndSink
(
freeMarkerConfig
urationFactoryBean
,
scriptObj
);
}
else
if
(
CommConstant
.
OUTPUT_TYPE_API
.
equals
(
outputType
))
{
}
else
if
(
CommConstant
.
OUTPUT_TYPE_API
.
equals
(
outputType
))
{
//产生api transform and sink
//产生api transform and sink
geneApiTransfAndSink
(
freeMarkerConfig
,
scriptObj
);
geneApiTransfAndSink
(
freeMarkerConfig
urationFactoryBean
,
scriptObj
);
}
}
//waterdrop script
//waterdrop script
geneWaterdropStr
(
freeMarkerConfig
);
geneWaterdropStr
(
freeMarkerConfig
urationFactoryBean
);
}
}
...
@@ -163,13 +164,13 @@ public class SqlParameters extends AbstractParameters {
...
@@ -163,13 +164,13 @@ public class SqlParameters extends AbstractParameters {
* @return void 返回类型
* @return void 返回类型
* @throws
* @throws
*/
*/
private
void
geneWaterdropStr
(
FreeMarkerConfigur
er
freeMarkerConfig
)
{
private
void
geneWaterdropStr
(
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
)
{
Map
<
String
,
String
>
waterdropModel
=
new
HashMap
<
String
,
String
>();
Map
<
String
,
String
>
waterdropModel
=
new
HashMap
<
String
,
String
>();
waterdropModel
.
put
(
"env"
,
env
);
waterdropModel
.
put
(
"env"
,
env
);
waterdropModel
.
put
(
"source"
,
source
);
waterdropModel
.
put
(
"source"
,
source
);
waterdropModel
.
put
(
"transform"
,
transform
);
waterdropModel
.
put
(
"transform"
,
transform
);
waterdropModel
.
put
(
"sink"
,
sink
);
waterdropModel
.
put
(
"sink"
,
sink
);
this
.
waterdropScript
=
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
WATERDROP_FTL
,
waterdropModel
,
freeMarkerConfig
);
this
.
waterdropScript
=
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
WATERDROP_FTL
,
waterdropModel
,
freeMarkerConfigurationFactoryBean
);
}
}
/**
/**
...
@@ -181,7 +182,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -181,7 +182,7 @@ public class SqlParameters extends AbstractParameters {
* @return void 返回类型
* @return void 返回类型
* @throws
* @throws
*/
*/
private
void
geneApiTransfAndSink
(
FreeMarkerConfigur
er
freeMarkerConfig
,
JSONObject
scriptObj
)
{
private
void
geneApiTransfAndSink
(
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
,
JSONObject
scriptObj
)
{
// 执行引擎是jdbc,不用生成waterdrop
// 执行引擎是jdbc,不用生成waterdrop
if
(
this
.
executioEngine
.
equals
(
CommConstant
.
EXECUTION_ENGINE_JDBC
))
{
if
(
this
.
executioEngine
.
equals
(
CommConstant
.
EXECUTION_ENGINE_JDBC
))
{
return
;
return
;
...
@@ -200,7 +201,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -200,7 +201,7 @@ public class SqlParameters extends AbstractParameters {
Map
<
String
,
String
>
transformMappingSqlModel
=
new
HashMap
<
String
,
String
>();
Map
<
String
,
String
>
transformMappingSqlModel
=
new
HashMap
<
String
,
String
>();
transformMappingSqlModel
.
put
(
"source_table_name"
,
"t"
);
transformMappingSqlModel
.
put
(
"source_table_name"
,
"t"
);
transformMappingSqlModel
.
put
(
"sql"
,
sqlStr
);
transformMappingSqlModel
.
put
(
"sql"
,
sqlStr
);
transform
=
transform
+
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
WATERDROP_FTL_TRANSFORM_SQL
,
transformMappingSqlModel
,
freeMarkerConfig
);
transform
=
transform
+
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
WATERDROP_FTL_TRANSFORM_SQL
,
transformMappingSqlModel
,
freeMarkerConfigurationFactoryBean
);
//sink
//sink
Map
<
String
,
String
>
sinkApiModel
=
new
HashMap
<
String
,
String
>();
Map
<
String
,
String
>
sinkApiModel
=
new
HashMap
<
String
,
String
>();
...
@@ -211,7 +212,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -211,7 +212,7 @@ public class SqlParameters extends AbstractParameters {
sinkApiModel
.
put
(
"signType"
,
apiObj
.
getString
(
"signType"
));
sinkApiModel
.
put
(
"signType"
,
apiObj
.
getString
(
"signType"
));
sinkApiModel
.
put
(
"authCode"
,
apiObj
.
getString
(
"authCode"
));
sinkApiModel
.
put
(
"authCode"
,
apiObj
.
getString
(
"authCode"
));
sinkApiModel
.
put
(
"salt"
,
apiObj
.
getString
(
"salt"
));
sinkApiModel
.
put
(
"salt"
,
apiObj
.
getString
(
"salt"
));
sink
=
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
WATERDROP_FTL_SINK_API
,
sinkApiModel
,
freeMarkerConfig
);
sink
=
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
WATERDROP_FTL_SINK_API
,
sinkApiModel
,
freeMarkerConfigurationFactoryBean
);
}
}
/**
/**
...
@@ -223,7 +224,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -223,7 +224,7 @@ public class SqlParameters extends AbstractParameters {
* @return void 返回类型
* @return void 返回类型
* @throws
* @throws
*/
*/
private
void
geneTopicTransfAndSink
(
FreeMarkerConfigur
er
freeMarkerConfig
,
JSONObject
scriptObj
)
{
private
void
geneTopicTransfAndSink
(
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
,
JSONObject
scriptObj
)
{
// 执行引擎是jdbc,不用生成waterdrop
// 执行引擎是jdbc,不用生成waterdrop
if
(
this
.
executioEngine
.
equals
(
CommConstant
.
EXECUTION_ENGINE_JDBC
))
{
if
(
this
.
executioEngine
.
equals
(
CommConstant
.
EXECUTION_ENGINE_JDBC
))
{
return
;
return
;
...
@@ -242,7 +243,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -242,7 +243,7 @@ public class SqlParameters extends AbstractParameters {
kafkaModel
.
put
(
"source_table_name"
,
"t"
);
kafkaModel
.
put
(
"source_table_name"
,
"t"
);
kafkaModel
.
put
(
"topic"
,
topicObj
.
getString
(
"topic"
));
kafkaModel
.
put
(
"topic"
,
topicObj
.
getString
(
"topic"
));
kafkaModel
.
put
(
"broker"
,
topicObj
.
getString
(
"server"
));
kafkaModel
.
put
(
"broker"
,
topicObj
.
getString
(
"server"
));
sink
=
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
WATERDROP_FTL_SINK_KAFKA
,
kafkaModel
,
freeMarkerConfig
);
sink
=
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
WATERDROP_FTL_SINK_KAFKA
,
kafkaModel
,
freeMarkerConfigurationFactoryBean
);
}
}
/**
/**
...
@@ -256,7 +257,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -256,7 +257,7 @@ public class SqlParameters extends AbstractParameters {
* @throws
* @throws
*/
*/
private
void
geneTableTransfAndSink
(
DmpSyncingDatasourceDao
dmpSyncingDatasourceDao
,
private
void
geneTableTransfAndSink
(
DmpSyncingDatasourceDao
dmpSyncingDatasourceDao
,
FreeMarkerConfigur
er
freeMarkerConfig
,
JSONObject
scriptObj
,
String
publicKey
)
{
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
,
JSONObject
scriptObj
,
String
publicKey
)
{
JSONObject
tableObj
=
scriptObj
.
getJSONObject
(
"table"
);
JSONObject
tableObj
=
scriptObj
.
getJSONObject
(
"table"
);
...
@@ -306,7 +307,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -306,7 +307,7 @@ public class SqlParameters extends AbstractParameters {
Map
<
String
,
String
>
transformMappingSqlModel
=
new
HashMap
<
String
,
String
>();
Map
<
String
,
String
>
transformMappingSqlModel
=
new
HashMap
<
String
,
String
>();
transformMappingSqlModel
.
put
(
"source_table_name"
,
"t"
);
transformMappingSqlModel
.
put
(
"source_table_name"
,
"t"
);
transformMappingSqlModel
.
put
(
"sql"
,
sqlStr
);
transformMappingSqlModel
.
put
(
"sql"
,
sqlStr
);
transform
=
transform
+
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
WATERDROP_FTL_TRANSFORM_SQL
,
transformMappingSqlModel
,
freeMarkerConfig
);
transform
=
transform
+
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
WATERDROP_FTL_TRANSFORM_SQL
,
transformMappingSqlModel
,
freeMarkerConfigurationFactoryBean
);
//sink
//sink
...
@@ -329,7 +330,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -329,7 +330,7 @@ public class SqlParameters extends AbstractParameters {
sinkJdbcModel
.
put
(
"user"
,
user
);
sinkJdbcModel
.
put
(
"user"
,
user
);
sinkJdbcModel
.
put
(
"password"
,
password
);
sinkJdbcModel
.
put
(
"password"
,
password
);
sinkJdbcModel
.
put
(
"dbtable"
,
targetTableName
);
sinkJdbcModel
.
put
(
"dbtable"
,
targetTableName
);
sink
=
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
WATERDROP_FTL_SINK_JDBC
,
sinkJdbcModel
,
freeMarkerConfig
);
sink
=
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
WATERDROP_FTL_SINK_JDBC
,
sinkJdbcModel
,
freeMarkerConfigurationFactoryBean
);
}
}
}
}
...
@@ -342,7 +343,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -342,7 +343,7 @@ public class SqlParameters extends AbstractParameters {
* @return void 返回类型
* @return void 返回类型
* @throws
* @throws
*/
*/
private
void
geneHdfsTransfAndSink
(
FreeMarkerConfigur
er
freeMarkerConfig
,
JSONObject
scriptObj
)
{
private
void
geneHdfsTransfAndSink
(
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
,
JSONObject
scriptObj
)
{
// 执行引擎是jdbc,不用生成waterdrop
// 执行引擎是jdbc,不用生成waterdrop
if
(
this
.
executioEngine
.
equals
(
CommConstant
.
EXECUTION_ENGINE_JDBC
))
{
if
(
this
.
executioEngine
.
equals
(
CommConstant
.
EXECUTION_ENGINE_JDBC
))
{
return
;
return
;
...
@@ -363,7 +364,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -363,7 +364,7 @@ public class SqlParameters extends AbstractParameters {
hdfsModel
.
put
(
"source_table_name"
,
"t"
);
hdfsModel
.
put
(
"source_table_name"
,
"t"
);
hdfsModel
.
put
(
"path"
,
hdfsDir
);
hdfsModel
.
put
(
"path"
,
hdfsDir
);
hdfsModel
.
put
(
"save_mode"
,
"overwrite"
);
hdfsModel
.
put
(
"save_mode"
,
"overwrite"
);
sink
=
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
WATERDROP_FTL_SINK_HDFS
,
hdfsModel
,
freeMarkerConfig
);
sink
=
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
WATERDROP_FTL_SINK_HDFS
,
hdfsModel
,
freeMarkerConfigurationFactoryBean
);
}
}
/**
/**
...
@@ -374,7 +375,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -374,7 +375,7 @@ public class SqlParameters extends AbstractParameters {
* @return void 返回类型
* @return void 返回类型
* @throws
* @throws
*/
*/
private
void
geneConsoleTransfAndSink
(
FreeMarkerConfigur
er
freeMarkerConfig
)
{
private
void
geneConsoleTransfAndSink
(
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
)
{
//执行引擎是jdbc,不用生成waterdrop
//执行引擎是jdbc,不用生成waterdrop
if
(
this
.
executioEngine
.
equals
(
CommConstant
.
EXECUTION_ENGINE_JDBC
))
{
if
(
this
.
executioEngine
.
equals
(
CommConstant
.
EXECUTION_ENGINE_JDBC
))
{
return
;
return
;
...
@@ -387,7 +388,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -387,7 +388,7 @@ public class SqlParameters extends AbstractParameters {
//sink
//sink
Map
<
String
,
String
>
stdoutModel
=
new
HashMap
<
String
,
String
>();
Map
<
String
,
String
>
stdoutModel
=
new
HashMap
<
String
,
String
>();
stdoutModel
.
put
(
"source_table_name"
,
"t"
);
stdoutModel
.
put
(
"source_table_name"
,
"t"
);
sink
=
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
WATERDROP_FTL_SINK_CONSOLE
,
stdoutModel
,
freeMarkerConfig
);
sink
=
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
WATERDROP_FTL_SINK_CONSOLE
,
stdoutModel
,
freeMarkerConfigurationFactoryBean
);
}
}
/**
/**
...
@@ -401,7 +402,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -401,7 +402,7 @@ public class SqlParameters extends AbstractParameters {
* @throws
* @throws
*/
*/
private
void
generatorSourceStr
(
DmpSyncingDatasourceDao
dmpSyncingDatasourceDao
,
private
void
generatorSourceStr
(
DmpSyncingDatasourceDao
dmpSyncingDatasourceDao
,
FreeMarkerConfigur
er
freeMarkerConfig
,
String
publicKey
,
JSONObject
scriptObj
)
{
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
,
String
publicKey
,
JSONObject
scriptObj
)
{
Integer
sourceId
=
scriptObj
.
getInteger
(
"sourceId"
);
Integer
sourceId
=
scriptObj
.
getInteger
(
"sourceId"
);
DmpSyncingDatasource
dmpSyncingDatasource
=
dmpSyncingDatasourceDao
.
queryById
(
sourceId
);
DmpSyncingDatasource
dmpSyncingDatasource
=
dmpSyncingDatasourceDao
.
queryById
(
sourceId
);
...
@@ -437,15 +438,15 @@ public class SqlParameters extends AbstractParameters {
...
@@ -437,15 +438,15 @@ public class SqlParameters extends AbstractParameters {
jdbcModel
.
put
(
"result_table_name"
,
"t"
);
jdbcModel
.
put
(
"result_table_name"
,
"t"
);
jdbcModel
.
put
(
"user"
,
user
);
jdbcModel
.
put
(
"user"
,
user
);
jdbcModel
.
put
(
"password"
,
password
);
jdbcModel
.
put
(
"password"
,
password
);
this
.
source
=
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_SOURCE_JDBC
,
jdbcModel
,
this
.
source
=
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_SOURCE_JDBC
,
jdbcModel
,
freeMarkerConfig
);
freeMarkerConfig
urationFactoryBean
);
}
else
if
(
this
.
sourceBaseDataSource
.
getMyDbType
()
==
MyDbType
.
Hive
)
{
}
else
if
(
this
.
sourceBaseDataSource
.
getMyDbType
()
==
MyDbType
.
Hive
)
{
Map
<
String
,
String
>
hiveModel
=
new
HashMap
<
String
,
String
>();
Map
<
String
,
String
>
hiveModel
=
new
HashMap
<
String
,
String
>();
hiveModel
.
put
(
"catalogImplementation"
,
"hive"
);
hiveModel
.
put
(
"catalogImplementation"
,
"hive"
);
hiveModel
.
put
(
"pre_sql"
,
"select * from hive_db.hive_table"
);
hiveModel
.
put
(
"pre_sql"
,
"select * from hive_db.hive_table"
);
hiveModel
.
put
(
"result_table_name"
,
"table_view"
);
hiveModel
.
put
(
"result_table_name"
,
"table_view"
);
this
.
source
=
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_SOURCE_HIVE
,
hiveModel
,
this
.
source
=
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_SOURCE_HIVE
,
hiveModel
,
freeMarkerConfig
);
freeMarkerConfig
urationFactoryBean
);
}
else
{
}
else
{
logger
.
info
(
"waterdrow,不支持的数据源类型"
);
logger
.
info
(
"waterdrow,不支持的数据源类型"
);
throw
new
RuntimeException
(
"waterdrow,不支持的数据源"
);
throw
new
RuntimeException
(
"waterdrow,不支持的数据源"
);
...
@@ -460,7 +461,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -460,7 +461,7 @@ public class SqlParameters extends AbstractParameters {
* @return void 返回类型
* @return void 返回类型
* @throws
* @throws
*/
*/
private
void
generatorEnvStr
(
FreeMarkerConfigur
er
freeMarkerConfig
)
{
private
void
generatorEnvStr
(
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
)
{
//如果执行引擎选择的事jdbc,不用生成env
//如果执行引擎选择的事jdbc,不用生成env
if
(
this
.
executioEngine
.
equals
(
CommConstant
.
EXECUTION_ENGINE_JDBC
))
{
if
(
this
.
executioEngine
.
equals
(
CommConstant
.
EXECUTION_ENGINE_JDBC
))
{
return
;
return
;
...
@@ -468,7 +469,7 @@ public class SqlParameters extends AbstractParameters {
...
@@ -468,7 +469,7 @@ public class SqlParameters extends AbstractParameters {
Map
<
String
,
String
>
envModel
=
new
HashMap
<
String
,
String
>();
Map
<
String
,
String
>
envModel
=
new
HashMap
<
String
,
String
>();
envModel
.
put
(
"sparkappname"
,
"Waterdrop"
);
envModel
.
put
(
"sparkappname"
,
"Waterdrop"
);
this
.
env
=
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
WATERDROP_FTL_ENV
,
envModel
,
freeMarkerConfig
);
this
.
env
=
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
WATERDROP_FTL_ENV
,
envModel
,
freeMarkerConfigurationFactoryBean
);
}
}
public
String
getScript
()
{
public
String
getScript
()
{
...
...
src/main/java/com/jz/dmp/cmdexectool/scheduler/common/task/sync/SyncParameters.java
View file @
8e1fa3fc
...
@@ -12,6 +12,7 @@ import com.jz.dmp.cmdexectool.scheduler.common.enums.MyDbType;
...
@@ -12,6 +12,7 @@ import com.jz.dmp.cmdexectool.scheduler.common.enums.MyDbType;
import
com.jz.dmp.cmdexectool.scheduler.common.process.ResourceInfo
;
import
com.jz.dmp.cmdexectool.scheduler.common.process.ResourceInfo
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.AbstractParameters
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.AbstractParameters
;
import
com.jz.dmp.cmdexectool.scheduler.dao.datasource.MyBaseDataSource
;
import
com.jz.dmp.cmdexectool.scheduler.dao.datasource.MyBaseDataSource
;
import
org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
...
@@ -85,7 +86,7 @@ public class SyncParameters extends AbstractParameters {
...
@@ -85,7 +86,7 @@ public class SyncParameters extends AbstractParameters {
*/
*/
private
MyBaseDataSource
targetBaseDataSource
;
private
MyBaseDataSource
targetBaseDataSource
;
public
SyncParameters
(
String
script
,
DmpProjectConfigInfoDto
projectConfigInfoDto
,
DmpSyncingDatasourceDao
dmpSyncingDatasourceDao
,
FreeMarkerConfigur
er
freeMarkerConfig
,
String
publicKey
)
throws
Exception
{
public
SyncParameters
(
String
script
,
DmpProjectConfigInfoDto
projectConfigInfoDto
,
DmpSyncingDatasourceDao
dmpSyncingDatasourceDao
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfig
,
String
publicKey
)
throws
Exception
{
source
=
""
;
source
=
""
;
env
=
""
;
env
=
""
;
sink
=
""
;
sink
=
""
;
...
@@ -99,7 +100,7 @@ public class SyncParameters extends AbstractParameters {
...
@@ -99,7 +100,7 @@ public class SyncParameters extends AbstractParameters {
//evn
//evn
Map
<
String
,
String
>
envModel
=
new
HashMap
<
String
,
String
>();
Map
<
String
,
String
>
envModel
=
new
HashMap
<
String
,
String
>();
envModel
.
put
(
"sparkappname"
,
"Waterdrop"
);
envModel
.
put
(
"sparkappname"
,
"Waterdrop"
);
env
=
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_ENV
,
envModel
,
freeMarkerConfig
);
env
=
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_ENV
,
envModel
,
freeMarkerConfig
);
//target information
//target information
Integer
targetDbId
=
Integer
.
valueOf
((
String
)
targetObj
.
get
(
"targetDbId"
));
Integer
targetDbId
=
Integer
.
valueOf
((
String
)
targetObj
.
get
(
"targetDbId"
));
...
@@ -175,25 +176,25 @@ public class SyncParameters extends AbstractParameters {
...
@@ -175,25 +176,25 @@ public class SyncParameters extends AbstractParameters {
waterdropModel
.
put
(
"source"
,
source
);
waterdropModel
.
put
(
"source"
,
source
);
waterdropModel
.
put
(
"transform"
,
transform
);
waterdropModel
.
put
(
"transform"
,
transform
);
waterdropModel
.
put
(
"sink"
,
sink
);
waterdropModel
.
put
(
"sink"
,
sink
);
waterdropScript
=
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL
,
waterdropModel
,
freeMarkerConfig
);
waterdropScript
=
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL
,
waterdropModel
,
freeMarkerConfig
);
}
}
private
void
getSinkKudu
(
DmpSyncingDatasource
targetDatasource
,
String
targetTable
,
FreeMarkerConfigur
er
freeMarkerConfig
)
{
private
void
getSinkKudu
(
DmpSyncingDatasource
targetDatasource
,
String
targetTable
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfig
)
{
Map
kuduModel
=
new
HashMap
();
Map
kuduModel
=
new
HashMap
();
kuduModel
.
put
(
"kuduMaster"
,
targetDatasource
.
getHost
()
+
":"
+
targetDatasource
.
getPort
());
//主机名
kuduModel
.
put
(
"kuduMaster"
,
targetDatasource
.
getHost
()
+
":"
+
targetDatasource
.
getPort
());
//主机名
kuduModel
.
put
(
"result_table_name"
,
targetTable
);
//spark生成的临时表名
kuduModel
.
put
(
"result_table_name"
,
targetTable
);
//spark生成的临时表名
source
=
source
+
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_SOURCE_SFTP
,
kuduModel
,
freeMarkerConfig
);
source
=
source
+
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_SOURCE_SFTP
,
kuduModel
,
freeMarkerConfig
);
}
}
private
void
getSourceKudu
(
DmpSyncingDatasource
dmpSyncingDatasource
,
String
[]
registerTableName
,
FreeMarkerConfigur
er
freeMarkerConfig
,
int
i
)
{
private
void
getSourceKudu
(
DmpSyncingDatasource
dmpSyncingDatasource
,
String
[]
registerTableName
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfig
,
int
i
)
{
String
tableName
=
registerTableName
[
i
];
String
tableName
=
registerTableName
[
i
];
Map
kuduModel
=
new
HashMap
();
Map
kuduModel
=
new
HashMap
();
kuduModel
.
put
(
"kuduMaster"
,
dmpSyncingDatasource
.
getHost
()
+
":"
+
dmpSyncingDatasource
.
getPort
());
//主机名
kuduModel
.
put
(
"kuduMaster"
,
dmpSyncingDatasource
.
getHost
()
+
":"
+
dmpSyncingDatasource
.
getPort
());
//主机名
kuduModel
.
put
(
"result_table_name"
,
tableName
);
//spark生成的临时表名
kuduModel
.
put
(
"result_table_name"
,
tableName
);
//spark生成的临时表名
source
=
source
+
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_SOURCE_SFTP
,
kuduModel
,
freeMarkerConfig
);
source
=
source
+
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_SOURCE_SFTP
,
kuduModel
,
freeMarkerConfig
);
}
}
private
void
getSinkSftp
(
String
targetTable
,
DmpSyncingDatasource
targetDatasource
,
String
publicKey
,
Map
<
String
,
Object
>
targetObj
,
FreeMarkerConfigur
er
freeMarkerConfig
)
{
private
void
getSinkSftp
(
String
targetTable
,
DmpSyncingDatasource
targetDatasource
,
String
publicKey
,
Map
<
String
,
Object
>
targetObj
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfig
)
{
Map
sftpModel
=
new
HashMap
();
Map
sftpModel
=
new
HashMap
();
sftpModel
.
put
(
"host"
,
targetDatasource
.
getHost
());
//主机名
sftpModel
.
put
(
"host"
,
targetDatasource
.
getHost
());
//主机名
sftpModel
.
put
(
"user"
,
targetDatasource
.
getUserName
());
//用户
sftpModel
.
put
(
"user"
,
targetDatasource
.
getUserName
());
//用户
...
@@ -206,10 +207,10 @@ public class SyncParameters extends AbstractParameters {
...
@@ -206,10 +207,10 @@ public class SyncParameters extends AbstractParameters {
sftpModel
.
put
(
"path"
,
targetObj
.
get
(
"sourceFtpDir"
));
//文件路径
sftpModel
.
put
(
"path"
,
targetObj
.
get
(
"sourceFtpDir"
));
//文件路径
}
}
sftpModel
.
put
(
"result_table_name"
,
targetTable
);
//spark生成的临时表名
sftpModel
.
put
(
"result_table_name"
,
targetTable
);
//spark生成的临时表名
source
=
source
+
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_SOURCE_SFTP
,
sftpModel
,
freeMarkerConfig
);
source
=
source
+
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_SOURCE_SFTP
,
sftpModel
,
freeMarkerConfig
);
}
}
private
void
getSyncTransform
(
List
<
Map
<
String
,
Object
>>
mappingObj
,
FreeMarkerConfigur
er
freeMarkerConfig
,
String
source_table_name
)
{
private
void
getSyncTransform
(
List
<
Map
<
String
,
Object
>>
mappingObj
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfig
,
String
source_table_name
)
{
String
sourceField
=
""
;
String
sourceField
=
""
;
String
targetField
=
""
;
String
targetField
=
""
;
for
(
Map
<
String
,
Object
>
item
:
mappingObj
)
{
for
(
Map
<
String
,
Object
>
item
:
mappingObj
)
{
...
@@ -224,7 +225,7 @@ public class SyncParameters extends AbstractParameters {
...
@@ -224,7 +225,7 @@ public class SyncParameters extends AbstractParameters {
.
append
(
" $t "
);
.
append
(
" $t "
);
sqlModel
.
put
(
"sql"
,
sql
);
sqlModel
.
put
(
"sql"
,
sql
);
sqlModel
.
put
(
"source_table_name"
,
source_table_name
);
sqlModel
.
put
(
"source_table_name"
,
source_table_name
);
transform
=
transform
+
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_TRANSFORM_SQL
,
sqlModel
,
freeMarkerConfig
);
transform
=
transform
+
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_TRANSFORM_SQL
,
sqlModel
,
freeMarkerConfig
);
Map
targetModel
=
new
HashMap
();
Map
targetModel
=
new
HashMap
();
StringBuilder
targetSql
=
new
StringBuilder
()
StringBuilder
targetSql
=
new
StringBuilder
()
...
@@ -234,10 +235,10 @@ public class SyncParameters extends AbstractParameters {
...
@@ -234,10 +235,10 @@ public class SyncParameters extends AbstractParameters {
.
append
(
" $t "
);
.
append
(
" $t "
);
targetModel
.
put
(
"sql"
,
targetSql
);
targetModel
.
put
(
"sql"
,
targetSql
);
targetModel
.
put
(
"source_table_name"
,
source_table_name
);
targetModel
.
put
(
"source_table_name"
,
source_table_name
);
transform
=
transform
+
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_TRANSFORM_SQL
,
targetModel
,
freeMarkerConfig
);
transform
=
transform
+
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_TRANSFORM_SQL
,
targetModel
,
freeMarkerConfig
);
}
}
private
void
getsourceElasticsearch
(
String
[]
registerTableName
,
DmpSyncingDatasource
dmpSyncingDatasource
,
FreeMarkerConfigur
er
freeMarkerConfig
,
int
i
)
{
private
void
getsourceElasticsearch
(
String
[]
registerTableName
,
DmpSyncingDatasource
dmpSyncingDatasource
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfig
,
int
i
)
{
//source
//source
String
tableName
=
registerTableName
[
i
];
String
tableName
=
registerTableName
[
i
];
List
<
String
>
list
=
new
ArrayList
<>();
List
<
String
>
list
=
new
ArrayList
<>();
...
@@ -248,19 +249,19 @@ public class SyncParameters extends AbstractParameters {
...
@@ -248,19 +249,19 @@ public class SyncParameters extends AbstractParameters {
jdbcModel
.
put
(
"result_table_name"
,
tableName
);
jdbcModel
.
put
(
"result_table_name"
,
tableName
);
//jdbcModel.put("index", "");
//jdbcModel.put("index", "");
//jdbcModel.put("name_age", "");
//jdbcModel.put("name_age", "");
source
=
source
+
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_SOURCE_ELASTICSEARCH
,
jdbcModel
,
freeMarkerConfig
);
source
=
source
+
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_SOURCE_ELASTICSEARCH
,
jdbcModel
,
freeMarkerConfig
);
}
}
public
void
getSourceHive
(
Map
<
String
,
String
>
envModel
,
FreeMarkerConfigur
er
freeMarkerConfig
,
String
[]
registerTableName
,
int
i
)
{
public
void
getSourceHive
(
Map
<
String
,
String
>
envModel
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfig
,
String
[]
registerTableName
,
int
i
)
{
//source
//source
String
tableName
=
registerTableName
[
i
];
String
tableName
=
registerTableName
[
i
];
Map
hiveModel
=
new
HashMap
();
Map
hiveModel
=
new
HashMap
();
hiveModel
.
put
(
"pre_sql"
,
" select * from "
+
tableName
);
hiveModel
.
put
(
"pre_sql"
,
" select * from "
+
tableName
);
hiveModel
.
put
(
"result_table_name"
,
tableName
);
hiveModel
.
put
(
"result_table_name"
,
tableName
);
source
=
source
+
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_SOURCE_JDBC
,
hiveModel
,
freeMarkerConfig
);
source
=
source
+
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_SOURCE_JDBC
,
hiveModel
,
freeMarkerConfig
);
}
}
public
void
getJdbcSource
(
DmpSyncingDatasource
dmpSyncingDatasource
,
String
[]
registerTableName
,
FreeMarkerConfigur
er
freeMarkerConfig
,
String
publicKey
,
int
i
,
Map
<
String
,
Object
>
sourceObj
)
{
public
void
getJdbcSource
(
DmpSyncingDatasource
dmpSyncingDatasource
,
String
[]
registerTableName
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfig
,
String
publicKey
,
int
i
,
Map
<
String
,
Object
>
sourceObj
)
{
String
tableName
=
registerTableName
[
i
];
String
tableName
=
registerTableName
[
i
];
//for (String tableName : registerTableName) {
//for (String tableName : registerTableName) {
Map
jdbcModel
=
new
HashMap
();
Map
jdbcModel
=
new
HashMap
();
...
@@ -275,11 +276,11 @@ public class SyncParameters extends AbstractParameters {
...
@@ -275,11 +276,11 @@ public class SyncParameters extends AbstractParameters {
//jdbcModel.put("numPartitions", "");
//jdbcModel.put("numPartitions", "");
//jdbcModel.put("lowerBound", "");
//jdbcModel.put("lowerBound", "");
//jdbcModel.put("upperBound", "");
//jdbcModel.put("upperBound", "");
source
=
source
+
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_SOURCE_JDBC
,
jdbcModel
,
freeMarkerConfig
);
source
=
source
+
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_SOURCE_JDBC
,
jdbcModel
,
freeMarkerConfig
);
//}
//}
}
}
public
void
getJdbcSink
(
DmpSyncingDatasource
targetDatasource
,
Map
<
String
,
Object
>
targetObj
,
FreeMarkerConfigur
er
freeMarkerConfig
,
String
publicKey
,
String
source_table_name
)
{
public
void
getJdbcSink
(
DmpSyncingDatasource
targetDatasource
,
Map
<
String
,
Object
>
targetObj
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfig
,
String
publicKey
,
String
source_table_name
)
{
String
postImportStatement
=
String
.
valueOf
(
targetObj
.
get
(
"postImportStatement"
));
//导入后语句
String
postImportStatement
=
String
.
valueOf
(
targetObj
.
get
(
"postImportStatement"
));
//导入后语句
String
preImportStatement
=
String
.
valueOf
(
targetObj
.
get
(
"preImportStatement"
));
//导入前语句
String
preImportStatement
=
String
.
valueOf
(
targetObj
.
get
(
"preImportStatement"
));
//导入前语句
preStatements
=
new
ArrayList
<
String
>();
preStatements
=
new
ArrayList
<
String
>();
...
@@ -313,10 +314,10 @@ public class SyncParameters extends AbstractParameters {
...
@@ -313,10 +314,10 @@ public class SyncParameters extends AbstractParameters {
jdbcSinkModel
.
put
(
"user"
,
targetDatasource
.
getUserName
());
jdbcSinkModel
.
put
(
"user"
,
targetDatasource
.
getUserName
());
jdbcSinkModel
.
put
(
"password"
,
password
);
jdbcSinkModel
.
put
(
"password"
,
password
);
jdbcSinkModel
.
put
(
"source_table_name"
,
source_table_name
);
jdbcSinkModel
.
put
(
"source_table_name"
,
source_table_name
);
sink
=
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_SINK_JDBC
,
jdbcSinkModel
,
freeMarkerConfig
);
sink
=
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_SINK_JDBC
,
jdbcSinkModel
,
freeMarkerConfig
);
}
}
public
void
getSourceSftp
(
String
[]
registerTableName
,
DmpSyncingDatasource
dmpSyncingDatasource
,
String
publicKey
,
Map
<
String
,
Object
>
sourceObj
,
FreeMarkerConfigur
er
freeMarkerConfig
,
int
i
)
{
public
void
getSourceSftp
(
String
[]
registerTableName
,
DmpSyncingDatasource
dmpSyncingDatasource
,
String
publicKey
,
Map
<
String
,
Object
>
sourceObj
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfig
,
int
i
)
{
String
tableName
=
registerTableName
[
i
];
String
tableName
=
registerTableName
[
i
];
//for (String tableName : registerTableName) {
//for (String tableName : registerTableName) {
Map
sftpModel
=
new
HashMap
();
Map
sftpModel
=
new
HashMap
();
...
@@ -337,7 +338,7 @@ public class SyncParameters extends AbstractParameters {
...
@@ -337,7 +338,7 @@ public class SyncParameters extends AbstractParameters {
sftpModel
.
put
(
"path"
,
sourceObj
.
get
(
"sourceFtpDir"
).
toString
());
//文件路径
sftpModel
.
put
(
"path"
,
sourceObj
.
get
(
"sourceFtpDir"
).
toString
());
//文件路径
}
}
sftpModel
.
put
(
"result_table_name"
,
tableName
);
//spark生成的临时表名
sftpModel
.
put
(
"result_table_name"
,
tableName
);
//spark生成的临时表名
source
=
source
+
FreeMarkerUtils
.
freemakerJson
(
CommConstant
.
WATERDROP_FTL_SOURCE_SFTP
,
sftpModel
,
freeMarkerConfig
);
source
=
source
+
FreeMarkerUtils
.
freemaker
NoneWeb
Json
(
CommConstant
.
WATERDROP_FTL_SOURCE_SFTP
,
sftpModel
,
freeMarkerConfig
);
//}
//}
}
}
...
...
src/main/java/com/jz/dmp/cmdexectool/scheduler/common/task/unzipfile/UnzipfileParameters.java
View file @
8e1fa3fc
...
@@ -26,6 +26,7 @@ import java.util.HashMap;
...
@@ -26,6 +26,7 @@ import java.util.HashMap;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
import
org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
/**
/**
...
@@ -54,7 +55,7 @@ public class UnzipfileParameters extends AbstractParameters {
...
@@ -54,7 +55,7 @@ public class UnzipfileParameters extends AbstractParameters {
}
}
public
UnzipfileParameters
(
String
script
,
FreeMarkerConfigur
er
freeMarkerConfig
)
{
public
UnzipfileParameters
(
String
script
,
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
)
{
this
.
script
=
script
;
this
.
script
=
script
;
JSONObject
scriptObj
=
JSONObject
.
parseObject
(
script
);
JSONObject
scriptObj
=
JSONObject
.
parseObject
(
script
);
...
@@ -69,7 +70,7 @@ public class UnzipfileParameters extends AbstractParameters {
...
@@ -69,7 +70,7 @@ public class UnzipfileParameters extends AbstractParameters {
unzipfileModel
.
put
(
"src_dir"
,
srcDir
);
unzipfileModel
.
put
(
"src_dir"
,
srcDir
);
unzipfileModel
.
put
(
"des_dir"
,
desDir
);
unzipfileModel
.
put
(
"des_dir"
,
desDir
);
unzipfileModel
.
put
(
"type"
,
type
);
unzipfileModel
.
put
(
"type"
,
type
);
this
.
cmdScript
=
FreeMarkerUtils
.
freemaker
Json
(
CommConstant
.
FTL_UNZIPFILE
,
unzipfileModel
,
freeMarkerConfig
);
this
.
cmdScript
=
FreeMarkerUtils
.
freemaker
NoneWebJson
(
CommConstant
.
FTL_UNZIPFILE
,
unzipfileModel
,
freeMarkerConfigurationFactoryBean
);
}
}
...
...
src/main/java/com/jz/dmp/cmdexectool/scheduler/service/process/ProcessService.java
View file @
8e1fa3fc
...
@@ -21,13 +21,14 @@ import java.util.HashMap;
...
@@ -21,13 +21,14 @@ import java.util.HashMap;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.sync.SyncParameters
;
import
org.slf4j.Logger
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.stereotype.Component
;
import
org.springframework.stereotype.Component
;
import
org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean
;
import
org.springframework.util.CollectionUtils
;
import
org.springframework.util.CollectionUtils
;
//import org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer
;
import
com.alibaba.fastjson.JSONArray
;
import
com.alibaba.fastjson.JSONArray
;
...
@@ -49,6 +50,7 @@ import com.jz.dmp.cmdexectool.scheduler.common.task.ftp.FtpParameters;
...
@@ -49,6 +50,7 @@ import com.jz.dmp.cmdexectool.scheduler.common.task.ftp.FtpParameters;
import
com.jz.dmp.cmdexectool.scheduler.common.task.hdfs.HdfsParameters
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.hdfs.HdfsParameters
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.shell.ShellParameters
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.shell.ShellParameters
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlParameters
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlParameters
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.sync.SyncParameters
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.unzipfile.UnzipfileParameters
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.unzipfile.UnzipfileParameters
;
import
com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext
;
import
com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext
;
import
com.jz.dmp.cmdexectool.scheduler.server.worker.task.AbstractTask
;
import
com.jz.dmp.cmdexectool.scheduler.server.worker.task.AbstractTask
;
...
@@ -73,8 +75,10 @@ public class ProcessService {
...
@@ -73,8 +75,10 @@ public class ProcessService {
private
DmpProjectConfigInfoMapper
dmpProjectConfigInfoMapper
;
private
DmpProjectConfigInfoMapper
dmpProjectConfigInfoMapper
;
@Autowired
@Autowired
private
DmpSyncingDatasourceDao
dmpSyncingDatasourceDao
;
private
DmpSyncingDatasourceDao
dmpSyncingDatasourceDao
;
//@Autowired
//private FreeMarkerConfigurer freeMarkerConfigurer;
@Autowired
@Autowired
private
FreeMarkerConfigur
er
freeMarkerConfigurer
;
private
FreeMarkerConfigur
ationFactoryBean
freeMarkerConfigurationFactoryBean
;
/**
/**
* @Title: taskStart @Description: TODO(启动task) @param 参数 @return void
* @Title: taskStart @Description: TODO(启动task) @param 参数 @return void
...
@@ -212,37 +216,37 @@ public class ProcessService {
...
@@ -212,37 +216,37 @@ public class ProcessService {
break
;
break
;
case
sql:
case
sql:
SqlParameters
sqlParameters
=
new
SqlParameters
(
script
,
projectConfigInfoDto
,
dmpSyncingDatasourceDao
,
freeMarkerConfigur
er
,
publicKey
);
SqlParameters
sqlParameters
=
new
SqlParameters
(
script
,
projectConfigInfoDto
,
dmpSyncingDatasourceDao
,
freeMarkerConfigur
ationFactoryBean
,
publicKey
);
sqlParameters
.
setTaskAppId
(
taskAppId
);
sqlParameters
.
setTaskAppId
(
taskAppId
);
taskExecutionContext
=
new
TaskExecutionContext
(
sqlParameters
,
projectConfigInfoDto
);
taskExecutionContext
=
new
TaskExecutionContext
(
sqlParameters
,
projectConfigInfoDto
);
break
;
break
;
case
sync:
case
sync:
SyncParameters
sync
=
new
SyncParameters
(
script
,
projectConfigInfoDto
,
dmpSyncingDatasourceDao
,
freeMarkerConfigur
er
,
publicKey
);
SyncParameters
sync
=
new
SyncParameters
(
script
,
projectConfigInfoDto
,
dmpSyncingDatasourceDao
,
freeMarkerConfigur
ationFactoryBean
,
publicKey
);
sync
.
setTaskAppId
(
taskAppId
);
sync
.
setTaskAppId
(
taskAppId
);
taskExecutionContext
=
new
TaskExecutionContext
(
sync
,
projectConfigInfoDto
);
taskExecutionContext
=
new
TaskExecutionContext
(
sync
,
projectConfigInfoDto
);
break
;
break
;
case
subprocess:
case
subprocess:
break
;
break
;
case
ftp:
case
ftp:
FtpParameters
ftpParameters
=
new
FtpParameters
(
script
,
freeMarkerConfigur
er
);
FtpParameters
ftpParameters
=
new
FtpParameters
(
script
,
freeMarkerConfigur
ationFactoryBean
);
ftpParameters
.
setTaskAppId
(
taskAppId
);
ftpParameters
.
setTaskAppId
(
taskAppId
);
taskExecutionContext
=
new
TaskExecutionContext
(
ftpParameters
,
projectConfigInfoDto
);
taskExecutionContext
=
new
TaskExecutionContext
(
ftpParameters
,
projectConfigInfoDto
);
break
;
break
;
case
unzipFile:
case
unzipFile:
UnzipfileParameters
unzipfileParameters
=
new
UnzipfileParameters
(
script
,
freeMarkerConfigur
er
);
UnzipfileParameters
unzipfileParameters
=
new
UnzipfileParameters
(
script
,
freeMarkerConfigur
ationFactoryBean
);
unzipfileParameters
.
setScript
(
script
);
unzipfileParameters
.
setScript
(
script
);
unzipfileParameters
.
setTaskAppId
(
taskAppId
);
unzipfileParameters
.
setTaskAppId
(
taskAppId
);
taskExecutionContext
=
new
TaskExecutionContext
(
unzipfileParameters
,
projectConfigInfoDto
);
taskExecutionContext
=
new
TaskExecutionContext
(
unzipfileParameters
,
projectConfigInfoDto
);
break
;
break
;
case
docTrans:
case
docTrans:
DoctransParameters
doctransParameters
=
new
DoctransParameters
(
script
,
freeMarkerConfigur
er
);
DoctransParameters
doctransParameters
=
new
DoctransParameters
(
script
,
freeMarkerConfigur
ationFactoryBean
);
doctransParameters
.
setScript
(
script
);
doctransParameters
.
setScript
(
script
);
doctransParameters
.
setTaskAppId
(
taskAppId
);
doctransParameters
.
setTaskAppId
(
taskAppId
);
taskExecutionContext
=
new
TaskExecutionContext
(
doctransParameters
,
projectConfigInfoDto
);
taskExecutionContext
=
new
TaskExecutionContext
(
doctransParameters
,
projectConfigInfoDto
);
break
;
break
;
case
hdfs:
case
hdfs:
HdfsParameters
hdfsParameters
=
new
HdfsParameters
(
script
,
freeMarkerConfigur
er
);
HdfsParameters
hdfsParameters
=
new
HdfsParameters
(
script
,
freeMarkerConfigur
ationFactoryBean
);
hdfsParameters
.
setScript
(
script
);
hdfsParameters
.
setScript
(
script
);
hdfsParameters
.
setTaskAppId
(
taskAppId
);
hdfsParameters
.
setTaskAppId
(
taskAppId
);
taskExecutionContext
=
new
TaskExecutionContext
(
hdfsParameters
,
projectConfigInfoDto
);
taskExecutionContext
=
new
TaskExecutionContext
(
hdfsParameters
,
projectConfigInfoDto
);
...
...
src/test/java/com/jz/cmdexectool/test/task/shell/SQLCommandExecutorTest.java
View file @
8e1fa3fc
...
@@ -16,35 +16,12 @@
...
@@ -16,35 +16,12 @@
*/
*/
package
com
.
jz
.
cmdexectool
.
test
.
task
.
shell
;
package
com
.
jz
.
cmdexectool
.
test
.
task
.
shell
;
import
com.alibaba.fastjson.JSONObject
;
import
static
com
.
jz
.
dmp
.
cmdexectool
.
scheduler
.
common
.
Constants
.
HIVE_CONF
;
import
com.fasterxml.jackson.databind.node.ArrayNode
;
import
static
com
.
jz
.
dmp
.
cmdexectool
.
scheduler
.
common
.
Constants
.
PASSWORD
;
import
com.fasterxml.jackson.databind.node.ObjectNode
;
import
static
com
.
jz
.
dmp
.
cmdexectool
.
scheduler
.
common
.
Constants
.
SEMICOLON
;
import
com.jz.dmp.cmdexectool.ApiApplication
;
import
static
com
.
jz
.
dmp
.
cmdexectool
.
scheduler
.
common
.
Constants
.
USER
;
import
com.jz.dmp.cmdexectool.scheduler.common.Constants
;
import
static
com
.
jz
.
dmp
.
cmdexectool
.
scheduler
.
common
.
enums
.
DbType
.
HIVE
;
import
com.jz.dmp.cmdexectool.scheduler.common.enums.DbType
;
import
com.jz.dmp.cmdexectool.scheduler.common.enums.ExecutionStatus
;
import
com.jz.dmp.cmdexectool.scheduler.common.enums.MyDbType
;
import
com.jz.dmp.cmdexectool.scheduler.common.process.Property
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlBinds
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlType
;
import
com.jz.dmp.cmdexectool.scheduler.common.utils.CollectionUtils
;
import
com.jz.dmp.cmdexectool.scheduler.common.utils.CommonUtils
;
import
com.jz.dmp.cmdexectool.scheduler.common.utils.JSONUtils
;
import
com.jz.dmp.cmdexectool.scheduler.common.utils.ParameterUtils
;
import
com.jz.dmp.cmdexectool.scheduler.dao.datasource.BaseDataSource
;
import
com.jz.dmp.cmdexectool.scheduler.dao.datasource.DataSourceFactory
;
import
com.jz.dmp.cmdexectool.scheduler.dao.datasource.MyBaseDataSource
;
import
com.jz.dmp.cmdexectool.scheduler.dao.datasource.MySQLDataSource
;
import
com.jz.dmp.cmdexectool.scheduler.dao.utils.DatabaseUtils
;
import
com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext
;
import
com.jz.dmp.cmdexectool.scheduler.server.utils.ParamUtils
;
import
com.jz.dmp.cmdexectool.scheduler.server.worker.task.AbstractTask
;
import
com.jz.dmp.cmdexectool.scheduler.server.worker.task.TaskManager
;
import
com.jz.dmp.cmdexectool.scheduler.service.process.ProcessService
;
import
java.io.File
;
import
java.nio.file.Files
;
import
java.nio.file.Path
;
import
java.sql.Connection
;
import
java.sql.Connection
;
import
java.sql.DriverManager
;
import
java.sql.DriverManager
;
import
java.sql.PreparedStatement
;
import
java.sql.PreparedStatement
;
...
@@ -72,8 +49,28 @@ import org.springframework.beans.factory.annotation.Autowired;
...
@@ -72,8 +49,28 @@ import org.springframework.beans.factory.annotation.Autowired;
import
org.springframework.boot.test.context.SpringBootTest
;
import
org.springframework.boot.test.context.SpringBootTest
;
import
org.springframework.test.context.junit4.SpringRunner
;
import
org.springframework.test.context.junit4.SpringRunner
;
import
static
com
.
jz
.
dmp
.
cmdexectool
.
scheduler
.
common
.
Constants
.*;
import
com.fasterxml.jackson.databind.node.ArrayNode
;
import
static
com
.
jz
.
dmp
.
cmdexectool
.
scheduler
.
common
.
enums
.
DbType
.
HIVE
;
import
com.fasterxml.jackson.databind.node.ObjectNode
;
import
com.jz.dmp.cmdexectool.ApiApplication
;
import
com.jz.dmp.cmdexectool.scheduler.common.Constants
;
import
com.jz.dmp.cmdexectool.scheduler.common.enums.DbType
;
import
com.jz.dmp.cmdexectool.scheduler.common.enums.ExecutionStatus
;
import
com.jz.dmp.cmdexectool.scheduler.common.enums.MyDbType
;
import
com.jz.dmp.cmdexectool.scheduler.common.process.Property
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlBinds
;
import
com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlType
;
import
com.jz.dmp.cmdexectool.scheduler.common.utils.CollectionUtils
;
import
com.jz.dmp.cmdexectool.scheduler.common.utils.CommonUtils
;
import
com.jz.dmp.cmdexectool.scheduler.common.utils.JSONUtils
;
import
com.jz.dmp.cmdexectool.scheduler.common.utils.ParameterUtils
;
import
com.jz.dmp.cmdexectool.scheduler.dao.datasource.DataSourceFactory
;
import
com.jz.dmp.cmdexectool.scheduler.dao.datasource.MyBaseDataSource
;
import
com.jz.dmp.cmdexectool.scheduler.dao.utils.DatabaseUtils
;
import
com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext
;
import
com.jz.dmp.cmdexectool.scheduler.server.utils.ParamUtils
;
import
com.jz.dmp.cmdexectool.scheduler.server.worker.task.AbstractTask
;
import
com.jz.dmp.cmdexectool.scheduler.server.worker.task.TaskManager
;
import
com.jz.dmp.cmdexectool.scheduler.service.process.ProcessService
;
/**
/**
* python shell command executor test
* python shell command executor test
...
@@ -89,6 +86,11 @@ public class SQLCommandExecutorTest {
...
@@ -89,6 +86,11 @@ public class SQLCommandExecutorTest {
private
static
final
int
LIMIT
=
10000
;
private
static
final
int
LIMIT
=
10000
;
@Test
public
void
test22
()
{
System
.
out
.
println
(
"test"
);
}
//@Test
//@Test
public
void
test
()
{
public
void
test
()
{
try
{
try
{
...
@@ -121,11 +123,6 @@ public class SQLCommandExecutorTest {
...
@@ -121,11 +123,6 @@ public class SQLCommandExecutorTest {
}
}
//@Test
//@Test
public
void
test22
()
{
System
.
out
.
println
(
"test"
);
}
@Test
public
void
testJdbcHandler
()
{
public
void
testJdbcHandler
()
{
// set the name of the current thread
// set the name of the current thread
//String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, "test");
//String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, "test");
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment