Skip to content

J
jz-dmp-service
Commit be5c85f8 authored by mcb's avatar mcb
Browse files
Options

no message

parent f353dd2b
Hide whitespace changes
Inline Side-by-side
Showing with 109 additions and 91 deletions
src/main/java/com/jz/dmp/modules/service/impl/DmpDevelopTaskServiceImpl.java
View file @ be5c85f8
......@@ -128,7 +128,7 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
String _extract = (String) settingMap.get("extract");
String _extractExpression = (String) settingMap.get("extractExpression");
String _targetBucketCounts = (String) settingMap.get("targetBucketCounts");
String _errorLimitRecord = (String) settingMap.get("errorLimitRecord");
String _errorLimitRecord = (String) settingMap.get("errorLimitRecord");//错误记录数超过
String _executorMemory = (String) settingMap.get("executorMemory");
String _executorCores = (String) settingMap.get("executorCores");
String _totalExecutorCores = (String) settingMap.get("totalExecutorCores");
......@@ -136,6 +136,10 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
String _ftCount = (String) settingMap.get("ftCount");//分桶个数
String _separateMax = (String) settingMap.get("separateMax");//分桶字段最大值
String _separateMin = (String) settingMap.get("separateMin");//分桶字段最小值
String _syncRate = (String) settingMap.get("syncRate");//同步速率
String _maxConcurrency = (String) settingMap.get("maxConcurrency");//最大并发数
String _preImportStatement = (String) settingMap.get("preImportStatement");//导入前语句
String _postImportStatement = (String) settingMap.get("postImportStatement");//导入后语句
//******源数据******
Map<String, Object> readerMap = (Map<String, Object>) scriptMap.get("reader");
......@@ -288,6 +292,11 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
sb.append("<ft_count>").append(_ftCount).append("</ft_count>").append("\r\n");
sb.append("<separate_max>").append(_separateMax).append("</separate_max>").append("\r\n");
sb.append("<separate_min>").append(_separateMin).append("</separate_min>").append("\r\n");
sb.append("<syncRate>").append(_syncRate).append("</syncRate>").append("\r\n");
sb.append("<maxConcurrency>").append(_maxConcurrency).append("</maxConcurrency>").append("\r\n");
sb.append("<preImportStatement>").append(_preImportStatement).append("</preImportStatement>").append("\r\n");
sb.append("<postImportStatement>").append(_postImportStatement).append("</postImportStatement>").append("\r\n");
sb.append("<errorLimitRecord>").append(_errorLimitRecord).append("</errorLimitRecord>").append("\r\n");
sb.append("<executor_memory>").append(_executorMemory).append("</executor_memory>").append("\r\n");
sb.append("<executor_cores>").append(_executorCores).append("</executor_cores>").append("\r\n");
......
src/main/resources/lxTaskJson.json
View file @ be5c85f8
......@@ -80,130 +80,139 @@
{
"params":{
"version":"1.0",
"parentId":"509",
"mode":"0",
"projectId":"31",
"taskName":"dmp_demo_dmp_azkaban_exector_server_config",
"scripts":{
"setting":{
"extract":"incremental",
"extractExpression":"where 1=1",
"targetInsertMergeOverwrite":"insert",
"ftColumn":"",
"ftCount":"20",
"separateMax":"",
"separateMin":"",
"primaryKey":"on",
"partition":"on"
"params": {
"version": "1.0",
"parentId": "509",
"mode": "0",
"projectId": "31",
"taskName": "dmp_demo_dmp_azkaban_exector_server_config",
"scripts": {
"setting": {
"extract": "incremental",
"extractExpression": "where 1=1",
"targetInsertMergeOverwrite": "insert",
"ftColumn": "分桶字段",
"ftCount": "分桶个数",
"separateMax": "分桶字段最大值",
"separateMin": "分桶字段最小值",
"primaryKey": "主键",
"partition": "分区",
"postImportStatement": "导入后语句",
"preImportStatement": "导入前语句",
"errorLimitRecord": "错误记录数超过",
"maxConcurrency": "最大并发数",
"syncRate": "同步速率"
},
"reader":{
"dbConnection":"mysql_dmp_demo_test",
"fileType":"",
"sourceHdfsPath":"",
"sourceHdfsFile":"",
"sourceFtpDir":"",
"sourceFtpFile":"",
"sourceSkipFtpFile":"",
"sourceCsvDelimiter":"",
"sourceCsvHeader":"",
"sourceCsvCharset":"",
"sourceCsvQuote":"",
"sourceFtpLoadDate":"",
"registerTableName":"dmp_azkaban_exector_server_config",
"dayByDay":"false",
"column":[
"reader": {
"dbConnection": "mysql_dmp_demo_test",
"fileType": "",
"sourceHdfsPath": "",
"sourceHdfsFile": "",
"sourceFtpDir": "",
"sourceFtpFile": "",
"sourceSkipFtpFile": "",
"sourceCsvDelimiter": "",
"sourceCsvHeader": "",
"sourceCsvCharset": "",
"sourceCsvQuote": "",
"sourceFtpLoadDate": "",
"registerTableName": "dmp_azkaban_exector_server_config",
"dayByDay": "false",
"column": [
{
"name":"host",
"type":"VARCHAR"
"name": "host",
"type": "VARCHAR"
},
{
"name":"port",
"type":"VARCHAR"
"name": "port",
"type": "VARCHAR"
},
{
"name":"user_name",
"type":"VARCHAR"
"name": "user_name",
"type": "VARCHAR"
},
{
"name":"pass_word",
"type":"VARCHAR"
"name": "pass_word",
"type": "VARCHAR"
}
]
},
"writer":{
"targetDbConnection":"mysql_dmp_demo",
"targetTable":"dmp_azkaban_exector_server_config",
"targetFtpDir":"",
"targetFtpFile":"",
"targetCsvDelimiter":"",
"targetCsvHeader":"",
"targetCsvCharset":"",
"targetInsertMergeOverwrite":"insert",
"column":[
"writer": {
"targetDbConnection": "mysql_dmp_demo",
"targetTable": "dmp_azkaban_exector_server_config",
"targetFtpDir": "",
"targetFtpFile": "",
"targetCsvDelimiter": "",
"targetCsvHeader": "",
"targetCsvCharset": "",
"targetInsertMergeOverwrite": "insert",
"column": [
{
"name":"host",
"type":"VARCHAR",
"isPk":"1",
"isPt":"0",
"rules":[
{"method":"",
"type":""
"name": "host",
"type": "VARCHAR",
"isPk": "1",
"isPt": "0",
"rules": [
{
"method": "",
"type": ""
}
]
},
{
"name":"port",
"type":"VARCHAR",
"isPk":"0",
"isPt":"1",
"rules":[
{"method":"",
"type":""
"name": "port",
"type": "VARCHAR",
"isPk": "0",
"isPt": "1",
"rules": [
{
"method": "",
"type": ""
}
]
},
{
"name":"user_name",
"type":"VARCHAR",
"isPk":"0",
"isPt":"0",
"rules":[
{"method":"",
"type":""
"name": "user_name",
"type": "VARCHAR",
"isPk": "0",
"isPt": "0",
"rules": [
{
"method": "",
"type": ""
}
]
},
{
"name":"pass_word",
"type":"VARCHAR",
"isPk":"0",
"isPt":"0",
"rules":[
{"method":"",
"type":""
"name": "pass_word",
"type": "VARCHAR",
"isPk": "0",
"isPt": "0",
"rules": [
{
"method": "",
"type": ""
}
]
}
]
}
},
"treeId":669,
"taskRules":[
"treeId": 669,
"taskRules": [
{
"ruleId":"",
"ruleValue":{
"dv_fields":[
"ruleId": "",
"ruleValue": {
"dv_fields": [
{
"fieldName":""
"fieldName": ""
}
],
"dvTime":{
"timeField":"",
"timeValue":{
"startTime":"",
"endTime":""
"dvTime": {
"timeField": "",
"timeValue": {
"startTime": "",
"endTime": ""
}
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment