Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
J
jz-dmp-service
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
姚本章
jz-dmp-service
Commits
2504c4aa
Commit
2504c4aa
authored
Dec 30, 2020
by
mcb
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
no message
parent
14119a28
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
105 additions
and
33 deletions
+105
-33
SyncingDatasourceTypeDto.java
...roller/DataIntegration/bean/SyncingDatasourceTypeDto.java
+71
-13
DmpDevelopTaskServiceImpl.java
...z/dmp/modules/service/impl/DmpDevelopTaskServiceImpl.java
+26
-16
DmpSyncingDatasourceServiceImpl.java
...modules/service/impl/DmpSyncingDatasourceServiceImpl.java
+8
-4
DmpSyncingDatasourceTypeMapper.xml
...n/resources/mapper/dmp/DmpSyncingDatasourceTypeMapper.xml
+0
-0
No files found.
src/main/java/com/jz/dmp/modules/controller/DataIntegration/bean/SyncingDatasourceTypeDto.java
View file @
2504c4aa
...
@@ -36,16 +36,35 @@ public class SyncingDatasourceTypeDto implements Serializable {
...
@@ -36,16 +36,35 @@ public class SyncingDatasourceTypeDto implements Serializable {
*/
*/
@ApiModelProperty
(
value
=
"数据源分类名称"
)
@ApiModelProperty
(
value
=
"数据源分类名称"
)
private
String
datasourceCatename
;
private
String
datasourceCatename
;
/**
* jdbcUrl
*/
@ApiModelProperty
(
value
=
"jdbcUrl"
)
private
String
jdbcUrl
;
/**
* 密码
*/
@ApiModelProperty
(
value
=
"密码"
)
private
String
password
;
/**
* 账户
*/
@ApiModelProperty
(
value
=
"账户"
)
private
String
userName
;
@ApiModelProperty
(
value
=
"数据源描述"
)
private
String
datasourceDesc
;
@ApiModelProperty
(
value
=
"数据源名称"
)
private
String
datasourceName
;
@ApiModelProperty
(
value
=
"网络连接类型"
)
private
String
networkConnectionType
;
/**
/**
* 数据源类型
* 数据源类型
*/
*/
@ApiModelProperty
(
value
=
"数据源分类输入框属性json"
)
@ApiModelProperty
(
value
=
"数据源分类输入框属性json"
)
private
JSONObject
dbAttrs
;
private
JSONObject
dbAttrs
;
/**
* 数据源图标
*/
@ApiModelProperty
(
value
=
"数据源图标"
)
private
String
imgUrl
;
public
String
getDatasourceCatecode
()
{
public
String
getDatasourceCatecode
()
{
return
datasourceCatecode
;
return
datasourceCatecode
;
...
@@ -63,14 +82,6 @@ public class SyncingDatasourceTypeDto implements Serializable {
...
@@ -63,14 +82,6 @@ public class SyncingDatasourceTypeDto implements Serializable {
this
.
datasourceCatename
=
datasourceCatename
;
this
.
datasourceCatename
=
datasourceCatename
;
}
}
public
String
getImgUrl
()
{
return
imgUrl
;
}
public
void
setImgUrl
(
String
imgUrl
)
{
this
.
imgUrl
=
imgUrl
;
}
public
String
getDatasourceTypeId
()
{
public
String
getDatasourceTypeId
()
{
return
datasourceTypeId
;
return
datasourceTypeId
;
}
}
...
@@ -95,4 +106,51 @@ public class SyncingDatasourceTypeDto implements Serializable {
...
@@ -95,4 +106,51 @@ public class SyncingDatasourceTypeDto implements Serializable {
this
.
dbAttrs
=
dbAttrs
;
this
.
dbAttrs
=
dbAttrs
;
}
}
public
String
getJdbcUrl
()
{
return
jdbcUrl
;
}
public
void
setJdbcUrl
(
String
jdbcUrl
)
{
this
.
jdbcUrl
=
jdbcUrl
;
}
public
String
getPassword
()
{
return
password
;
}
public
void
setPassword
(
String
password
)
{
this
.
password
=
password
;
}
public
String
getUserName
()
{
return
userName
;
}
public
void
setUserName
(
String
userName
)
{
this
.
userName
=
userName
;
}
public
String
getDatasourceDesc
()
{
return
datasourceDesc
;
}
public
void
setDatasourceDesc
(
String
datasourceDesc
)
{
this
.
datasourceDesc
=
datasourceDesc
;
}
public
String
getDatasourceName
()
{
return
datasourceName
;
}
public
void
setDatasourceName
(
String
datasourceName
)
{
this
.
datasourceName
=
datasourceName
;
}
public
String
getNetworkConnectionType
()
{
return
networkConnectionType
;
}
public
void
setNetworkConnectionType
(
String
networkConnectionType
)
{
this
.
networkConnectionType
=
networkConnectionType
;
}
}
}
\ No newline at end of file
src/main/java/com/jz/dmp/modules/service/impl/DmpDevelopTaskServiceImpl.java
View file @
2504c4aa
...
@@ -65,7 +65,9 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
...
@@ -65,7 +65,9 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
return
submitSyncXml
(
task
.
getProjectId
(),
xmlContent
);
return
submitSyncXml
(
task
.
getProjectId
(),
xmlContent
);
}
}
/** 提交离线同步到指定projectId项目下(上传xml配置文件) */
/**
* 提交离线同步到指定projectId项目下(上传xml配置文件)
* */
private
JsonResult
submitSyncXml
(
long
projectId
,
String
xmlContent
)
{
private
JsonResult
submitSyncXml
(
long
projectId
,
String
xmlContent
)
{
String
_type_
=
XmlUtils
.
getPropertyValue
(
xmlContent
,
"type"
);
String
_type_
=
XmlUtils
.
getPropertyValue
(
xmlContent
,
"type"
);
String
_fileName_
=
XmlUtils
.
getPropertyValue
(
xmlContent
,
"name"
);
String
_fileName_
=
XmlUtils
.
getPropertyValue
(
xmlContent
,
"name"
);
...
@@ -108,6 +110,7 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
...
@@ -108,6 +110,7 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
public
String
convert2SyncXmlContent
(
DmpDevelopTask
body
)
throws
Exception
{
public
String
convert2SyncXmlContent
(
DmpDevelopTask
body
)
throws
Exception
{
StringBuilder
sb
=
new
StringBuilder
();
StringBuilder
sb
=
new
StringBuilder
();
long
treeId
=
body
.
getTreeId
();
long
treeId
=
body
.
getTreeId
();
//通过树节点id 查询任务信息
DmpDevelopTask
task
=
dmpDevelopTaskDao
.
selectTaskInfoByParam
(
treeId
);
DmpDevelopTask
task
=
dmpDevelopTaskDao
.
selectTaskInfoByParam
(
treeId
);
Map
<
String
,
Object
>
map
=
(
Map
<
String
,
Object
>)
JsonMapper
.
fromJsonString
(
task
.
getScript
(),
Map
.
class
);
Map
<
String
,
Object
>
map
=
(
Map
<
String
,
Object
>)
JsonMapper
.
fromJsonString
(
task
.
getScript
(),
Map
.
class
);
...
@@ -131,8 +134,9 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
...
@@ -131,8 +134,9 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
String
_separateMax
=(
String
)
settingMap
.
get
(
"separateMax"
);
//分桶字段最大值
String
_separateMax
=(
String
)
settingMap
.
get
(
"separateMax"
);
//分桶字段最大值
String
_separateMin
=(
String
)
settingMap
.
get
(
"separateMin"
);
//分桶字段最小值
String
_separateMin
=(
String
)
settingMap
.
get
(
"separateMin"
);
//分桶字段最小值
//******源数据******
Map
<
String
,
Object
>
readerMap
=
(
Map
<
String
,
Object
>)
scriptMap
.
get
(
"reader"
);
Map
<
String
,
Object
>
readerMap
=
(
Map
<
String
,
Object
>)
scriptMap
.
get
(
"reader"
);
String
_dbConnection
=
(
String
)
readerMap
.
get
(
"dbConnection"
);
String
_dbConnection
=
(
String
)
readerMap
.
get
(
"dbConnection"
);
//源数据库名称
String
_fileType
=
(
String
)
readerMap
.
get
(
"fileType"
);
String
_fileType
=
(
String
)
readerMap
.
get
(
"fileType"
);
String
_sourceHdfsPath
=
(
String
)
readerMap
.
get
(
"sourceHdfsPath"
);
String
_sourceHdfsPath
=
(
String
)
readerMap
.
get
(
"sourceHdfsPath"
);
String
_sourceHdfsFile
=
(
String
)
readerMap
.
get
(
"sourceHdfsFile"
);
String
_sourceHdfsFile
=
(
String
)
readerMap
.
get
(
"sourceHdfsFile"
);
...
@@ -144,34 +148,38 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
...
@@ -144,34 +148,38 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
String
_sourceCsvCharset
=
(
String
)
readerMap
.
get
(
"sourceCsvCharset"
);
String
_sourceCsvCharset
=
(
String
)
readerMap
.
get
(
"sourceCsvCharset"
);
String
_sourceCsvQuote
=
(
String
)
readerMap
.
get
(
"sourceCsvQuote"
);
String
_sourceCsvQuote
=
(
String
)
readerMap
.
get
(
"sourceCsvQuote"
);
String
_sourceFtpLoadDate
=
(
String
)
readerMap
.
get
(
"sourceFtpLoadDate"
);
String
_sourceFtpLoadDate
=
(
String
)
readerMap
.
get
(
"sourceFtpLoadDate"
);
String
_registerTableName
=
(
String
)
readerMap
.
get
(
"registerTableName"
);
String
_registerTableName
=
(
String
)
readerMap
.
get
(
"registerTableName"
);
//源数据库表名称
String
registerTableName_
=
(
String
)
readerMap
.
get
(
"registerTableName"
);
String
registerTableName_
=
(
String
)
readerMap
.
get
(
"registerTableName"
);
String
_dayByDay
=
(
String
)
readerMap
.
get
(
"dayByDay"
);
String
_dayByDay
=
(
String
)
readerMap
.
get
(
"dayByDay"
);
List
<
Map
<
String
,
Object
>>
_readerColumns
=
(
List
<
Map
<
String
,
Object
>>)
readerMap
.
get
(
"column"
);
List
<
Map
<
String
,
Object
>>
_readerColumns
=
(
List
<
Map
<
String
,
Object
>>)
readerMap
.
get
(
"column"
);
//源数据库表字段
//******目标数据******
Map
<
String
,
Object
>
writerMap
=
(
Map
<
String
,
Object
>)
scriptMap
.
get
(
"writer"
);
Map
<
String
,
Object
>
writerMap
=
(
Map
<
String
,
Object
>)
scriptMap
.
get
(
"writer"
);
String
_targetDbConnection
=
(
String
)
writerMap
.
get
(
"targetDbConnection"
);
String
_targetDbConnection
=
(
String
)
writerMap
.
get
(
"targetDbConnection"
);
//目标数据库
String
_targetTable
=
(
String
)
writerMap
.
get
(
"targetTable"
);
String
_targetTable
=
(
String
)
writerMap
.
get
(
"targetTable"
);
//目标数据库表名称
String
_targetFtpDir
=
(
String
)
writerMap
.
get
(
"targetFtpDir"
);
String
_targetFtpDir
=
(
String
)
writerMap
.
get
(
"targetFtpDir"
);
String
_targetFtpFile
=
(
String
)
writerMap
.
get
(
"targetFtpFile"
);
String
_targetFtpFile
=
(
String
)
writerMap
.
get
(
"targetFtpFile"
);
String
_targetCsvDelimiter
=
(
String
)
writerMap
.
get
(
"targetCsvDelimiter"
);
String
_targetCsvDelimiter
=
(
String
)
writerMap
.
get
(
"targetCsvDelimiter"
);
String
_targetCsvHeader
=
(
String
)
writerMap
.
get
(
"targetCsvHeader"
);
String
_targetCsvHeader
=
(
String
)
writerMap
.
get
(
"targetCsvHeader"
);
String
_targetCsvCharset
=
(
String
)
writerMap
.
get
(
"targetCsvCharset"
);
String
_targetCsvCharset
=
(
String
)
writerMap
.
get
(
"targetCsvCharset"
);
String
_targetInsertMergeOverwrite
=
(
String
)
writerMap
.
get
(
"targetInsertMergeOverwrite"
);
String
_targetInsertMergeOverwrite
=
(
String
)
writerMap
.
get
(
"targetInsertMergeOverwrite"
);
//同步策略
List
<
Map
<
String
,
Object
>>
_writerColumns
=
(
List
<
Map
<
String
,
Object
>>)
writerMap
.
get
(
"column"
);
List
<
Map
<
String
,
Object
>>
_writerColumns
=
(
List
<
Map
<
String
,
Object
>>)
writerMap
.
get
(
"column"
);
//目标数据库表字段
Integer
_projectId_
=
body
.
getProjectId
();
Integer
_projectId_
=
body
.
getProjectId
();
//项目id
Integer
_parentId_
=
(
Integer
)
map
.
get
(
"parentId"
);
Integer
_parentId_
=
(
Integer
)
map
.
get
(
"parentId"
);
//父节点id
String
_mode_
=
(
String
)
map
.
get
(
"mode"
);
String
_mode_
=
(
String
)
map
.
get
(
"mode"
);
String
_version_
=
(
String
)
map
.
get
(
"version"
);
String
_version_
=
(
String
)
map
.
get
(
"version"
);
String
_name_
=
(
String
)
map
.
get
(
"name"
);
String
_name_
=
(
String
)
map
.
get
(
"name"
);
//节点数名称
//根据项目id和数据源名称 查询数据信息
DmpSyncingDatasource
sDS
=
this
.
getDmpSyncingDatasource
(
_projectId_
,
_dbConnection
);
//源
DmpSyncingDatasource
tDS
=
this
.
getDmpSyncingDatasource
(
_projectId_
,
_targetDbConnection
);
//目标
DmpSyncingDatasource
sDS
=
this
.
getDmpSyncingDatasource
(
_projectId_
,
_dbConnection
);
DmpSyncingDatasource
tDS
=
this
.
getDmpSyncingDatasource
(
_projectId_
,
_targetDbConnection
);
if
(
null
==
sDS
||
null
==
tDS
||
null
==
sDS
.
getDatasourceType
()
||
null
==
tDS
.
getDatasourceType
())
if
(
null
==
sDS
||
null
==
tDS
||
null
==
sDS
.
getDatasourceType
()
||
null
==
tDS
.
getDatasourceType
())
throw
new
Exception
(
"同步数据来源或数据去向信息有误"
);
throw
new
Exception
(
"同步数据来源或数据去向信息有误"
);
DmpSyncingDatasourceType
sDST
=
dmpSyncingDatasourceTypeDao
.
queryById
(
sDS
.
getDatasourceType
());
//根据数据源类型ID 获取数据源类型
DmpSyncingDatasourceType
tDST
=
dmpSyncingDatasourceTypeDao
.
queryById
(
tDS
.
getDatasourceType
());
DmpSyncingDatasourceType
sDST
=
dmpSyncingDatasourceTypeDao
.
queryById
(
sDS
.
getDatasourceType
());
//源
DmpSyncingDatasourceType
tDST
=
dmpSyncingDatasourceTypeDao
.
queryById
(
tDS
.
getDatasourceType
());
//目标
String
_targetDbName_
=
this
.
getTargetDBname
(
tDS
);
String
_targetDbName_
=
this
.
getTargetDBname
(
tDS
);
String
_sourceDbName_
=
this
.
getTargetDBname
(
sDS
);
String
_sourceDbName_
=
this
.
getTargetDBname
(
sDS
);
...
@@ -288,7 +296,9 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
...
@@ -288,7 +296,9 @@ public class DmpDevelopTaskServiceImpl implements DmpDevelopTaskService {
}
}
return
sb
.
toString
();
return
sb
.
toString
();
}
}
/*
*根据项目id和数据源名称 查询数据信息
* */
private
DmpSyncingDatasource
getDmpSyncingDatasource
(
Integer
projectId
,
String
dsName
)
throws
Exception
{
private
DmpSyncingDatasource
getDmpSyncingDatasource
(
Integer
projectId
,
String
dsName
)
throws
Exception
{
DmpSyncingDatasource
ds
=
new
DmpSyncingDatasource
();
DmpSyncingDatasource
ds
=
new
DmpSyncingDatasource
();
ds
.
setDataStatus
(
"1"
);
ds
.
setDataStatus
(
"1"
);
...
...
src/main/java/com/jz/dmp/modules/service/impl/DmpSyncingDatasourceServiceImpl.java
View file @
2504c4aa
...
@@ -290,10 +290,14 @@ public class DmpSyncingDatasourceServiceImpl implements DmpSyncingDatasourceServ
...
@@ -290,10 +290,14 @@ public class DmpSyncingDatasourceServiceImpl implements DmpSyncingDatasourceServ
if
(
StringUtils
.
isNotEmpty
(
str
.
get
(
"datasourceTypeName"
).
toString
()))
{
if
(
StringUtils
.
isNotEmpty
(
str
.
get
(
"datasourceTypeName"
).
toString
()))
{
dto
.
setDatasourceTypeName
(
str
.
get
(
"datasourceTypeName"
).
toString
());
dto
.
setDatasourceTypeName
(
str
.
get
(
"datasourceTypeName"
).
toString
());
}
}
if
(
StringUtils
.
isNotEmpty
(
str
.
get
(
"imgUrl"
).
toString
()))
{
if
(
StringUtils
.
isNotEmpty
(
str
.
get
(
"dbAttrs"
).
toString
()))
{
//数据源字段属性json
dto
.
setImgUrl
(
str
.
get
(
"imgUrl"
).
toString
());
Map
<
String
,
Object
>
dbAttrs
=
JSONObject
.
parseObject
(
str
.
get
(
"dbAttrs"
).
toString
());
}
dto
.
setJdbcUrl
(
dbAttrs
.
get
(
"jdbcUrl"
).
toString
());
if
(
StringUtils
.
isNotEmpty
(
str
.
get
(
"dbAttrs"
).
toString
()))
{
dto
.
setPassword
(
dbAttrs
.
get
(
"password"
).
toString
());
dto
.
setDatasourceDesc
(
dbAttrs
.
get
(
"datasourceDesc"
).
toString
());
dto
.
setUserName
(
dbAttrs
.
get
(
"userName"
).
toString
());
dto
.
setDatasourceName
(
dbAttrs
.
get
(
"datasourceName"
).
toString
());
dto
.
setNetworkConnectionType
(
dbAttrs
.
get
(
"networkConnectionType"
).
toString
());
dto
.
setDbAttrs
(
JSONObject
.
parseObject
(
str
.
get
(
"dbAttrs"
).
toString
()));
dto
.
setDbAttrs
(
JSONObject
.
parseObject
(
str
.
get
(
"dbAttrs"
).
toString
()));
}
}
}
}
...
...
src/main/resources/mapper/dmp/DmpSyncingDatasourceType
Dao
.xml
→
src/main/resources/mapper/dmp/DmpSyncingDatasourceType
Mapper
.xml
View file @
2504c4aa
File moved
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment