Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
J
jz-dmp-service
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
姚本章
jz-dmp-service
Commits
ad3a98af
Commit
ad3a98af
authored
Jan 11, 2021
by
mcb
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
no message
parent
8e3f9c35
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
49 additions
and
36 deletions
+49
-36
SessionUtils.java
src/main/java/com/jz/common/utils/web/SessionUtils.java
+17
-2
RealTimeSyncController.java
...es/controller/DataIntegration/RealTimeSyncController.java
+0
-3
DmpRealtimeSyncInfoServiceImpl.java
.../modules/service/impl/DmpRealtimeSyncInfoServiceImpl.java
+31
-30
DmpRealtimeSyncInfoMapper.xml
src/main/resources/mapper/dmp/DmpRealtimeSyncInfoMapper.xml
+1
-1
No files found.
src/main/java/com/jz/common/utils/web/SessionUtils.java
View file @
ad3a98af
package
com
.
jz
.
common
.
utils
.
web
;
package
com
.
jz
.
common
.
utils
.
web
;
import
org.apache.commons.lang3.StringUtils
;
import
org.springframework.security.core.Authentication
;
import
org.springframework.security.core.Authentication
;
import
org.springframework.security.core.context.SecurityContextHolder
;
import
org.springframework.security.core.context.SecurityContextHolder
;
import
org.springframework.web.context.request.RequestContextHolder
;
import
org.springframework.web.context.request.RequestContextHolder
;
...
@@ -105,8 +106,22 @@ public class SessionUtils {
...
@@ -105,8 +106,22 @@ public class SessionUtils {
}
}
return
(
DmpMember
)
principal
;
return
(
DmpMember
)
principal
;
}
}
/**
/**
* @Title: getSessionUserId
* @Description: 获取登录用户ID
* @return String 返回类型
*/
public
static
String
getCurrentUserId
(){
String
userId
=
""
;
DmpMember
user
=
getSecurityUser
();
if
(
user
!=
null
){
userId
=
String
.
valueOf
(
user
.
getUserId
());
}
return
userId
;
}
/**
* @Title: getAuthentication
* @Title: getAuthentication
* @Description: TODO(获取authentication)
* @Description: TODO(获取authentication)
* @param @return 参数
* @param @return 参数
...
...
src/main/java/com/jz/dmp/modules/controller/DataIntegration/RealTimeSyncController.java
View file @
ad3a98af
...
@@ -228,8 +228,6 @@ public class RealTimeSyncController {
...
@@ -228,8 +228,6 @@ public class RealTimeSyncController {
if
(
StringUtils
.
isEmpty
(
params
.
get
(
"targetDataSourceId"
).
toString
()))
{
if
(
StringUtils
.
isEmpty
(
params
.
get
(
"targetDataSourceId"
).
toString
()))
{
return
new
JsonResult
(
ResultCode
.
PARAMS_ERROR
,
"目标数据源id不能为空!"
);
return
new
JsonResult
(
ResultCode
.
PARAMS_ERROR
,
"目标数据源id不能为空!"
);
}
}
boolean
flag
=
false
;
//异步提交
//异步提交
Thread
thread
=
new
Thread
(
new
Runnable
()
{
Thread
thread
=
new
Thread
(
new
Runnable
()
{
@Override
@Override
...
@@ -242,7 +240,6 @@ public class RealTimeSyncController {
...
@@ -242,7 +240,6 @@ public class RealTimeSyncController {
}
}
});
});
thread
.
start
();
thread
.
start
();
return
new
JsonResult
();
return
new
JsonResult
();
}
}
...
...
src/main/java/com/jz/dmp/modules/service/impl/DmpRealtimeSyncInfoServiceImpl.java
View file @
ad3a98af
...
@@ -9,6 +9,7 @@ import com.jz.common.page.PageInfoResponse;
...
@@ -9,6 +9,7 @@ import com.jz.common.page.PageInfoResponse;
import
com.jz.common.persistence.BaseService
;
import
com.jz.common.persistence.BaseService
;
import
com.jz.common.utils.realTime.DBUtil
;
import
com.jz.common.utils.realTime.DBUtil
;
import
com.jz.common.utils.realTime.RestClient
;
import
com.jz.common.utils.realTime.RestClient
;
import
com.jz.common.utils.web.SessionUtils
;
import
com.jz.dmp.modules.controller.DataIntegration.bean.DataSourceNameListDto
;
import
com.jz.dmp.modules.controller.DataIntegration.bean.DataSourceNameListDto
;
import
com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListDto
;
import
com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListDto
;
import
com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListReq
;
import
com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListReq
;
...
@@ -283,20 +284,20 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
...
@@ -283,20 +284,20 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
@Override
@Override
public
boolean
addRealTimeTask
(
Map
<
String
,
Object
>
params
)
throws
Exception
{
public
boolean
addRealTimeTask
(
Map
<
String
,
Object
>
params
)
throws
Exception
{
String
srcDataSourceId
=
params
.
get
(
"srcDataSourceId"
).
toString
();
// 来源id
String
srcDataSourceId
=
params
.
get
(
"srcDataSourceId"
).
toString
();
// 来源id
String
targetDataSourceId
=
(
String
)
params
.
get
(
"targetDataSourceId"
);
// 目标源id
String
targetDataSourceId
=
params
.
get
(
"targetDataSourceId"
).
toString
(
);
// 目标源id
Long
projectId
=
Long
.
valueOf
(
params
.
get
(
"projectId"
).
toString
());
//项目id
Long
projectId
=
Long
.
valueOf
(
params
.
get
(
"projectId"
).
toString
());
//项目id
String
connectorUrl
=
(
String
)
params
.
get
(
"connectorUrl"
);
//connectorUrl连接信息
String
connectorUrl
=
(
String
)
params
.
get
(
"connectorUrl"
);
//connectorUrl连接信息
//根据来源数据源id获取数据信息
//根据来源数据源id获取数据信息
RealTimeSyncDataSourceModel
sourceDbInfo
=
dmpRealtimeSyncInfoDao
.
querygSourceDbInfoById
(
srcDataSourceId
);
RealTimeSyncDataSourceModel
sourceDbInfo
=
dmpRealtimeSyncInfoDao
.
querygSourceDbInfoById
(
srcDataSourceId
);
if
(
sourceDbInfo
!
=
null
)
{
if
(
sourceDbInfo
=
=
null
)
{
throw
new
RuntimeException
(
"来源数据设置信息不存在!"
);
throw
new
RuntimeException
(
"来源数据设置信息不存在!"
);
}
}
sourceDbInfo
.
setPassword
(
new
BaseService
().
decode
(
sourceDbInfo
.
getPassword
(),
publicKey
));
sourceDbInfo
.
setPassword
(
new
BaseService
().
decode
(
sourceDbInfo
.
getPassword
(),
publicKey
));
//根据目标数据源id获取数据信息
//根据目标数据源id获取数据信息
RealTimeSyncDataSourceModel
targetDbInfo
=
dmpRealtimeSyncInfoDao
.
querygSourceDbInfoById
(
targetDataSourceId
);
RealTimeSyncDataSourceModel
targetDbInfo
=
dmpRealtimeSyncInfoDao
.
querygSourceDbInfoById
(
targetDataSourceId
);
if
(
targetDbInfo
!
=
null
)
{
if
(
targetDbInfo
=
=
null
)
{
throw
new
RuntimeException
(
"目标数据源设置信息不存在!"
);
throw
new
RuntimeException
(
"目标数据源设置信息不存在!"
);
}
}
targetDbInfo
.
setPassword
(
new
BaseService
().
decode
(
targetDbInfo
.
getPassword
(),
publicKey
));
targetDbInfo
.
setPassword
(
new
BaseService
().
decode
(
targetDbInfo
.
getPassword
(),
publicKey
));
...
@@ -326,12 +327,14 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
...
@@ -326,12 +327,14 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
}
}
//提交源到源的connector
//提交源到源的connector
Long
realtimeId
=
submitDatasource2DatasourceToConnector
(
projectId
,
sourceDbInfo
,
targetDbInfo
,
dmpProjectSystemInfo
,
connectorUrl
,
params
);
Long
realtimeId
=
submitDatasource2DatasourceToConnector
(
projectId
,
sourceDbInfo
,
targetDbInfo
,
dmpProjectSystemInfo
,
connectorUrl
,
params
);
//处理已选择的表信息
if
(
realtimeId
!=
null
)
{
submitNoSelectTable
(
realtimeId
,
projectId
,
sourceDbInfo
,
targetDbInfo
,
dmpProjectSystemInfo
,
connectorUrl
,
params
);
//处理已选择的表信息
submitNoSelectTable
(
realtimeId
,
projectId
,
sourceDbInfo
,
targetDbInfo
,
dmpProjectSystemInfo
,
connectorUrl
,
params
);
}
return
false
;
return
false
;
}
}
private
Long
submitDatasource2DatasourceToConnector
(
Long
projectId
,
RealTimeSyncDataSourceModel
sourceDbInfo
,
RealTimeSyncDataSourceModel
targetDbInfo
,
DmpProjectSystemInfo
dmpProjectSystemInfo
,
String
connectorUrl
,
Map
<
String
,
Object
>
params
)
{
private
Long
submitDatasource2DatasourceToConnector
(
Long
projectId
,
RealTimeSyncDataSourceModel
sourceDbInfo
,
RealTimeSyncDataSourceModel
targetDbInfo
,
DmpProjectSystemInfo
dmpProjectSystemInfo
,
String
connectorUrl
,
Map
<
String
,
Object
>
params
)
throws
Exception
{
Integer
srcDataSourceId
=
sourceDbInfo
.
getId
();
Integer
srcDataSourceId
=
sourceDbInfo
.
getId
();
Integer
targetDataSourceId
=
targetDbInfo
.
getId
();
Integer
targetDataSourceId
=
targetDbInfo
.
getId
();
...
@@ -379,7 +382,7 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
...
@@ -379,7 +382,7 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
}
}
}
}
}
}
logger
.
info
(
"################### 解析黑名单表--结束"
+
connectorBlacklistTables
+
" ################"
);
logger
.
info
(
"################### 解析黑名单表--结束
{}
"
+
connectorBlacklistTables
+
" ################"
);
//解析已选择表
//解析已选择表
String
connectorWhitelistTables
=
""
;
String
connectorWhitelistTables
=
""
;
...
@@ -400,7 +403,7 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
...
@@ -400,7 +403,7 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
}
}
}
}
}
}
logger
.
info
(
"################### 解析已选择表--结束 ################"
);
logger
.
info
(
"################### 解析已选择表--结束
{} "
+
connectorWhitelistTables
+
"
################"
);
Map
<
String
,
String
>
dataModelMap
=
new
HashMap
<>();
Map
<
String
,
String
>
dataModelMap
=
new
HashMap
<>();
...
@@ -435,10 +438,8 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
...
@@ -435,10 +438,8 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
queryParam
.
put
(
"sourceConnectorName"
,
sourceConnectorName
);
queryParam
.
put
(
"sourceConnectorName"
,
sourceConnectorName
);
Map
dataMap
=
dmpRealtimeSyncInfoDao
.
selecltRealtimeSyncInfoByParams
(
queryParam
);
Map
dataMap
=
dmpRealtimeSyncInfoDao
.
selecltRealtimeSyncInfoByParams
(
queryParam
);
//不存在同一个数据源的同步任务 没查到信息
//不存在同一个数据源的同步任务 没查到信息
if
(
dataMap
.
size
()
==
0
&&
dataMap
.
isEmpty
())
{
if
(
dataMap
==
null
)
{
logger
.
info
(
"######不存在同一个数据源的同步任务srcDataSourceId="
+
srcDataSourceId
+
",type="
+
1
+
",sourceConnectorName="
+
sourceConnectorName
+
",connectorUrl="
+
connectorUrl
+
"的sourceConnector,添加新的"
);
dataModelMap
.
put
(
"name"
,
sourceDbInfo
.
getDatasourceName
()
+
"_"
+
sourceName
+
"-"
+
sourceDbInfo
.
getDbName
());
//source connector name
dataModelMap
.
put
(
"name"
,
sourceDbInfo
.
getDatasourceName
()
+
"_"
+
sourceName
+
"-"
+
sourceDbInfo
.
getDbName
());
//source connector name
String
dataSource2DataSourceJsonStr
=
freemakerJson
(
"source"
,
dataModelMap
);
//使用freemaker模板生成 kafka connector 请求参数
String
dataSource2DataSourceJsonStr
=
freemakerJson
(
"source"
,
dataModelMap
);
//使用freemaker模板生成 kafka connector 请求参数
Map
<
String
,
Object
>
dataSource2DataSourceResult
=
RestClient
.
post
(
connectorUrl
,
dataSource2DataSourceJsonStr
);
Map
<
String
,
Object
>
dataSource2DataSourceResult
=
RestClient
.
post
(
connectorUrl
,
dataSource2DataSourceJsonStr
);
...
@@ -465,6 +466,7 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
...
@@ -465,6 +466,7 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
saveBody
.
setConnectorJobId
(
connectorJobId
);
saveBody
.
setConnectorJobId
(
connectorJobId
);
saveBody
.
setConnectorJsonData
(
dataSource2DataSourceJsonStr
);
saveBody
.
setConnectorJsonData
(
dataSource2DataSourceJsonStr
);
saveBody
.
setCreateTime
(
new
Date
());
saveBody
.
setCreateTime
(
new
Date
());
saveBody
.
setCrePerson
(
SessionUtils
.
getCurrentUserId
());
dmpRealtimeSyncInfoDao
.
insert
(
saveBody
);
dmpRealtimeSyncInfoDao
.
insert
(
saveBody
);
realtiemId
=
Long
.
valueOf
(
saveBody
.
getId
());
realtiemId
=
Long
.
valueOf
(
saveBody
.
getId
());
logger
.
info
(
"###################保存实时同步任务--结束 ################"
);
logger
.
info
(
"###################保存实时同步任务--结束 ################"
);
...
@@ -496,7 +498,7 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
...
@@ -496,7 +498,7 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
StringWriter
stringWriter
=
new
StringWriter
();
StringWriter
stringWriter
=
new
StringWriter
();
try
{
try
{
Template
template
=
freeMarkerConfig
.
getConfiguration
().
getTemplate
(
"realtime-sync-connector/"
+
type
+
"_connector.ftl"
);
Template
template
=
freeMarkerConfig
.
getConfiguration
().
getTemplate
(
type
+
"_connector.ftl"
);
if
(
template
!=
null
)
{
if
(
template
!=
null
)
{
try
{
try
{
template
.
process
(
dataModel
,
stringWriter
);
template
.
process
(
dataModel
,
stringWriter
);
...
@@ -539,10 +541,10 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
...
@@ -539,10 +541,10 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
* @author Bellamy
* @author Bellamy
* @since 2021-01-08
* @since 2021-01-08
*/
*/
private
JsonResult
submitNoSelectTable
(
Long
realtimeId
,
Long
projectId
,
RealTimeSyncDataSourceModel
sourceDbInfo
,
RealTimeSyncDataSourceModel
targetDbInfo
,
DmpProjectSystemInfo
dmpProjectSystemInfo
,
String
connectorUrl
,
Map
<
String
,
Object
>
params
)
{
private
JsonResult
submitNoSelectTable
(
Long
realtimeId
,
Long
projectId
,
RealTimeSyncDataSourceModel
sourceDbInfo
,
RealTimeSyncDataSourceModel
targetDbInfo
,
DmpProjectSystemInfo
dmpProjectSystemInfo
,
String
connectorUrl
,
Map
<
String
,
Object
>
params
)
throws
Exception
{
logger
.
info
(
"###################处理已选择的表信息################"
);
logger
.
info
(
"###################处理已选择的表信息################"
);
/*
//源数据源到数据源同步connector信息 ----开始------
/*
//源数据源到数据源同步connector信息 ----开始------
Map<String, String> tableToTableConfigArg = new HashMap<>();
Map<String, String> tableToTableConfigArg = new HashMap<>();
//默认配置参数 --{kudu master 的地址} 10.0.108.61,10.0.108.62
//默认配置参数 --{kudu master 的地址} 10.0.108.61,10.0.108.62
String endpoint = targetDbInfo.getEndpoint();
String endpoint = targetDbInfo.getEndpoint();
...
@@ -573,26 +575,25 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
...
@@ -573,26 +575,25 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
tableToTableConfigArg.put("hdfsNamenodePrincipal", targetDbInfo.getAccessId());
tableToTableConfigArg.put("hdfsNamenodePrincipal", targetDbInfo.getAccessId());
tableToTableConfigArg.put("keytabLocation", targetDbInfo.getAccessKey());
tableToTableConfigArg.put("keytabLocation", targetDbInfo.getAccessKey());
}
}
tableToTableConfigArg.put("connectorSecurityFlag", (String) params.get("connectorSecurityFlag"));
tableToTableConfigArg.put("connectorSecurityFlag", (String) params.get("connectorSecurityFlag"));
*/
//已选择的表信息
//已选择的表信息
List
<
Map
<
String
,
String
>>
tables
=
(
List
<
Map
<
String
,
String
>>)
params
.
get
(
"tables"
);
List
<
Map
<
String
,
String
>>
tables
=
(
List
<
Map
<
String
,
String
>>)
params
.
get
(
"tables"
);
for (Map<String, String> str : tables) {
if
(
tables
.
size
()
>
0
&&
tables
!=
null
)
{
str.put("targetDbNameAndTableName", targetDbInfo.getDbName() + "." + str.get("targetTableName")); //目标库和表名称
for
(
Map
<
String
,
String
>
str
:
tables
)
{
str.putAll(tableToTableConfigArg);
DmpRealtimeSyncSelectTable
selectTable
=
new
DmpRealtimeSyncSelectTable
();
selectTable
.
setCrePerson
(
SessionUtils
.
getCurrentUserId
());
DmpRealtimeSyncSelectTable selectTable = new DmpRealtimeSyncSelectTable();
selectTable
.
setCreTime
(
new
Date
());
//selectTable.setCrePerson();
selectTable
.
setSrcTableName
(
str
.
get
(
"sourceTableName"
));
selectTable.setCreTime(new Date());
selectTable
.
setTargetTableName
(
str
.
get
(
"targetTableName"
));
selectTable.setSrcTableName(str.get("sourceTableName"));
selectTable
.
setRealtimeId
(
realtimeId
);
selectTable.setTargetTableName(str.get("targetTableName"));
selectTable
.
setDesensitizationField
(
str
.
get
(
"desensitizationField"
));
selectTable.setRealtimeId(realtimeId);
selectTable
.
setPkName
(
str
.
get
(
"pkName"
));
selectTable.setDesensitizationField(str.get("desensitizationField"));
selectTable
.
setArithmetic
(
str
.
get
(
"arithmetic"
));
selectTable.setPkName(str.get("pkName"));
dmpRealtimeSyncInfoDao
.
insertRealtimeSelectTable
(
selectTable
);
selectTable.setArithmetic(str.get("arithmetic"));
}
dmpRealtimeSyncInfoDao.insertRealtimeSelectTable(selectTable);
}
}
logger.info("###################处理已选择的表信息--结束################");
*/
logger
.
info
(
"###################处理已选择的表信息--结束################"
);
return
new
JsonResult
();
return
new
JsonResult
();
}
}
}
}
\ No newline at end of file
src/main/resources/mapper/dmp/DmpRealtimeSyncInfoMapper.xml
View file @
ad3a98af
...
@@ -375,7 +375,7 @@
...
@@ -375,7 +375,7 @@
ds.host,
ds.host,
ds.port,
ds.port,
ds.endpoint,
ds.endpoint,
ds
,
access_key as accessKey,
ds
.
access_key as accessKey,
ds.access_id as accessId,
ds.access_id as accessId,
dsdt.datasource_type as datasourceTypeName,
dsdt.datasource_type as datasourceTypeName,
dsdt.driver_class_name as driverName
dsdt.driver_class_name as driverName
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment