Skip to content

J
jz-dmp-service
Commit aa80f0d8 authored by mcb's avatar mcb
Browse files
Options

no message

parent 610a5c1b
Hide whitespace changes
Inline Side-by-side
Showing with 816 additions and 478 deletions
src/main/java/com/jz/common/exception/GlobalExceptionHandler.java
View file @ aa80f0d8
...@@ -28,9 +28,9 @@ public class GlobalExceptionHandler { ...@@ -28,9 +28,9 @@ public class GlobalExceptionHandler {
private static Logger logger = LoggerFactory.getLogger(GlobalExceptionHandler.class); private static Logger logger = LoggerFactory.getLogger(GlobalExceptionHandler.class);
/** /**
* 403 - Bad Request * 缺少请求参数
*/ */
@ResponseStatus(HttpStatus.BAD_REQUEST) @ResponseStatus(HttpStatus.OK)
@ExceptionHandler(MissingServletRequestParameterException.class) @ExceptionHandler(MissingServletRequestParameterException.class)
public JsonResult handleMissingServletRequestParameterException(MissingServletRequestParameterException e) { public JsonResult handleMissingServletRequestParameterException(MissingServletRequestParameterException e) {
logger.error("缺少请求参数", e); logger.error("缺少请求参数", e);
...@@ -48,9 +48,9 @@ public class GlobalExceptionHandler { ...@@ -48,9 +48,9 @@ public class GlobalExceptionHandler {
} }
/** /**
* 403 - Bad Request * 参数验证失败
*/ */
@ResponseStatus(HttpStatus.BAD_REQUEST) @ResponseStatus(HttpStatus.OK)
@ExceptionHandler(MethodArgumentNotValidException.class) @ExceptionHandler(MethodArgumentNotValidException.class)
public JsonResult handleMethodArgumentNotValidException(MethodArgumentNotValidException e) { public JsonResult handleMethodArgumentNotValidException(MethodArgumentNotValidException e) {
logger.error("参数验证失败", e); logger.error("参数验证失败", e);
...@@ -78,11 +78,11 @@ public class GlobalExceptionHandler { ...@@ -78,11 +78,11 @@ public class GlobalExceptionHandler {
return new JsonResult(ResultCode.PARAMS_ERROR, message); return new JsonResult(ResultCode.PARAMS_ERROR, message);
} }
/** /**
* 403 - Bad Request * 403 - Bad Request
*/ */
@ResponseStatus(HttpStatus.BAD_REQUEST) @ResponseStatus(HttpStatus.OK)
@ExceptionHandler(ValidationException.class) @ExceptionHandler(ValidationException.class)
public JsonResult handleValidationException(ValidationException e) { public JsonResult handleValidationException(ValidationException e) {
logger.error("参数验证失败", e); logger.error("参数验证失败", e);
...@@ -95,7 +95,7 @@ public class GlobalExceptionHandler { ...@@ -95,7 +95,7 @@ public class GlobalExceptionHandler {
@ResponseStatus(HttpStatus.METHOD_NOT_ALLOWED) @ResponseStatus(HttpStatus.METHOD_NOT_ALLOWED)
@ExceptionHandler(HttpRequestMethodNotSupportedException.class) @ExceptionHandler(HttpRequestMethodNotSupportedException.class)
public JsonResult handleHttpRequestMethodNotSupportedException(HttpRequestMethodNotSupportedException e) { public JsonResult handleHttpRequestMethodNotSupportedException(HttpRequestMethodNotSupportedException e) {
logger.error("不支持当前请求方�?", e); logger.error("不支持当前请求方式", e);
return new JsonResult(ResultCode.NOT_SUPPORTED, "request_method_not_supported:"); return new JsonResult(ResultCode.NOT_SUPPORTED, "request_method_not_supported:");
} }
...@@ -116,7 +116,7 @@ public class GlobalExceptionHandler { ...@@ -116,7 +116,7 @@ public class GlobalExceptionHandler {
@ExceptionHandler(ServiceException.class) @ExceptionHandler(ServiceException.class)
public JsonResult handleServiceException(ServiceException e) { public JsonResult handleServiceException(ServiceException e) {
logger.error("业务逻辑异常", e); logger.error("业务逻辑异常", e);
return new JsonResult(ResultCode.INTERNAL_SERVER_ERROR, "业务逻辑异常�?"); return new JsonResult(ResultCode.INTERNAL_SERVER_ERROR, "业务逻辑异常!");
} }
/** /**
...@@ -126,7 +126,7 @@ public class GlobalExceptionHandler { ...@@ -126,7 +126,7 @@ public class GlobalExceptionHandler {
@ExceptionHandler(Exception.class) @ExceptionHandler(Exception.class)
public JsonResult handleException(Exception e) { public JsonResult handleException(Exception e) {
logger.error("通用逻辑异常", e); logger.error("通用逻辑异常", e);
return new JsonResult(ResultCode.INTERNAL_SERVER_ERROR, "参数错误"); return new JsonResult(ResultCode.INTERNAL_SERVER_ERROR, "通用逻辑异常");
} }
/** /**
...@@ -135,7 +135,7 @@ public class GlobalExceptionHandler { ...@@ -135,7 +135,7 @@ public class GlobalExceptionHandler {
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
@ExceptionHandler(DataIntegrityViolationException.class) @ExceptionHandler(DataIntegrityViolationException.class)
public JsonResult handleException(DataIntegrityViolationException e) { public JsonResult handleException(DataIntegrityViolationException e) {
logger.error("操作数据库出现异�?:", e); logger.error("操作数据库出现异常:", e);
return new JsonResult(ResultCode.NOT_SUPPORTED, "操作数据库出现异常：字段重复、有外键关联�?"); return new JsonResult(ResultCode.NOT_SUPPORTED, "操作数据库出现异常：字段重复、有外键关联�?");
} }
} }
src/main/java/com/jz/common/utils/realTime/DBUtil.java
View file @ aa80f0d8
//package com.jz.common.utils.realTime; package com.jz.common.utils.realTime;
//
//import com.jz.dmp.web.ui.modules.dmp.model.task.realtime.RealTimeSyncColumnModel; import com.jz.dmp.modules.model.RealTimeSyncColumnModel;
//import com.jz.dmp.web.ui.modules.dmp.model.task.realtime.RealTimeSyncDataSourceModel; import com.jz.dmp.modules.model.RealTimeSyncDataSourceModel;
//import com.jz.dmp.web.ui.modules.dmp.model.task.realtime.RealTimeSyncTableModel; import com.jz.dmp.modules.model.RealTimeSyncTableModel;
//import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
//import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
//
//import java.sql.*; import java.sql.*;
//import java.util.ArrayList; import java.util.ArrayList;
//import java.util.HashMap; import java.util.HashMap;
//import java.util.List; import java.util.List;
//import java.util.Map; import java.util.Map;
//
//public class DBUtil { public class DBUtil {
//
//
// /**
// /** * 创建连接
// * 创建连接 *
// * @param driverName 驱动名称 * @param driverName 驱动名称
// * @param jdbcUrl 连接字符串 * @param jdbcUrl 连接字符串
// * @param username 用户名 * @param username 用户名
// * @param password 密码 * @param password 密码
// * @return 获取的连接 * @return 获取的连接
// * @throws ClassNotFoundException * @throws ClassNotFoundException
// * @throws SQLException * @throws SQLException
// */ */
// public static Connection openConn(RealTimeSyncDataSourceModel realTimeSyncDataSourceModel) throws Exception { public static Connection openConn(RealTimeSyncDataSourceModel realTimeSyncDataSourceModel) throws Exception {
// String driverName = realTimeSyncDataSourceModel.getDriverName(); String driverName = realTimeSyncDataSourceModel.getDriverName(); //jdbc驱动名称
// String jdbcUrl = realTimeSyncDataSourceModel.getJdbcUrl(); String jdbcUrl = realTimeSyncDataSourceModel.getJdbcUrl(); // jdbc 连接
// String username = realTimeSyncDataSourceModel.getUserName(); String username = realTimeSyncDataSourceModel.getUserName(); //用户
// String password = realTimeSyncDataSourceModel.getPassword(); String password = realTimeSyncDataSourceModel.getPassword(); //密码
// if (!StringUtils.isEmpty(driverName)){ if (!StringUtils.isEmpty(driverName)) {
// Class.forName(driverName); Class.forName(driverName);
// } }
// return DriverManager.getConnection(jdbcUrl, username, password); return DriverManager.getConnection(jdbcUrl, username, password);
// } }
//
// // 关闭数据库连接 // 关闭数据库连接
// public static void closeConn(Connection conn) { public static void closeConn(Connection conn) {
// try { try {
// if (conn != null) { if (conn != null) {
// conn.close(); conn.close();
// } }
// } catch (Exception e) { } catch (Exception e) {
// e.printStackTrace(); e.printStackTrace();
// } }
// } }
//
//
// /** /**
// * 获取数据库里的元数据信息 * 获取数据库里的元数据信息
// * @param connection 创建的数据库连接 *
// * @return * @param connection 创建的数据库连接
// * @throws Exception * @return
// */ * @throws Exception
// public static DatabaseMetaData getMetaData(Connection connection) throws Exception { */
// if (connection != null) { public static DatabaseMetaData getMetaData(Connection connection) throws Exception {
// return connection.getMetaData(); if (connection != null) {
// } return connection.getMetaData();
// return null; }
// } return null;
// }
//
// /**
// * 获取数据源里的所有表详细信息 /**
// * @param realTimeSyncDataSourceModel 根据数据源Id获取的数据源信息 * 获取数据源里的所有表详细信息
// * @param toQueryTableName 要查询的表明 如果为空,默认查询所有表 *
// * @param isContainColumnInfo 是否包含列信息 * @param realTimeSyncDataSourceModel 根据数据源Id获取的数据源信息
// * @param tableToTableSyncTasks 当前数据源已经同步过的表到表的任务 * @param toQueryTableName 要查询的表明 如果为空,默认查询所有表
// * @param blacklistTables 黑名单表 * @param isContainColumnInfo 是否包含列信息
// * @return * @param tableToTableSyncTasksMap 数据源对应的表详细信息
// */ * @param selectTablesName 选中的表名称
// public static List<RealTimeSyncTableModel> getDataSourceTables(RealTimeSyncDataSourceModel realTimeSyncDataSourceModel, String toQueryTableName, boolean isContainColumnInfo, Map<String, Map<String, String>> tableToTableSyncTasksMap, String blacklistTables, String selectTablesName, Map<String, String> setttingBlacklistTableMap ){ * @param blacklistTables 选择的黑名单表
// Connection connection = null; * @param setttingBlacklistTableMap 来源数据源的黑名单表
// try { * @return
// connection = openConn(realTimeSyncDataSourceModel); */
// DatabaseMetaData metaData = connection.getMetaData(); public static List<RealTimeSyncTableModel> getDataSourceTables(RealTimeSyncDataSourceModel realTimeSyncDataSourceModel, String toQueryTableName, boolean isContainColumnInfo,
// return getDataSourceTables(metaData, toQueryTableName, isContainColumnInfo,tableToTableSyncTasksMap,blacklistTables,selectTablesName,setttingBlacklistTableMap); Map<String, Map<String, String>> tableToTableSyncTasksMap, String blacklistTables, String selectTablesName, Map<String, String> setttingBlacklistTableMap) {
// } catch (Exception e) { Connection connection = null;
// e.printStackTrace(); try {
// }finally { //创建JDBC连接
// closeConn(connection); connection = openConn(realTimeSyncDataSourceModel);
// } DatabaseMetaData metaData = connection.getMetaData();
// //获取数据源里的表信息
// return null; return getDataSourceTables(metaData, toQueryTableName, isContainColumnInfo, tableToTableSyncTasksMap, blacklistTables, selectTablesName, setttingBlacklistTableMap);
// } } catch (Exception e) {
// e.printStackTrace();
// /** } finally {
// * 获取数据源里的表信息 closeConn(connection);
// * @param metaData 数据库源数据 }
// * @param toQueryTableName 要查询的表明 如果为空,默认查询所有表
// * @param isContainColumnInfo 是否包含列信息 return null;
// * @param tableToTableSyncTasksMap 当前表已经存在的任务 }
// * @param blacklistTables 前台设置后 传递到后台的黑名单表 ，如果在黑名单内 不返回当前表的信息
// * @param selectTablesName 选择的要同步的表 /**
// * @return * 获取数据源里的表信息
// * @throws Exception *
// */ * @param metaData 数据库源数据
// public static List<RealTimeSyncTableModel> getDataSourceTables(DatabaseMetaData metaData, String toQueryTableName, boolean isContainColumnInfo, Map<String, Map<String, String>> tableToTableSyncTasksMap, String blacklistTables, String selectTablesName, Map<String, String> setttingBlacklistTableMap ) throws Exception{ * @param toQueryTableName 要查询的表明 如果为空,默认查询所有表
// * @param isContainColumnInfo 是否包含列信息
// List<RealTimeSyncTableModel> dataSourceTables = null; * @param tableToTableSyncTasksMap 当前表已经存在的任务
// if (metaData != null) { * @param blacklistTables 前台设置后 传递到后台的黑名单表 ，如果在黑名单内 不返回当前表的信息
// String tableNamePattern = "%"; * @param selectTablesName 选择的要同步的表
// if (!StringUtils.isEmpty(toQueryTableName)) { * @return
// tableNamePattern = toQueryTableName+"%"; * @throws Exception
// } */
// ResultSet tables = metaData.getTables(null, "%", tableNamePattern, new String[] {"TABLE"}); public static List<RealTimeSyncTableModel> getDataSourceTables(DatabaseMetaData metaData, String toQueryTableName, boolean isContainColumnInfo, Map<String, Map<String, String>> tableToTableSyncTasksMap, String blacklistTables, String selectTablesName, Map<String, String> setttingBlacklistTableMap) throws Exception {
// if (tables.wasNull()) {
// return dataSourceTables; List<RealTimeSyncTableModel> dataSourceTables = null;
// } if (metaData != null) {
// String tableNamePattern = "%";
// //选择查询的表 if (!StringUtils.isEmpty(toQueryTableName)) {
// Map<String, String> selectTableMap = null; tableNamePattern = toQueryTableName + "%";
// }
// if (!StringUtils.isEmpty(selectTablesName)) { ResultSet tables = metaData.getTables(null, "%", tableNamePattern, new String[]{"TABLE"});
// selectTableMap = new HashMap<>(); if (tables.wasNull()) {
// String [] selectTableArr = selectTablesName.split(","); return dataSourceTables;
// for (String selectTableName : selectTableArr) { }
// selectTableMap.put(selectTableName, selectTableName);
// } //选择查询的表
// } Map<String, String> selectTableMap = null;
//
// //返回信息中不包含的表 if (!StringUtils.isEmpty(selectTablesName)) {
// Map<String, String> notIncludeTableMap = null; selectTableMap = new HashMap<>();
// //判断表是否在黑名单表 String[] selectTableArr = selectTablesName.split(",");
// if (!StringUtils.isEmpty(blacklistTables)) { for (String selectTableName : selectTableArr) {
// notIncludeTableMap = new HashMap<>(); selectTableMap.put(selectTableName, selectTableName);
// String [] blacklistTablesArr = blacklistTables.split(","); }
// for (String blacklistTable : blacklistTablesArr) { }
// notIncludeTableMap.put(blacklistTable, blacklistTable);
// } //返回信息中不包含的表
// } Map<String, String> notIncludeTableMap = null;
// //判断表是否在黑名单表
// dataSourceTables = new ArrayList<>(); if (!StringUtils.isEmpty(blacklistTables)) {
// RealTimeSyncTableModel realTimeSyncTableModel = null; notIncludeTableMap = new HashMap<>();
// while (tables.next()) { String[] blacklistTablesArr = blacklistTables.split(",");
// for (String blacklistTable : blacklistTablesArr) {
// //表名 notIncludeTableMap.put(blacklistTable, blacklistTable);
// String tableName = tables.getString("TABLE_NAME"); }
// }
// String schema = tables.getString("TABLE_SCHEM");
// //sqlServer 过滤sys表 只要dbo的表 dataSourceTables = new ArrayList<>();
// if ("sys".equals(schema)) { RealTimeSyncTableModel realTimeSyncTableModel = null;
// continue; while (tables.next()) {
// }
// //表名
// //已经选择的表 String tableName = tables.getString("TABLE_NAME");
// if (!CollectionUtils.isEmpty(selectTableMap) && !selectTableMap.containsKey(tableName)) {
// //当前表不在选择的表内 不继续构造信息 String schema = tables.getString("TABLE_SCHEM");
// continue; //sqlServer 过滤sys表 只要dbo的表
// } if ("sys".equals(schema)) {
// continue;
// //选择的黑名单 }
// if (!CollectionUtils.isEmpty(notIncludeTableMap) && notIncludeTableMap.containsKey(tableName)) {
// notIncludeTableMap.remove(tableName); //已经选择的表
// continue; if (!CollectionUtils.isEmpty(selectTableMap) && !selectTableMap.containsKey(tableName)) {
// } //当前表不在选择的表内 不继续构造信息
// continue;
// }
// realTimeSyncTableModel = new RealTimeSyncTableModel();
// realTimeSyncTableModel.setTableName(tableName); //选择的黑名单
// //判断表的记录在同步任务中是否存在记录,如果已经存在,说明同步过 if (!CollectionUtils.isEmpty(notIncludeTableMap) && notIncludeTableMap.containsKey(tableName)) {
// String desensitizationField = null ; notIncludeTableMap.remove(tableName);
// if (!CollectionUtils.isEmpty(tableToTableSyncTasksMap) && tableToTableSyncTasksMap.containsKey(tableName)) { continue;
// realTimeSyncTableModel.setIsSubmited(true); }
// //desensitization_field,脱敏字段
// //算法 arithmetic
// Map<String, String> tempMap = tableToTableSyncTasksMap.get(tableName); realTimeSyncTableModel = new RealTimeSyncTableModel();
// desensitizationField = tempMap.get("desensitization_field"); realTimeSyncTableModel.setTableName(tableName);
// String arithmetic = tempMap.get("arithmetic"); //判断表的记录在同步任务中是否存在记录,如果已经存在,说明同步过
// realTimeSyncTableModel.setDesensitizationField(desensitizationField); String desensitizationField = null;
// realTimeSyncTableModel.setArithmetic(arithmetic); if (!CollectionUtils.isEmpty(tableToTableSyncTasksMap) && tableToTableSyncTasksMap.containsKey(tableName)) {
// } realTimeSyncTableModel.setIfSubmited(true);
// //desensitization_field,脱敏字段
// //判断表是否在设置的黑名单中 如果有，说明已经在黑名单中 设置成true，返回给前端 //算法 arithmetic
// if (!CollectionUtils.isEmpty(setttingBlacklistTableMap) && setttingBlacklistTableMap.containsKey(tableName)) { Map<String, String> tempMap = tableToTableSyncTasksMap.get(tableName);
// realTimeSyncTableModel.setIsBlacklist(true); desensitizationField = tempMap.get("desensitization_field");
// } String arithmetic = tempMap.get("arithmetic");
// realTimeSyncTableModel.setDesensitizationField(desensitizationField);
// //主键 realTimeSyncTableModel.setArithmetic(arithmetic);
// String pkName = getPrimaryKeyColumnByTableName(metaData, tableName); }
// realTimeSyncTableModel.setPkName(pkName);
// //判断表是否在设置的黑名单中 如果有，说明已经在黑名单中 设置成true，返回给前端
// //是否包含列信息 if (!CollectionUtils.isEmpty(setttingBlacklistTableMap) && setttingBlacklistTableMap.containsKey(tableName)) {
// if (isContainColumnInfo) { realTimeSyncTableModel.setIfBlacklist(true);
// Map<String, String > desensitizationFieldMap = null; }
// if (!StringUtils.isEmpty(desensitizationField)) {
// desensitizationFieldMap = new HashMap<>(); //主键
// String [] desensitizationFieldArr = desensitizationField.split(","); String pkName = getPrimaryKeyColumnByTableName(metaData, tableName);
// for (String desensitizationFieldName : desensitizationFieldArr) { realTimeSyncTableModel.setPkName(pkName);
// selectTableMap.put(desensitizationFieldName, desensitizationFieldName);
// } //是否包含列信息
// } if (isContainColumnInfo) {
// realTimeSyncTableModel.setColumnInfo(getColumnInfoByTableName(metaData, tableName,desensitizationFieldMap)); Map<String, String> desensitizationFieldMap = null;
// } if (!StringUtils.isEmpty(desensitizationField)) {
// dataSourceTables.add(realTimeSyncTableModel); desensitizationFieldMap = new HashMap<>();
// } String[] desensitizationFieldArr = desensitizationField.split(",");
// for (String desensitizationFieldName : desensitizationFieldArr) {
// } selectTableMap.put(desensitizationFieldName, desensitizationFieldName);
// return dataSourceTables; }
// } }
// realTimeSyncTableModel.setColumnInfo(getColumnInfoByTableName(metaData, tableName, desensitizationFieldMap));
// /** }
// * 获取当前表里 设置主键的列名 dataSourceTables.add(realTimeSyncTableModel);
// * @param metaData }
// * @param tableName
// * @return }
// * @throws Exception return dataSourceTables;
// */ }
// public static String getPrimaryKeyColumnByTableName(DatabaseMetaData metaData,String tableName) throws Exception {
// if (metaData == null) { /**
// return null; * 获取当前表里 设置主键的列名
// } *
// * @param metaData
// ResultSet pkResultSet = metaData.getPrimaryKeys(null, null, tableName); * @param tableName
// * @return
// String primaryKey = ""; * @throws Exception
// while (pkResultSet.next()) { */
// primaryKey += pkResultSet.getString("COLUMN_NAME")+","; public static String getPrimaryKeyColumnByTableName(DatabaseMetaData metaData, String tableName) throws Exception {
// } if (metaData == null) {
// return null;
// if (StringUtils.isEmpty(primaryKey)){ }
// //没有设置主键 但是设置了唯一索引
// ResultSet indexResultSet = metaData.getIndexInfo(null, null, "yyy3", true, false); ResultSet pkResultSet = metaData.getPrimaryKeys(null, null, tableName);
// while (indexResultSet.next()){
// primaryKey += indexResultSet.getString("COLUMN_NAME")+","; String primaryKey = "";
// } while (pkResultSet.next()) {
// } primaryKey += pkResultSet.getString("COLUMN_NAME") + ",";
// }
// if (!StringUtils.isEmpty(primaryKey)) {
// primaryKey = primaryKey.substring(0, primaryKey.length() - 1); if (StringUtils.isEmpty(primaryKey)) {
// } //没有设置主键 但是设置了唯一索引
// ResultSet indexResultSet = metaData.getIndexInfo(null, null, "yyy3", true, false);
// return primaryKey; while (indexResultSet.next()) {
// } primaryKey += indexResultSet.getString("COLUMN_NAME") + ",";
// }
// /** }
// * 获取当前表的所有列 以及是字符串的列
// * @param metaData if (!StringUtils.isEmpty(primaryKey)) {
// * @param tableName primaryKey = primaryKey.substring(0, primaryKey.length() - 1);
// * @throws Exception }
// */
// public static RealTimeSyncColumnModel getColumnInfoByTableName(DatabaseMetaData metaData, String tableName, Map<String, String> desensitizationFieldMap) throws Exception { return primaryKey;
// RealTimeSyncColumnModel realTimeSyncColumnModel = null ; }
// ResultSet columnResult = metaData.getColumns(null, "%", tableName,"%");
// if (!columnResult.wasNull()) { /**
// realTimeSyncColumnModel = new RealTimeSyncColumnModel(); * 获取当前表的所有列 以及是字符串的列
// List<String> allColumns = new ArrayList<>(); *
// * @param metaData
// * @param tableName
// List<Map<String, Object>> strColumns = new ArrayList<>(); * @throws Exception
// Map<String, Object> tempMap = null ; */
// public static RealTimeSyncColumnModel getColumnInfoByTableName(DatabaseMetaData metaData, String tableName, Map<String, String> desensitizationFieldMap) throws Exception {
// while (columnResult.next()) { RealTimeSyncColumnModel realTimeSyncColumnModel = null;
// //列名 ResultSet columnResult = metaData.getColumns(null, "%", tableName, "%");
// String columnName = columnResult.getString("COLUMN_NAME"); if (!columnResult.wasNull()) {
// allColumns.add(columnName); realTimeSyncColumnModel = new RealTimeSyncColumnModel();
// //列类型 List<String> allColumns = new ArrayList<>();
// String columnType = columnResult.getString("TYPE_NAME");
// if (isStringColumn(columnType)) {
// tempMap = new HashMap<>(); List<Map<String, Object>> strColumns = new ArrayList<>();
// tempMap.put("columnName", columnName); Map<String, Object> tempMap = null;
// if (!CollectionUtils.isEmpty(desensitizationFieldMap) && desensitizationFieldMap.containsKey(columnName)) {
// tempMap.put("isDesensitization", true); while (columnResult.next()) {
// } //列名
// strColumns.add(tempMap); String columnName = columnResult.getString("COLUMN_NAME");
// } allColumns.add(columnName);
// } //列类型
// realTimeSyncColumnModel.setAllColumns(null); String columnType = columnResult.getString("TYPE_NAME");
// realTimeSyncColumnModel.setStrColumns(strColumns); if (isStringColumn(columnType)) {
// realTimeSyncColumnModel.setTableName(tableName); tempMap = new HashMap<>();
// } tempMap.put("columnName", columnName);
// return realTimeSyncColumnModel; if (!CollectionUtils.isEmpty(desensitizationFieldMap) && desensitizationFieldMap.containsKey(columnName)) {
// } tempMap.put("isDesensitization", true);
// }
// /** strColumns.add(tempMap);
// * 判断列是否是字符串类型的列 }
// * @param columnType }
// * @return realTimeSyncColumnModel.setAllColumns(null);
// */ realTimeSyncColumnModel.setStrColumns(strColumns);
// private static boolean isStringColumn(String columnType) { realTimeSyncColumnModel.setTableName(tableName);
// columnType = columnType.toLowerCase(); }
// if (columnType.equals("varchar") return realTimeSyncColumnModel;
// || columnType.equals("nvarchar") }
// || columnType.equals("char")
// || columnType.equals("tinytext") /**
// || columnType.equals("text") * 判断列是否是字符串类型的列
// || columnType.equals("mediumtext") *
// || columnType.equals("longtext") * @param columnType
// ) { * @return
// return true ; */
// } private static boolean isStringColumn(String columnType) {
// return false; columnType = columnType.toLowerCase();
// } if (columnType.equals("varchar")
// || columnType.equals("nvarchar")
// || columnType.equals("char")
// public static void main(String[] args) { || columnType.equals("tinytext")
// || columnType.equals("text")
// String aa = "aa,bb,"; || columnType.equals("mediumtext")
// || columnType.equals("longtext")
// System.err.println(aa.substring(0,aa.length()-1)); ) {
// return true;
// String driverName = "com.mysql.jdbc.Driver"; }
// String jdbcUrl = "jdbc:mysql://10.0.53.179:3306/testunique"; return false;
// String username = "root"; }
// String password = "jz@2018";
//
// RealTimeSyncDataSourceModel realTimeSyncDataSourceModel = new RealTimeSyncDataSourceModel(); public static void main(String[] args) {
// realTimeSyncDataSourceModel.setDriverName(driverName);
// realTimeSyncDataSourceModel.setJdbcUrl(jdbcUrl); String aa = "aa,bb,";
// realTimeSyncDataSourceModel.setUserName(username);
// realTimeSyncDataSourceModel.setPassword(password); System.err.println(aa.substring(0, aa.length() - 1));
//
// String blacklistTables = ""; String driverName = "com.mysql.jdbc.Driver";
// Map<String, Map<String, String>> map = null; String jdbcUrl = "jdbc:mysql://10.0.53.179:3306/testunique";
// Connection connection = null ; String username = "root";
// try { String password = "jz@2018";
// connection = openConn(realTimeSyncDataSourceModel);
// DatabaseMetaData metaData = getMetaData(connection); RealTimeSyncDataSourceModel realTimeSyncDataSourceModel = new RealTimeSyncDataSourceModel();
// realTimeSyncDataSourceModel.setDriverName(driverName);
// realTimeSyncDataSourceModel.setJdbcUrl(jdbcUrl);
// ResultSet indexResultSet = metaData.getIndexInfo(null, null, "yyy3", true, false); realTimeSyncDataSourceModel.setUserName(username);
// realTimeSyncDataSourceModel.setPassword(password);
// String primaryKey = "";
// while (indexResultSet.next()){ String blacklistTables = "";
// primaryKey += indexResultSet.getString("COLUMN_NAME")+","; Map<String, Map<String, String>> map = null;
// } Connection connection = null;
// try {
// if (!StringUtils.isEmpty(primaryKey)) { connection = openConn(realTimeSyncDataSourceModel);
// primaryKey = primaryKey.substring(0, primaryKey.length() - 1); DatabaseMetaData metaData = getMetaData(connection);
// }
// System.err.println(primaryKey);
// ResultSet indexResultSet = metaData.getIndexInfo(null, null, "yyy3", true, false);
// /*ResultSet resultSet = metaData.getColumns(null,"%","yyy3","%");
// while (resultSet.next()){ String primaryKey = "";
// System.err.println(resultSet.getString("COLUMN_NAME") +"--"+resultSet.getString("TYPE_NAME")); while (indexResultSet.next()) {
// }*/ primaryKey += indexResultSet.getString("COLUMN_NAME") + ",";
// }
// String key = getPrimaryKeyColumnByTableName(metaData,"yyy3");
// if (!StringUtils.isEmpty(primaryKey)) {
// System.err.println("key--"+key); primaryKey = primaryKey.substring(0, primaryKey.length() - 1);
// }
// } catch (Exception e) { System.err.println(primaryKey);
// // TODO Auto-generated catch block
// e.printStackTrace(); /*ResultSet resultSet = metaData.getColumns(null,"%","yyy3","%");
// }finally { while (resultSet.next()){
// closeConn(connection); System.err.println(resultSet.getString("COLUMN_NAME") +"--"+resultSet.getString("TYPE_NAME"));
// } }*/
// }
// String key = getPrimaryKeyColumnByTableName(metaData, "yyy3");
//
//} System.err.println("key--" + key);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
closeConn(connection);
}
}
}
src/main/java/com/jz/dmp/modules/controller/DataIntegration/RealTimeSyncController.java
View file @ aa80f0d8
...@@ -9,6 +9,7 @@ import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListDto; ...@@ -9,6 +9,7 @@ import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListDto;
import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListReq; import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListReq;
import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeTableInfoReq; import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeTableInfoReq;
import com.jz.dmp.modules.model.DmpRealtimeSyncInfo; import com.jz.dmp.modules.model.DmpRealtimeSyncInfo;
import com.jz.dmp.modules.model.RealTimeSyncModel;
import com.jz.dmp.modules.service.DmpRealtimeSyncInfoService; import com.jz.dmp.modules.service.DmpRealtimeSyncInfoService;
import io.swagger.annotations.Api; import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam; import io.swagger.annotations.ApiImplicitParam;
...@@ -18,6 +19,7 @@ import org.apache.commons.lang3.StringUtils; ...@@ -18,6 +19,7 @@ import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
...@@ -50,7 +52,7 @@ public class RealTimeSyncController { ...@@ -50,7 +52,7 @@ public class RealTimeSyncController {
*/ */
@ApiOperation(value = "实时同步任务列表分页查询", notes = "实时同步任务列表分页查询") @ApiOperation(value = "实时同步任务列表分页查询", notes = "实时同步任务列表分页查询")
@PostMapping(value = "/realTimeSyncListPage") @PostMapping(value = "/realTimeSyncListPage")
public PageInfoResponse<RealTimeSyncListDto> getDataSourceListPage(@RequestBody RealTimeSyncListReq req, HttpServletRequest httpRequest) throws Exception { public PageInfoResponse<RealTimeSyncListDto> getDataSourceListPage(@RequestBody @Validated RealTimeSyncListReq req, HttpServletRequest httpRequest) throws Exception {
PageInfoResponse<RealTimeSyncListDto> pageInfo = new PageInfoResponse<RealTimeSyncListDto>(); PageInfoResponse<RealTimeSyncListDto> pageInfo = new PageInfoResponse<RealTimeSyncListDto>();
if (StringUtils.isEmpty(req.getProjectId())) { if (StringUtils.isEmpty(req.getProjectId())) {
pageInfo.setMessage("项目id不能为空！"); pageInfo.setMessage("项目id不能为空！");
...@@ -141,22 +143,40 @@ public class RealTimeSyncController { ...@@ -141,22 +143,40 @@ public class RealTimeSyncController {
} }
/** /**
* 根据数据源id获取表详细信息 * 新增--选择源表信息
* *
* @return * @return
* @author Bellamy * @author Bellamy
* @since 2021-01-06 * @since 2021-01-06
*/ */
@ApiOperation(value = "新增--获取表信息", notes = "新增--获取表信息") @ApiOperation(value = "新增--选择源表信息", notes = "新增--选择源表信息")
@PostMapping(value = "/getSourceDbTableList") @PostMapping(value = "/getSourceDbTableList")
public JsonResult getTableInfo(@RequestBody RealTimeTableInfoReq req, HttpServletRequest httpRequest) throws Exception { public JsonResult<RealTimeSyncModel> getTableInfo(@RequestBody @Validated RealTimeTableInfoReq req, HttpServletRequest httpRequest) throws Exception {
if (StringUtils.isEmpty(req.getProjectId())) { if (StringUtils.isEmpty(req.getProjectId())) {
return new JsonResult(ResultCode.PARAMS_ERROR, "项目id不能为空！"); return new JsonResult(ResultCode.PARAMS_ERROR, "项目id不能为空！");
} }
if (StringUtils.isEmpty(req.getSrcDatasourceId())) { if (StringUtils.isEmpty(req.getSrcDatasourceId())) {
return new JsonResult(ResultCode.PARAMS_ERROR, "来源数据源id不能为空！"); return new JsonResult(ResultCode.PARAMS_ERROR, "来源数据源id不能为空！");
} }
JsonResult jsonResult = dmpRealtimeSyncInfoService.queryTableInfoByParams(req); JsonResult<RealTimeSyncModel> jsonResult = dmpRealtimeSyncInfoService.queryTableInfoByParams(req);
return jsonResult;
}
/**
* 新增--选择目标信息
*
* @return
* @author Bellamy
* @since 2021-01-07
*/
@ApiOperation(value = "新增--选择目标信息", notes = "新增--选择目标信息")
@GetMapping(value = "/targetDatasourceInfo")
@ApiImplicitParam(name = "projectId", value = "项目id",required = true)
public JsonResult getTargetDatasourceInfo(@RequestParam String projectId) throws Exception {
if (StringUtils.isEmpty(projectId)) {
return new JsonResult(ResultCode.PARAMS_ERROR, "项目id不能为空！");
}
JsonResult jsonResult = dmpRealtimeSyncInfoService.selectTargetDatasourceInfo(projectId);
return jsonResult; return jsonResult;
} }
......
src/main/java/com/jz/dmp/modules/dao/DmpRealtimeSyncInfoDao.java
View file @ aa80f0d8
...@@ -5,6 +5,7 @@ import com.jz.dmp.modules.controller.DataIntegration.bean.DataSourceNameListDto; ...@@ -5,6 +5,7 @@ import com.jz.dmp.modules.controller.DataIntegration.bean.DataSourceNameListDto;
import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListDto; import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListDto;
import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListReq; import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListReq;
import com.jz.dmp.modules.model.DmpRealtimeSyncInfo; import com.jz.dmp.modules.model.DmpRealtimeSyncInfo;
import com.jz.dmp.modules.model.RealTimeSyncDataSourceModel;
import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Param;
import java.util.List; import java.util.List;
...@@ -109,7 +110,7 @@ public interface DmpRealtimeSyncInfoDao { ...@@ -109,7 +110,7 @@ public interface DmpRealtimeSyncInfoDao {
* @author Bellamy * @author Bellamy
* @since 2021-01-06 * @since 2021-01-06
*/ */
Map querygSourceDbInfoById(@Param("srcDataSourceId") String srcDataSourceId) throws Exception; RealTimeSyncDataSourceModel querygSourceDbInfoById(@Param("srcDataSourceId") String srcDataSourceId) throws Exception;
List<Map> queryRealTimeInfoByDataSourceId(@Param("srcDatasourceId") String srcDatasourceId, @Param("targetDatasourceId") String targetDatasourceId); List<Map> queryRealTimeInfoByDataSourceId(@Param("srcDatasourceId") String srcDatasourceId, @Param("targetDatasourceId") String targetDatasourceId);
......
src/main/java/com/jz/dmp/modules/model/RealTimeSyncColumnModel.java 0 → 100644
View file @ aa80f0d8
package com.jz.dmp.modules.model;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
/**
* 实时同步表信息 列级别的信息
*/
@ApiModel(value = "实时同步表下列级别的信息", description = "实时同步表下列级别的信息")
public class RealTimeSyncColumnModel implements Serializable {
private static final long serialVersionUID = 7171169593984549127L;
/**
* 表名
*/
@ApiModelProperty(value = "表名")
private String tableName;
/**
* 当前表的所有列
*/
@ApiModelProperty(value = "当前表的所有列")
private List<String> allColumns;
/**
* 当前表是字符串类型的列
*/
@ApiModelProperty(value = "当前表是字符串类型的列")
private List<Map<String, Object>> strColumns;
public String getTableName() {
return tableName;
}
public void setTableName(String tableName) {
this.tableName = tableName;
}
public List<String> getAllColumns() {
return allColumns;
}
public void setAllColumns(List<String> allColumns) {
this.allColumns = allColumns;
}
public List<Map<String, Object>> getStrColumns() {
return strColumns;
}
public void setStrColumns(List<Map<String, Object>> strColumns) {
this.strColumns = strColumns;
}
}
src/main/java/com/jz/dmp/modules/model/RealTimeSyncDataSourceModel.java 0 → 100644
View file @ aa80f0d8
package com.jz.dmp.modules.model;
import java.util.Date;
/**
* 实时同步表信息 列级别的信息
*/
public class RealTimeSyncDataSourceModel {
/**
* 数据源id
*/
private Integer id;
/**
* 数据源名称
*/
private String datasourceName;
/**
* 数据源描述
*/
private String datasourceDesc;
/**
* 数据源类型
*/
private Integer datasourceType;
/**
* jdbc 连接字符串
*/
private String jdbcUrl;
/**
* 数据库名称
*/
private String dbName;
/**
* 数据库用户名
*/
private String userName;
/**
* 数据库密码
*/
private String password;
/**
* 数据库主机地址
*/
private String host;
/**
* 数据库端口
*/
private String port;
/**
* 项目id
*/
private Integer projectId;
/**
* jdbc驱动名称
*/
private String driverName;
/**
* 数据源类型名称
*/
private String dataSourceTypeName;
public Integer getDatasourceType() {
return datasourceType;
}
public void setDatasourceType(Integer datasourceType) {
this.datasourceType = datasourceType;
}
public String getJdbcUrl() {
return jdbcUrl;
}
public void setJdbcUrl(String jdbcUrl) {
this.jdbcUrl = jdbcUrl;
}
public String getDbName() {
return dbName;
}
public void setDbName(String dbName) {
this.dbName = dbName;
}
public String getUserName() {
return userName;
}
public void setUserName(String userName) {
this.userName = userName;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public String getPort() {
return port;
}
public void setPort(String port) {
this.port = port;
}
public Integer getProjectId() {
return projectId;
}
public void setProjectId(Integer projectId) {
this.projectId = projectId;
}
public String getDriverName() {
return driverName;
}
public void setDriverName(String driverName) {
this.driverName = driverName;
}
public String getDataSourceTypeName() {
return dataSourceTypeName;
}
public void setDataSourceTypeName(String dataSourceTypeName) {
this.dataSourceTypeName = dataSourceTypeName;
}
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public String getDatasourceName() {
return datasourceName;
}
public void setDatasourceName(String datasourceName) {
this.datasourceName = datasourceName;
}
public String getDatasourceDesc() {
return datasourceDesc;
}
public void setDatasourceDesc(String datasourceDesc) {
this.datasourceDesc = datasourceDesc;
}
}
src/main/java/com/jz/dmp/modules/model/RealTimeSyncModel.java
View file @ aa80f0d8
package com.jz.dmp.modules.model; package com.jz.dmp.modules.model;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.List; import java.util.List;
@ApiModel(value = "新增--新增选择源表返回参数", description = "新增--新增选择源表返回参数")
public class RealTimeSyncModel { public class RealTimeSyncModel {
/** /**
* 数据源Id * 来源数据源id
*/ */
private Integer dataSourceId; @ApiModelProperty(value = "来源数据源id")
private Integer srcdatasourceId;
/** /**
* 数据源名称 * 来源数据源名称
*/ */
@ApiModelProperty(value = "来源数据源名称")
private String dataSourceName; private String dataSourceName;
/** /**
* 数据库名称 * 数据库名称
*/ */
@ApiModelProperty(value = "数据库名称")
private String dbName ; private String dbName ;
/** /**
* 数据源下的表信息 * 来源数据源下的表信息
*/ */
@ApiModelProperty(value = "来源数据源下的表信息")
private List<RealTimeSyncTableModel> tables; private List<RealTimeSyncTableModel> tables;
public Integer getSrcdatasourceId() {
public Integer getDataSourceId() { return srcdatasourceId;
return dataSourceId;
} }
public void setSrcdatasourceId(Integer srcdatasourceId) {
public void setDataSourceId(Integer dataSourceId) { this.srcdatasourceId = srcdatasourceId;
this.dataSourceId = dataSourceId;
} }
public String getDataSourceName() { public String getDataSourceName() {
return dataSourceName; return dataSourceName;
} }
......
src/main/java/com/jz/dmp/modules/model/RealTimeSyncTableModel.java
View file @ aa80f0d8
package com.jz.dmp.modules.model; package com.jz.dmp.modules.model;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.io.Serializable;
/** /**
* 实时同步表信息 表级别的信息 * 实时同步表信息 表级别的信息
*/ */
public class RealTimeSyncTableModel { @ApiModel(value = "实时同步表级别的信息", description = "实时同步表级别的信息")
public class RealTimeSyncTableModel implements Serializable {
private static final long serialVersionUID = -5882129191251102234L;
/** /**
* 表名 * 表名
*/ */
@ApiModelProperty(value = "表名")
private String tableName; private String tableName;
/** /**
* 主键名称 * 主键名称
*/ */
@ApiModelProperty(value = "主键名称")
private String pkName; private String pkName;
/** /**
* 是否提交同步过同步 * 是否提交同步过同步
*/ */
@ApiModelProperty(value = "是否提交同步过同步")
private boolean isSubmited; private boolean ifSubmited;
/** /**
* 是否是黑名单 * 是否是黑名单
*/ */
private boolean isBlacklist; @ApiModelProperty(value = "是否是黑名单")
private boolean ifBlacklist;
/** /**
* 脱敏字段 * 脱敏字段
*/ */
@ApiModelProperty(value = "脱敏字段")
private String desensitizationField; private String desensitizationField;
/** /**
* 脱敏算法 * 脱敏算法
*/ */
@ApiModelProperty(value = "脱敏算法")
private String arithmetic; private String arithmetic;
/** /**
* 表下的所有列信息 * 表下的所有列信息
*/ */
//private RealTimeSyncColumnModel columnInfo; @ApiModelProperty(value = "表下的所有列信息")
private RealTimeSyncColumnModel columnInfo;
public String getTableName() { public String getTableName() {
...@@ -58,44 +71,43 @@ public class RealTimeSyncTableModel { ...@@ -58,44 +71,43 @@ public class RealTimeSyncTableModel {
this.pkName = pkName; this.pkName = pkName;
} }
/*public RealTimeSyncColumnModel getColumnInfo() { public String getDesensitizationField() {
return columnInfo; return desensitizationField;
} }
public void setColumnInfo(RealTimeSyncColumnModel columnInfo) { public void setDesensitizationField(String desensitizationField) {
this.columnInfo = columnInfo; this.desensitizationField = desensitizationField;
}
*/
public boolean isSubmited() {
return isSubmited;
} }
public void setIsSubmited(boolean isSubmited) { public String getArithmetic() {
this.isSubmited = isSubmited; return arithmetic;
} }
public boolean isBlacklist() { public void setArithmetic(String arithmetic) {
return isBlacklist; this.arithmetic = arithmetic;
} }
public void setIsBlacklist(boolean isBlacklist) { public RealTimeSyncColumnModel getColumnInfo() {
this.isBlacklist = isBlacklist; return columnInfo;
} }
public String getDesensitizationField() { public void setColumnInfo(RealTimeSyncColumnModel columnInfo) {
return desensitizationField; this.columnInfo = columnInfo;
} }
public void setDesensitizationField(String desensitizationField) { public boolean isIfSubmited() {
this.desensitizationField = desensitizationField; return ifSubmited;
} }
public String getArithmetic() { public void setIfSubmited(boolean ifSubmited) {
return arithmetic; this.ifSubmited = ifSubmited;
} }
public void setArithmetic(String arithmetic) { public boolean isIfBlacklist() {
this.arithmetic = arithmetic; return ifBlacklist;
} }
public void setIfBlacklist(boolean ifBlacklist) {
this.ifBlacklist = ifBlacklist;
}
} }
src/main/java/com/jz/dmp/modules/service/DmpRealtimeSyncInfoService.java
View file @ aa80f0d8
...@@ -7,6 +7,7 @@ import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListDto; ...@@ -7,6 +7,7 @@ import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListDto;
import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListReq; import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListReq;
import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeTableInfoReq; import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeTableInfoReq;
import com.jz.dmp.modules.model.DmpRealtimeSyncInfo; import com.jz.dmp.modules.model.DmpRealtimeSyncInfo;
import com.jz.dmp.modules.model.RealTimeSyncModel;
import java.util.List; import java.util.List;
...@@ -83,11 +84,20 @@ public interface DmpRealtimeSyncInfoService { ...@@ -83,11 +84,20 @@ public interface DmpRealtimeSyncInfoService {
JsonResult<List<DataSourceNameListDto>> queryDatasourceNameList(String projectId, String type) throws Exception; JsonResult<List<DataSourceNameListDto>> queryDatasourceNameList(String projectId, String type) throws Exception;
/** /**
* 根据数据源id获取表详细信息 * 新增--选择源表信息
* *
* @return * @return
* @author Bellamy * @author Bellamy
* @since 2021-01-06 * @since 2021-01-06
*/ */
JsonResult queryTableInfoByParams(RealTimeTableInfoReq req) throws Exception; JsonResult<RealTimeSyncModel> queryTableInfoByParams(RealTimeTableInfoReq req) throws Exception;
/**
* 新增--选择目标信息
*
* @return
* @author Bellamy
* @since 2021-01-07
*/
JsonResult selectTargetDatasourceInfo(String projectId) throws Exception;
} }
\ No newline at end of file
src/main/java/com/jz/dmp/modules/service/impl/DmpRealtimeSyncInfoServiceImpl.java
View file @ aa80f0d8
package com.jz.dmp.modules.service.impl; package com.jz.dmp.modules.service.impl;
import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo; import com.github.pagehelper.PageInfo;
import com.jz.common.constant.JsonResult; import com.jz.common.constant.JsonResult;
import com.jz.common.constant.ResultCode; import com.jz.common.constant.ResultCode;
import com.jz.common.page.PageInfoResponse; import com.jz.common.page.PageInfoResponse;
import com.jz.common.persistence.BaseService;
import com.jz.common.utils.realTime.DBUtil;
import com.jz.dmp.modules.controller.DataIntegration.bean.DataSourceNameListDto; import com.jz.dmp.modules.controller.DataIntegration.bean.DataSourceNameListDto;
import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListDto; import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListDto;
import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListReq; import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeSyncListReq;
import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeTableInfoReq; import com.jz.dmp.modules.controller.DataIntegration.bean.RealTimeTableInfoReq;
import com.jz.dmp.modules.dao.DmpProjectDao;
import com.jz.dmp.modules.dao.DmpRealtimeSyncInfoDao; import com.jz.dmp.modules.dao.DmpRealtimeSyncInfoDao;
import com.jz.dmp.modules.model.DmpRealtimeSyncInfo; import com.jz.dmp.modules.model.*;
import com.jz.dmp.modules.service.DmpRealtimeSyncInfoService; import com.jz.dmp.modules.service.DmpRealtimeSyncInfoService;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
...@@ -35,9 +38,15 @@ import java.util.regex.Pattern; ...@@ -35,9 +38,15 @@ import java.util.regex.Pattern;
@Transactional @Transactional
public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoService { public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoService {
@Value("${spring.public-key}")
private String publicKey;
@Autowired @Autowired
private DmpRealtimeSyncInfoDao dmpRealtimeSyncInfoDao; private DmpRealtimeSyncInfoDao dmpRealtimeSyncInfoDao;
@Autowired
private DmpProjectDao dmpProjectDao;
/** /**
* 通过ID查询单条数据 * 通过ID查询单条数据
* *
...@@ -159,19 +168,19 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic ...@@ -159,19 +168,19 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
} }
/** /**
* 根据数据源id获取表详细信息 * 新增--选择源表信息
* *
* @return * @return
* @author Bellamy * @author Bellamy
* @since 2021-01-06 * @since 2021-01-06
*/ */
@Override @Override
public JsonResult queryTableInfoByParams(RealTimeTableInfoReq req) throws Exception { public JsonResult<RealTimeSyncModel> queryTableInfoByParams(RealTimeTableInfoReq req) throws Exception {
Map<String, Map<String, String>> paramsMap = null; Map<String, Map<String, String>> paramsMap = null;
//根据数据源id获取数据信息 //根据来源数据源id获取数据信息
Map sourceDbInfo = dmpRealtimeSyncInfoDao.querygSourceDbInfoById(req.getSrcDatasourceId()); RealTimeSyncDataSourceModel sourceDbInfo = dmpRealtimeSyncInfoDao.querygSourceDbInfoById(req.getSrcDatasourceId());
sourceDbInfo.setPassword(new BaseService().decode(sourceDbInfo.getPassword(), publicKey));
//数据源对应的表详细信息 //数据源对应的表详细信息 判断表的记录在同步任务中是否存在记录,如果已经存在,说明同步过
List<Map> list = dmpRealtimeSyncInfoDao.queryRealTimeInfoByDataSourceId(req.getSrcDatasourceId(), req.getTargetDatasourceId()); List<Map> list = dmpRealtimeSyncInfoDao.queryRealTimeInfoByDataSourceId(req.getSrcDatasourceId(), req.getTargetDatasourceId());
if (list.size() > 0 && list != null) { if (list.size() > 0 && list != null) {
paramsMap = new HashMap<>(); paramsMap = new HashMap<>();
...@@ -181,7 +190,7 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic ...@@ -181,7 +190,7 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
} }
} }
Map<String, String> blackMapSetting = null; Map<String, String> blackMapSetting = null;
//查询源数据源的黑名单表 //查询来源数据源的黑名单表
Map blackTableMap = dmpRealtimeSyncInfoDao.queryBlackTableByDataSourceId(req.getSrcDatasourceId()); Map blackTableMap = dmpRealtimeSyncInfoDao.queryBlackTableByDataSourceId(req.getSrcDatasourceId());
if (blackTableMap.size() > 0 && blackTableMap != null) { if (blackTableMap.size() > 0 && blackTableMap != null) {
String blacklistTable = blackTableMap.get("blacklistTable").toString(); String blacklistTable = blackTableMap.get("blacklistTable").toString();
...@@ -195,7 +204,29 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic ...@@ -195,7 +204,29 @@ public class DmpRealtimeSyncInfoServiceImpl implements DmpRealtimeSyncInfoServic
} }
} }
} }
//DBUtil.getDataSourceTables(sourceDbInfo,req.getToQueryTableName(),true,paramsMap,req.getBlacklistTables(),req.getSelectTablesName(),blackMapSetting); //来源数据源下的表信息
return null; List<RealTimeSyncTableModel> dataSourceTables = DBUtil.getDataSourceTables(sourceDbInfo, req.getToQueryTableName(), true, paramsMap, req.getBlacklistTables(), req.getSelectTablesName(), blackMapSetting);
RealTimeSyncModel realTimeSyncModel = new RealTimeSyncModel();
realTimeSyncModel.setSrcdatasourceId(Integer.valueOf(req.getSrcDatasourceId()));
realTimeSyncModel.setDataSourceName(sourceDbInfo.getDatasourceName());
realTimeSyncModel.setDbName(sourceDbInfo.getDbName());
realTimeSyncModel.setTables(dataSourceTables); //来源数据源下的表信息
return new JsonResult(realTimeSyncModel);
}
/**
* 新增--选择目标信息
*
* @return
* @author Bellamy
* @since 2021-01-07
*/
@Override
public JsonResult selectTargetDatasourceInfo(String projectId) throws Exception {
DmpProjectSystemInfo dmpProjectSystemInfo = dmpProjectDao.queryProjectSystemInfo(Long.valueOf(projectId));
String kafkaConnectUrl = dmpProjectSystemInfo.getKafkaConnectorUrl(); //kafka 连接信息
String [] arr = kafkaConnectUrl.split(",");
return new JsonResult(arr);
} }
} }
\ No newline at end of file
src/main/resources/mapper/dmp/DmpProjectMapper.xml
View file @ aa80f0d8
...@@ -459,54 +459,54 @@ ...@@ -459,54 +459,54 @@
<select id="queryProjectSystemInfo" parameterType="java.lang.Long" resultType="com.jz.dmp.modules.model.DmpProjectSystemInfo"> <select id="queryProjectSystemInfo" parameterType="java.lang.Long" resultType="com.jz.dmp.modules.model.DmpProjectSystemInfo">
SELECT SELECT
ID, ID,
PROJECT_ID, PROJECT_ID,
KERBEROS_ISENABLE, KERBEROS_ISENABLE,
KERBEROS_KRB5_CONF, KERBEROS_KRB5_CONF,
KERBEROS_JAAS_CONF, KERBEROS_JAAS_CONF,
KERBEROS_KEYTAB_CONF, KERBEROS_KEYTAB_CONF,
KERBEROS_KEYTAB_USER, KERBEROS_KEYTAB_USER,
KERBEROS_FQDN, KERBEROS_FQDN,
KERBEROS_SPARK_JAAS_CONF, KERBEROS_SPARK_JAAS_CONF,
KERBEROS_JAAS_CLIENT_NAME, KERBEROS_JAAS_CLIENT_NAME,
SHELL_CMD_SERVER, SHELL_CMD_SERVER,
SHELL_CMD_USER, SHELL_CMD_USER,
SHELL_CMD_PASSWORD, SHELL_CMD_PASSWORD,
SHELL_CMD_SUBMIT_SYCNING, SHELL_CMD_SUBMIT_SYCNING,
SHELL_CMD_QUERY_STATUS, SHELL_CMD_QUERY_STATUS,
SHELL_CMD_STOP_SYCNING, SHELL_CMD_STOP_SYCNING,
SHELL_CMD_CAT_LOG, SHELL_CMD_CAT_LOG,
SHELL_FTP_DOWNLOAD_DIR, SHELL_FTP_DOWNLOAD_DIR,
HDFS_HTTP_PATH, HDFS_HTTP_PATH,
HDFS_USER_NAME, HDFS_USER_NAME,
HDFS_SYNCING_PATH, HDFS_SYNCING_PATH,
IMPALA_JDBC_URL, IMPALA_JDBC_URL,
IMPALA_SHELL_URL, IMPALA_SHELL_URL,
AZKABAN_MONITOR_URL, AZKABAN_MONITOR_URL,
AZKABAN_EXECTOR_SHELL_EXEC, AZKABAN_EXECTOR_SHELL_EXEC,
AZKABAN_EXECTOR_SQL_EXEC, AZKABAN_EXECTOR_SQL_EXEC,
AZKABAN_EXECTOR_XML_EXEC, AZKABAN_EXECTOR_XML_EXEC,
AZKABAN_EXECTOR_SQL_PATH, AZKABAN_EXECTOR_SQL_PATH,
AZKABAN_EXECTOR_SHELL_PATH, AZKABAN_EXECTOR_SHELL_PATH,
AZKABAN_EXECTOR_SHELL_EXPORT_DATA, AZKABAN_EXECTOR_SHELL_EXPORT_DATA,
AZKABAN_LOCAL_TASK_FILE_PATH, AZKABAN_LOCAL_TASK_FILE_PATH,
KAFKA_CONNECTOR_URL, KAFKA_CONNECTOR_URL,
KAFKA_BOOTSTRAP_SERVERS, KAFKA_BOOTSTRAP_SERVERS,
KAFKA_MONITOR_URL, KAFKA_MONITOR_URL,
KAFKA_OFFSET_UPDATE_SHELL, KAFKA_OFFSET_UPDATE_SHELL,
KAFKA_SCHEMA_REGISTER_URL, KAFKA_SCHEMA_REGISTER_URL,
KAFKA_INFLUX_URL, KAFKA_INFLUX_URL,
KAFKA_INFLUX_USER_NAME, KAFKA_INFLUX_USER_NAME,
KAFKA_INFLUX_PASSWORD, KAFKA_INFLUX_PASSWORD,
SHELL_SFTP_PORT, SHELL_SFTP_PORT,
SPARK_HIVE_METASTORE_URIS, SPARK_HIVE_METASTORE_URIS,
SPARK_YARN_QUEUE, SPARK_YARN_QUEUE,
SPARK_DEFAULT_EXECUTOR_MEMORY, SPARK_DEFAULT_EXECUTOR_MEMORY,
SPARK_DEFAULT_EXECUTOR_CORES, SPARK_DEFAULT_EXECUTOR_CORES,
SPARK_DEFAULT_TOTAL_EXECUTOR_CORES, SPARK_DEFAULT_TOTAL_EXECUTOR_CORES,
SPARK_STATISTICS_SOURCE_DATA_SWITCH, SPARK_STATISTICS_SOURCE_DATA_SWITCH,
ATLAS_MONITOR_URL, ATLAS_MONITOR_URL,
OPENAPI_INTERFACE_URL OPENAPI_INTERFACE_URL
FROM dmp_project_system_info FROM dmp_project_system_info
WHERE data_status = '1' and PROJECT_ID = #{projectId} WHERE data_status = '1' and PROJECT_ID = #{projectId}
</select> </select>
......
src/main/resources/mapper/dmp/DmpRealtimeSyncInfoMapper.xml
View file @ aa80f0d8
...@@ -364,22 +364,23 @@ ...@@ -364,22 +364,23 @@
</select> </select>
<!--根据数据源id获取数据信息--> <!--根据数据源id获取数据信息-->
<select id="querygSourceDbInfoById" parameterType="map" resultType="map"> <select id="querygSourceDbInfoById" parameterType="map" resultType="com.jz.dmp.modules.model.RealTimeSyncDataSourceModel">
select select
ds.id as id, ds.id as id,
ds.datasource_name as datasourceName,
ds.jdbc_url as jdbcUrl, ds.jdbc_url as jdbcUrl,
ds.user_name as userName, ds.user_name as userName,
ds.password, ds.password,
ds.db_name as dbName, ds.db_name as dbName,
dsdt.datasource as datasourceTypeName, dsdt.datasource_type as datasourceTypeName,
dsdt.driver_class_name as driverClassName, dsdt.driver_class_name as driverName
dsdt.datasource,
from dmp_syncing_datasource ds from dmp_syncing_datasource ds
inner join dmp_syncing_datasource_type dsdt on ds.datasource_type = dsdt.id inner join dmp_syncing_datasource_type dsdt on ds.datasource_type = dsdt.id
where 1=1 and ds.data_status = '1' dsd.ID = #{srcDataSourceId} where 1=1 and ds.data_status = '1'
and ds.ID = #{srcDataSourceId}
</select> </select>
<select id="queryListById11" resultType="java.util.Map"> <select id="queryRealTimeInfoByDataSourceId" resultType="java.util.Map">
select select
src_table_name as srcTableName, src_table_name as srcTableName,
connector_job_id as connectorJobId, connector_job_id as connectorJobId,
...@@ -387,8 +388,9 @@ ...@@ -387,8 +388,9 @@
arithmetic arithmetic
from dmp_realtime_sync_info from dmp_realtime_sync_info
where 1=1 and type =2 where 1=1 and type =2
<if test="targetDataSourceId != null"> and target_datasource_id = #{targetDataSourceId} </if> and src_datasource_id = #{srcDatasourceId}
<if test="srcTableName != null"> and src_table_name = #{srcTableName} </if> <if test="targetDatasourceId != null"> and target_datasource_id = #{targetDatasourceId} </if>
<!--<if test="srcTableName != null"> and src_table_name = #{srcTableName} </if>-->
</select> </select>
<!-- 查询源数据源的黑名单表 --> <!-- 查询源数据源的黑名单表 -->
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment