Skip to content

J
jz-dmp-service
Commit a6f92fb6 authored by mcb's avatar mcb
Browse files
Options

任务 立即运行

parent a81458f4
Hide whitespace changes
Inline Side-by-side
Showing with 4985 additions and 3 deletions
pom.xml
View file @ a6f92fb6
...@@ -242,6 +242,11 @@ ...@@ -242,6 +242,11 @@
<artifactId>kudu-client</artifactId> <artifactId>kudu-client</artifactId>
<version>1.1.0</version> <version>1.1.0</version>
</dependency> </dependency>
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
<version>1.3.1</version>
</dependency>
</dependencies> </dependencies>
<build> <build>
......
src/main/java/com/jz/agent/client/DmpDsAgentClient.java 0 → 100644
View file @ a6f92fb6
package com.jz.agent.client;
import com.jz.dmp.agent.DmpAgentResult;
import com.jz.dmp.modules.model.DmpAgentDatasourceInfo;
public interface DmpDsAgentClient {
//连接
public DmpAgentResult testConnect(DmpAgentDatasourceInfo ds);
//获取数据库表
public DmpAgentResult getTableNameList(DmpAgentDatasourceInfo ds);
//获取表字段名
public DmpAgentResult getTableColumnList(DmpAgentDatasourceInfo ds, String tableName);
//获取表数据预览
public DmpAgentResult previewData(DmpAgentDatasourceInfo ds, String tableName);
}
src/main/java/com/jz/agent/client/DmpDsAgentClientBuilder.java 0 → 100644
View file @ a6f92fb6
package com.jz.agent.client;
import com.jz.agent.client.service.*;
import com.jz.dmp.modules.model.DmpAgentDatasourceInfo;
public class DmpDsAgentClientBuilder {
public static DmpDsAgentClient build(DmpAgentDatasourceInfo ds) {
switch (ds.getDatasourceType()) {
case "MYSQL":
return new DmpMysqlAgentClientService();
case "SQLSERVER":
return new DmpSqlServerAgentClientService();
case "POSTGRESQL":
return new DmpPostgreSqlAgentClientService();
case "ORACLE":
return new DmpOracleAgentClientService();
case "DM":
return null;
case "DB2":
return new DmpDB2AgentClientService();
case "HIVE":
return new DmpHiveAgentClientService();
case "IMPALA":
return null;
case "KUDU":
return new DmpKuduAgentClientService();
// return SpringUtils.getBean(DmpKuduAgentClientService.class);
case "HDFS":
return new DmpHdfsAgentClientService();
case "FTP":
return new DmpFtpAgentClientService();
case "ELASTICSEARCH":
return new DmpElasticsearchAgentClientService();
case "MONGODB":
return null;
case "MEMCACHE":
return null;
case "REDIS":
return null;
case "HBASE":
return null;
case "LOGHUB":
return null;
case "KAFKA":
return null;
case "ROCKETMQ":
return null;
case "DEFAULT":
return new DmpHiveAgentClientService();
default:
break;
}
return null;
}
}
src/main/java/com/jz/agent/client/service/DmpDB2AgentClientService.java 0 → 100644
View file @ a6f92fb6
package com.jz.agent.client.service;
import com.google.gson.Gson;
import com.jz.agent.client.DmpDsAgentClient;
import com.jz.agent.utils.JDBCUtils;
import com.jz.dmp.agent.DmpAgentResult;
import com.jz.dmp.agent.ResultCode;
import com.jz.dmp.modules.model.DmpAgentDatasourceInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class DmpDB2AgentClientService implements DmpDsAgentClient {
private Gson gson = new Gson();
@SuppressWarnings("unused")
private static Logger logger = LoggerFactory.getLogger(DmpDB2AgentClientService.class);
@SuppressWarnings("unused")
public DmpAgentResult testConnect(DmpAgentDatasourceInfo ds) {
Connection conn = null;
try {
Class.forName(ds.getDriverClassName());
conn = DriverManager.getConnection(
ds.getJdbcUrl(),
ds.getUserName(),
ds.getPassword()
);
conn.close();
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(true));
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "DB2连通性异常：" + e.getMessage());
} finally {
try {
if (conn != null)
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
@SuppressWarnings({"unchecked", "rawtypes"})
public DmpAgentResult getTableNameList(DmpAgentDatasourceInfo ds) {
List result = new ArrayList();
String sql = "select * from information_schema.tables where table_schema = '" + this.getTargetDBName(ds) + "'";
Connection conn = null;
ResultSet rs;
PreparedStatement ps;
try {
conn = JDBCUtils.getConnections(ds.getDriverClassName(), ds.getJdbcUrl(), ds.getUserName(), ds.getPassword());
ps = conn.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
result.add(rs.getString("TABLE_NAME").toLowerCase());
}
} catch (SQLException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "DB2获取表列表异常：" + e.getMessage());
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "DB2获取表列表异常：" + e.getMessage());
} finally {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(result));
}
private String getTargetDBName(DmpAgentDatasourceInfo ds) {
String url = ds.getJdbcUrl();
if (url != null && !"".equals(url.trim())) {
String[] arr1 = url.split("/");
String ls = arr1[arr1.length - 1];
String[] arr2 = ls.split("\\?");
return arr2[0];
} else {
return "";
}
}
@SuppressWarnings({"rawtypes", "unchecked", "unused"})
public DmpAgentResult getTableColumnList(DmpAgentDatasourceInfo ds, String tableName) {
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
String sql = "select * from information_schema.columns where table_name = '" + tableName + "'";
Connection conn = null;
ResultSet rs;
PreparedStatement ps;
try {
conn = JDBCUtils.getConnections(ds.getDriverClassName(), ds.getJdbcUrl(), ds.getUserName(), ds.getPassword());
ps = conn.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
Map map = new HashMap();
String colName = rs.getString("COLUMN_NAME");
String dbType = rs.getString("DATA_TYPE");
String remarks = rs.getString("COLUMN_COMMENT");
Integer size = 0;
Integer characterLength = (int) rs.getDouble("CHARACTER_MAXIMUM_LENGTH");
Integer columnSize = (int) rs.getDouble("NUMERIC_PRECISION");
Integer dicimalDigits = (int) rs.getDouble("NUMERIC_SCALE");
if (characterLength != null) {
size = characterLength;
} else {
size = columnSize;
}
map.put("name", colName.toLowerCase());
map.put("type", dbType.toUpperCase());
map.put("size", size);
map.put("scale", dicimalDigits);
map.put("comment", remarks);
list.add(map);
}
} catch (SQLException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "DB2获取表字段信息异常：" + e.getMessage());
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "DB2获取表字段信息异常：" + e.getMessage());
} finally {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
public DmpAgentResult previewData(DmpAgentDatasourceInfo ds, String tableName) {
Map<String, Object> map = new HashMap<String, Object>();
Connection conn = null;
ResultSet rs;
PreparedStatement ps;
List<String> headerList = new ArrayList<String>();
try {
conn = JDBCUtils.getConnections(ds.getDriverClassName(), ds.getJdbcUrl(), ds.getUserName(), ds.getPassword());
String sql = "select * from " + tableName + " limit 5";
String sql2 = "select * from information_schema.columns where table_name='" + tableName + "'";
ps = conn.prepareStatement(sql2);
rs = ps.executeQuery();
while (rs.next()) {
String colName = rs.getString("COLUMN_NAME");
headerList.add(colName);
}
map.put("header", headerList);
ps = conn.prepareStatement(sql);
rs = ps.executeQuery();
List<List<Object>> resultList = new ArrayList<List<Object>>();
while (rs.next()) {
List<Object> row = new ArrayList<Object>();
for (int i = 1; i <= headerList.size(); i++) {
row.add(rs.getObject(i));
}
resultList.add(row);
}
map.put("result", resultList);
} catch (SQLException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "DB2数据预览异常：" + e.getMessage());
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "DB2数据预览异常：" + e.getMessage());
} finally {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(map));
}
}
src/main/java/com/jz/agent/client/service/DmpElasticsearchAgentClientService.java 0 → 100644
View file @ a6f92fb6
package com.jz.agent.client.service;
import com.google.gson.Gson;
import com.jz.agent.client.DmpDsAgentClient;
import com.jz.dmp.agent.DmpAgentResult;
import com.jz.dmp.agent.ResultCode;
import com.jz.dmp.modules.model.DmpAgentDatasourceInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class DmpElasticsearchAgentClientService implements DmpDsAgentClient {
private Gson gson = new Gson();
@SuppressWarnings("unused")
private static Logger logger = LoggerFactory.getLogger(DmpElasticsearchAgentClientService.class);
@SuppressWarnings("unused")
public DmpAgentResult testConnect(DmpAgentDatasourceInfo ds) {
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(false));
}
@SuppressWarnings({"unchecked", "rawtypes"})
public DmpAgentResult getTableNameList(DmpAgentDatasourceInfo ds) {
List<String> list = new ArrayList<String>();
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
@SuppressWarnings({"rawtypes", "unchecked", "unused"})
public DmpAgentResult getTableColumnList(DmpAgentDatasourceInfo ds, String tableName) {
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
public DmpAgentResult previewData(DmpAgentDatasourceInfo ds, String tableName) {
Map<String, Object> map = new HashMap<String, Object>();
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(map));
}
}
src/main/java/com/jz/agent/client/service/DmpFtpAgentClientService.java 0 → 100644
View file @ a6f92fb6
package com.jz.agent.client.service;
import com.google.gson.Gson;
import com.jcraft.jsch.*;
import com.jz.agent.client.DmpDsAgentClient;
import com.jz.dmp.agent.DmpAgentResult;
import com.jz.dmp.agent.ResultCode;
import com.jz.dmp.modules.model.DmpAgentDatasourceInfo;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
public class DmpFtpAgentClientService implements DmpDsAgentClient {
private Gson gson = new Gson();
private static Logger logger = LoggerFactory.getLogger(DmpFtpAgentClientService.class);
@SuppressWarnings("unused")
public DmpAgentResult testConnect(DmpAgentDatasourceInfo ds) {
if ("ftp".equals(ds.getProtocol())) {
return testFtpConnect(ds);
} else {
return testSftpConnect(ds);
}
}
public DmpAgentResult getTableNameList(DmpAgentDatasourceInfo ds) {
List<String> list = new ArrayList<String>();
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
@SuppressWarnings("unchecked")
public DmpAgentResult getTableColumnList(DmpAgentDatasourceInfo ds, String filePath) {
Object o = this.getColumnAndResult(ds, filePath);
if(o instanceof Map) {
Map<String, Object> m = (Map<String, Object>) o;
List<Map<String, Object>> l = (List<Map<String, Object>>) m.get("column");
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(l));
} else {
return new DmpAgentResult(ResultCode.EXCEPTION, gson.toJson(o));
}
}
@SuppressWarnings({ "unused"})
public DmpAgentResult previewData(DmpAgentDatasourceInfo ds, String filePath) {
Object o = this.getColumnAndResult(ds, filePath);
if(o instanceof Map) {
Map<String, Object> m = (Map<String, Object>) o;
m.remove("column");
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(m));
} else {
return new DmpAgentResult(ResultCode.EXCEPTION, gson.toJson(o));
}
}
@SuppressWarnings({ "unused", "unchecked" })
private Object getColumnAndResult(DmpAgentDatasourceInfo ds, String filePath) {
Map<String, Object> map = new HashMap<String, Object>();
String remotePath = "";
String fileName = "";
if(StringUtils.isNotBlank(filePath)) {
try {
remotePath = filePath.substring(0, filePath.lastIndexOf("/"));
fileName = filePath.substring(filePath.lastIndexOf("/")+1, filePath.length());
} catch(Exception e) {
return new DmpAgentResult(ResultCode.PARAMS_ERROR, "参数错误：filePath="+filePath);
}
} else {
return new DmpAgentResult(ResultCode.PARAMS_ERROR, "参数错误：filePath="+filePath);
}
String splitSymbol = ds.getDelimiter();
String fileType = ds.getFileType();
int columnNum = 0;
InputStream is = null;
if ("ftp".equals(ds.getProtocol())) {
boolean result = false;
FTPClient ftp = new FTPClient();
try {
int reply;
ftp.connect(ds.getHost(), Integer.valueOf(ds.getPort()).intValue());
ftp.login(ds.getUserName(), ds.getPassword());
reply = ftp.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
ftp.disconnect();
return new DmpAgentResult(ResultCode.INTERNAL_SERVER_ERROR, "FTP不能连接");
}
ftp.changeWorkingDirectory(remotePath);
FTPFile[] fs = ftp.listFiles();
for (FTPFile ff : fs) {
if (ff.getName().equals(fileName)) {
//File localFile = new File(localPath + "\\" + ff.getName());
/*OutputStream os = new FileOutputStream(localFile);
ftp.retrieveFile(ff.getName(), os);*/
is = ftp.retrieveFileStream(filePath);
StringBuilder sb = new StringBuilder();
int rowCnt = 0;
int rowNum = 6;
byte[] bs = new byte[1024*50];
int len = -1;
List<String> header = new ArrayList<String>();
List<String> type = new ArrayList<String>();
while((len = is.read(bs)) != -1){
sb.append(new String(bs,0,len));
/*Pattern p = Pattern.compile("\\s*|\t|\r\n|\r|\n");
Matcher m = p.matcher(sb.toString());
while(m.find()) {
System.out.println(m.group());
}*/
//System.out.println(m.groupCount());
//char[] c = System.getProperty("line.separator").toCharArray();
//System.out.println(System.getProperty("line.separator").toCharArray());
String s = sb.toString();
List<Object[]> list = new ArrayList<Object[]>();
boolean flag = true;
String enterLine = "\n";
if(s.indexOf("\r\n") > -1) {
enterLine = "\r\n";
} else if(s.indexOf("\r") > -1) {
enterLine = "\r";
} else if(s.indexOf("\n") > -1) {
enterLine = "\n";
} else {
flag = false;
}
while(rowCnt < rowNum && flag) {
int i1 = s.indexOf(enterLine);
String tmp = s.substring(0, i1);
if("json".equalsIgnoreCase(fileType)) {
Map<String, Object> m = gson.fromJson(tmp, Map.class);
if(rowCnt == 0) {
for(Map.Entry<String, Object> me : m.entrySet()) {
header.add(me.getKey());
type.add(this.getObjectType(me.getValue()));
columnNum++;
}
}
Object[] row = new Object[columnNum];
int i = 0;
for(String rn : header) {
row[i] = m.get(rn);
i++;
}
list.add(row);
} else {
if(StringUtils.isBlank(splitSymbol)) {
if(tmp.indexOf(",") > -1) {
splitSymbol = ",";
} else if(tmp.indexOf("\t") > -1) {
splitSymbol = "\t";
} else if(tmp.indexOf(" ") > -1) {
splitSymbol = " ";
}
}
String[] tArr = tmp.split(splitSymbol);
list.add(tArr);
columnNum = tArr.length;
}
s = s.substring(i1+2, s.length());
rowCnt++;
}
if(rowCnt == rowNum) {
map.put("result", list);
break;
} else {
rowCnt = 0;
rowNum = 6;
}
}
//os.close();
break;
}
}
ftp.logout();
} catch (IOException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "FTP读取信息异常："+e.getMessage());
} finally {
if (ftp.isConnected()) {
try {
ftp.disconnect();
} catch (IOException ioe) {
}
}
if(is != null)
try {
is.close();
} catch (IOException e) {
e.printStackTrace();
}
}
} else {
String host = ds.getHost();
int port = Integer.valueOf(ds.getPort()).intValue();
String username = ds.getUserName();
String password = ds.getPassword();
ChannelSftp sftp = null;
Session session = null;
try {
JSch jsch = new JSch();
session = jsch.getSession(username, host, port);
if (password != null) {
session.setPassword(password);
}
Properties config = new Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
Channel channel = session.openChannel("sftp");
channel.connect();
sftp = (ChannelSftp) channel;
if (remotePath != null && !"".equals(remotePath)) {
sftp.cd(remotePath);
}
is = sftp.get(fileName);
StringBuilder sb = new StringBuilder();
int rowCnt = 0;
int rowNum = 6;
byte[] bs = new byte[1024*50];
int len = -1;
List<String> header = new ArrayList<String>();
List<String> type = new ArrayList<String>();
while((len = is.read(bs)) != -1){
sb.append(new String(bs,"UTF-8"));
String s = sb.toString();
s = s.replaceAll("\\ufeff", "");
List<Object[]> list = new ArrayList<Object[]>();
boolean flag = true;
String enterLine = "\n";
if(s.indexOf("\r\n") > -1) {
enterLine = "\r\n";
} else if(s.indexOf("\r") > -1) {
enterLine = "\r";
} else if(s.indexOf("\n") > -1) {
enterLine = "\n";
} else {
flag = false;
}
while(rowCnt < rowNum && flag) {
int i1 = s.indexOf(enterLine);
String tmp = s.substring(0, i1);
if("json".equalsIgnoreCase(fileType)) {
Map<String, Object> m = gson.fromJson(tmp, Map.class);
if(rowCnt == 0) {
for(Map.Entry<String, Object> me : m.entrySet()) {
header.add(me.getKey());
type.add(this.getObjectType(me.getValue()));
columnNum++;
}
}
Object[] row = new Object[columnNum];
int i = 0;
for(String rn : header) {
row[i] = m.get(rn);
i++;
}
list.add(row);
} else {
if(StringUtils.isBlank(splitSymbol)) {
if(tmp.indexOf(",") > -1) {
splitSymbol = ",";
} else if(tmp.indexOf("\t") > -1) {
splitSymbol = "\t";
} else if(tmp.indexOf(" ") > -1) {
splitSymbol = " ";
}
}
String[] tArr = tmp.split(splitSymbol);
list.add(tArr);
columnNum = tArr.length;
}
s = s.substring(i1+2, s.length());
rowCnt++;
}
if(rowCnt == rowNum) {
map.put("result", list);
break;
} else {
rowCnt = 0;
rowNum = 3;
}
}
List<Map<String, Object>> l2 = new ArrayList<Map<String, Object>>();
List<String> l3 = new ArrayList<String>();
if("json".equalsIgnoreCase(fileType)) {
for(int i = 0; i < header.size(); i++) {
Map<String, Object> m = new HashMap<String, Object>();
m.put("key", i);
m.put("name", header.get(i));
m.put("type", type.get(i));
l2.add(m);
l3.add(header.get(i).toUpperCase());
}
} else {
if("1".equals(ds.getIsHaveHeader())){
List<String[]> l = (List<String[]>) map.get("result");
String[] arr = l.get(0);
for(int i = 0; i < columnNum; i++) {
Map<String, Object> m = new HashMap<String, Object>();
m.put("key", i);
m.put("name", arr[i]);
m.put("type", "STRING");
l2.add(m);
l3.add(arr[i].toUpperCase());
}
map.put("result", l.subList(1, l.size()));
} else {
for(int i = 0; i < columnNum; i++) {
Map<String, Object> m = new HashMap<String, Object>();
m.put("key", i);
m.put("name", "第"+i+"列");
m.put("type", "STRING");
l2.add(m);
l3.add("第"+i+"列");
}
}
}
map.put("header", l3);
map.put("column", l2);
} catch (JSchException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "SFTP读取信息异常："+e.getMessage());
} catch (SftpException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "SFTP读取信息异常："+e.getMessage());
} catch (IOException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "SFTP读取信息异常："+e.getMessage());
} finally {
if (sftp != null) {
if (sftp.isConnected()) {
sftp.disconnect();
}
}
if (session != null) {
if (session.isConnected()) {
session.disconnect();
}
}
}
}
return map;
}
private String getObjectType(Object obj) {
String t = null;
if(obj instanceof Long) {
t = "LONG";
} else if(obj instanceof Double) {
t = "DOUBLE";
} else if(obj instanceof Boolean) {
t = "BOOLEAN";
} else if(obj instanceof Date) {
t = "DATE";
} else {
t = "STRING";
}
return t;
}
private DmpAgentResult testFtpConnect(DmpAgentDatasourceInfo ds) {
String host = ds.getHost();
int port = Integer.valueOf(ds.getPort()).intValue();
String username = ds.getUserName();
String password = ds.getPassword();
boolean result = false;
FTPClient ftp = new FTPClient();
try {
int reply;
ftp.connect(host, port);
ftp.login(username, password);
reply = ftp.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
result = false;
} else {
result = true;
}
ftp.disconnect();
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(result));
} catch (IOException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.SUCCESS, "FTP连通性测试异常："+e.getMessage());
} finally {
if (ftp.isConnected()) {
try {
ftp.disconnect();
} catch (IOException ioe) {
}
}
}
}
private DmpAgentResult testSftpConnect(DmpAgentDatasourceInfo ds) {
ChannelSftp sftp = null;
Session session = null;
String host = ds.getHost();
int port = Integer.valueOf(ds.getPort()).intValue();
String username = ds.getUserName();
String password = ds.getPassword();
try {
JSch jsch = new JSch();
session = jsch.getSession(username, host, port);
if (password != null) {
session.setPassword(password);
}
Properties config = new Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
Channel channel = session.openChannel("sftp");
channel.connect();
sftp = (ChannelSftp) channel;
return new DmpAgentResult(ResultCode.SUCCESS, "true");
} catch (JSchException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.SUCCESS, "SFTP连通性测试异常："+e.getMessage());
} finally {
if (sftp != null) {
if (sftp.isConnected()) {
sftp.disconnect();
}
}
if (session != null) {
if (session.isConnected()) {
session.disconnect();
}
}
}
}
@SuppressWarnings("unused")
private void unzip() {
}
}
src/main/java/com/jz/agent/client/service/DmpHdfsAgentClientService.java 0 → 100644
View file @ a6f92fb6
package com.jz.agent.client.service;
import com.google.gson.Gson;
import com.jz.agent.client.DmpDsAgentClient;
import com.jz.agent.config.DmpDsAgentConstants;
import com.jz.dmp.agent.DmpAgentResult;
import com.jz.dmp.agent.ResultCode;
import com.jz.dmp.modules.model.DmpAgentDatasourceInfo;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
// HDFS没有提供界面操作接口
public class DmpHdfsAgentClientService implements DmpDsAgentClient {
private Gson gson = new Gson();
private static Logger logger = LoggerFactory.getLogger(DmpHdfsAgentClientService.class);
@SuppressWarnings("unused")
public DmpAgentResult testConnect(DmpAgentDatasourceInfo ds) {
FileSystem fileSystem = null;
try {
/*String defaultFs = ds.getDefaultFs();
if(StringUtils.isBlank(defaultFs) && StringUtils.isNotBlank(ds.getJdbcUrl())) {
defaultFs = ds.getJdbcUrl();
}*/
//Configuration configuration = initConfiguration(this.getHadoopConfFilePath(defaultFs));
//if(DmpDsAgentConstants.IS_ENABLED_KERBEROS) JDBCUtils.initKerberosENV4HDFS(configuration);
Configuration configuration = setHdfsConfiguration(ds);
if(configuration == null) {
return new DmpAgentResult(ResultCode.PARAMS_ERROR, "HDFS参数有误");
}
fileSystem = FileSystem.get(configuration);
fileSystem.getUsed();
return new DmpAgentResult(ResultCode.SUCCESS, "true");
} catch (IOException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "测试连通性出错："+e.getMessage());
} finally {
if(fileSystem != null)
try {
fileSystem.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
private Configuration setHdfsConfiguration(DmpAgentDatasourceInfo ds) {
logger.info("DmpAgentDatasourceInfo: "+gson.toJson(ds));
Configuration conf = new Configuration();
//conf.set("hadoop.security.authentication",ds.getHdfsUserName());
conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
String defaultFs = ds.getDefaultFs();
if(StringUtils.isBlank(defaultFs) && StringUtils.isNotBlank(ds.getJdbcUrl())) {
defaultFs = ds.getJdbcUrl();
}
conf.set("fs.defaultFS", defaultFs);
logger.info("fs.defaultFS: "+defaultFs);
if(ds.getKerberosIsenable() != null && "true".equals(ds.getKerberosIsenable())) {
conf.set("hadoop.security.authentication","kerberos");
System.setProperty("java.security.krb5.conf",ds.getKerberosKrb5Conf());
//System.setProperty("java.security.auth.login.config",ds.getKerberosJaasConf());
logger.info("hadoop.security.authentication: "+"kerberos");
logger.info("java.security.krb5.conf: "+ds.getKerberosKrb5Conf());
}
UserGroupInformation.setConfiguration(conf);
logger.info("kerberos.is.enabled: "+ds.getKerberosIsenable());
if(ds.getKerberosIsenable() != null && "true".equals(ds.getKerberosIsenable())) {
try {
logger.info("hdfs.user.name: "+ds.getHdfsUserName());
logger.info("hdfs.auth.path: "+ds.getHdfsAuthPath());
UserGroupInformation.loginUserFromKeytab(ds.getHdfsUserName(), ds.getHdfsAuthPath());
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
return conf;
}
public DmpAgentResult getTableNameList(DmpAgentDatasourceInfo ds) {
List<String> list = new ArrayList<String>();
FileSystem fileSystem = null;
try {
String defaultFs = ds.getDefaultFs();
if(StringUtils.isBlank(defaultFs) && StringUtils.isNotBlank(ds.getJdbcUrl())) {
defaultFs = ds.getJdbcUrl();
}
//Configuration configuration = initConfiguration(this.getHadoopConfFilePath(defaultFs));
//if(DmpDsAgentConstants.IS_ENABLED_KERBEROS) JDBCUtils.initKerberosENV4HDFS(configuration);
Configuration configuration = setHdfsConfiguration(ds);
if(configuration == null) {
return new DmpAgentResult(ResultCode.PARAMS_ERROR, "HDFS参数有误");
}
fileSystem = FileSystem.get(configuration);
String filePath = ds.getTargetFileName();
Path srcPath = new Path(filePath);
if(fileSystem.exists(srcPath) && fileSystem.isDirectory(srcPath)) {
RemoteIterator<LocatedFileStatus> ri = fileSystem.listFiles(srcPath, true);
while(ri.hasNext()) {
LocatedFileStatus lfs = ri.next();
System.out.println(lfs.getPath().toString());
list.add(lfs.getPath().toString());
}
}
} catch (IOException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "HDFS获取列表出错："+e.getMessage());
} finally {
if(fileSystem != null) {
try {
fileSystem.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
@SuppressWarnings("unchecked")
public DmpAgentResult getTableColumnList(DmpAgentDatasourceInfo ds, String targetName) {
Object o = this.getColumnAndResult(ds, targetName);
//List<Map<String, Object>> retMap = (List<Map<String, Object>>) map.get("column");
//return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(retMap));
if(o instanceof DmpAgentResult) {
return new DmpAgentResult(ResultCode.EXCEPTION, gson.toJson(o));
} else {
Map<String, Object> map = (Map<String, Object>) o;
List<Map<String, Object>> retMap = (List<Map<String, Object>>) map.get("column");
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(retMap));
}
}
public DmpAgentResult previewData(DmpAgentDatasourceInfo ds, String targetName) {
//Map<String, Object> m = this.getColumnAndResult(ds, targetName);
//m.remove("column");
//return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(m));
Object o = this.getColumnAndResult(ds, targetName);
if(o instanceof DmpAgentResult) {
return new DmpAgentResult(ResultCode.EXCEPTION, gson.toJson(o));
} else {
Map<String, Object> map = (Map<String, Object>) o;
map.remove("column");
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(map));
}
}
@SuppressWarnings("unchecked")
private Map<String, Object> getColumnAndResult(DmpAgentDatasourceInfo ds, String filePath) {
Map<String, Object> map = new HashMap<String, Object>();
InputStream is = null;
FileSystem fileSystem = null;
Path srcPath = new Path(filePath);
String splitSymbol = ds.getDelimiter();
String fileType = ds.getFileType();
int columnNum = 0;
try {
String defaultFs = ds.getDefaultFs();
if(StringUtils.isBlank(defaultFs) && StringUtils.isNotBlank(ds.getJdbcUrl())) {
defaultFs = ds.getJdbcUrl();
}
//Configuration configuration = initConfiguration(this.getHadoopConfFilePath(defaultFs));
//if(DmpDsAgentConstants.IS_ENABLED_KERBEROS) JDBCUtils.initKerberosENV4HDFS(configuration);
Configuration configuration = setHdfsConfiguration(ds);
if(configuration == null) {
return map;
}
fileSystem = FileSystem.get(configuration);
is = fileSystem.open(srcPath);
//IOUtils.copyBytes(is, System.out, 4096, false);
StringBuilder sb = new StringBuilder();
int rowCnt = 0;
int rowNum = 6;
byte[] bs = new byte[1024*50];
int len = -1;
List<String> header = new ArrayList<String>();
List<String> type = new ArrayList<String>();
while((len = is.read(bs)) != -1){
sb.append(new String(bs,0,len));
String s = sb.toString();
List<Object[]> list = new ArrayList<Object[]>();
boolean flag = true;
String enterLine = "\n";
if(s.indexOf("\r\n") > -1) {
enterLine = "\r\n";
} else if(s.indexOf("\r") > -1) {
enterLine = "\r";
} else if(s.indexOf("\n") > -1) {
enterLine = "\n";
} else {
flag = false;
}
while(rowCnt < rowNum && flag) {
int i1 = s.indexOf(enterLine);
String tmp = s.substring(0, i1);
if("json".equalsIgnoreCase(fileType)) {
Map<String, Object> m = gson.fromJson(tmp, Map.class);
if(rowCnt == 0) {
for(Map.Entry<String, Object> me : m.entrySet()) {
header.add(me.getKey());
type.add(this.getObjectType(me.getValue()));
columnNum++;
}
}
Object[] row = new Object[columnNum];
int i = 0;
for(String rn : header) {
row[i] = m.get(rn);
i++;
}
list.add(row);
} else {
if(StringUtils.isBlank(splitSymbol)) {
if(tmp.indexOf(",") > -1) {
splitSymbol = ",";
} else if(tmp.indexOf("\t") > -1) {
splitSymbol = "\t";
} else if(tmp.indexOf(" ") > -1) {
splitSymbol = " ";
}
}
String[] tArr = tmp.split(splitSymbol);
list.add(tArr);
columnNum = tArr.length;
}
s = s.substring(i1+2, s.length());
rowCnt++;
}
if(rowCnt == rowNum) {
map.put("result", list);
break;
} else {
rowCnt = 0;
rowNum = 6;
}
}
List<Map<String, Object>> l2 = new ArrayList<Map<String, Object>>();
List<String> l3 = new ArrayList<String>();
if("json".equalsIgnoreCase(fileType)) {
for(int i = 0; i < header.size(); i++) {
Map<String, Object> m = new HashMap<String, Object>();
m.put("key", i);
m.put("name", header.get(i));
m.put("type", type.get(i));
l2.add(m);
l3.add(header.get(i).toUpperCase());
}
} else {
if("1".equals(ds.getIsHaveHeader())){
List<String[]> l = (List<String[]>) map.get("result");
String[] arr = l.get(0);
for(int i = 0; i < columnNum; i++) {
Map<String, Object> m = new HashMap<String, Object>();
m.put("key", i);
m.put("name", arr[i]);
m.put("type", "STRING");
l2.add(m);
l3.add(arr[i].toUpperCase());
}
map.put("result", l.subList(1, l.size()));
} else {
for(int i = 0; i < columnNum; i++) {
Map<String, Object> m = new HashMap<String, Object>();
m.put("key", i);
m.put("name", "第"+i+"列");
m.put("type", "STRING");
l2.add(m);
l3.add("第"+i+"列");
}
}
}
map.put("header", l3);
map.put("column", l2);
} catch (IOException e) {
e.printStackTrace();
} finally {
if (is != null) {
try {
is.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(fileSystem != null) {
try {
fileSystem.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return map;
}
private String getObjectType(Object obj) {
String t = null;
if(obj instanceof Long) {
t = "LONG";
} else if(obj instanceof Double) {
t = "DOUBLE";
} else if(obj instanceof Boolean) {
t = "BOOLEAN";
} else if(obj instanceof Date) {
t = "DATE";
} else {
t = "STRING";
}
return t;
}
public String getHadoopConfFilePath(String nameNodeAddr) {
String defaultFs = nameNodeAddr;
defaultFs = defaultFs.replace("hdfs://", "");
int idx = defaultFs.indexOf(":");
String hadoopConfIp = defaultFs.substring(0, idx);
return DmpDsAgentConstants.HADOOP_CONF_FILE_PATH + hadoopConfIp + "/hadoop-conf";
}
/**
* 将该任务对应的表数据转换为xml文件 并上传到hdfs指定路径（/jz/etlprocesses/guide_xml/目标库名/）
* Xml名称：目标库名_目标表名（等同于xml文件中name标签的值）
*/
public DmpAgentResult submitSycingConfig(DmpAgentDatasourceInfo ds, String xml) {
FileSystem fileSystem = null;
try {
//Configuration configuration = initConfiguration(DmpDsAgentConstants.HADOOP_DEFAULT_CONF_FILE_PATH);
//if(DmpDsAgentConstants.IS_ENABLED_KERBEROS) JDBCUtils.initKerberosENV4HDFS(configuration);
Configuration configuration = setHdfsConfiguration(ds);
if(configuration == null) {
return new DmpAgentResult(ResultCode.PARAMS_ERROR, "HDFS参数有误");
}
fileSystem = FileSystem.get(configuration);
/*
mkdir(fileSystem, DmpDsAgentConstants.SUBMIT_SYNCING_CONFIG_HDFS_PATH
+ ds.getTargetDbName() + "/");
createFile(fileSystem, DmpDsAgentConstants.SUBMIT_SYNCING_CONFIG_HDFS_PATH
+ ds.getTargetDbName() + "/" + ds.getTargetFileName(), xml);
*/
String hasSeparate = ds.getHdfsSyncingPath().substring(ds.getHdfsSyncingPath().length() - 1);
String pathSup = hasSeparate.equals("/") ? "" : "/";
String mkdirPath = ds.getHdfsSyncingPath() + pathSup + ds.getProjectId() + "/" + ds.getTargetDbName() + "/";
logger.info("mkdir: " + mkdirPath);
mkdir(fileSystem, mkdirPath);
logger.info("createFile : " + mkdirPath + ds.getTargetFileName());
logger.info("file content : " + xml);
createFile(fileSystem, mkdirPath + ds.getTargetFileName(), xml);
return new DmpAgentResult(ResultCode.SUCCESS, "true");
} catch (IOException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "上传出错："+e.getMessage());
} finally {
if(fileSystem != null)
try {
fileSystem.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
public static Configuration initConfiguration(String confPath) {
Configuration configuration = new Configuration();
System.out.println(confPath + File.separator + "core-site.xml");
configuration.addResource(new Path(confPath + File.separator + "core-site.xml"));
configuration.addResource(new Path(confPath + File.separator + "hdfs-site.xml"));
configuration.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
System.setProperty("HADOOP_USER_NAME", DmpDsAgentConstants.AGENT_HADOOP_USER_NAME);
return configuration;
}
public static void mkdir(FileSystem fs, String path) {
try {
Path srcPath = new Path(path);
if (fs.exists(srcPath)) {
System.out.println("目录已存在");
return;
}
boolean isok = fs.mkdirs(srcPath);
if (isok) {
System.out.println("create dir ok!");
} else {
System.out.println("create dir failure");
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void uploadFile(FileSystem fs, String src, String dst) {
try {
Path srcPath = new Path(src);
Path dstPath = new Path(dst);
fs.copyFromLocalFile(false, srcPath, dstPath);
FileStatus[] fileStatus = fs.listStatus(dstPath);
for (FileStatus file : fileStatus) {
System.out.println(file.getPath());
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void createFile(FileSystem fs, String dst, String contents) {
try {
Path path = new Path(dst);
FSDataOutputStream fsDataOutputStream = fs.create(path);
fsDataOutputStream.write(contents.getBytes());
fsDataOutputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public static void readFile(FileSystem fs, String filePath) {
try {
Path srcPath = new Path(filePath);
InputStream in = null;
in = fs.open(srcPath);
IOUtils.copyBytes(in, System.out, 4096, false); //复制到标准输出流
} catch (IOException e) {
e.printStackTrace();
}
}
}
src/main/java/com/jz/agent/client/service/DmpHiveAgentClientService.java 0 → 100644
View file @ a6f92fb6
package com.jz.agent.client.service;
import com.google.gson.Gson;
import com.jz.agent.client.DmpDsAgentClient;
import com.jz.agent.utils.JDBCUtils;
import com.jz.dmp.agent.DmpAgentResult;
import com.jz.dmp.agent.ResultCode;
import com.jz.dmp.modules.model.DmpAgentDatasourceInfo;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class DmpHiveAgentClientService implements DmpDsAgentClient {
private Gson gson = new Gson();
private static Logger logger = LoggerFactory.getLogger(DmpHiveAgentClientService.class);
public DmpAgentResult testConnect(DmpAgentDatasourceInfo ds) {
Connection connection = null;
String jdbcUrl = ds.getJdbcUrl();
try {
jdbcUrl += securityConnect(ds);
Class.forName(ds.getDriverClassName());
connection = DriverManager.getConnection(jdbcUrl);
connection.close();
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(true));
} catch (Exception e) {
logger.error("HIVE连通性异常", e);
return new DmpAgentResult(ResultCode.EXCEPTION, "HIVE连通性异常：" + e.getMessage());
} finally {
JDBCUtils.disconnect(connection, null, null);
}
}
public DmpAgentResult getTableNameList(DmpAgentDatasourceInfo ds) {
List<String> list = new ArrayList();
Connection connection = null;
PreparedStatement ps = null;
ResultSet rs = null;
String jdbcUrl = ds.getJdbcUrl();
String sql = "show tables";
try {
jdbcUrl += securityConnect(ds);
Class.forName(ds.getDriverClassName());
connection = DriverManager.getConnection(jdbcUrl);
ps = connection.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
list.add(rs.getString("tab_name"));
}
} catch (Exception e) {
logger.error("HIVE获取表列表异常", e);
return new DmpAgentResult(ResultCode.EXCEPTION, "HIVE获取表列表异常：" + e.getMessage());
} finally {
JDBCUtils.disconnect(connection, rs, ps);
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
public DmpAgentResult getTableColumnList(DmpAgentDatasourceInfo ds, String tableName) {
List<Map<String, Object>> list = new ArrayList();
Connection connection = null;
PreparedStatement ps = null;
ResultSet rs = null;
String jdbcUrl = ds.getJdbcUrl();
String sql = "DESCRIBE " + tableName;
try {
jdbcUrl += securityConnect(ds);
Class.forName(ds.getDriverClassName());
connection = DriverManager.getConnection(jdbcUrl);
ps = connection.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
Map<String, Object> m = new HashMap<String, Object>();
String col_name = rs.getString("col_name");
String data_type = rs.getString("data_type");
if(!"".equals(col_name) && data_type != null) {
m.put("name", rs.getString("col_name").toLowerCase());
m.put("type", rs.getString("data_type").toUpperCase());
m.put("comment", rs.getString("comment"));
list.add(m);
}
}
} catch (Exception e) {
logger.error("HIVE获取表属性异常", e);
return new DmpAgentResult(ResultCode.EXCEPTION, "HIVE获取表属性异常：" + e.getMessage());
} finally {
JDBCUtils.disconnect(connection, rs, ps);
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
public DmpAgentResult previewData(DmpAgentDatasourceInfo ds, String tableName) {
Map<String, Object> map = new HashMap();
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(map));
}
/**
* 安全连接并返回jdbc连接url的安全请求后缀
*
* @param ds
* @return jdbc连接url的安全请求后缀
*/
private String securityConnect(DmpAgentDatasourceInfo ds) throws IOException {
String jdbcUrlSuffix = "";
// 与kudu大体无异，不同于安全校验
if ("true".equalsIgnoreCase(ds.getKerberosIsenable())) {
String principal = ds.getAccessId();
String hiveKeytabPath = ds.getAccessKey(); // D:/keytab/hive.keytab
String hiveUser = ds.getEndpoint(); // hive@CRNETTEST.COM
jdbcUrlSuffix = ";principal=" + principal;
System.setProperty("java.security.krb5.conf", ds.getKerberosKrb5Conf());
System.setProperty("java.security.auth.login.config", ds.getKerberosJaasConf());
Configuration conf = new Configuration();
conf.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(conf);
UserGroupInformation.loginUserFromKeytab(hiveUser, hiveKeytabPath);
}
return jdbcUrlSuffix;
}
public static void main(String[] args) {
Connection connection = null;
ResultSet rs = null;
PreparedStatement ps = null;
String JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver";
String CONNECT_URL = "jdbc:hive2://10.0.53.177:10000/tmp;principal=hive/jtjzvdra116@CRNETTEST.COM";
// String sql = "show tables";
String sql = "DESCRIBE rzf_report_rzfyinliu_yp";
try {
System.setProperty("java.security.krb5.conf", "D:/keytab/krb5.conf");
System.setProperty("java.security.auth.login.config", "D:/keytab/jaas.conf");
Configuration conf = new Configuration();
conf.set("hadoop.security.authentication", "Kerberos");
UserGroupInformation.setConfiguration(conf);
UserGroupInformation.loginUserFromKeytab("hive@CRNETTEST.COM", "D:/keytab/hive.keytab");
Class.forName(JDBC_DRIVER);
connection = DriverManager.getConnection(CONNECT_URL);
ps = connection.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
// System.out.println(rs.getString("tab_name"));
System.out.print(rs.getString("col_name").toLowerCase() + " = ");
System.out.print(rs.getString("data_type").toUpperCase() + " = ");
System.out.println(rs.getString("comment") + " = ");
}
} catch (Exception e) {
e.printStackTrace();
} finally {
JDBCUtils.disconnect(connection, rs, ps);
}
}
}
src/main/java/com/jz/agent/client/service/DmpImpalaAgentClientService.java 0 → 100644
View file @ a6f92fb6
package com.jz.agent.client.service;
import com.google.gson.Gson;
import com.jz.agent.client.DmpDsAgentClient;
import com.jz.agent.config.DmpDsAgentConstants;
import com.jz.agent.utils.JDBCUtils;
import com.jz.dmp.agent.DmpAgentResult;
import com.jz.dmp.agent.ResultCode;
import com.jz.dmp.modules.model.DmpAgentDatasourceInfo;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.security.PrivilegedAction;
import java.sql.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@SuppressWarnings("unused")
public class DmpImpalaAgentClientService implements DmpDsAgentClient {
private static String JDBC_DRIVER = "com.cloudera.impala.jdbc41.Driver";
private Gson gson = new Gson();
private static Logger logger = LoggerFactory.getLogger(DmpImpalaAgentClientService.class);
static {
try {
Class.forName(JDBC_DRIVER);
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
}
protected String CONNECTION_URL;
public DmpAgentResult testConnect(DmpAgentDatasourceInfo ds) {
/*boolean flag = false;
Connection connection = null;
try {
connection = getConnection(ds);
if(connection != null) flag = true;
} catch (Exception e) {
e.printStackTrace();
} finally {
if(connection != null)
JDBCUtils.disconnect(connection, null, null);
}
return flag;*/
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(true));
}
private Connection getConnection(DmpAgentDatasourceInfo ds) {
Connection connection = null;
ResultSet rs = null;
PreparedStatement ps = null;
CONNECTION_URL = ds.getJdbcUrl();
try {
if(DmpDsAgentConstants.IS_ENABLED_KERBEROS) {
UserGroupInformation loginUser = JDBCUtils.initKerberosENV4IMPALA();
CONNECTION_URL = CONNECTION_URL + ";AuthMech=1;KrbRealm="
+ DmpDsAgentConstants.KERBEROS_KEYTAB_ACCOUNT_IMPALA
+ ";KrbHostFQDN=" + ds.getAccessId() + ";KrbServiceName=impala";
connection = (Connection) loginUser.doAs(new PrivilegedAction<Object>(){
public Object run() {
Connection connection = null;
ResultSet rs = null;
PreparedStatement ps = null;
try {
Class.forName(JDBC_DRIVER);
connection = DriverManager.getConnection(CONNECTION_URL);
} catch (Exception e) {
e.printStackTrace();
} finally {
JDBCUtils.disconnect(connection, rs, ps);
}
return connection;
}
});
} else {
connection = DriverManager.getConnection(CONNECTION_URL);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if(rs != null)
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
if(ps != null)
try {
ps.close();
} catch (SQLException e) {
e.printStackTrace();
}
if(connection != null)
try {
connection.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return connection;
}
public DmpAgentResult getTableNameList(DmpAgentDatasourceInfo ds) {
String sql = "show tables"; // like '*likename*'
Connection conn = null;
ResultSet rs = null;
PreparedStatement ps = null;
List<String> list = new ArrayList<String>();
try {
conn = getConnection(ds);
ps = conn.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
list.add(rs.getString(1).toLowerCase());
}
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "Impala获取表列表异常："+e.getMessage());
} finally {
if(rs != null)
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
if(ps != null)
try {
ps.close();
} catch (SQLException e) {
e.printStackTrace();
}
if(conn != null)
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
public DmpAgentResult getTableColumnList(DmpAgentDatasourceInfo ds, String tableName) {
String sql = "desc " + tableName;
Connection conn = null;
ResultSet rs = null;
PreparedStatement ps = null;
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
try {
conn = getConnection(ds);
ps = conn.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
Map<String, Object> row = new HashMap<String, Object>();
row.put("name", rs.getString(1).toLowerCase());
row.put("type", rs.getString(2).toUpperCase());
row.put("comment", rs.getString(3));
list.add(row);
}
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "Impala获取表字段信息异常："+e.getMessage());
} finally {
if(rs != null)
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
if(ps != null)
try {
ps.close();
} catch (SQLException e) {
e.printStackTrace();
}
if(conn != null)
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
@SuppressWarnings("resource")
public DmpAgentResult previewData(DmpAgentDatasourceInfo ds, String tableName) {
Map<String, Object> map = new HashMap<String, Object>();
String sql = "select * from " + tableName + " limit 5";
String sql1 = "desc " + tableName;
Connection conn = null;
ResultSet rs = null;
PreparedStatement ps = null;
try {
conn = getConnection(ds);
ps = conn.prepareStatement(sql1);
rs = ps.executeQuery();
List<String> headerList = new ArrayList<String>();
while (rs.next()) {
headerList.add(rs.getString(1).toLowerCase());
}
map.put("header", headerList);
ps = conn.prepareStatement(sql);
rs = ps.executeQuery();
List<List<Object>> resultList = new ArrayList<List<Object>>();
while (rs.next()) {
List<Object> row = new ArrayList<Object>();
for (int i = 1; i <= headerList.size(); i++) {
row.add(rs.getObject(i));
}
resultList.add(row);
}
map.put("result", resultList);
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "IMPALA数据预览异常："+e.getMessage());
} finally {
if(rs != null)
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
if(ps != null)
try {
ps.close();
} catch (SQLException e) {
e.printStackTrace();
}
if(conn != null)
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(map));
}
}
src/main/java/com/jz/agent/client/service/DmpKuduAgentClientService.java 0 → 100644
View file @ a6f92fb6
package com.jz.agent.client.service;
import com.google.gson.Gson;
import com.jz.agent.client.DmpDsAgentClient;
import com.jz.agent.utils.JDBCUtils;
import com.jz.dmp.agent.DmpAgentResult;
import com.jz.dmp.agent.ResultCode;
import com.jz.dmp.modules.model.DmpAgentDatasourceInfo;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.kudu.ColumnSchema;
import org.apache.kudu.Type;
import org.apache.kudu.client.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class DmpKuduAgentClientService implements DmpDsAgentClient {
private Gson gson = new Gson();
private static Logger logger = LoggerFactory.getLogger(DmpKuduAgentClientService.class);
public DmpAgentResult testConnect(DmpAgentDatasourceInfo ds) {
Connection connection = null;
String jdbcUrl = ds.getJdbcUrl();
try {
jdbcUrl += securityConnect(ds);
Class.forName(ds.getDriverClassName());
connection = DriverManager.getConnection(jdbcUrl);
connection.close();
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(true));
} catch (Exception e) {
logger.error("KUDU连通性异常", e);
return new DmpAgentResult(ResultCode.EXCEPTION, "KUDU连通性异常：" + e.getMessage());
} finally {
JDBCUtils.disconnect(connection, null, null);
}
}
public DmpAgentResult getTableNameList(DmpAgentDatasourceInfo ds) {
List<String> list = new ArrayList();
Connection connection = null;
PreparedStatement ps = null;
ResultSet rs = null;
String jdbcUrl = ds.getJdbcUrl();
String sql = "show tables";
try {
jdbcUrl += securityConnect(ds);
Class.forName(ds.getDriverClassName());
connection = DriverManager.getConnection(jdbcUrl);
ps = connection.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
list.add(rs.getString("name"));
}
} catch (Exception e) {
logger.error("KUDU获取表列表异常", e);
return new DmpAgentResult(ResultCode.EXCEPTION, "KUDU获取表列表异常：" + e.getMessage());
} finally {
JDBCUtils.disconnect(connection, rs, ps);
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
public DmpAgentResult getTableColumnList(DmpAgentDatasourceInfo ds, String tableName) {
List<Map<String, Object>> list = new ArrayList();
Connection connection = null;
PreparedStatement ps = null;
ResultSet rs = null;
String jdbcUrl = ds.getJdbcUrl();
String sql = "DESCRIBE " + tableName;
try {
jdbcUrl += securityConnect(ds);
Class.forName(ds.getDriverClassName());
connection = DriverManager.getConnection(jdbcUrl);
ps = connection.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
Map<String, Object> m = new HashMap<String, Object>();
m.put("name", rs.getString("name").toLowerCase());
m.put("type", convertColumnType(rs.getString("type").toUpperCase()));
m.put("comment", rs.getString("comment"));
list.add(m);
}
} catch (Exception e) {
logger.error("KUDU获取表属性异常", e);
return new DmpAgentResult(ResultCode.EXCEPTION, "KUDU获取表属性异常：" + e.getMessage());
} finally {
JDBCUtils.disconnect(connection, rs, ps);
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
public DmpAgentResult previewData(DmpAgentDatasourceInfo ds, String tableName) {
Map<String, Object> map = new HashMap();
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(map));
}
/**
* 安全连接并返回jdbc连接url的安全请求后缀
*
* @param ds
* @return jdbc连接url的安全请求后缀
*/
private String securityConnect(DmpAgentDatasourceInfo ds) {
String jdbcUrlSuffix = "";
if ("true".equalsIgnoreCase(ds.getKerberosIsenable())) {
// 不从系统配置中取kerberosFqdn
jdbcUrlSuffix = ";AuthMech=1;KrbHostFQDN=" + ds.getAccessId() + ";KrbServiceName=impala";
System.setProperty("java.security.krb5.conf", ds.getKerberosKrb5Conf());
System.setProperty("java.security.auth.login.config", ds.getKerberosJaasConf());
Configuration conf = new Configuration();
conf.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(conf);
}
return jdbcUrlSuffix;
}
private String convertColumnType(String type) {
// KUDU开启INT BIGINT DOUBLE SMALLINT STRING
if ("INT32".equals(type) || "INT8".equals(type) || "INT".equals(type)) {
type = "INT";
} else if ("INT64".equals(type) || "BIGINT".equals(type)) {
type = "BIGINT";
} else if ("FLOAT".equals(type) || "DOUBLE".equals(type)) {
type = "DOUBLE";
} else if ("SMALLINT".equals(type)) {
// 不做变更
} else {
type = "STRING";
}
return type;
}
public DmpAgentResult previewData_bak(DmpAgentDatasourceInfo ds, String tableName) {
Map<String, Object> map = new HashMap<String, Object>();
String kuduClients = ds.getEndpoint();
KuduClient client = null;
KuduSession session = null;
//List<Map<String, Object>> headerList1 = new ArrayList<Map<String, Object>>();
List<String> headerList = new ArrayList<String>();
List<List<Object>> resultList = new ArrayList<List<Object>>();
try {
/*if (DmpDsAgentConstants.IS_ENABLED_KERBEROS)
JDBCUtils.initKerberosENV4KUDU();*/
client = new KuduClient.KuduClientBuilder(kuduClients).build();
session = client.newSession();
KuduTable table = client.openTable(tableName);
List<ColumnSchema> schema = table.getSchema().getColumns();
List<String> projectColumns = new ArrayList<String>(1);
for (ColumnSchema cs : schema) {
headerList.add(cs.getName().toLowerCase());
projectColumns.add(cs.getName());
}
map.put("header", headerList);
//ScanRequestPB srp = new ScanRequestPB(false);
KuduScanner scanner = client.newScannerBuilder(table).setProjectedColumnNames(projectColumns).build();
while (scanner.hasMoreRows()) {
RowResultIterator results = scanner.nextRows();
int i = 0;
while (results.hasNext()) {
RowResult result = results.next();
Map<String, Object> m = new HashMap<String, Object>();
List<Object> l = new ArrayList<Object>();
for (int j = 0; j < projectColumns.size(); j++) {
if (result.getColumnType(j) == Type.STRING) {
m.put(projectColumns.get(j), result.getString(j));
l.add(result.getString(j));
} else if (result.getColumnType(j) == Type.BINARY) {
m.put(projectColumns.get(j), result.getBinary(j));
l.add(result.getBinary(j));
} else if (result.getColumnType(j) == Type.BOOL) {
m.put(projectColumns.get(j), result.getBoolean(j));
l.add(result.getBoolean(j));
} else if (result.getColumnType(j) == Type.DOUBLE) {
m.put(projectColumns.get(j), result.getDouble(j));
l.add(result.getDouble(j));
} else if (result.getColumnType(j) == Type.FLOAT) {
m.put(projectColumns.get(j), result.getFloat(j));
l.add(result.getFloat(j));
} else if (result.getColumnType(j) == Type.INT32) {
m.put(projectColumns.get(j), result.getInt(j));
l.add(result.getInt(j));
} else if (result.getColumnType(j) == Type.INT64) {
m.put(projectColumns.get(j), result.getLong(j));
l.add(result.getLong(j));
} else if (result.getColumnType(j) == Type.INT8) {
m.put(projectColumns.get(j), result.getShort(j));
l.add(result.getShort(j));
}
resultList.add(l);
}
if (i == 5) break;
i++;
}
break;
}
map.put("result", resultList);
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "KUDU数据预览异常：" + e.getMessage());
} finally {
try {
if (session != null && !session.isClosed())
session.close();
} catch (KuduException e) {
e.printStackTrace();
}
if (client != null)
try {
client.close();
} catch (KuduException e) {
e.printStackTrace();
}
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(map));
}
public static void main(String[] args) {
Connection connection = null;
ResultSet rs = null;
PreparedStatement ps = null;
String JDBC_DRIVER = "com.cloudera.impala.jdbc41.Driver";
// String CONNECT_URL = "jdbc:impala://10.0.53.177:21050/kudu_demo_imp";
String CONNECT_URL = "jdbc:impala://10.0.53.177:21050/tmp";
// String sql = "show tables";
String sql = "CREATE TABLE tmp.test (" +
"id INT NOT NULL," +
"member_id BIGINT NULL," +
"small_id SMALLINT NULL," +
"double_c DOUBLE NULL," +
"etl_create_time STRING NULL," +
"etl_update_time STRING NULL," +
"PRIMARY KEY (id)" +
") PARTITION BY HASH (id) PARTITIONS 2 STORED AS KUDU";
// String sql = "DESCRIBE hht_impala_1";
try {
/*CONNECT_URL += ";AuthMech=1;KrbHostFQDN=jtjzvdra116;KrbServiceName=impala";
System.setProperty("java.security.krb5.conf", "D:/keytab/krb5.conf");
System.setProperty("java.security.auth.login.config", "D:/keytab/jaas.conf");
Configuration conf = new Configuration();
conf.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(conf);*/
Class.forName(JDBC_DRIVER);
connection = DriverManager.getConnection(CONNECT_URL);
ps = connection.prepareStatement(sql);
ps.execute();
// rs = ps.executeQuery();
/*while (rs.next()) {
System.out.println(rs.getString("name"));
System.out.print(rs.getString("name").toLowerCase() + " = ");
System.out.print(rs.getString("type").toUpperCase() + " = ");
System.out.println(rs.getString("comment") + " = ");
}*/
} catch (Exception e) {
e.printStackTrace();
} finally {
JDBCUtils.disconnect(connection, rs, ps);
}
}
}
src/main/java/com/jz/agent/client/service/DmpMysqlAgentClientService.java 0 → 100644
View file @ a6f92fb6
package com.jz.agent.client.service;
import com.google.gson.Gson;
import com.jz.agent.client.DmpDsAgentClient;
import com.jz.agent.utils.JDBCUtils;
import com.jz.common.utils.StringUtils;
import com.jz.dmp.agent.DmpAgentResult;
import com.jz.dmp.agent.ResultCode;
import com.jz.dmp.modules.model.DmpAgentDatasourceInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class DmpMysqlAgentClientService implements DmpDsAgentClient {
private Gson gson = new Gson();
@SuppressWarnings("unused")
private static Logger logger = LoggerFactory.getLogger(DmpMysqlAgentClientService.class);
public DmpAgentResult testConnect(DmpAgentDatasourceInfo ds) {
Connection conn = null;
try {
Class.forName(ds.getDriverClassName());
conn = DriverManager.getConnection(
ds.getJdbcUrl(),
ds.getUserName(),
ds.getPassword()
);
conn.close();
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(true));
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "MYSQL连通性异常："+e.getMessage());
} finally {
try {
if(conn != null)
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
@SuppressWarnings({ "unchecked", "rawtypes" })
public DmpAgentResult getTableNameList(DmpAgentDatasourceInfo ds) {
List result = new ArrayList();
String sql = "select * from information_schema.tables where table_schema = '"+this.getTargetDBname(ds)+"'";
Connection conn = null;
ResultSet rs = null;
PreparedStatement ps = null;
try {
conn = JDBCUtils.getConnections(ds.getDriverClassName(), ds.getJdbcUrl(), ds.getUserName(), ds.getPassword());
ps = conn.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
result.add(rs.getString("TABLE_NAME").toLowerCase());
}
} catch (SQLException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "MYSQL获取表列表异常："+e.getMessage());
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "MYSQL获取表列表异常："+e.getMessage());
} finally {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(result));
}
private String getTargetDBname(DmpAgentDatasourceInfo ds) {
String url = ds.getJdbcUrl();
if(url != null && !"".equals(url.trim())) {
String[] arr1 = url.split("/");
String ls = arr1[arr1.length-1];
String[] arr2 = ls.split("\\?");
return arr2[0];
} else {
return "";
}
}
@SuppressWarnings({ "rawtypes", "unchecked" })
public DmpAgentResult getTableColumnList(DmpAgentDatasourceInfo ds, String tableName) {
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
String dbName = "";
if (StringUtils.hasLength(ds.getDbName()))
dbName = " table_schema = '" + ds.getDbName() + "' and ";
String sql = "select * from information_schema.columns where " + dbName + "table_name = '" + tableName + "'";
Connection conn = null;
ResultSet rs;
PreparedStatement ps;
try {
conn = JDBCUtils.getConnections(ds.getDriverClassName(), ds.getJdbcUrl(), ds.getUserName(), ds.getPassword());
ps = conn.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
Map map = new HashMap();
String colName = rs.getString("COLUMN_NAME");
String dbType = rs.getString("DATA_TYPE");
String remarks = rs.getString("COLUMN_COMMENT");
Integer size= 0;
Integer characterLength = (int) rs.getDouble("CHARACTER_MAXIMUM_LENGTH");
Integer columnSize = (int) rs.getDouble("NUMERIC_PRECISION");
Integer dicimalDigits = (int) rs.getDouble("NUMERIC_SCALE");
if(characterLength != null) {
size = characterLength;
} else {
size = columnSize;
}
map.put("name", colName.toLowerCase());
map.put("type", dbType.toUpperCase());
map.put("size", size);
map.put("scale", dicimalDigits);
map.put("comment", remarks);
list.add(map);
}
} catch (SQLException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "MYSQL获取表字段信息异常："+e.getMessage());
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "MYSQL获取表字段信息异常："+e.getMessage());
} finally {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
public DmpAgentResult previewData(DmpAgentDatasourceInfo ds, String tableName) {
Map<String, Object> map = new HashMap<String, Object>();
Connection conn = null;
ResultSet rs;
PreparedStatement ps;
List<String> headerList = new ArrayList<String>();
try {
conn = JDBCUtils.getConnections(ds.getDriverClassName(), ds.getJdbcUrl(), ds.getUserName(), ds.getPassword());
String sql = "select * from " + tableName + " limit 5";
String sql2 = "select * from information_schema.columns where table_name = '"+tableName+"'";
ps = conn.prepareStatement(sql2);
rs = ps.executeQuery();
while (rs.next()) {
String colName = rs.getString("COLUMN_NAME");
headerList.add(colName);
}
map.put("header", headerList);
ps = conn.prepareStatement(sql);
rs = ps.executeQuery();
List<List<Object>> resultList = new ArrayList<List<Object>>();
while (rs.next()) {
List<Object> row = new ArrayList<Object>();
for(int i = 1; i <= headerList.size(); i++) {
row.add(rs.getObject(i));
}
resultList.add(row);
}
map.put("result", resultList);
} catch (SQLException e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "MYSQL数据预览异常："+e.getMessage());
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "MYSQL数据预览异常："+e.getMessage());
} finally {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(map));
}
}
src/main/java/com/jz/agent/client/service/DmpOracleAgentClientService.java 0 → 100644
View file @ a6f92fb6
package com.jz.agent.client.service;
import com.google.gson.Gson;
import com.jz.agent.client.DmpDsAgentClient;
import com.jz.agent.utils.DbInfoUtil;
import com.jz.dmp.agent.DmpAgentResult;
import com.jz.dmp.agent.ResultCode;
import com.jz.dmp.modules.model.DmpAgentDatasourceInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
public class DmpOracleAgentClientService implements DmpDsAgentClient {
private Gson gson = new Gson();
private static Logger logger = LoggerFactory.getLogger(DmpOracleAgentClientService.class);
@SuppressWarnings("unused")
public DmpAgentResult testConnect(DmpAgentDatasourceInfo ds) {
Connection conn = null;
try {
Class.forName(ds.getDriverClassName());
conn = DriverManager.getConnection(
ds.getJdbcUrl(),
ds.getUserName(),
ds.getPassword()
);
conn.close();
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(true));
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "oracle连通性异常："+e.getMessage());
} finally {
try {
if(conn != null)
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
public DmpAgentResult getTableNameList(DmpAgentDatasourceInfo ds) {
List<String> list = DbInfoUtil.getTableNameList(ds.getDriverClassName(), ds.getJdbcUrl(), ds.getUserName(), ds.getPassword());
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
@SuppressWarnings({ "rawtypes" })
public DmpAgentResult getTableColumnList(DmpAgentDatasourceInfo ds, String tableName) {
List list = DbInfoUtil.getTableInfo(ds.getDriverClassName(), ds.getJdbcUrl(), ds.getUserName(), ds.getPassword(), tableName);
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
@SuppressWarnings("unchecked")
public DmpAgentResult previewData(DmpAgentDatasourceInfo ds, String tableName) {
Map<String, Object> map = DbInfoUtil.getTableLimitData(ds.getDriverClassName(), ds.getJdbcUrl(), ds.getUserName(), ds.getPassword(), tableName);
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(map));
}
}
src/main/java/com/jz/agent/client/service/DmpPostgreSqlAgentClientService.java 0 → 100644
View file @ a6f92fb6
package com.jz.agent.client.service;
import com.google.gson.Gson;
import com.jz.agent.client.DmpDsAgentClient;
import com.jz.dmp.agent.DmpAgentResult;
import com.jz.dmp.agent.ResultCode;
import com.jz.dmp.modules.model.DmpAgentDatasourceInfo;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class DmpPostgreSqlAgentClientService implements DmpDsAgentClient {
private Gson gson = new Gson();
private static Logger logger = LoggerFactory.getLogger(DmpPostgreSqlAgentClientService.class);
@SuppressWarnings("unused")
public DmpAgentResult testConnect(DmpAgentDatasourceInfo ds) {
Connection conn = null;
try {
Class.forName(ds.getDriverClassName());
conn = DriverManager.getConnection(ds.getJdbcUrl(), ds.getUserName(), ds.getPassword());
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(true));
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "POSTGRESQL连通性异常："+e.getMessage());
} finally {
try {
if(conn != null)
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
public DmpAgentResult getTableNameList(DmpAgentDatasourceInfo ds) {
String sql = "select relname as tabname from pg_class where relkind='r' and relname not like 'pg_%' and relname not like 'sql_%' order by relname";
Connection conn = null;
PreparedStatement ps = null;
ResultSet rs = null;
List<String> list = new ArrayList<String>();
try {
Class.forName("org.postgresql.Driver");// "org.postgresql.Driver"
conn = DriverManager.getConnection(ds.getJdbcUrl(), ds.getUserName(), ds.getPassword());
// "jdbc:postgresql://192.168.204.102:5432/test", "postgres", "postgres"
ps = conn.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
list.add(rs.getString(1).toLowerCase());
}
} catch (Exception e) {
e.printStackTrace();
return new DmpAgentResult(ResultCode.EXCEPTION, "POSTGRESQL获取表列表异常："+e.getMessage());
} finally {
if(rs != null)
try {
rs.close();
} catch (SQLException e1) {
e1.printStackTrace();
}
if(ps != null)
try {
ps.close();
} catch (SQLException e1) {
e1.printStackTrace();
}
try {
if(conn != null)
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return new DmpAgentResult(ResultCode.SUCCESS, gson.toJson(list));
}
public DmpAgentResult getTableColumnList(DmpAgentDatasourceInfo ds, String tableName) {
String sql = "SELECT col_description(a.attrelid,a.attnum) as comment,format_type(a.atttypid,a.atttypmod) as type,a.attname as name, a.attnotnull as notnull FROM pg_class as c,pg_attribute as a where c.relname = '"
+ tableName + "' and a.attrelid = c.oid and a.attnum>0";
Connection conn = null;
PreparedStatement ps = null;
ResultSet rs = null;
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
try {
Class.forName(ds.getDriverClassName());// "org.postgresql.Driver"
conn = DriverManager.getConnection(ds.getJdbcUrl(), ds.getUserName(), ds.getPassword());
// "jdbc:postgresql://192.168.204.102:5432/test", "postgres", "postgres"
ps = conn.prepareStatement(sql);
rs = ps.executeQuery();
while (rs.next()) {
Map<String, Object> row = new HashMap<String, Object>();
String type = rs.getString(2).toUpperCase();
if(StringUtils.isNotBlank(type) && type.indexOf("(") > -1) {
int idx1 = type.indexOf("(");
int idx2 = type.indexOf(")");
String t = type.substring(0, idx1);
String ts = type.substring(idx1+1, idx2);
if(ts.indexOf(",") > -1) {
String[] strs = ts.split(",");
row.put("size", new Integer(strs[0]));
row.put("scale", new Integer(strs[1]));
} else {
row.put("size", new Integer(ts));
row.put("scale", 0);
}
type = t;
}
row.put("name", rs.getString(3).toLowerCase());
row.put("type", type);