Skip to content

J
jz-dmp-service
Commit b07a4416 authored by sml's avatar sml
Browse files
Options

Merge branch 'dmp_dev' of http://gitlab.ioubuy.cn/yaobenzhang/jz-dmp-service.git into dmp_dev

parents 2fe650ef 778d7deb
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 218 additions and 181 deletions
src/main/java/com/jz/common/enums/DelFlagEnum.java
View file @ b07a4416
......@@ -28,6 +28,18 @@ public enum DelFlagEnum {
this.value = value;
}
public static DelFlagEnum get(String code) {
if (code == null) {
return null;
}
for (DelFlagEnum status : values()) {
if (status.getCode().equalsIgnoreCase(code)) {
return status;
}
}
return null;
}
public String getCode() {
return code;
}
......@@ -43,16 +55,4 @@ public enum DelFlagEnum {
public void setValue(String value) {
this.value = value;
}
public static DelFlagEnum get(String code) {
if (code == null) {
return null;
}
for (DelFlagEnum status : values()) {
if (status.getCode().equalsIgnoreCase(code)) {
return status;
}
}
return null;
}
}
src/main/java/com/jz/common/utils/AzkabanApiUtils2.java
View file @ b07a4416
......@@ -9,9 +9,9 @@ import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.FileSystemResource;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.StringRedisSerializer;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
......@@ -22,7 +22,6 @@ import org.springframework.web.client.RestTemplate;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.support.spring.FastJsonRedisSerializer;
import com.google.gson.Gson;
import com.jz.common.utils.web.HttpClientUtils;
import com.jz.common.utils.web.SessionUtils;
......@@ -44,21 +43,19 @@ public class AzkabanApiUtils2 {
private String userName;
private String password;
private RedisTemplate redisTemplate;
private static final StringRedisSerializer stringRedisSerializer = new StringRedisSerializer();
private static final FastJsonRedisSerializer fastJsonRedisSerializer = new FastJsonRedisSerializer(SSOUserInfo.class);
@Autowired
RedisTemplate<String,SSOUserInfo> redisTemplate;
public AzkabanApiUtils2(String azkabanServerUrl, String userName, String password, RedisTemplate<String,SSOUserInfo> redisTemplate) {
this(azkabanServerUrl, redisTemplate);
public AzkabanApiUtils2(String azkabanServerUrl, String userName, String password) {
this(azkabanServerUrl);
this.userName = userName;
this.password = password;
}
public AzkabanApiUtils2(String azkabanServerUrl, RedisTemplate<String,SSOUserInfo> redisTemplate) {
public AzkabanApiUtils2(String azkabanServerUrl) {
this.azkabanServerUrl = azkabanServerUrl;
this.userName = "admin";
this.password = "admin";
this.redisTemplate = redisTemplate;
}
/**
......@@ -104,18 +101,16 @@ public class AzkabanApiUtils2 {
LOGGER.error(azkabanServerUrl+"-----"+linkedMultiValueMap+" sessionId 为空");
throw new RuntimeException("登陆失败");
}*/
String sessionId = SessionUtils.getSession().getId();
String sessionId = SessionUtils.getSession().getId(); //"dcfc608c-c58a-45b7-adc7-9902b652496e";
//String sessionId = "f0d06f4a-874c-4dfc-8959-101b6add6bf5";
//通过redis方式登录Azkaban
String redisKey = "spring:sessions:sessions:"+sessionId;
SSOUserInfo ssoUserInfo = (SSOUserInfo) redisTemplate.opsForValue().get(redisKey);
if (ssoUserInfo == null) {
redisTemplate.setKeySerializer(stringRedisSerializer);
redisTemplate.setValueSerializer(fastJsonRedisSerializer);
redisTemplate.opsForValue().set(redisKey, getSSOuserInfo());
SSOUserInfo ssoUserInfo = redisTemplate.opsForValue().get(redisKey);
if (ssoUserInfo==null) {
redisTemplate.opsForValue().set(redisKey, getSSOuserInfo());
}
System.err.println("----sessionId="+sessionId);
return sessionId; //SessionUtils.getSession().getId();
}
......@@ -657,7 +652,7 @@ public class AzkabanApiUtils2 {
}
public static void main(String[] args) throws Exception {
AzkabanApiUtils2 azkabanApiUtils = new AzkabanApiUtils2("http://119.23.32.151:8083", null);
AzkabanApiUtils2 azkabanApiUtils = new AzkabanApiUtils2("http://119.23.32.151:8083");
boolean dw_test = azkabanApiUtils.checkFlowExists("dw_test", "123");
System.err.println(dw_test);
}
......
src/main/java/com/jz/common/utils/FlowParseTool.java
View file @ b07a4416
......@@ -10,7 +10,6 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.data.redis.core.RedisTemplate;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
......@@ -24,7 +23,6 @@ import com.jz.dmp.modules.model.DmpNavigationTree;
import com.jz.dmp.modules.model.DmpProject;
import com.jz.dmp.modules.model.DmpProjectSystemInfo;
import com.jz.dmp.modules.model.DmpWorkFlowSubmitDetails;
import com.jz.dmp.modules.model.SSOUserInfo;
import com.jz.dmp.modules.service.DmpDevelopTaskService;
import com.jz.dmp.modules.service.DmpNavigationTreeService;
import com.jz.dmp.modules.service.DmpWorkFlowSubmitDetailsService;
......@@ -61,8 +59,6 @@ public class FlowParseTool {
private DmpWorkFlowSubmitDetailsService dmpWorkFlowSubmitDetailsService;
private RedisTemplate<String,SSOUserInfo> redisTemplate;
/**
* 流程属性
*/
......@@ -110,14 +106,12 @@ public class FlowParseTool {
DmpProjectSystemInfo publishedToProjectSystemInfo,
DmpDevelopTaskService dmpDevelopTaskService,
DmpNavigationTreeService dmpNavigationTreeService,
DmpWorkFlowSubmitDetailsService dmpWorkFlowSubmitDetailsService,
RedisTemplate<String,SSOUserInfo> redisTemplate) {
DmpWorkFlowSubmitDetailsService dmpWorkFlowSubmitDetailsService) {
this(flowPro, dmpWorkFlowSubmitDetailsService);
this.publishedToProject = publishedToProject;
//this.publishedToProjectSystemInfo = publishedToProjectSystemInfo;
this.dmpDevelopTaskService = dmpDevelopTaskService;
this.dmpNavigationTreeService = dmpNavigationTreeService;
this.redisTemplate = redisTemplate;
}
/**
......@@ -132,14 +126,12 @@ public class FlowParseTool {
DmpProjectConfigInfoDto dmpProjectConfigInfoDto,
DmpDevelopTaskService dmpDevelopTaskService,
DmpNavigationTreeService dmpNavigationTreeService,
DmpWorkFlowSubmitDetailsService dmpWorkFlowSubmitDetailsService,
RedisTemplate<String,SSOUserInfo> redisTemplate) {
DmpWorkFlowSubmitDetailsService dmpWorkFlowSubmitDetailsService) {
this(flowPro, dmpWorkFlowSubmitDetailsService);
this.publishedToProject = publishedToProject;
this.dmpProjectConfigInfoDto = dmpProjectConfigInfoDto;
this.dmpDevelopTaskService = dmpDevelopTaskService;
this.dmpNavigationTreeService = dmpNavigationTreeService;
this.redisTemplate = redisTemplate;
}
private void parse() {
......@@ -524,7 +516,7 @@ public class FlowParseTool {
//上次zip包到azkaban
String localTaskZipAbsolutePath = localTaskZipPath + "/" + localZipTargetFileName;
String azkabanApiUrl = dmpProjectConfigInfoDto.getDmpPublicConfigInfoDto().getAzkabanMonitorUrl();
AzkabanApiUtils2 azkabanApiUtils = new AzkabanApiUtils2(azkabanApiUrl, redisTemplate);
AzkabanApiUtils2 azkabanApiUtils = new AzkabanApiUtils2(azkabanApiUrl);
return azkabanApiUtils.loginCreateProjectuploadZipAndSchedule("jz_workflow_new_" + publishedToProjectId, publishedToProject.getProjectDesc(), localTaskZipAbsolutePath, flowPro);
}
......@@ -661,7 +653,7 @@ public class FlowParseTool {
String subProcessFlowName = flowNode.getScript();
//检查子流程是否存在 todo
String azkabanApiUrl = dmpProjectConfigInfoDto.getDmpPublicConfigInfoDto().getAzkabanMonitorUrl();
AzkabanApiUtils2 azkabanApiUtils = new AzkabanApiUtils2(azkabanApiUrl, redisTemplate);
AzkabanApiUtils2 azkabanApiUtils = new AzkabanApiUtils2(azkabanApiUrl);
boolean flowExists = azkabanApiUtils.checkFlowExists("jz_workflow_" + flowPro.getPublishedToProjectId(), subProcessFlowName);
if (!flowExists) {
throw new RuntimeException("节点:" + flowNode.getNodeName() + "设置的子流程:" + subProcessFlowName + "不存在,请先发布" + subProcessFlowName);
......
src/main/java/com/jz/dmp/modules/controller/DataIntegration/OfflineSynchController.java
View file @ b07a4416
......@@ -5,7 +5,6 @@ import com.jz.common.constant.ResultCode;
import com.jz.common.page.BasePageBean;
import com.jz.common.page.PageInfoResponse;
import com.jz.dmp.modules.controller.DataIntegration.bean.*;
import com.jz.dmp.modules.controller.dataService.bean.SoureTableColumnsReq;
import com.jz.dmp.modules.service.DmpSyncingDatasourceTypeService;
import com.jz.dmp.modules.service.OfflineSynchService;
import io.swagger.annotations.Api;
......@@ -244,7 +243,14 @@ public class OfflineSynchController {
@ApiOperation(value = "保存离线任务数据", notes = "保存离线任务数据")
@PostMapping(value = "/addSyncTask")
public JsonResult addSyncTask(@RequestBody @Validated SyncDmpTaskAddReq syncDmpTaskAddReq) throws Exception {
JsonResult list = offlineSynchService.addSyncTask(syncDmpTaskAddReq);
JsonResult list = new JsonResult();
try {
list = offlineSynchService.addSyncTask(syncDmpTaskAddReq);
} catch (Exception e) {
list.setCode(ResultCode.INTERNAL_SERVER_ERROR);
list.setMessage(e.getMessage());
e.printStackTrace();
}
return list;
}
......
src/main/java/com/jz/dmp/modules/dao/DmpDevelopTaskDao.java
View file @ b07a4416
......@@ -44,22 +44,32 @@ public interface DmpDevelopTaskDao {
List<DataDevTaskListDto> queryTaskTreeInfo(Map params) throws Exception;
/**条件查询任务开发
* @param param
* @return
* @throws Exception
*/
public List<DmpDevelopTask> findList(Map<String, Object> param)throws Exception;
/**
* @Title: get
* @Description: TODO(主键获取对象)
* @param @param id
* @param @return
* @param @throws Exception 参数
* @return DmpDevelopTask 返回类型
* @throws
*/
public DmpDevelopTask get(Long id)throws Exception;
/**
* 条件查询任务开发
*
* @param param
* @return
* @throws Exception
*/
public List<DmpDevelopTask> findList(Map<String, Object> param) throws Exception;
/**
* @param @param id
* @param @return
* @param @throws Exception 参数
* @return DmpDevelopTask 返回类型
* @throws
* @Title: get
* @Description: TODO(主键获取对象)
*/
public DmpDevelopTask get(Long id) throws Exception;
/**
* 根据主键查询
*
* @return
* @author Bellamy
* @since 2021-02-01
*/
DmpDevelopTask selectTaskById(@Param("taskId") String taskId) throws Exception;
}
src/main/java/com/jz/dmp/modules/model/DmpDevelopTaskHistory.java
View file @ b07a4416
......@@ -176,6 +176,8 @@ public class DmpDevelopTaskHistory implements Serializable{
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date updateTime;
private byte[] data;
public Integer getId() {
return id;
}
......@@ -375,4 +377,12 @@ public class DmpDevelopTaskHistory implements Serializable{
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
public byte[] getData() {
return data;
}
public void setData(byte[] data) {
this.data = data;
}
}
src/main/java/com/jz/dmp/modules/model/DmpWorkFlowSubmitDetails.java
View file @ b07a4416
......@@ -10,8 +10,7 @@ import java.util.Date;
public class DmpWorkFlowSubmitDetails implements Serializable {
private static final long serialVersionUID = 1L;
private Long id;
private Long scheduleProjectId;
private String scheduleFlowName;
private String nodeName;
......@@ -24,16 +23,8 @@ public class DmpWorkFlowSubmitDetails implements Serializable {
private Date createdTime;
private String createTimeStr;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public Long getScheduleProjectId() {
public Long getScheduleProjectId() {
return scheduleProjectId;
}
......
src/main/java/com/jz/dmp/modules/service/impl/DmpDevelopTaskServiceImpl.java
View file @ b07a4416
......@@ -129,7 +129,8 @@ public class DmpDevelopTaskServiceImpl extends BaseService implements DmpDevelop
System.out.println(xmlContent);
} catch (Exception e) {
logger.error("封装离线同步xml内容出错：" + e.getMessage(), e);
return new JsonResult(ResultCode.PARAMS_ERROR);
throw new RuntimeException(e.getMessage());
//return JsonResult.error(ResultCode.INTERNAL_SERVER_ERROR,e.getMessage());
}
return submitSyncXml(task.getProjectId(), xmlContent);
}
......@@ -250,7 +251,7 @@ public class DmpDevelopTaskServiceImpl extends BaseService implements DmpDevelop
DmpSyncingDatasource tDS = this.getDmpSyncingDatasource(_projectId_, _targetDbConnection);//目标
if (null == sDS || null == tDS || null == sDS.getDatasourceType() || null == tDS.getDatasourceType())
throw new Exception("同步数据来源或数据去向信息有误");
throw new RuntimeException("同步数据来源或数据去向信息有误");
//根据数据源类型ID 获取数据源类型
DmpSyncingDatasourceType sDST = dmpSyncingDatasourceTypeDao.queryById(sDS.getDatasourceType());//源
DmpSyncingDatasourceType tDST = dmpSyncingDatasourceTypeDao.queryById(tDS.getDatasourceType());//目标
......
src/main/java/com/jz/dmp/modules/service/impl/FlowServiceImpl.java
View file @ b07a4416
......@@ -4,7 +4,6 @@ import java.util.Date;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
......@@ -22,7 +21,6 @@ import com.jz.dmp.modules.dao.projconfig.DmpProjectConfigInfoMapper;
import com.jz.dmp.modules.model.DmpProject;
import com.jz.dmp.modules.model.DmpProjectSystemInfo;
import com.jz.dmp.modules.model.DmpWorkFlowSubmitDetails;
import com.jz.dmp.modules.model.SSOUserInfo;
import com.jz.dmp.modules.service.DmpDevelopTaskService;
import com.jz.dmp.modules.service.DmpNavigationTreeService;
import com.jz.dmp.modules.service.DmpProjectService;
......@@ -59,9 +57,6 @@ public class FlowServiceImpl implements FlowService {
@Autowired
private DmpProjectConfigInfoService dmpProjectConfigInfoService;
@Autowired
private RedisTemplate redisTemplate;
/**
*工作流发布
......@@ -94,8 +89,7 @@ public class FlowServiceImpl implements FlowService {
dmpProjectConfigInfoDto,
dmpDevelopTaskService,
dmpNavigationTreeService,
dmpWorkFlowSubmitDetailsService,
redisTemplate
dmpWorkFlowSubmitDetailsService
);
//保存发布信息
......
src/main/resources/mapper/dmp/DmpDevelopTaskMapper.xml
View file @ b07a4416
......@@ -221,4 +221,12 @@
SELECT <include refid="FIND_ALL_COLUMN" /> FROM dmp_develop_task WHERE tree_id = #{id}
</select>
<select id="selectTaskById" resultType="com.jz.dmp.modules.model.DmpDevelopTask">
select
<include refid="FIND_ALL_COLUMN" />
from dmp_develop_task
where 1=1 and data_status='1'
and id=#{taskId}
</select>
</mapper>
\ No newline at end of file
src/main/resources/templates/lxTaskJson.json
View file @ b07a4416
......@@ -81,7 +81,7 @@
{
"params": {
"version": "1.0",
"version": "1.0", //版本
"treeId": 669,
//"parentId": "509",
"mode": "0",
......@@ -97,8 +97,8 @@
"ftCount": "分桶个数",
"separateMax": "分桶字段最大值",
"separateMin": "分桶字段最小值",
"primaryKey": "主键",
"partition": "分区",
//"primaryKey": "主键",
//"partition": "分区",
"postImportStatement": "导入后语句",
"preImportStatement": "导入前语句",
"errorLimitRecord": "错误记录数超过",
......@@ -106,7 +106,8 @@
//"syncRate": "同步速率",
"executorMemory":"1", //分配任务内存
"executorCores":"1", //单executor的cpu数
"totalExecutorCores":"1" //总executor的cpu数
"totalExecutorCores":"1", //总executor的cpu数
"fieldMapping":""//字段映射关系
},
"reader": {
"dbConnection": "mysql_dmp_demo_test", //来源名称
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment