Commit 25ec3bd8 authored by sml's avatar sml

代码提交

parent 44f4be8f
......@@ -35,5 +35,13 @@ public class CommConstant {
public static final String TASK_TYPE_OFFLINE = "2";// 离线任务
public static final String TASK_TYPE_DEVSHELL = "3";// SHELL任务
public static final String TASK_TYPE_DEVSQL = "4";// SQL任务
/***************************************************/
//SQL任务输出类型
public static final String OUTPUT_TYPE_CONSOLE = "console";//console
public static final String OUTPUT_TYPE_HDFS = "hdfs";//hdfs
public static final String OUTPUT_TYPE_TABLE = "table";//table
public static final String OUTPUT_TYPE_TOPIC = "topic";//topic
public static final String OUTPUT_TYPE_API = "api";//api
}
package com.jz.dmp.cmdexectool.controller.bean;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.jz.dmp.cmdexectool.entity.DmpComputEngine;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
/**计算引擎表Dto
* @author ybz
*
*/
@ApiModel(value = "计算引擎表Dto", description = "计算引擎表Dto")
@JsonIgnoreProperties(value = {"handler"})
public class DmpComputEngineDto extends DmpComputEngine {
/**
*
*/
private static final long serialVersionUID = 1L;
@ApiModelProperty(value = "计算引擎参数列表")
private List<DmpComputEngineParamDto> paramDtos;
public List<DmpComputEngineParamDto> getParamDtos() {
return paramDtos;
}
public void setParamDtos(List<DmpComputEngineParamDto> paramDtos) {
this.paramDtos = paramDtos;
}
}
package com.jz.dmp.cmdexectool.controller.bean;
import com.jz.dmp.cmdexectool.entity.DmpComputEngineParam;
import io.swagger.annotations.ApiModel;
/**计算引擎参数表Dto
* @author ybz
*
*/
@ApiModel(value = "计算引擎参数表Dto", description = "计算引擎参数表Dto")
public class DmpComputEngineParamDto extends DmpComputEngineParam {
}
......@@ -10,5 +10,25 @@ import io.swagger.annotations.ApiModel;
*/
@ApiModel(value = "任务开发Dto", description = "任务开发Dto")
public class DmpDevelopTaskDto extends DmpDevelopTask {
private String name;
private Integer projectId;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Integer getProjectId() {
return projectId;
}
public void setProjectId(Integer projectId) {
this.projectId = projectId;
}
}
package com.jz.dmp.cmdexectool.controller.bean;
import com.jz.dmp.cmdexectool.entity.DmpProjectConfigEngine;
import io.swagger.annotations.ApiModel;
/**项目配置计算引擎关系表Dto
* @author ybz
*
*/
@ApiModel(value = "项目配置计算引擎关系表Dto", description = "项目配置计算引擎关系表Dto")
public class DmpProjectConfigEngineDto extends DmpProjectConfigEngine {
}
package com.jz.dmp.cmdexectool.controller.bean;
import com.jz.dmp.cmdexectool.entity.DmpProjectConfigInfo;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
/**项目配置表Dto
* @author ybz
*
*/
/**
* @ClassName: DmpProjectConfigInfoDto
* @Description: TODO(这里用一句话描述这个类的作用)
* @author ybz
* @date 2021年2月4日
*
*/
@ApiModel(value = "项目配置表Dto", description = "项目配置表Dto")
public class DmpProjectConfigInfoDto extends DmpProjectConfigInfo {
/**
*
*/
private static final long serialVersionUID = 1L;
@ApiModelProperty(value = "公共配置")
private DmpPublicConfigInfoDto dmpPublicConfigInfoDto;
public DmpPublicConfigInfoDto getDmpPublicConfigInfoDto() {
return dmpPublicConfigInfoDto;
}
public void setDmpPublicConfigInfoDto(DmpPublicConfigInfoDto dmpPublicConfigInfoDto) {
this.dmpPublicConfigInfoDto = dmpPublicConfigInfoDto;
}
}
package com.jz.dmp.cmdexectool.controller.bean;
import com.jz.dmp.cmdexectool.entity.DmpProjectEngineParam;
import io.swagger.annotations.ApiModel;
/**计算引擎项目参数表Dto
* @author ybz
*
*/
@ApiModel(value = "计算引擎项目参数表Dto", description = "计算引擎项目参数表Dto")
public class DmpProjectEngineParamDto extends DmpProjectEngineParam {
}
package com.jz.dmp.cmdexectool.controller.bean;
import com.jz.dmp.cmdexectool.entity.DmpPublicConfigInfo;
import io.swagger.annotations.ApiModel;
/**公共配置表Dto
* @author ybz
*
*/
@ApiModel(value = "公共配置表Dto", description = "公共配置表Dto")
public class DmpPublicConfigInfoDto extends DmpPublicConfigInfo {
}
......@@ -74,63 +74,4 @@ public class MyDmpDevelopTaskConverter {
return taskInstance;
}
/**
* @Title: dto2execcontext
* @Description: TODO(将task的job转换成taskExecutionContext)
* @param @param dmpDevelopTaskDto
* @param @param jobId
* @param @return 参数
* @return TaskExecutionContext 返回类型
* @throws
*/
public TaskExecutionContext dto2execcontext(DmpDevelopTaskDto dmpDevelopTaskDto, String jobId) {
TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
JobType jobType = null;
if (StringUtils.isEmpty(jobId)) {
jobType = JobType.valueOf(jobId);
}
String taskType = dmpDevelopTaskDto.getTaskType();
if (taskType.equals(CommConstant.TASK_TYPE_DEVSHELL)) {
jobType = JobType.SHELL;
}else if (taskType.equals(CommConstant.TASK_TYPE_DEVSQL)) {
jobType = JobType.SQL;
}else if (taskType.equals(CommConstant.TASK_TYPE_OFFLINE)) {
jobType = JobType.SYNC;
}
switch (jobType) {
case SHELL:
taskExecutionContext = ShellParameters.getTaskExecutionContext(dmpDevelopTaskDto);
break;
case SQL:
break;
case SYNC:
break;
case SUBPROCESS:
break;
case FTP:
break;
case UNZIPFILE:
break;
case DOCTRANS:
break;
case HDFS:
break;
default:
break;
}
return taskExecutionContext;
}
}
package com.jz.dmp.cmdexectool.entity;
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.Date;
import java.util.List;
/**计算引擎表
* @author ybz
*
*/
@ApiModel(value = "计算引擎表", description = "计算引擎表")
public class DmpComputEngine implements Serializable{
/**
*
*/
private static final long serialVersionUID = 1L;
/**
* 主键
*/
@ApiModelProperty(value = "主键")
private Integer engineId;
/**
* 引擎名称
*/
@ApiModelProperty(value = "引擎名称")
private String engineName;
/**
* 备注
*/
@ApiModelProperty(value = "备注")
private String remark;
/**
* 数据状态(0:删除,1,未删除)
*/
@ApiModelProperty(value = "数据状态(0:删除,1,未删除)")
private String dataStatus;
/**
* 创建用户ID
*/
@ApiModelProperty(value = "创建用户ID")
private Integer createUserId;
/**
* 创建时间
*/
@ApiModelProperty(value = "创建时间")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date createTime;
/**
* 修改用户ID
*/
@ApiModelProperty(value = "修改用户ID")
private Integer updateUserId;
/**
* 修改时间
*/
@ApiModelProperty(value = "修改时间")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date updateTime;
@ApiModelProperty(value = "计算引擎参数")
private List<DmpComputEngineParam> params;
public Integer getEngineId() {
return engineId;
}
public void setEngineId(Integer engineId) {
this.engineId = engineId;
}
public String getEngineName() {
return engineName;
}
public void setEngineName(String engineName) {
this.engineName = engineName;
}
public String getRemark() {
return remark;
}
public void setRemark(String remark) {
this.remark = remark;
}
public String getDataStatus() {
return dataStatus;
}
public void setDataStatus(String dataStatus) {
this.dataStatus = dataStatus;
}
public Integer getCreateUserId() {
return createUserId;
}
public void setCreateUserId(Integer createUserId) {
this.createUserId = createUserId;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public Integer getUpdateUserId() {
return updateUserId;
}
public void setUpdateUserId(Integer updateUserId) {
this.updateUserId = updateUserId;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
public List<DmpComputEngineParam> getParams() {
return params;
}
public void setParams(List<DmpComputEngineParam> params) {
this.params = params;
}
}
package com.jz.dmp.cmdexectool.entity;
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.Date;
/**计算引擎参数表
* @author ybz
*
*/
@ApiModel(value = "计算引擎参数表", description = "计算引擎参数表")
public class DmpComputEngineParam implements Serializable{
/**
*
*/
private static final long serialVersionUID = 1L;
/**
* 主键
*/
@ApiModelProperty(value = "主键")
private Integer paramId;
/**
* 引擎主键
*/
@ApiModelProperty(value = "引擎主键")
private Integer engineId;
/**
* 参数名称
*/
@ApiModelProperty(value = "参数名称")
private String paramName;
/**
* 类型(0:引擎公共参数,1:引擎私配置参数)
*/
@ApiModelProperty(value = "类型(0:引擎公共参数,1:引擎私配置参数)")
private String paramType;
/**
* 参数值
*/
@ApiModelProperty(value = "参数值")
private String paramValue;
/**
* 备注
*/
@ApiModelProperty(value = "备注")
private String remark;
/**
* 数据状态(0:删除,1,未删除)
*/
@ApiModelProperty(value = "数据状态(0:删除,1,未删除)")
private String dataStatus;
/**
* 创建用户ID
*/
@ApiModelProperty(value = "创建用户ID")
private Integer createUserId;
/**
* 创建时间
*/
@ApiModelProperty(value = "创建时间")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date createTime;
/**
* 修改用户ID
*/
@ApiModelProperty(value = "修改用户ID")
private Integer updateUserId;
/**
* 修改时间
*/
@ApiModelProperty(value = "修改时间")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date updateTime;
public Integer getParamId() {
return paramId;
}
public void setParamId(Integer paramId) {
this.paramId = paramId;
}
public Integer getEngineId() {
return engineId;
}
public void setEngineId(Integer engineId) {
this.engineId = engineId;
}
public String getParamName() {
return paramName;
}
public void setParamName(String paramName) {
this.paramName = paramName;
}
public String getParamType() {
return paramType;
}
public void setParamType(String paramType) {
this.paramType = paramType;
}
public String getParamValue() {
return paramValue;
}
public void setParamValue(String paramValue) {
this.paramValue = paramValue;
}
public String getRemark() {
return remark;
}
public void setRemark(String remark) {
this.remark = remark;
}
public String getDataStatus() {
return dataStatus;
}
public void setDataStatus(String dataStatus) {
this.dataStatus = dataStatus;
}
public Integer getCreateUserId() {
return createUserId;
}
public void setCreateUserId(Integer createUserId) {
this.createUserId = createUserId;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public Integer getUpdateUserId() {
return updateUserId;
}
public void setUpdateUserId(Integer updateUserId) {
this.updateUserId = updateUserId;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
}
package com.jz.dmp.cmdexectool.entity;
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.Date;
import java.util.List;
/**项目配置计算引擎关系表
* @author ybz
*
*/
@ApiModel(value = "项目配置计算引擎关系表", description = "项目配置计算引擎关系表")
public class DmpProjectConfigEngine implements Serializable{
/**
*
*/
private static final long serialVersionUID = 1L;
/**
* 主键
*/
@ApiModelProperty(value = "主键")
private Integer configEngineId;
/**
* 项目主键
*/
@ApiModelProperty(value = "项目主键")
private Integer projectId;
/**
* 引擎主键
*/
@ApiModelProperty(value = "引擎主键")
private Integer engineId;
/**
* 备注
*/
@ApiModelProperty(value = "备注")
private String remark;
/**
* 数据状态(0:删除,1,未删除)
*/
@ApiModelProperty(value = "数据状态(0:删除,1,未删除)")
private String dataStatus;
/**
* 创建用户ID
*/
@ApiModelProperty(value = "创建用户ID")
private Integer createUserId;
/**
* 创建时间
*/
@ApiModelProperty(value = "创建时间")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date createTime;
/**
* 修改用户ID
*/
@ApiModelProperty(value = "修改用户ID")
private Integer updateUserId;
/**
* 修改时间
*/
@ApiModelProperty(value = "修改时间")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date updateTime;
@ApiModelProperty(value = "选择引擎私有配置参数")
private List<DmpProjectEngineParam> dmpProjectEngineParams;
public Integer getConfigEngineId() {
return configEngineId;
}
public void setConfigEngineId(Integer configEngineId) {
this.configEngineId = configEngineId;
}
public Integer getProjectId() {
return projectId;
}
public void setProjectId(Integer projectId) {
this.projectId = projectId;
}
public Integer getEngineId() {
return engineId;
}
public void setEngineId(Integer engineId) {
this.engineId = engineId;
}
public String getRemark() {
return remark;
}
public void setRemark(String remark) {
this.remark = remark;
}
public String getDataStatus() {
return dataStatus;
}
public void setDataStatus(String dataStatus) {
this.dataStatus = dataStatus;
}
public Integer getCreateUserId() {
return createUserId;
}
public void setCreateUserId(Integer createUserId) {
this.createUserId = createUserId;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public Integer getUpdateUserId() {
return updateUserId;
}
public void setUpdateUserId(Integer updateUserId) {
this.updateUserId = updateUserId;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
public List<DmpProjectEngineParam> getDmpProjectEngineParams() {
return dmpProjectEngineParams;
}
public void setDmpProjectEngineParams(List<DmpProjectEngineParam> dmpProjectEngineParams) {
this.dmpProjectEngineParams = dmpProjectEngineParams;
}
}
package com.jz.dmp.cmdexectool.entity;
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.Date;
import java.util.List;
/**项目配置表
* @author ybz
*
*/
@ApiModel(value = "项目配置表", description = "项目配置表")
public class DmpProjectConfigInfo implements Serializable{
/**
*
*/
private static final long serialVersionUID = 1L;
/**
* 主键
*/
@ApiModelProperty(value = "主键")
private Integer projectConfigId;
/**
* 项目主键
*/
@ApiModelProperty(value = "项目主键")
private Integer projectId;
/**
* 备注
*/
@ApiModelProperty(value = "备注")
private String remark;
/**
* 数据状态(0:删除,1,未删除)
*/
@ApiModelProperty(value = "数据状态(0:删除,1,未删除)")
private String dataStatus;
/**
* 创建用户ID
*/
@ApiModelProperty(value = "创建用户ID")
private Integer createUserId;
/**
* 创建时间
*/
@ApiModelProperty(value = "创建时间")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date createTime;
/**
* 修改用户ID
*/
@ApiModelProperty(value = "修改用户ID")
private Integer updateUserId;
/**
* 修改时间
*/
@ApiModelProperty(value = "修改时间")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date updateTime;
@ApiModelProperty(value = "选择的计算引擎")
private List<DmpProjectConfigEngine> dmpProjectConfigEngines;
public Integer getProjectConfigId() {
return projectConfigId;
}
public void setProjectConfigId(Integer projectConfigId) {
this.projectConfigId = projectConfigId;
}
public Integer getProjectId() {
return projectId;
}
public void setProjectId(Integer projectId) {
this.projectId = projectId;
}
public String getRemark() {
return remark;
}
public void setRemark(String remark) {
this.remark = remark;
}
public String getDataStatus() {
return dataStatus;
}
public void setDataStatus(String dataStatus) {
this.dataStatus = dataStatus;
}
public Integer getCreateUserId() {
return createUserId;
}
public void setCreateUserId(Integer createUserId) {
this.createUserId = createUserId;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public Integer getUpdateUserId() {
return updateUserId;
}
public void setUpdateUserId(Integer updateUserId) {
this.updateUserId = updateUserId;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
public List<DmpProjectConfigEngine> getDmpProjectConfigEngines() {
return dmpProjectConfigEngines;
}
public void setDmpProjectConfigEngines(List<DmpProjectConfigEngine> dmpProjectConfigEngines) {
this.dmpProjectConfigEngines = dmpProjectConfigEngines;
}
}
package com.jz.dmp.cmdexectool.entity;
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.Date;
/**计算引擎项目参数表
* @author ybz
*
*/
@ApiModel(value = "计算引擎项目参数表", description = "计算引擎项目参数表")
public class DmpProjectEngineParam implements Serializable{
/**
*
*/
private static final long serialVersionUID = 1L;
/**
* 主键
*/
@ApiModelProperty(value = "主键")
private Integer projectParamId;
/**
* 项目配置引擎关系主键
*/
@ApiModelProperty(value = "项目配置引擎关系主键")
private Integer configEngineId;
/**
* 引擎参数主键
*/
@ApiModelProperty(value = "引擎参数主键")
private Integer paramId;
/**
* 参数名称
*/
@ApiModelProperty(value = "参数名称")
private String paramName;
/**
* 参数值
*/
@ApiModelProperty(value = "参数值")
private String paramValue;
/**
* 备注
*/
@ApiModelProperty(value = "备注")
private String remark;
/**
* 数据状态(0:删除,1,未删除)
*/
@ApiModelProperty(value = "数据状态(0:删除,1,未删除)")
private String dataStatus;
/**
* 创建用户ID
*/
@ApiModelProperty(value = "创建用户ID")
private Integer createUserId;
/**
* 创建时间
*/
@ApiModelProperty(value = "创建时间")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date createTime;
/**
* 修改用户ID
*/
@ApiModelProperty(value = "修改用户ID")
private Integer updateUserId;
/**
* 修改时间
*/
@ApiModelProperty(value = "修改时间")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date updateTime;
public Integer getProjectParamId() {
return projectParamId;
}
public void setProjectParamId(Integer projectParamId) {
this.projectParamId = projectParamId;
}
public Integer getConfigEngineId() {
return configEngineId;
}
public void setConfigEngineId(Integer configEngineId) {
this.configEngineId = configEngineId;
}
public Integer getParamId() {
return paramId;
}
public void setParamId(Integer paramId) {
this.paramId = paramId;
}
public String getParamName() {
return paramName;
}
public void setParamName(String paramName) {
this.paramName = paramName;
}
public String getParamValue() {
return paramValue;
}
public void setParamValue(String paramValue) {
this.paramValue = paramValue;
}
public String getRemark() {
return remark;
}
public void setRemark(String remark) {
this.remark = remark;
}
public String getDataStatus() {
return dataStatus;
}
public void setDataStatus(String dataStatus) {
this.dataStatus = dataStatus;
}
public Integer getCreateUserId() {
return createUserId;
}
public void setCreateUserId(Integer createUserId) {
this.createUserId = createUserId;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public Integer getUpdateUserId() {
return updateUserId;
}
public void setUpdateUserId(Integer updateUserId) {
this.updateUserId = updateUserId;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
}
package com.jz.dmp.cmdexectool.mapper;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.annotations.Param;
import com.jz.dmp.cmdexectool.controller.bean.DmpComputEngineDto;
import com.jz.dmp.cmdexectool.entity.DmpComputEngine;
/**计算引擎表 mapper
* @author ybz
*
*/
public interface DmpComputEngineMapper {
/**新增计算引擎表
* @param dmpComputEngine
* @return
* @throws Exception
*/
public int insert(DmpComputEngine dmpComputEngine)throws Exception;
/**选择性增加计算引擎表
* @param dmpComputEngine
* @return
* @throws Exception
*/
public int insertSelective(DmpComputEngine dmpComputEngine)throws Exception;
/**主键修改计算引擎表
* @param dmpComputEngine
* @return
* @throws Exception
*/
public int updateByPrimaryKey(DmpComputEngine dmpComputEngine)throws Exception;
/**选择性修改计算引擎表
* @param dmpComputEngine
* @return
* @throws Exception
*/
public int updateByPrimaryKeySelective(DmpComputEngine dmpComputEngine)throws Exception;
/**主键查询计算引擎表
* @param engineId
* @return
* @throws Exception
*/
public DmpComputEngine selectByPrimaryKey(Integer engineId)throws Exception;
/**主键删除计算引擎表
* @param engineId
* @return
* @throws Exception
*/
public int deleteByPrimaryKey(Integer engineId)throws Exception;
/**主键软删除计算引擎表
* @param engineId
* @return
* @throws Exception
*/
public int softDeleteByPrimaryKey(Integer engineId)throws Exception;
/**主键删除计算引擎表
* @param engineId
* @return
* @throws Exception
*/
public int delete(Map<String, Object> param)throws Exception;
/**主键软删除计算引擎表
* @param engineId
* @return
* @throws Exception
*/
public int softDelete(Map<String, Object> param)throws Exception;
/**条件查询计算引擎表
* @param param
* @return
* @throws Exception
*/
public List<DmpComputEngineDto> findList(Map<String, Object> param)throws Exception;
/**主键查询计算引擎表
* @param engineId
* @return
* @throws Exception
*/
public DmpComputEngineDto findById(Integer engineId)throws Exception;
/**批量新增计算引擎表
* @param dmpComputEngines
* @throws Exception
*/
public void insertBatch(List<DmpComputEngine> dmpComputEngines)throws Exception;
/**
* @Title: deleteByIds
* @Description: TODO(批量删除)
* @param @param idList
* @param @throws Exception 参数
* @return void 返回类型
* @throws
*/
public void deleteByIds(@Param("idList")List<Integer> idList)throws Exception;
/**
* @Title: softDeleteByIds
* @Description: TODO(批量软删除)
* @param @param idList
* @param @throws Exception 参数
* @return void 返回类型
* @throws
*/
public void softDeleteByIds(@Param("idList")List<Integer> idList)throws Exception;
}
package com.jz.dmp.cmdexectool.mapper;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.annotations.Param;
import com.jz.dmp.cmdexectool.controller.bean.DmpComputEngineParamDto;
import com.jz.dmp.cmdexectool.entity.DmpComputEngineParam;
/**计算引擎参数表 mapper
* @author ybz
*
*/
public interface DmpComputEngineParamMapper {
/**新增计算引擎参数表
* @param dmpComputEngineParam
* @return
* @throws Exception
*/
public int insert(DmpComputEngineParam dmpComputEngineParam)throws Exception;
/**选择性增加计算引擎参数表
* @param dmpComputEngineParam
* @return
* @throws Exception
*/
public int insertSelective(DmpComputEngineParam dmpComputEngineParam)throws Exception;
/**主键修改计算引擎参数表
* @param dmpComputEngineParam
* @return
* @throws Exception
*/
public int updateByPrimaryKey(DmpComputEngineParam dmpComputEngineParam)throws Exception;
/**选择性修改计算引擎参数表
* @param dmpComputEngineParam
* @return
* @throws Exception
*/
public int updateByPrimaryKeySelective(DmpComputEngineParam dmpComputEngineParam)throws Exception;
/**主键查询计算引擎参数表
* @param paramId
* @return
* @throws Exception
*/
public DmpComputEngineParam selectByPrimaryKey(Integer paramId)throws Exception;
/**主键删除计算引擎参数表
* @param paramId
* @return
* @throws Exception
*/
public int deleteByPrimaryKey(Integer paramId)throws Exception;
/**主键软删除计算引擎参数表
* @param paramId
* @return
* @throws Exception
*/
public int softDeleteByPrimaryKey(Integer paramId)throws Exception;
/**主键删除计算引擎参数表
* @param paramId
* @return
* @throws Exception
*/
public int delete(Map<String, Object> param)throws Exception;
/**主键软删除计算引擎参数表
* @param paramId
* @return
* @throws Exception
*/
public int softDelete(Map<String, Object> param)throws Exception;
/**条件查询计算引擎参数表
* @param param
* @return
* @throws Exception
*/
public List<DmpComputEngineParamDto> findList(Map<String, Object> param)throws Exception;
/**主键查询计算引擎参数表
* @param paramId
* @return
* @throws Exception
*/
public DmpComputEngineParamDto findById(Integer paramId)throws Exception;
/**批量新增计算引擎参数表
* @param dmpComputEngineParams
* @throws Exception
*/
public void insertBatch(List<DmpComputEngineParam> dmpComputEngineParams)throws Exception;
/**
* @Title: deleteByIds
* @Description: TODO(批量删除)
* @param @param idList
* @param @throws Exception 参数
* @return void 返回类型
* @throws
*/
public void deleteByIds(@Param("idList")List<Integer> idList)throws Exception;
/**
* @Title: softDeleteByIds
* @Description: TODO(批量软删除)
* @param @param idList
* @param @throws Exception 参数
* @return void 返回类型
* @throws
*/
public void softDeleteByIds(@Param("idList")List<Integer> idList)throws Exception;
}
package com.jz.dmp.cmdexectool.mapper;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.annotations.Param;
import com.jz.dmp.cmdexectool.controller.bean.DmpProjectConfigEngineDto;
import com.jz.dmp.cmdexectool.entity.DmpProjectConfigEngine;
/**项目配置计算引擎关系表 mapper
* @author ybz
*
*/
public interface DmpProjectConfigEngineMapper {
/**新增项目配置计算引擎关系表
* @param dmpProjectConfigEngine
* @return
* @throws Exception
*/
public int insert(DmpProjectConfigEngine dmpProjectConfigEngine)throws Exception;
/**选择性增加项目配置计算引擎关系表
* @param dmpProjectConfigEngine
* @return
* @throws Exception
*/
public int insertSelective(DmpProjectConfigEngine dmpProjectConfigEngine)throws Exception;
/**主键修改项目配置计算引擎关系表
* @param dmpProjectConfigEngine
* @return
* @throws Exception
*/
public int updateByPrimaryKey(DmpProjectConfigEngine dmpProjectConfigEngine)throws Exception;
/**选择性修改项目配置计算引擎关系表
* @param dmpProjectConfigEngine
* @return
* @throws Exception
*/
public int updateByPrimaryKeySelective(DmpProjectConfigEngine dmpProjectConfigEngine)throws Exception;
/**主键查询项目配置计算引擎关系表
* @param configEngineId
* @return
* @throws Exception
*/
public DmpProjectConfigEngine selectByPrimaryKey(Integer configEngineId)throws Exception;
/**主键删除项目配置计算引擎关系表
* @param configEngineId
* @return
* @throws Exception
*/
public int deleteByPrimaryKey(Integer configEngineId)throws Exception;
/**主键软删除项目配置计算引擎关系表
* @param configEngineId
* @return
* @throws Exception
*/
public int softDeleteByPrimaryKey(Integer configEngineId)throws Exception;
/**主键删除项目配置计算引擎关系表
* @param configEngineId
* @return
* @throws Exception
*/
public int delete(Map<String, Object> param)throws Exception;
/**主键软删除项目配置计算引擎关系表
* @param configEngineId
* @return
* @throws Exception
*/
public int softDelete(Map<String, Object> param)throws Exception;
/**条件查询项目配置计算引擎关系表
* @param param
* @return
* @throws Exception
*/
public List<DmpProjectConfigEngineDto> findList(Map<String, Object> param)throws Exception;
/**主键查询项目配置计算引擎关系表
* @param configEngineId
* @return
* @throws Exception
*/
public DmpProjectConfigEngineDto findById(Integer configEngineId)throws Exception;
/**批量新增项目配置计算引擎关系表
* @param dmpProjectConfigEngines
* @throws Exception
*/
public void insertBatch(List<DmpProjectConfigEngine> dmpProjectConfigEngines)throws Exception;
/**
* @Title: deleteByIds
* @Description: TODO(批量删除)
* @param @param idList
* @param @throws Exception 参数
* @return void 返回类型
* @throws
*/
public void deleteByIds(@Param("idList")List<Integer> idList)throws Exception;
/**
* @Title: softDeleteByIds
* @Description: TODO(批量软删除)
* @param @param idList
* @param @throws Exception 参数
* @return void 返回类型
* @throws
*/
public void softDeleteByIds(@Param("idList")List<Integer> idList)throws Exception;
}
package com.jz.dmp.cmdexectool.mapper;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.annotations.Param;
import com.jz.dmp.cmdexectool.controller.bean.DmpProjectConfigInfoDto;
import com.jz.dmp.cmdexectool.entity.DmpProjectConfigInfo;
/**项目配置表 mapper
* @author ybz
*
*/
public interface DmpProjectConfigInfoMapper {
/**新增项目配置表
* @param dmpProjectConfigInfo
* @return
* @throws Exception
*/
public int insert(DmpProjectConfigInfo dmpProjectConfigInfo)throws Exception;
/**选择性增加项目配置表
* @param dmpProjectConfigInfo
* @return
* @throws Exception
*/
public int insertSelective(DmpProjectConfigInfo dmpProjectConfigInfo)throws Exception;
/**主键修改项目配置表
* @param dmpProjectConfigInfo
* @return
* @throws Exception
*/
public int updateByPrimaryKey(DmpProjectConfigInfo dmpProjectConfigInfo)throws Exception;
/**选择性修改项目配置表
* @param dmpProjectConfigInfo
* @return
* @throws Exception
*/
public int updateByPrimaryKeySelective(DmpProjectConfigInfo dmpProjectConfigInfo)throws Exception;
/**主键查询项目配置表
* @param projectConfigId
* @return
* @throws Exception
*/
public DmpProjectConfigInfo selectByPrimaryKey(Integer projectConfigId)throws Exception;
/**主键删除项目配置表
* @param projectConfigId
* @return
* @throws Exception
*/
public int deleteByPrimaryKey(Integer projectConfigId)throws Exception;
/**主键软删除项目配置表
* @param projectConfigId
* @return
* @throws Exception
*/
public int softDeleteByPrimaryKey(Integer projectConfigId)throws Exception;
/**主键删除项目配置表
* @param projectConfigId
* @return
* @throws Exception
*/
public int delete(Map<String, Object> param)throws Exception;
/**主键软删除项目配置表
* @param projectConfigId
* @return
* @throws Exception
*/
public int softDelete(Map<String, Object> param)throws Exception;
/**条件查询项目配置表
* @param param
* @return
* @throws Exception
*/
public List<DmpProjectConfigInfoDto> findList(Map<String, Object> param)throws Exception;
/**主键查询项目配置表
* @param projectConfigId
* @return
* @throws Exception
*/
public DmpProjectConfigInfoDto findById(Integer projectConfigId)throws Exception;
/**批量新增项目配置表
* @param dmpProjectConfigInfos
* @throws Exception
*/
public void insertBatch(List<DmpProjectConfigInfo> dmpProjectConfigInfos)throws Exception;
/**
* @Title: deleteByIds
* @Description: TODO(批量删除)
* @param @param idList
* @param @throws Exception 参数
* @return void 返回类型
* @throws
*/
public void deleteByIds(@Param("idList")List<Integer> idList)throws Exception;
/**
* @Title: softDeleteByIds
* @Description: TODO(批量软删除)
* @param @param idList
* @param @throws Exception 参数
* @return void 返回类型
* @throws
*/
public void softDeleteByIds(@Param("idList")List<Integer> idList)throws Exception;
}
package com.jz.dmp.cmdexectool.mapper;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.annotations.Param;
import com.jz.dmp.cmdexectool.controller.bean.DmpProjectEngineParamDto;
import com.jz.dmp.cmdexectool.entity.DmpProjectEngineParam;
/**计算引擎项目参数表 mapper
* @author ybz
*
*/
public interface DmpProjectEngineParamMapper {
/**新增计算引擎项目参数表
* @param dmpProjectEngineParam
* @return
* @throws Exception
*/
public int insert(DmpProjectEngineParam dmpProjectEngineParam)throws Exception;
/**选择性增加计算引擎项目参数表
* @param dmpProjectEngineParam
* @return
* @throws Exception
*/
public int insertSelective(DmpProjectEngineParam dmpProjectEngineParam)throws Exception;
/**主键修改计算引擎项目参数表
* @param dmpProjectEngineParam
* @return
* @throws Exception
*/
public int updateByPrimaryKey(DmpProjectEngineParam dmpProjectEngineParam)throws Exception;
/**选择性修改计算引擎项目参数表
* @param dmpProjectEngineParam
* @return
* @throws Exception
*/
public int updateByPrimaryKeySelective(DmpProjectEngineParam dmpProjectEngineParam)throws Exception;
/**主键查询计算引擎项目参数表
* @param projectParamId
* @return
* @throws Exception
*/
public DmpProjectEngineParam selectByPrimaryKey(Integer projectParamId)throws Exception;
/**主键删除计算引擎项目参数表
* @param projectParamId
* @return
* @throws Exception
*/
public int deleteByPrimaryKey(Integer projectParamId)throws Exception;
/**主键软删除计算引擎项目参数表
* @param projectParamId
* @return
* @throws Exception
*/
public int softDeleteByPrimaryKey(Integer projectParamId)throws Exception;
/**主键删除计算引擎项目参数表
* @param projectParamId
* @return
* @throws Exception
*/
public int delete(Map<String, Object> param)throws Exception;
/**主键软删除计算引擎项目参数表
* @param projectParamId
* @return
* @throws Exception
*/
public int softDelete(Map<String, Object> param)throws Exception;
/**条件查询计算引擎项目参数表
* @param param
* @return
* @throws Exception
*/
public List<DmpProjectEngineParamDto> findList(Map<String, Object> param)throws Exception;
/**主键查询计算引擎项目参数表
* @param projectParamId
* @return
* @throws Exception
*/
public DmpProjectEngineParamDto findById(Integer projectParamId)throws Exception;
/**批量新增计算引擎项目参数表
* @param dmpProjectEngineParams
* @throws Exception
*/
public void insertBatch(List<DmpProjectEngineParam> dmpProjectEngineParams)throws Exception;
/**
* @Title: deleteByIds
* @Description: TODO(批量删除)
* @param @param idList
* @param @throws Exception 参数
* @return void 返回类型
* @throws
*/
public void deleteByIds(@Param("idList")List<Integer> idList)throws Exception;
/**
* @Title: softDeleteByIds
* @Description: TODO(批量软删除)
* @param @param idList
* @param @throws Exception 参数
* @return void 返回类型
* @throws
*/
public void softDeleteByIds(@Param("idList")List<Integer> idList)throws Exception;
}
package com.jz.dmp.cmdexectool.mapper;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.annotations.Param;
import com.jz.dmp.cmdexectool.controller.bean.DmpPublicConfigInfoDto;
import com.jz.dmp.cmdexectool.entity.DmpPublicConfigInfo;
/**公共配置表 mapper
* @author ybz
*
*/
public interface DmpPublicConfigInfoMapper {
/**新增公共配置表
* @param dmpPublicConfigInfo
* @return
* @throws Exception
*/
public int insert(DmpPublicConfigInfo dmpPublicConfigInfo)throws Exception;
/**选择性增加公共配置表
* @param dmpPublicConfigInfo
* @return
* @throws Exception
*/
public int insertSelective(DmpPublicConfigInfo dmpPublicConfigInfo)throws Exception;
/**主键修改公共配置表
* @param dmpPublicConfigInfo
* @return
* @throws Exception
*/
public int updateByPrimaryKey(DmpPublicConfigInfo dmpPublicConfigInfo)throws Exception;
/**选择性修改公共配置表
* @param dmpPublicConfigInfo
* @return
* @throws Exception
*/
public int updateByPrimaryKeySelective(DmpPublicConfigInfo dmpPublicConfigInfo)throws Exception;
/**主键查询公共配置表
* @param publicConfigId
* @return
* @throws Exception
*/
public DmpPublicConfigInfo selectByPrimaryKey(Integer publicConfigId)throws Exception;
/**主键删除公共配置表
* @param publicConfigId
* @return
* @throws Exception
*/
public int deleteByPrimaryKey(Integer publicConfigId)throws Exception;
/**主键软删除公共配置表
* @param publicConfigId
* @return
* @throws Exception
*/
public int softDeleteByPrimaryKey(Integer publicConfigId)throws Exception;
/**主键删除公共配置表
* @param publicConfigId
* @return
* @throws Exception
*/
public int delete(Map<String, Object> param)throws Exception;
/**主键软删除公共配置表
* @param publicConfigId
* @return
* @throws Exception
*/
public int softDelete(Map<String, Object> param)throws Exception;
/**条件查询公共配置表
* @param param
* @return
* @throws Exception
*/
public List<DmpPublicConfigInfoDto> findList(Map<String, Object> param)throws Exception;
/**主键查询公共配置表
* @param publicConfigId
* @return
* @throws Exception
*/
public DmpPublicConfigInfoDto findById(Integer publicConfigId)throws Exception;
/**批量新增公共配置表
* @param dmpPublicConfigInfos
* @throws Exception
*/
public void insertBatch(List<DmpPublicConfigInfo> dmpPublicConfigInfos)throws Exception;
/**
* @Title: deleteByIds
* @Description: TODO(批量删除)
* @param @param idList
* @param @throws Exception 参数
* @return void 返回类型
* @throws
*/
public void deleteByIds(@Param("idList")List<Integer> idList)throws Exception;
/**
* @Title: softDeleteByIds
* @Description: TODO(批量软删除)
* @param @param idList
* @param @throws Exception 参数
* @return void 返回类型
* @throws
*/
public void softDeleteByIds(@Param("idList")List<Integer> idList)throws Exception;
}
package com.jz.dmp.cmdexectool.mapper;
import com.jz.dmp.cmdexectool.entity.DmpSyncingDatasource;
/**
* 数据源配置(DmpSyncingDatasource)表数据库访问层
*
* @author Bellamy
* @since 2020-12-24 14:09:59
*/
public interface DmpSyncingDatasourceDao {
/**
* 通过ID查询单条数据
*
* @param id 主键
* @return 实例对象
*/
DmpSyncingDatasource queryById(Integer id);
}
\ No newline at end of file
......@@ -31,9 +31,6 @@ public abstract class AbstractParameters implements IParameters {
@Override
public abstract boolean checkParameters();
@Override
public abstract List<ResourceInfo> getResourceFilesList();
/**
* local parameters
*/
......
......@@ -32,6 +32,8 @@ public class ShellParameters extends AbstractParameters {
* shell script
*/
private String script;
private String taskAppId;
/**
* resource list
......@@ -46,6 +48,14 @@ public class ShellParameters extends AbstractParameters {
this.script = script;
}
public String getTaskAppId() {
return taskAppId;
}
public void setTaskAppId(String taskAppId) {
this.taskAppId = taskAppId;
}
public List<ResourceInfo> getResourceList() {
return resourceList;
}
......@@ -72,7 +82,7 @@ public class ShellParameters extends AbstractParameters {
* @return TaskExecutionContext 返回类型
* @throws
*/
public static TaskExecutionContext getTaskExecutionContext(DmpDevelopTaskDto dmpDevelopTaskDto) {
public TaskExecutionContext getTaskExecutionContext() {
TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
// taskName;
......@@ -82,7 +92,7 @@ public class ShellParameters extends AbstractParameters {
// host;
// executePath;
taskExecutionContext.setExecutePath("E:/test");
//taskExecutionContext.setExecutePath("E:/test");
// logPath;
// taskJson;
......@@ -92,12 +102,12 @@ public class ShellParameters extends AbstractParameters {
// projectId;
// taskParams;
taskExecutionContext.setTaskParams(dmpDevelopTaskDto.getScript());
taskExecutionContext.setTaskParams(script);
// envFile;
// taskAppId
taskExecutionContext.setTaskAppId("test");
taskExecutionContext.setTaskAppId(taskAppId);
return taskExecutionContext;
}
......
......@@ -16,199 +16,206 @@
*/
package com.jz.dmp.cmdexectool.scheduler.common.task.sql;
import com.alibaba.fastjson.JSONObject;
import com.jz.dmp.cmdexectool.common.constant.CommConstant;
import com.jz.dmp.cmdexectool.controller.bean.DmpProjectConfigInfoDto;
import com.jz.dmp.cmdexectool.entity.DmpSyncingDatasource;
import com.jz.dmp.cmdexectool.mapper.DmpSyncingDatasourceDao;
import com.jz.dmp.cmdexectool.scheduler.common.process.ResourceInfo;
import com.jz.dmp.cmdexectool.scheduler.common.task.AbstractParameters;
import org.apache.commons.lang.StringUtils;
import com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Sql/Hql parameter
*/
public class SqlParameters extends AbstractParameters {
/**
* data source type,eg MYSQL, POSTGRES, HIVE ...
*/
private String type;
/**
* datasource id
*/
private int datasource;
/**
* sql
*/
private String sql;
/**
* sql type
* 0 query
* 1 NON_QUERY
*/
private int sqlType;
/**
* udf list
*/
private String udfs;
/**
* show type
* 0 TABLE
* 1 TEXT
* 2 attachment
* 3 TABLE+attachment
*/
private String showType;
/**
* SQL connection parameters
*/
private String connParams;
/**
* Pre Statements
*/
private List<String> preStatements;
/**
* Post Statements
*/
private List<String> postStatements;
/**
* title
*/
private String title;
/**
* receivers
*/
private String receivers;
/**
* receivers cc
*/
private String receiversCc;
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public int getDatasource() {
return datasource;
}
public void setDatasource(int datasource) {
this.datasource = datasource;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
public String getUdfs() {
return udfs;
}
public void setUdfs(String udfs) {
this.udfs = udfs;
}
public int getSqlType() {
return sqlType;
}
public void setSqlType(int sqlType) {
this.sqlType = sqlType;
}
public String getShowType() {
return showType;
}
public void setShowType(String showType) {
this.showType = showType;
}
public String getConnParams() {
return connParams;
}
public void setConnParams(String connParams) {
this.connParams = connParams;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getReceivers() {
return receivers;
}
public void setReceivers(String receivers) {
this.receivers = receivers;
}
public String getReceiversCc() {
return receiversCc;
}
public void setReceiversCc(String receiversCc) {
this.receiversCc = receiversCc;
}
public List<String> getPreStatements() {
return preStatements;
}
public void setPreStatements(List<String> preStatements) {
this.preStatements = preStatements;
}
public List<String> getPostStatements() {
return postStatements;
}
public void setPostStatements(List<String> postStatements) {
this.postStatements = postStatements;
}
@Override
public boolean checkParameters() {
return datasource != 0 && StringUtils.isNotEmpty(type) && StringUtils.isNotEmpty(sql);
}
@Override
public List<ResourceInfo> getResourceFilesList() {
return new ArrayList<>();
}
@Override
public String toString() {
return "SqlParameters{" +
"type='" + type + '\'' +
", datasource=" + datasource +
", sql='" + sql + '\'' +
", sqlType=" + sqlType +
", udfs='" + udfs + '\'' +
", showType='" + showType + '\'' +
", connParams='" + connParams + '\'' +
", title='" + title + '\'' +
", receivers='" + receivers + '\'' +
", receiversCc='" + receiversCc + '\'' +
", preStatements=" + preStatements +
", postStatements=" + postStatements +
'}';
}
/**
* shell script
*/
private String script;
private String taskAppId;
/**
* 数据源相关配置
*/
private Map<String, Object> source;
/**
* 目标相关配置
*/
private Map<String, Object> sink;
/**
* 环境相关配置
*/
private Map<String, Object> env;
/**
* ETL相关配置
*/
private Map<String, Object> transform;
/**
* resource list
*/
private List<ResourceInfo> resourceList;
public SqlParameters(String script, DmpProjectConfigInfoDto projectConfigInfoDto, DmpSyncingDatasourceDao dmpSyncingDatasourceDao) {
this.script = script;
JSONObject scriptObj = JSONObject.parseObject(script);
String outputType = scriptObj.getString("outputType");
String sqlScript = scriptObj.getString("sqlScript");
//evn
env.put("spark.app.name", "Waterdrop");
//source
Integer sourceId = scriptObj.getInteger("sourceId");
DmpSyncingDatasource dmpSyncingDatasource = dmpSyncingDatasourceDao.queryById(sourceId);
Map<String, Object> jdbcMap = new HashMap<String, Object>();
jdbcMap.put("driver", "com.mysql.jdbc.Driver");
jdbcMap.put("url", dmpSyncingDatasource.getJdbcUrl());
//jdbcMap.put("table", value);
//jdbcMap.put("result_table_name", value);
//jdbcMap.put("user", dmpSyncingDatasource.getUserName());
//jdbcMap.put("password", dmpSyncingDatasource.getPassword());
//jdbcMap.put("", value);
//jdbcMap.put("", value);
//jdbcMap.put("", value);
//jdbcMap.put("", value);
source.put("jdbc", jdbcMap);
if (CommConstant.OUTPUT_TYPE_CONSOLE.equals(outputType)) {
//transform
Map<String, Object> sqlMap = new HashMap<String, Object>();
sqlMap.put("sql", sqlScript);
transform.put("sql", sqlMap);
//sink
Map<String, Object> stdoutMap = new HashMap<String, Object>();
stdoutMap.put("limit", 10);
stdoutMap.put("serializer", "json");
sink.put("stdout", stdoutMap);
}else if (CommConstant.OUTPUT_TYPE_HDFS.equals(outputType)) {
//transform
//sink
}else if (CommConstant.OUTPUT_TYPE_TABLE.equals(outputType)) {
}else if (CommConstant.OUTPUT_TYPE_TOPIC.equals(outputType)) {
}else if (CommConstant.OUTPUT_TYPE_API.equals(outputType)) {
}
}
public String getScript() {
return script;
}
public void setScript(String script) {
this.script = script;
}
public String getTaskAppId() {
return taskAppId;
}
public void setTaskAppId(String taskAppId) {
this.taskAppId = taskAppId;
}
public Map<String, Object> getSource() {
return source;
}
public void setSource(Map<String, Object> source) {
this.source = source;
}
public Map<String, Object> getSink() {
return sink;
}
public void setSink(Map<String, Object> sink) {
this.sink = sink;
}
public Map<String, Object> getEnv() {
return env;
}
public void setEnv(Map<String, Object> env) {
this.env = env;
}
public Map<String, Object> getTransform() {
return transform;
}
public void setTransform(Map<String, Object> transform) {
this.transform = transform;
}
public List<ResourceInfo> getResourceList() {
return resourceList;
}
public void setResourceList(List<ResourceInfo> resourceList) {
this.resourceList = resourceList;
}
@Override
public boolean checkParameters() {
return script != null && !script.isEmpty();
}
@Override
public List<ResourceInfo> getResourceFilesList() {
return resourceList;
}
/**
* @Title: getTaskExecutionContext @Description:
* TODO(获取TaskExecutionContext) @param @param dmpDevelopTaskDto @param @return
* 参数 @return TaskExecutionContext 返回类型 @throws
*/
public TaskExecutionContext getTaskExecutionContext() {
TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
// taskName;
// taskType;
taskExecutionContext.setTaskType(CommConstant.WORK_TYPE_SQL);
// host;
// executePath;
// taskExecutionContext.setExecutePath("E:/test");
// logPath;
// taskJson;
// globalParams;
// projectId;
// taskParams;
taskExecutionContext.setTaskParams(JSONObject.toJSONString(this));
// envFile;
// taskAppId
taskExecutionContext.setTaskAppId(taskAppId);
return taskExecutionContext;
}
}
......@@ -27,6 +27,8 @@ import com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext;
//import com.jz.dmp.cmdexectool.scheduler.server.worker.task.processdure.ProcedureTask;
//import com.jz.dmp.cmdexectool.scheduler.server.worker.task.python.PythonTask;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.shell.ShellTask;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.sql.SqlTask;
//import com.jz.dmp.cmdexectool.scheduler.server.worker.task.spark.SparkTask;
//import com.jz.dmp.cmdexectool.scheduler.server.worker.task.sql.SqlTask;
//import com.jz.dmp.cmdexectool.scheduler.server.worker.task.sqoop.SqoopTask;
......@@ -59,8 +61,8 @@ public class TaskManager {
return new ShellTask(taskExecutionContext, logger);
//case PROCEDURE:
// return new ProcedureTask(taskExecutionContext, logger);
//case SQL:
// return new SqlTask(taskExecutionContext, logger);
case SQL:
return new SqlTask(taskExecutionContext, logger);
//case MR:
// return new MapReduceTask(taskExecutionContext, logger);
//case SPARK:
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jz.dmp.cmdexectool.scheduler.server.worker.task.sql;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.Map;
import java.util.Set;
import org.slf4j.Logger;
import com.jz.dmp.cmdexectool.scheduler.common.Constants;
import com.jz.dmp.cmdexectool.scheduler.common.enums.CommandType;
import com.jz.dmp.cmdexectool.scheduler.common.process.Property;
import com.jz.dmp.cmdexectool.scheduler.common.task.AbstractParameters;
import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlParameters;
import com.jz.dmp.cmdexectool.scheduler.common.utils.DateUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.JSONUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.OSUtils;
import com.jz.dmp.cmdexectool.scheduler.common.utils.ParameterUtils;
import com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext;
import com.jz.dmp.cmdexectool.scheduler.server.utils.ParamUtils;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.AbstractTask;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.CommandExecuteResult;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.ShellCommandExecutor;
/**
* sql task
*/
public class SqlTask extends AbstractTask {
/**
* sql parameters
*/
private SqlParameters sqlParameters;
/**
* shell command executor
*/
private ShellCommandExecutor shellCommandExecutor;
/**
* taskExecutionContext
*/
private TaskExecutionContext taskExecutionContext;
/**
* constructor
*
* @param taskExecutionContext taskExecutionContext
* @param logger logger
*/
public SqlTask(TaskExecutionContext taskExecutionContext, Logger logger) {
super(taskExecutionContext, logger);
this.taskExecutionContext = taskExecutionContext;
this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle, taskExecutionContext, logger);
}
@Override
public void init() {
logger.info("sql task params {}", taskExecutionContext.getTaskParams());
sqlParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SqlParameters.class);
if (!sqlParameters.checkParameters()) {
throw new RuntimeException("shell task params is not valid");
}
}
@Override
public void handle() throws Exception {
try {
// construct process
CommandExecuteResult commandExecuteResult = shellCommandExecutor.run(buildCommand());
setExitStatusCode(commandExecuteResult.getExitStatusCode());
setAppIds(commandExecuteResult.getAppIds());
setProcessId(commandExecuteResult.getProcessId());
} catch (Exception e) {
logger.error("sql task error", e);
setExitStatusCode(Constants.EXIT_CODE_FAILURE);
throw e;
}
}
@Override
public void cancelApplication(boolean cancelApplication) throws Exception {
// cancel process
shellCommandExecutor.cancelApplication();
}
/**
* create command
*
* @return file name
* @throws Exception exception
*/
private String buildCommand() throws Exception {
// generate scripts
String fileName = String.format("%s/%s_node.%s", taskExecutionContext.getExecutePath(),
taskExecutionContext.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh");
Path path = new File(fileName).toPath();
if (Files.exists(path)) {
return fileName;
}
String script = sqlParameters.getScript().replaceAll("\\r\\n", "\n");
/**
* combining local and global parameters
*/
Map<String, Property> paramsMap = ParamUtils.convert(
ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()),
taskExecutionContext.getDefinedParams(), sqlParameters.getLocalParametersMap(),
CommandType.of(taskExecutionContext.getCmdTypeIfComplement()), taskExecutionContext.getScheduleTime());
if (paramsMap != null) {
script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
}
// new
// replace variable TIME with $[YYYYmmddd...] in shell file when history run job
// and batch complement job
if (paramsMap != null) {
if (taskExecutionContext.getScheduleTime() != null) {
String dateTime = DateUtils.format(taskExecutionContext.getScheduleTime(),
Constants.PARAMETER_FORMAT_TIME);
Property p = new Property();
p.setValue(dateTime);
p.setProp(Constants.PARAMETER_SHECDULE_TIME);
paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
}
script = ParameterUtils.convertParameterPlaceholders2(script, ParamUtils.convert(paramsMap));
}
sqlParameters.setScript(script);
logger.info("raw script : {}", sqlParameters.getScript());
logger.info("task execute path : {}", taskExecutionContext.getExecutePath());
Set<PosixFilePermission> perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X);
FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
if (OSUtils.isWindows()) {
// Files.createFile(path);
File file = path.toFile();
File parentFile = file.getParentFile();
if (!parentFile.exists()) {
parentFile.mkdirs();
}
file.createNewFile();
} else {
Files.createFile(path, attr);
}
Files.write(path, sqlParameters.getScript().getBytes(), StandardOpenOption.APPEND);
return fileName;
}
@Override
public AbstractParameters getParameters() {
return sqlParameters;
}
}
......@@ -23,18 +23,28 @@ import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.aop.ThrowsAdvice;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.jz.dmp.cmdexectool.common.constant.CommConstant;
import com.jz.dmp.cmdexectool.controller.bean.DmpDevelopTaskDto;
import com.jz.dmp.cmdexectool.controller.bean.DmpDevelopTaskHistoryDto;
import com.jz.dmp.cmdexectool.controller.bean.DmpProjectConfigInfoDto;
import com.jz.dmp.cmdexectool.controller.bean.MyDmpDevelopTaskConverter;
import com.jz.dmp.cmdexectool.controller.bean.MyDmpDevelopTaskHistoryConverter;
import com.jz.dmp.cmdexectool.mapper.DmpDevelopTaskHistoryMapper;
import com.jz.dmp.cmdexectool.mapper.DmpDevelopTaskMapper;
import com.jz.dmp.cmdexectool.mapper.DmpProjectConfigInfoMapper;
import com.jz.dmp.cmdexectool.mapper.DmpSyncingDatasourceDao;
import com.jz.dmp.cmdexectool.scheduler.common.Constants;
import com.jz.dmp.cmdexectool.scheduler.common.enums.ExecutionStatus;
import com.jz.dmp.cmdexectool.scheduler.common.enums.JobType;
import com.jz.dmp.cmdexectool.scheduler.common.task.shell.ShellParameters;
import com.jz.dmp.cmdexectool.scheduler.common.task.sql.SqlParameters;
import com.jz.dmp.cmdexectool.scheduler.dao.entity.TaskInstance;
import com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.AbstractTask;
......@@ -52,6 +62,10 @@ public class ProcessService {
private DmpDevelopTaskMapper dmpDevelopTaskMapper;
@Autowired
private DmpDevelopTaskHistoryMapper dmpDevelopTaskHistoryMapper;
@Autowired
private DmpProjectConfigInfoMapper dmpProjectConfigInfoMapper;
@Autowired
private DmpSyncingDatasourceDao dmpSyncingDatasourceDao;
/**
* @Title: taskStart @Description: TODO(启动task) @param 参数 @return void
......@@ -136,9 +150,103 @@ public class ProcessService {
dmpDevelopTaskDto = MyDmpDevelopTaskHistoryConverter.INSTANCE().historyDto2taskDto(dmpDevelopTaskHistoryDtos.get(0));
}
taskExecutionContext = MyDmpDevelopTaskConverter.INSTANCE().dto2execcontext(dmpDevelopTaskDto, jobId);
taskExecutionContext = dto2execcontext(dmpDevelopTaskDto, jobId);
return taskExecutionContext;
}
/**
* @Title: dto2execcontext
* @Description: TODO(将task的job转换成taskExecutionContext)
* @param @param dmpDevelopTaskDto
* @param @param jobId
* @param @return 参数
* @return TaskExecutionContext 返回类型
* @throws
*/
private TaskExecutionContext dto2execcontext(DmpDevelopTaskDto dmpDevelopTaskDto, String jobId)throws Exception {
TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
Map<String, Object> param =new HashMap<String, Object>();
param.put("projectId", dmpDevelopTaskDto.getProjectId());
List<DmpProjectConfigInfoDto> dtos = dmpProjectConfigInfoMapper.findList(param);
if (CollectionUtils.isEmpty(dtos)) {
throw new RuntimeException("项目没有配置信息,请联系管理员");
}
DmpProjectConfigInfoDto projectConfigInfoDto = dtos.get(0);
JobType jobType = null;
String script = "";
String taskAppId = "";
String taskType = dmpDevelopTaskDto.getTaskType();//任务类型
if (taskType.equals(CommConstant.TASK_TYPE_DEVSHELL)) {
jobType = JobType.SHELL;
script = dmpDevelopTaskDto.getScript();
taskAppId = dmpDevelopTaskDto.getName();
}else if (taskType.equals(CommConstant.TASK_TYPE_DEVSQL)) {
jobType = JobType.SQL;
script = dmpDevelopTaskDto.getScript();
taskAppId = dmpDevelopTaskDto.getName();
}else if (taskType.equals(CommConstant.TASK_TYPE_OFFLINE)) {
jobType = JobType.SYNC;
script = dmpDevelopTaskDto.getScript();
taskAppId = dmpDevelopTaskDto.getName();
}else if (taskType.equals(CommConstant.TASK_TYPE_DEVELOP)) {
String scriptStr = dmpDevelopTaskDto.getScript();
JSONObject scriptObj = JSONObject.parseObject(scriptStr);
JSONArray array = scriptObj.getJSONArray("nodes");
for (int i = 0; i < array.size(); i++) {
JSONObject jObject = array.getJSONObject(i);
String lable = jObject.getString("label");
if (jobId.equals(lable)) {
jobType = JobType.valueOf(jObject.getString("taskType"));//job类型
script = jObject.toJSONString();
taskAppId = lable;
break;
}
}
}
switch (jobType) {
case SHELL:
ShellParameters shellParameters = new ShellParameters();
shellParameters.setScript(script);
shellParameters.setTaskAppId(taskAppId);
taskExecutionContext = shellParameters.getTaskExecutionContext();
break;
case SQL:
SqlParameters sqlParameters = new SqlParameters(script, projectConfigInfoDto, dmpSyncingDatasourceDao);
sqlParameters.setTaskAppId(taskAppId);
taskExecutionContext = sqlParameters.getTaskExecutionContext();
break;
case SYNC:
break;
case SUBPROCESS:
break;
case FTP:
break;
case UNZIPFILE:
break;
case DOCTRANS:
break;
case HDFS:
break;
default:
break;
}
return taskExecutionContext;
}
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.jz.dmp.cmdexectool.mapper.DmpSyncingDatasourceDao">
<resultMap type="com.jz.dmp.cmdexectool.entity.DmpSyncingDatasource" id="DmpSyncingDatasourceMap">
<result property="id" column="ID" jdbcType="INTEGER"/>
<result property="datasourceType" column="DATASOURCE_TYPE" jdbcType="INTEGER"/>
<result property="datasourceName" column="DATASOURCE_NAME" jdbcType="VARCHAR"/>
<result property="datasourceDesc" column="DATASOURCE_DESC" jdbcType="VARCHAR"/>
<result property="jdbcUrl" column="JDBC_URL" jdbcType="VARCHAR"/>
<result property="dbName" column="DB_NAME" jdbcType="VARCHAR"/>
<result property="userName" column="USER_NAME" jdbcType="VARCHAR"/>
<result property="password" column="PASSWORD" jdbcType="VARCHAR"/>
<result property="endpoint" column="ENDPOINT" jdbcType="VARCHAR"/>
<result property="bucket" column="BUCKET" jdbcType="VARCHAR"/>
<result property="accessId" column="ACCESS_ID" jdbcType="VARCHAR"/>
<result property="accessKey" column="ACCESS_KEY" jdbcType="VARCHAR"/>
<result property="protocol" column="PROTOCOL" jdbcType="VARCHAR"/>
<result property="host" column="HOST" jdbcType="VARCHAR"/>
<result property="port" column="PORT" jdbcType="VARCHAR"/>
<result property="defaultFs" column="DEFAULT_FS" jdbcType="VARCHAR"/>
<result property="tableSchema" column="TABLE_SCHEMA" jdbcType="VARCHAR"/>
<result property="dataStatus" column="DATA_STATUS" jdbcType="VARCHAR"/>
<result property="createUserId" column="CREATE_USER_ID" jdbcType="VARCHAR"/>
<result property="createTime" column="CREATE_TIME" jdbcType="TIMESTAMP"/>
<result property="updateUserId" column="UPDATE_USER_ID" jdbcType="VARCHAR"/>
<result property="updateTime" column="UPDATE_TIME" jdbcType="TIMESTAMP"/>
<result property="projectId" column="PROJECT_ID" jdbcType="INTEGER"/>
</resultMap>
<!--查询单个-->
<select id="queryById" resultMap="DmpSyncingDatasourceMap">
select
ID, DATASOURCE_TYPE, DATASOURCE_NAME, DATASOURCE_DESC, JDBC_URL, DB_NAME, USER_NAME, PASSWORD, ENDPOINT, BUCKET, ACCESS_ID, ACCESS_KEY, PROTOCOL, HOST, PORT, DEFAULT_FS, TABLE_SCHEMA, DATA_STATUS, CREATE_USER_ID, CREATE_TIME, UPDATE_USER_ID, UPDATE_TIME, PROJECT_ID
from dmp_syncing_datasource
where ID = #{id}
</select>
</mapper>
\ No newline at end of file
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jz.cmdexectool.test.task.shell;
import com.jz.dmp.cmdexectool.ApiApplication;
import com.jz.dmp.cmdexectool.scheduler.common.Constants;
import com.jz.dmp.cmdexectool.scheduler.common.enums.ExecutionStatus;
import com.jz.dmp.cmdexectool.scheduler.server.entity.TaskExecutionContext;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.AbstractTask;
import com.jz.dmp.cmdexectool.scheduler.server.worker.task.TaskManager;
import com.jz.dmp.cmdexectool.scheduler.service.process.ProcessService;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
/**
* python shell command executor test
*/
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ApiApplication.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class SQLCommandExecutorTest {
private static final Logger logger = LoggerFactory.getLogger(SQLCommandExecutorTest.class);
@Autowired
private ProcessService processService;
@Test
public void test() {
try {
TaskExecutionContext taskExecutionContext = processService.findTaskExecutionContextById(460,null,true);
AbstractTask task = TaskManager.newTask(taskExecutionContext);
logger.info("task info : {}", task);
// job init
task.init();
// job handle
task.handle();
ExecutionStatus status = ExecutionStatus.SUCCESS;
if (task.getExitStatusCode() == Constants.EXIT_CODE_SUCCESS){
status = ExecutionStatus.SUCCESS;
}else if (task.getExitStatusCode() == Constants.EXIT_CODE_KILL){
status = ExecutionStatus.KILL;
}else {
status = ExecutionStatus.FAILURE;
}
logger.info(status.toString());
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
logger.info(e.getMessage());
}
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment