diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java index 8b94a3a26..37e376ffe 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java @@ -1341,6 +1341,7 @@ public Message queryDataSourceDBTableFieldsMapping(HttpServletRequest request, F field.setFieldEditable(!"HIVE".equals(vo.getSinkTypeId()) && !"ELASTICSEARCH".equals(vo.getSinkTypeId())); } message.data("sinkFields", sinkFields); + // field mapping deduction List> deductions = new ArrayList<>(); List left = sourceFields; @@ -1351,16 +1352,20 @@ public Message queryDataSourceDBTableFieldsMapping(HttpServletRequest request, F right = sourceFields; exchanged = true; } - for (int i = 0; i < right.size(); i ++){ - LOGGER.info("rightElement is:" + right.get(i).getName()); - DataSourceDbTableColumnDTO leftElement = left.get(i % left.size()); - DataSourceDbTableColumnDTO rightElement = right.get(i); - Map deduction = new HashMap<>(); - deduction.put("source", exchanged ? rightElement : leftElement); - deduction.put("sink", exchanged ? leftElement : rightElement); - deduction.put("deleteEnable", true); - deductions.add(deduction); + + // source size and sink size must not be null + if (!Objects.isNull(left) && left.size() > 0) { + for (int i = 0; i < right.size(); i ++){ + DataSourceDbTableColumnDTO leftElement = left.get(i % left.size()); + DataSourceDbTableColumnDTO rightElement = right.get(i); + Map deduction = new HashMap<>(); + deduction.put("source", exchanged ? rightElement : leftElement); + deduction.put("sink", exchanged ? leftElement : rightElement); + deduction.put("deleteEnable", true); + deductions.add(deduction); + } } + message.data("deductions", deductions); message.data("transformEnable", true); diff --git a/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml b/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml index 1f389e7f8..37d6498ae 100644 --- a/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml @@ -28,7 +28,6 @@ provided org.slf4j slf4j-api - ch.qos.logback @@ -61,7 +60,6 @@ commons-net 3.3 - diff --git a/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml b/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml index 1d5d9f058..39f66786b 100644 --- a/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml +++ b/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml @@ -91,6 +91,10 @@ zookeeper org.apache.zookeeper + + org.pentaho + pentaho-aggdesigner-algorithm + diff --git a/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml b/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml index c53736de7..24efb6cf1 100644 --- a/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml @@ -100,6 +100,10 @@ geronimo-jaspic_1.0_spec org.apache.geronimo.specs + + org.pentaho + pentaho-aggdesigner-algorithm + @@ -111,6 +115,12 @@ org.apache.hive.hcatalog hive-hcatalog-core ${hive.version} + + + org.pentaho + pentaho-aggdesigner-algorithm + + xalan diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java index d29c3724b..da50f76d1 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java @@ -307,7 +307,7 @@ public Message deleteJob(@PathVariable("id") Long id, HttpServletRequest request public Message getJob(HttpServletRequest request, @PathVariable("id") Long id) { Message response = Message.ok(); try { - LOG.info("Request88888: {}", request); + LOG.info("Request: {}", request); String userName = SecurityFilter.getLoginUsername(request); if (!hasAuthority(userName, jobInfoService.getJob(id, true))) { diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java index 261d7c2a7..254d5d6d0 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java @@ -90,7 +90,7 @@ public Message createJob( Long id = null; id = jobInfoService.createJob(exchangisJobVo).getId(); response.data("id", id); - LOG.info("id6666: {}", id); + LOG.info("job id is: {}", id); } catch (Exception e){ String message = "Fail to create dss job: " + exchangisJobVo.getJobName() +" (创建DSS任务失败)"; LOG.error(message, e); @@ -177,9 +177,9 @@ else if (!hasAuthority(userName, jobInfoService.getJob(id , true))) { @RequestMapping( value = "/execute/{id}", method = RequestMethod.POST) public Message executeJob(@PathVariable("id") Long id, HttpServletRequest request, @RequestBody Map params) { try { - LOG.info("start to parse params8909"); + LOG.info("start to parse params"); String paramString = BDPJettyServerHelper.jacksonJson().writeValueAsString(params); - LOG.error("paramString999879: {}", paramString); + LOG.error("paramString: {}", paramString); } catch (JsonProcessingException e) { LOG.error("parse execute content error: {}", e.getMessage()); } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java index c9b217545..ac71bf043 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java @@ -185,7 +185,7 @@ public ExchangisLaunchedTaskMetricsVo getLaunchedTaskMetrics(String taskId, Stri @Override public boolean hasExecuteJobAuthority(String jobExecutionId, String userName) { String jobUser = this.launchedJobDao.searchLaunchedJob(jobExecutionId).getCreateUser(); - LOG.info("Job user is99999:{}, reuquest user is: {}", jobUser, userName); + LOG.info("Job user is:{}, reuquest user is: {}", jobUser, userName); return hasExecuteJobAuthority(this.launchedJobDao.searchLaunchedJob(jobExecutionId) , userName); } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java index b3983e9e3..1b6fdc568 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java @@ -352,10 +352,10 @@ private void setSqoop(Long projectId, Map> moduleIdsMap, boole LOG.error("Occur error while tranform class", e.getMessage()); } - LOG.info("sqoopStr99999:{}", sqoopStr); + LOG.info("sqoopStr:{}", sqoopStr); LOG.info("ExchangisJobVo sqoop: {}", job.getContent()); LOG.info("getCreateTime: {}", job.getId()); - LOG.info("executorUser999: {}", job.getExecuteUser()); + LOG.info("executorUser: {}", job.getExecuteUser()); sqoops.add(job); } exportedProject.setSqoops(sqoops); @@ -390,7 +390,7 @@ private void setDatax(Long projectId, Map> moduleIdsMap, boole LOG.info("dataxStr:{}", dataxStr); LOG.info("ExchangisJobVo sqoop: {}", job.getContent()); LOG.info("getCreateTime: {}", job.getId()); - LOG.info("executorUser999: {}", job.getExecuteUser()); + LOG.info("executorUser: {}", job.getExecuteUser()); dataxs.add(job); } exportedProject.setDataxes(dataxs); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java index 7d67cffca..37ba84083 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java @@ -179,10 +179,9 @@ public void addSqoopTask (List sqoops, List pr idCatalog.getSqoop().put(oldId, existingId); throw new ExchangisJobServerException(31101, "Already exits duplicated job name(存在重复任务名称) jobName is:" + "[" + sqoop.getJobName() + "]"); } else { - //sqoop.setJobName("hahaha"); LOG.info("Sqoop job content is: {}, Modify user is: {}, jobType is: {}", sqoop.getContent(), sqoop.getExecuteUser(), sqoop.getJobType()); ExchangisJobVo jobVo = jobInfoService.createJob(sqoop); - LOG.info("oldId: {}, newid: {}, jobName0000000: {}", sqoop.getId(), jobVo.getId(), jobVo.getJobName()); + LOG.info("oldId: {}, newid: {}, jobName: {}", sqoop.getId(), jobVo.getId(), jobVo.getJobName()); idCatalog.getSqoop().put(oldId, jobVo.getId()); } } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/ExchangisHttpUtils.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/ExchangisHttpUtils.java index 3d4c9507c..c45f40e12 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/ExchangisHttpUtils.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/ExchangisHttpUtils.java @@ -51,7 +51,7 @@ public static InternalResponseRef getResponseRef(WorkspaceRequestRef requestRef, } LOG.info("User {} try to request Exchangis with url {} and labels {}.", httpAction.getUser(), httpAction.getURL(), requestRef.getDSSLabels().get(0).getValue().get("DSSEnv")); HttpResult httpResult = ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, httpAction); - LOG.info("responseBody00000:{}", httpResult.getResponseBody()); + LOG.info("responseBody:{}", httpResult.getResponseBody()); InternalResponseRef responseRef = ResponseRef.newInternalBuilder().setResponseBody(httpResult.getResponseBody()).build(); if (responseRef.isFailed()){ throw new ExternalOperationFailedException(95011, responseRef.getErrorMsg()); diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java index 13b9515d7..0884fa49f 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java @@ -220,7 +220,7 @@ public Message deleteProject(HttpServletRequest request, @PathVariable("id") Lon return Message.error("Cannot delete the outer project (无法删除来自 " + domain + " 的外部项目)"); } projectService.deleteProject(id); - return ExchangisProjectRestfulUtils.dealOk("删除工程成功888"); + return ExchangisProjectRestfulUtils.dealOk("删除工程成功"); } catch (Exception t) { LOG.error("Failed to delete project for user {}", username, t); return Message.error("Failed to delete project (删除工程失败)"); diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java index 03664bc31..b9b39574e 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java @@ -122,7 +122,7 @@ public Message deleteProject(HttpServletRequest request, @PathVariable("name") S // return Message.error("Cannot delete the outer project (无法删除来自 " + domain + " 的外部项目)"); // } projectService.deleteProjectByName(name); - return ExchangisProjectRestfulUtils.dealOk("删除工程成功777"); + return ExchangisProjectRestfulUtils.dealOk("删除工程成功"); } catch (Exception t) { LOG.error("Failed to delete project for user {}", username, t); return Message.error("Failed to delete project (删除工程失败)");