Bug correction in FC&L4tranSMART for Postgres

FCL4tranSMART/Postgres/README

@@ -0,0 +1,35 @@
+To use the ETL, the following modifications have to be applied to database:
+
+ALTER TABLE tm_cz.cz_job_error ALTER COLUMN error_number TYPE varchar(30);
+alter table tm_wz.wt_trial_nodes add column "data_type" VARCHAR(20);
+create sequence i2b2demodata.seq_patient_num start with 1000000200;
+create sequence i2b2demodata.concept_id start with 200;
+update i2b2metadata.table_access set c_hlevel=(select c_hlevel from i2b2metadata.i2b2  where c_fullname='\Public Studies\') where c_fullname='\Public Studies\';
+alter table deapp.de_gpl_info add column release_nbr numeric;
+alter table deapp.de_subject_sample_mapping add column partition_id numeric;
+alter table deapp.de_subject_microarray_data add column partition_id numeric;
+create sequence deapp.seq_mrna_partition_id;
+ALTER TABLE tm_wz.wt_subject_microarray_logs ALTER COLUMN log_intensity TYPE numeric; 
+ALTER TABLE tm_wz.wt_subject_mrna_probeset ALTER COLUMN intensity_value TYPE numeric; 
+alter table tm_wz.wt_subject_microarray_logs add column raw_intensity numeric;
+alter table i2b2demodata.observation_fact alter column instance_num drop not null;
+CREATE OR REPLACE FUNCTION tm_cz._final_median(anyarray)
+   RETURNS numeric AS
+$$
+   SELECT AVG(val)
+   FROM (
+     SELECT val
+     FROM unnest($1) val
+     ORDER BY 1
+     LIMIT 2 - MOD(array_upper($1, 1), 2)
+     OFFSET CEIL(array_upper($1, 1) / 2.0) - 1
+   ) sub;
+$$
+LANGUAGE 'sql' IMMUTABLE;
+
+CREATE AGGREGATE tm_cz.median(anyelement) (
+  SFUNC=array_append,
+  STYPE=anyarray,
+  FINALFUNC=tm_cz._final_median,
+  INITCOND='{}'
+);

FCL4tranSMART/Postgres/code/fr.sanofi.fcl4transmart/src/fr/sanofi/fcl4transmart/controllers/RetrieveData.java

@@ -791,11 +791,19 @@ public static int getGeneProbeNumber(String study){
 			Statement stmt = con.createStatement();
 			//ResultSet rs=stmt.executeQuery("select count(distinct probeset_id) from de_subject_microarray_data where trial_name='"+study.toUpperCase()+"'");
 
-			ResultSet rs=stmt.executeQuery("select count(distinct probeset_id) from deapp.de_subject_microarray_data where assay_id in("+
-					"select assay_id from deapp.de_subject_sample_mapping where trial_name='"+study.toUpperCase()+"');");
+			int partition=-1;
+			ResultSet rs=stmt.executeQuery("select distinct partition_id from deapp.de_subject_sample_mapping where trial_name='"+study.toUpperCase()+"'");
 
 			if(rs.next()){
-				n=rs.getInt(1);
+				partition=rs.getInt(1);
+			}
+			if(partition!=-1){
+				rs=stmt.executeQuery("select count(distinct probeset_id) from deapp.de_subject_microarray_data_"+partition+" where assay_id in("+
+						"select assay_id from deapp.de_subject_sample_mapping where trial_name='"+study.toUpperCase()+"');");
+
+				if(rs.next()){
+					n=rs.getInt(1);
+				}
 			}
 			con.close();
 		}
@@ -807,7 +815,7 @@ public static int getGeneProbeNumber(String study){
 		return n;
 	}
 	/**
-	 *Returns the probe count for a gene expression study
+	 *Returns the tags associated with a given study
 	 */	
 	public static Vector<Vector<String>> getTags(String topNode){
 		Vector<Vector<String>> tags=new Vector<Vector<String>>();

FCL4tranSMART/Postgres/code/fr.sanofi.fcl4transmart/src/fr/sanofi/fcl4transmart/controllers/listeners/clinicalData/ClinicalMonitoringController.java

@@ -52,7 +52,7 @@ public boolean kettleSucceed(){
 		if(this.logFile!=null){
 			try{
 				BufferedReader br = new BufferedReader(new FileReader(this.logFile));
-				Pattern pattern=Pattern.compile(".*Finished job entry \\[run i2b2_load_clinical_data\\] \\(result=\\[true\\]\\)");
+				Pattern pattern=Pattern.compile(".*\\[run i2b2_load_clinical_data\\] \\(.*=\\[true\\]\\)");
 				String line="";
 				while ((line=br.readLine())!=null){
 					Matcher matcher=pattern.matcher(line);
@@ -124,7 +124,7 @@ public String proceduresError(){
 			if(rs.next()){
 				procedureErrors=rs.getString("ERROR_MESSAGE");
 			}
-			if(procedureErrors.compareTo("User-Defined Exception")==0){
+			if(procedureErrors.compareTo("User-Defined Exception")==0 || procedureErrors.compareTo("Application raised error")==0){
 				rs=stmt.executeQuery("select STEP_DESC from CZ_JOB_AUDIT where JOB_ID="+String.valueOf(jobId)+" and SEQ_ID in(select max(SEQ_ID)-1 from CZ_JOB_AUDIT where JOB_ID="+String.valueOf(jobId)+")");
 				if(rs.next()){
 					procedureErrors=rs.getString("STEP_DESC");

FCL4tranSMART/Postgres/code/fr.sanofi.fcl4transmart/src/fr/sanofi/fcl4transmart/controllers/listeners/clinicalData/LoadClinicalDataListener.java

@@ -278,4 +278,4 @@ public void writeLog(String text)
 			ioe.printStackTrace();
 		}
 	}
-}
+}

FCL4tranSMART/Postgres/code/fr.sanofi.fcl4transmart/src/fr/sanofi/fcl4transmart/controllers/listeners/geneExpression/GeneMonitoringController.java

@@ -51,7 +51,7 @@ public boolean kettleSucceed(){
 		if(this.logFile!=null){
 			try{
 				BufferedReader br = new BufferedReader(new FileReader(this.logFile));
-				Pattern pattern=Pattern.compile(".*Finished job entry \\[run i2b2_process_mrna_data\\].*", Pattern.DOTALL);
+				Pattern pattern=Pattern.compile(".*\\[run i2b2_process_mrna_data\\].*", Pattern.DOTALL);
 				String line="";
 				while ((line=br.readLine())!=null){
 					Matcher matcher=pattern.matcher(line);
@@ -123,7 +123,7 @@ public String proceduresError(){
 			if(rs.next()){
 				procedureErrors=rs.getString("ERROR_MESSAGE");
 			}
-			if(procedureErrors.compareTo("User-Defined Exception")==0){
+			if(procedureErrors.compareTo("User-Defined Exception")==0 || procedureErrors.compareTo("Application raised error")==0){
 				rs=stmt.executeQuery("select STEP_DESC from CZ_JOB_AUDIT where JOB_ID="+String.valueOf(jobId)+" and SEQ_ID in(select max(SEQ_ID)-1 from CZ_JOB_AUDIT where JOB_ID="+String.valueOf(jobId)+")");
 				if(rs.next()){
 					procedureErrors=rs.getString("STEP_DESC");

FCL4tranSMART/Postgres/code/fr.sanofi.fcl4transmart/src/fr/sanofi/fcl4transmart/controllers/listeners/geneExpression/GeneQCController.java

@@ -59,15 +59,24 @@ public HashMap<String, Double> getDbValues(String probeId){
 			while(rs.next()){
 				samples.add(rs.getString("sample_cd"));
 			}
-			for(String sample: samples){
-				//rs=stmt.executeQuery("select raw_intensity from de_subject_microarray_data where probeset_id in (select probeset_id from de_mrna_annotation where probe_id='"+probeId+"') and patient_id in(select patient_id from de_subject_sample_mapping where trial_name='"+this.dataType.getStudy().toString().toUpperCase()+"' and sample_cd='"+sample+"')");
-				//
-
-				rs=stmt.executeQuery("select raw_intensity from deapp.de_subject_microarray_data where probeset_id in (select probeset_id from deapp.de_mrna_annotation where probe_id='"+probeId+"') and assay_id in("+
-				"select assay_id from deapp.de_subject_sample_mapping where trial_name='"+this.dataType.getStudy().toString().toUpperCase()+"' and sample_cd='"+sample+"')");
-
-				if(rs.next()){
-					dbValues.put(sample, rs.getDouble("raw_intensity"));
+			int partition=-1;
+			rs=stmt.executeQuery("select distinct partition_id from deapp.de_subject_sample_mapping where trial_name='"+this.dataType.getStudy().toString().toUpperCase()+"'");
+
+			if(rs.next()){
+				partition=rs.getInt(1);
+			}
+			if(partition!=-1){
+
+				for(String sample: samples){
+					//rs=stmt.executeQuery("select raw_intensity from de_subject_microarray_data where probeset_id in (select probeset_id from de_mrna_annotation where probe_id='"+probeId+"') and patient_id in(select patient_id from de_subject_sample_mapping where trial_name='"+this.dataType.getStudy().toString().toUpperCase()+"' and sample_cd='"+sample+"')");
+					//
+
+					rs=stmt.executeQuery("select raw_intensity from deapp.de_subject_microarray_data_"+partition+" where probeset_id in (select probeset_id from deapp.de_mrna_annotation where probe_id='"+probeId+"') and assay_id in("+
+					"select assay_id from deapp.de_subject_sample_mapping where trial_name='"+this.dataType.getStudy().toString().toUpperCase()+"' and sample_cd='"+sample+"')");
+
+					if(rs.next()){
+						dbValues.put(sample, rs.getDouble("raw_intensity"));
+					}
 				}
 			}
 			con.close();

FCL4tranSMART/Postgres/code/fr.sanofi.fcl4transmart/src/fr/sanofi/fcl4transmart/controllers/listeners/geneExpression/LoadAnnotationListener.java

@@ -19,6 +19,10 @@
 import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.Statement;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.apache.commons.io.FileUtils;
@@ -74,6 +78,15 @@ public void handleEvent(Event event) {
 			this.loadAnnotationUI.displayMessage("Please provide an annotation title");
 			return;
 		}
+		if(this.loadAnnotationUI.getAnnotationDate()!=null && this.loadAnnotationUI.getAnnotationDate().compareTo("")!=0){
+			try{
+				DateFormat df=new SimpleDateFormat("yyyy/MM/dd");
+				Date date=(Date)df.parse(this.loadAnnotationUI.getAnnotationDate());
+			}catch(ParseException pe){
+				this.loadAnnotationUI.displayMessage("The annotation date is not valid");
+				return;
+			}
+		}
 		String[] pathSplited=this.loadAnnotationUI.getPathToFile().split(File.separator);
 		this.pathToFile="";
 		for(int i=0; i<pathSplited.length-2; i++){
@@ -245,7 +258,7 @@ public void run() {
 		thread.start();
 		this.loadAnnotationUI.waitForThread();
 		WorkPart.updateSteps();
-		WorkPart.updateFiles();
+		WorkPart.addFiles(this.dataType);
 	}
 
 	/**
@@ -259,7 +272,7 @@ public void run() {
 			BufferedWriter output = new BufferedWriter(fw);
 			output.write(text);		
 			output.close();
-			((GeneExpressionData)dataType).setLogFile(log);
+			((GeneExpressionData)dataType).setAnnotationLogFile(log);
 		}
 		catch(IOException ioe){
 			loadAnnotationUI.setMessage("File error: "+ioe.getLocalizedMessage());

FCL4tranSMART/Postgres/code/fr.sanofi.fcl4transmart/src/fr/sanofi/fcl4transmart/controllers/listeners/geneExpression/LoadGeneExpressionDataListener.java

@@ -315,7 +315,7 @@ public void run() {
 		}.start();
 		this.loadDataUI.waitForThread();
 		WorkPart.updateSteps();
-		WorkPart.updateFiles();
+		WorkPart.addFiles(this.dataType);
 	}
 	/**
 	 *Writes a string corresponding to Kettle log in a log file

FCL4tranSMART/Postgres/code/fr.sanofi.fcl4transmart/src/fr/sanofi/fcl4transmart/model/classes/dataType/GeneExpressionData.java

@@ -1,12 +1,12 @@
 /*******************************************************************************
- * Copyright (c) 2012 Sanofi-Aventis Recherche et Développement.
+ * Copyright (c) 2012 Sanofi-Aventis Recherche et D�veloppement.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the GNU Public License v3.0
  * which accompanies this distribution, and is available at
  * http://www.gnu.org/licenses/gpl.html
  * 
  * Contributors:
- *    Sanofi-Aventis Recherche et Développement - initial API and implementation
+ *    Sanofi-Aventis Recherche et D�veloppement - initial API and implementation
  ******************************************************************************/
 package fr.sanofi.fcl4transmart.model.classes.dataType;
 
@@ -145,4 +145,10 @@ public File getLogFile(){
 	public void setLogFile(File logFile){
 		this.logFile=logFile;
 	}
+	public File getAnnotationLogFile(){
+		return this.annotationLogFile;
+	}
+	public void setAnnotationLogFile(File logFile){
+		this.annotationLogFile=logFile;
+	}
 }

FCL4tranSMART/Postgres/code/fr.sanofi.fcl4transmart/src/fr/sanofi/fcl4transmart/model/classes/workUI/geneExpression/LoadAnnotationUI.java

@@ -1,12 +1,12 @@
 /*******************************************************************************
- * Copyright (c) 2012 Sanofi-Aventis Recherche et Développement.
+ * Copyright (c) 2012 Sanofi-Aventis Recherche et D�veloppement.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the GNU Public License v3.0
  * which accompanies this distribution, and is available at
  * http://www.gnu.org/licenses/gpl.html
  * 
  * Contributors:
- *    Sanofi-Aventis Recherche et Développement - initial API and implementation
+ *    Sanofi-Aventis Recherche et D�veloppement - initial API and implementation
  ******************************************************************************/
 package fr.sanofi.fcl4transmart.model.classes.workUI.geneExpression;
 
@@ -204,16 +204,17 @@ public void handleEvent(Event event) {
 		Label lab=new Label(load, SWT.NONE);
 
 		Label dateLabel=new Label(load, SWT.NONE);
-		dateLabel.setText("Date: ");
+		dateLabel.setText("Date (optional): ");
 		this.annotationDateField=new Text(load, SWT.BORDER);
 		this.annotationDateField.setEditable(editable);
 		gridData=new GridData();
 		gridData.widthHint=100;
 		this.annotationDateField.setLayoutData(gridData);
 		Label lab2=new Label(load, SWT.NONE);
+		lab2.setText("Format: \"yyyy/MM/dd\"");
 
 		Label releaseLabel=new Label(load, SWT.NONE);
-		releaseLabel.setText("Release: ");
+		releaseLabel.setText("Release (optional): ");
 		this.annotationReleaseField=new Text(load, SWT.BORDER);
 		this.annotationReleaseField.setEditable(editable);
 		gridData=new GridData();

FCL4tranSMART/Postgres/code/fr.sanofi.fcl4transmart/src/fr/sanofi/fcl4transmart/ui/parts/UsedFilesPart.java

@@ -1,12 +1,12 @@
 /*******************************************************************************
- * Copyright (c) 2012 Sanofi-Aventis Recherche et Développement.
+ * Copyright (c) 2012 Sanofi-Aventis Recherche et Développement.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the GNU Public License v3.0
  * which accompanies this distribution, and is available at
  * http://www.gnu.org/licenses/gpl.html
  * 
  * Contributors:
- *    Sanofi-Aventis Recherche et Développement - initial API and implementation
+ *    Sanofi-Aventis Recherche et Développement - initial API and implementation
  ******************************************************************************/
 package fr.sanofi.fcl4transmart.ui.parts;
 
@@ -135,4 +135,4 @@ void eventReceived(@Optional @UIEventTopic("newStudy/*") String string) {
 			}
 		}
 	} 
-}
+}

FCL4tranSMART/Postgres/code/fr.sanofi.fcl4transmart/src/fr/sanofi/fcl4transmart/ui/parts/WorkPart.java

@@ -1,12 +1,12 @@
 /*******************************************************************************
- * Copyright (c) 2012 Sanofi-Aventis Recherche et Développement.
+ * Copyright (c) 2012 Sanofi-Aventis Recherche et Développement.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the GNU Public License v3.0
  * which accompanies this distribution, and is available at
  * http://www.gnu.org/licenses/gpl.html
  * 
  * Contributors:
- *    Sanofi-Aventis Recherche et Développement - initial API and implementation
+ *    Sanofi-Aventis Recherche et Développement - initial API and implementation
  ******************************************************************************/
 package fr.sanofi.fcl4transmart.ui.parts;
 
@@ -112,4 +112,4 @@ public static void updateAll(){
 	public static Display display(){
 		return display;
 	}
-}
+}