Skip to content

Commit

Permalink
v
Browse files Browse the repository at this point in the history
  • Loading branch information
@MecBoc
MecBoc committed Dec 6, 2019
1 parent fea7e65 commit 389235a
Show file tree
Hide file tree
Showing 5 changed files with 80 additions and 52 deletions.
26 changes: 10 additions & 16 deletions RScripts/selemix/IS2_selemix.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,23 +91,17 @@ library("SeleMix")


#stima completa
is2_mlest <- function(workset,params, s=S,x=X,y=Y,z=Z, ...) {
is2_mlest <- function(workset,wsparams,s=S,x=X,y=Y,z=Z, ...) {
#environment check
print(params)
print("----")
print(x)
print("----")
print(y)
print("----")
print(z)
stdout <- vector('character')
con <- textConnection('stdout', 'wr', local = TRUE)


print(wsparams)
# stdout <- vector('character')
#con <- textConnection('stdout', 'wr', local = TRUE)

if(missing(y)) stop('iSS Error: Missing TARGET Variable(s)')
y <- matrix(as.numeric(workset[,Y]),ncol=length(Y),nrow=nrow(workset))
if(!missing(x)) x <- matrix(as.numeric(workset[,X]),ncol=length(X),nrow=nrow(workset))


print("------- #parameter check")
#parameter check
if(!exists("model")) model="LN" #da controllare
if(!exists("t.outl")) t.outl=0.5
Expand Down Expand Up @@ -142,14 +136,14 @@ is2_mlest <- function(workset,params, s=S,x=X,y=Y,z=Z, ...) {
roles <- list (P= predname, O="outlier", M=names(mod), G=names(report))
r_out<-data.frame(out,out1)
rolesgroup <- list (P= names(r_out), G= c("M","G"))
result <-list( out=r_out, roles= roles,rolesgroup= rolesgroup, mod=mod, report = report, log = stdout)
result <-list( out=r_out, roles= roles,rolesgroup= rolesgroup, mod=mod, report = report, log = 'stdout')

#roles <- list (FS=names(r_out))
#rolesgroup <- list (FS= c("FS"))
#result <-list( out=r_out, roles= roles,rolesgroup= rolesgroup, var_est = var_est, log = stdout)

sink()
close(con)
#sink()
#close(con)
return(result)
}

Expand Down
4 changes: 2 additions & 2 deletions db/is2-insert-selemix.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ INSERT INTO `is2_step_instance` (`ID`, `METHOD`, `DESCR`, `LABEL`, `APP_SERVICE_
-- PARAMETER
-- -----------------------------------------------------
INSERT INTO `is2_parameter` (`ID`, `NAME`, `DESCR`, `DEFAULT_VAL`, `JSON_TEMPLATE`) VALUES (101,'MODEL','DATA MODEL',NULL,'{ "data": [], "schema": {"items": {"properties": { "lambda": {"required": true,"title": "Lambda","description":"Estimated value for the variance inflation factor","type": "number", "default":3,"minimum": 0.01,"maximum": 10 },"w": {"required": true,"title": "W","description":"Estimated value for the proportion of contaminated data","type": "number","default":0.05,"minimum": 0.01,"maximum": 1 }, "B": {"required": true,"title": "B","description":"Matrix of estimated regression coefficients","type": "number"}, "sigma": {"required": true,"title": "sigma","description":"Estimated covariance matrix","type": "number"} },"type": "object"},"type": "array" }, "options": {"type": "array","showActionsColumn": false,"hideAddItemsBtn": true }}');
INSERT INTO `is2_parameter` (`ID`, `NAME`, `DESCR`, `DEFAULT_VAL`, `JSON_TEMPLATE`) VALUES (102,'INPUT PARAMETERS','INPUT PARAMETERS',NULL,'{"data": [],"schema": { "properties": {"graph": {"required": true,"title": "Graph","description": "Activates graphic output","type": "number","default": 0,"minimum": 0.01,"maximum": 10},"model": {"required": true,"title": "Model","description": "Data Distribution: LN lognormal / N Normal","default": "LN"},"tot": {"title": "Tot","description": "Estimates of originals vector for the target variables"},"t.sel": {"title": "t.sel","description": "Optional vector of threshold values for selective edinting on the target variables"},"t.outl": {"required": true,"title": "t.outl","description": "Threshold value for posterior probabilities of identifying outliers","type": "number","default": 0.05,"minimum": 0.01,"maximum": 10},"eps": {"required": true,"title": "eps","description": "Tolerance for the log-likelihood convergence","type": "number","default": 0.0000001,"minimum": 0.0000001,"maximum": 1},"lambda.fix": {"required": true,"title": "lambda.fix","description": "TRUE if w is known","type": "number","default": 0,"maximum": 1 },"w.fix": {"required": true,"title": "w.fix","description": "TRUE if w is known","type": "number","default": 0,"minimum": 0.01,"maximum": 1}},"type": "object" }}');
INSERT INTO `is2_parameter` (`ID`, `NAME`, `DESCR`, `DEFAULT_VAL`, `JSON_TEMPLATE`) VALUES (103,'OUTPUT PARAMETERS','OUTPUT PARAMETERS - INFO REPORT',NULL,NULL);
INSERT INTO `is2_parameter` (`ID`, `NAME`, `DESCR`, `DEFAULT_VAL`, `JSON_TEMPLATE`) VALUES (102,'INPUT_PARAMETERS','INPUT PARAMETERS',NULL,'{"data": [],"schema": { "properties": {"graph": {"required": true,"title": "Graph","description": "Activates graphic output","type": "number","default": 0,"minimum": 0.01,"maximum": 10},"model": {"required": true,"title": "Model","description": "Data Distribution: LN lognormal / N Normal","default": "LN"},"tot": {"title": "Tot","description": "Estimates of originals vector for the target variables"},"t.sel": {"title": "t.sel","description": "Optional vector of threshold values for selective edinting on the target variables"},"t.outl": {"required": true,"title": "t.outl","description": "Threshold value for posterior probabilities of identifying outliers","type": "number","default": 0.05,"minimum": 0.01,"maximum": 10},"eps": {"required": true,"title": "eps","description": "Tolerance for the log-likelihood convergence","type": "number","default": 0.0000001,"minimum": 0.0000001,"maximum": 1},"lambda.fix": {"required": true,"title": "lambda.fix","description": "TRUE if w is known","type": "number","default": 0,"maximum": 1 },"w.fix": {"required": true,"title": "w.fix","description": "TRUE if w is known","type": "number","default": 0,"minimum": 0.01,"maximum": 1}},"type": "object" }}');
INSERT INTO `is2_parameter` (`ID`, `NAME`, `DESCR`, `DEFAULT_VAL`, `JSON_TEMPLATE`) VALUES (103,'OUTPUT_PARAMETERS','OUTPUT PARAMETERS - INFO REPORT',NULL,NULL);

-- -----------------------------------------------------
-- ROLE
Expand Down
91 changes: 61 additions & 30 deletions src/main/java/it/istat/is2/workflow/engine/EngineR.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,22 +64,6 @@
@Service
public class EngineR implements EngineService {

public static final String IS2_RULESET = "ruleset";
public static final String IS2_RESULTSET = "sel_out";
public static final String IS2_WORKSET = "workset";
public static final String IS2_RUOLI_VAR = "role_var";
public static final String IS2_RUOLI_VAR_OUTPUT = "role_var_out";
public static final String IS2_RUOLI_INPUT = "role_in";
public static final String IS2_RUOLI_OUTPUT = "ruol_out";
public static final String IS2_PARAMETRI = "params";
public static final String IS2_MODELLO = "model";
public static final String IS2_RUOLO_SKIP_N = "N";
public static final String IS2_RESULT_RUOLI = "roles";
public static final String IS2_RESULT_ROLES_GROUPS = "rolesgroup";
public static final String IS2_RESULT_OUTPUT = "out";
public static final String IS2_RESULT_PARAM = "mod";
public static final String IS2_RESULT_REPORT = "report"; // aggiunto componente dei parametri di uscita
public static final String IS2_R_NA_VALUE = "NA"; //

@Autowired
AppRoleDao ruoloDao;
Expand Down Expand Up @@ -140,9 +124,10 @@ public void init(DataProcessing dataProcessing, StepInstance stepInstance) throw
this.fileScriptR = stepInstance.getAppService().getSource();
prepareEnv();
createConnection(serverRHost, serverRPort);
bindInputColumns(worksetVariabili, EngineR.IS2_WORKSET);
bindInputColumnsParams(parametriMap, EngineR.IS2_PARAMETRI);
bindInputColumns(ruleset, EngineR.IS2_RULESET);
bindInputColumns(worksetVariabili,WORKSET);
bindInputColumns(parametriMap, PARAMETERS);
//bindInputColumnsParams(parametriMap, PARAMETERS);
bindInputColumns(ruleset, EngineR.RULESET);
setRuoli(ruoliVariabileNome);

}
Expand Down Expand Up @@ -236,6 +221,7 @@ public void bindInputColumnsParams(LinkedHashMap<String, ArrayList<String>> work
listaCampi += key + ",";
listaCampiLabel += "'" + key + "',";
connection.assign(key, arrX);

}
listaCampi = listaCampi.substring(0, listaCampi.length() - 1);
listaCampiLabel = listaCampiLabel.substring(0, listaCampiLabel.length() - 1);
Expand All @@ -246,7 +232,7 @@ public void bindInputColumnsParams(LinkedHashMap<String, ArrayList<String>> work
String namecols = ((size > 1) ? "col" : "") + "names(" + varR + ") = c(" + listaCampiLabel + ")";
// String exec = "colnames(" + varR + ") = c(" + listaCampi + ")";
Logger.getRootLogger().debug("Bind input columns names " + namecols);
connection.eval(namecols);
//connection.eval(namecols);

}
}
Expand All @@ -256,7 +242,7 @@ public void bindInputColumnsParams(LinkedHashMap<String, ArrayList<String>> work
private boolean isNumeric(String[] arrX) {
// TODO Auto-generated method stub
for (String elem : arrX) {
if (elem.isEmpty() || IS2_R_NA_VALUE.equalsIgnoreCase(elem))
if (elem.isEmpty() || R_NA_VALUE.equalsIgnoreCase(elem))
continue;
try {
Double.parseDouble(elem);
Expand Down Expand Up @@ -311,9 +297,9 @@ public void doAction() throws RserveException {
String fname = stepInstance.getMethod();
// mlest <- ml.est (workset, y=Y,";
// Aggiunto il workset e params nella lista degli argomenti della funzione (by paolinux)
istruzione = IS2_RESULTSET + " <- " + fname + "( " + IS2_WORKSET + ",";
if (!parametriMap.isEmpty()) istruzione += IS2_PARAMETRI + ", ";
if (!ruleset.isEmpty()) istruzione += IS2_RULESET + ", ";
istruzione = RESULTSET + " <- " + fname + "( " + WORKSET + ",";
if (!parametriMap.isEmpty()) istruzione += PARAMETERS + ",";
if (!ruleset.isEmpty()) istruzione += RULESET + ", ";

for (Map.Entry<String, ArrayList<String>> entry : ruoliVariabileNome.entrySet()) {
String codiceRuolo = entry.getKey();
Expand Down Expand Up @@ -535,11 +521,11 @@ public void prepareEnv() {
public void processOutput() throws Exception {
// TODO Auto-generated method stub

getGenericoOutput(worksetOut, EngineR.IS2_RESULTSET, EngineR.IS2_RESULT_OUTPUT);
getGenericoOutput(ruoliOutputStep, EngineR.IS2_RESULTSET, EngineR.IS2_RESULT_RUOLI);
getGenericoOutput(parametriOutput, EngineR.IS2_RESULTSET, EngineR.IS2_RESULT_PARAM);
getGenericoOutput(parametriOutput, EngineR.IS2_RESULTSET, EngineR.IS2_RESULT_REPORT);
getRolesGroup(ruoliGruppoOutputStep, EngineR.IS2_RESULTSET, EngineR.IS2_RESULT_ROLES_GROUPS);
getGenericoOutput(worksetOut, RESULTSET, RESULT_OUTPUT);
getGenericoOutput(ruoliOutputStep, RESULTSET, RESULT_RUOLI);
getGenericoOutput(parametriOutput, RESULTSET, RESULT_PARAM);
getGenericoOutput(parametriOutput, RESULTSET, RESULT_REPORT);
getRolesGroup(ruoliGruppoOutputStep, RESULTSET,RESULT_ROLES_GROUPS);
writeLogScriptR();
saveOutputDB();

Expand All @@ -566,7 +552,7 @@ private void saveOutputDB() {
String ruolo = ruoliOutputStepInversa.get(nomeW);
String ruoloGruppo = ruoliGruppoOutputStep.get(ruolo);
if (ruolo == null) {
ruolo = EngineR.IS2_RUOLO_SKIP_N;
ruolo = RUOLO_SKIP_N;
}
if (ruoloGruppo == null) {
ruoloGruppo = RUOLO_SKIP_N;
Expand Down Expand Up @@ -602,6 +588,51 @@ private void saveOutputDB() {
stepRuntimeDao.save(stepRuntime);
}


// save output Parameter DB
for (Map.Entry<String, ArrayList<String>> entry : parametriOutput.entrySet()) {
String nomeW = entry.getKey();
ArrayList<String> value = entry.getValue();
StepRuntime stepRuntime;
String ruolo = ruoliOutputStepInversa.get(nomeW);
String ruoloGruppo = ruoliGruppoOutputStep.get(ruolo);
if (ruolo == null) {
ruolo = RUOLO_SKIP_N;
}
if (ruoloGruppo == null) {
ruoloGruppo = RUOLO_SKIP_N;
}
AppRole sxRuolo = ruoliAllMap.get(ruolo);
AppRole sxRuoloGruppo = ruoliAllMap.get(ruoloGruppo);

stepRuntime = Utility.retrieveStepRuntime(dataMap, nomeW, sxRuolo);

if (stepRuntime != null) { // update

stepRuntime.getWorkset().setContents(value);
stepRuntime.setTypeIO(new TypeIO(IS2Const.TYPE_IO_OUTPUT));
} else {
stepRuntime = new StepRuntime();
stepRuntime.setDataProcessing(dataProcessing);
stepRuntime.setTypeIO(new TypeIO(IS2Const.TYPE_IO_OUTPUT));
stepRuntime.setDataType(new DataTypeCls(IS2Const.DATA_TYPE_PARAMETER));
stepRuntime.setAppRole(sxRuolo);
stepRuntime.setRoleGroup(sxRuoloGruppo);
stepRuntime.setOrderCode(sxRuolo.getOrder());
Workset workset = new Workset();
workset.setName(nomeW.replaceAll("\\.", "_"));
workset.setDataType(new DataTypeCls(IS2Const.DATA_TYPE_VARIABLE));
ArrayList<StepRuntime> l = new ArrayList<>();
l.add(stepRuntime);
workset.setStepRuntimes(l);
workset.setContents(value);
workset.setContentSize(workset.getContents().size());
stepRuntime.setWorkset(workset);
}

stepRuntimeDao.save(stepRuntime);
}


}

Expand Down
9 changes: 6 additions & 3 deletions src/main/java/it/istat/is2/workflow/engine/EngineService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,20 @@ public interface EngineService {
public static final String RUOLI_VAR_OUTPUT = "role_var_out";
public static final String RUOLI_INPUT = "role_in";
public static final String RUOLI_OUTPUT = "ruol_out";
public static final String PARAMETRI = "params";
public static final String PARAMETERS = "wsparams";
public static final String MODELLO = "model";
public static final String RUOLO_SKIP_N = "N";
public static final String RESULT_RUOLI = "roles";
public static final String RESULT_OUTPUT = "out";
public static final String RESULT_PARAM = "mod";
public static final String RESULT_REPORT = "report"; // aggiunto componente dei parametri di uscita

public static final String RULESET = "ruleset";
public static final String R_PARAMETRI = "wsparams";
public static final String RESULT_ROLES_GROUPS = "rolesgroup";
public static final String R_NA_VALUE = "NA"; //

public void init(DataProcessing elaborazione, StepInstance stepInstance) throws Exception;
public void init() throws Exception;
public void init() throws Exception;
public void doAction()throws Exception;
public void processOutput() throws Exception;
public void destroy();
Expand Down
2 changes: 1 addition & 1 deletion src/main/resources/public/js/workflow/batch.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ function pollJobs() {
$('#' + this.idProcesso).addClass("success");
console.log(this.idProcesso + '- ' + old_status + ' '
+ this.status)
if (old_status.startsWith("START")
if (old_status.startsWith("START1")
&& old_status != this.status)
window.location.reload();
} else {
Expand Down

0 comments on commit 389235a

Please sign in to comment.