From b8e95b7a1655cfff343e57a8f5203af00ec0c11e Mon Sep 17 00:00:00 2001 From: Andre Marquand Date: Mon, 7 Nov 2022 14:43:51 +0100 Subject: [PATCH 1/2] fixed transfer bug for blr --- pcntoolkit/normative.py | 14 +++-- pcntoolkit/normative_model/norm_blr.py | 79 +++++++++++++++----------- 2 files changed, 56 insertions(+), 37 deletions(-) diff --git a/pcntoolkit/normative.py b/pcntoolkit/normative.py index 52e2d24b..8a8fbb70 100755 --- a/pcntoolkit/normative.py +++ b/pcntoolkit/normative.py @@ -1015,11 +1015,17 @@ def transfer(covfile, respfile, testcov=None, testresp=None, maskfile=None, print("Warning: redundant batch effect parameterisation. Using HBR syntax") yhat, s2 = nm.predict(Xte, X, Y[:, i], - adaptcovfile = covfile, - adaptrespfile = respfile, - adaptvargroupfile = trbefile, - testvargroupfile = tsbefile, + adaptcov = X, + adaptresp = Y[:, i], + adaptvargroup = batch_effects_train, + testvargroup = batch_effects_test, **kwargs) + # yhat, s2 = nm.predict(Xte, X, Y[:, i], + # adaptcovfile = covfile, + # adaptrespfile = respfile, + # adaptvargroupfile = trbefile, + # testvargroupfile = tsbefile, + # **kwargs) if testcov is not None: if outscaler == 'standardize': diff --git a/pcntoolkit/normative_model/norm_blr.py b/pcntoolkit/normative_model/norm_blr.py index 7205efd5..814e0b08 100644 --- a/pcntoolkit/normative_model/norm_blr.py +++ b/pcntoolkit/normative_model/norm_blr.py @@ -170,59 +170,72 @@ def predict(self, Xs, X=None, y=None, **kwargs): theta = self.theta # always use the estimated coefficients # remove from kwargs to avoid downstream problems kwargs.pop('theta', None) - - Phis = create_poly_basis(Xs, self._model_order) if X is None: - Phi =None + Phi = None else: Phi = create_poly_basis(X, self._model_order) # process variance groups for the test data - if 'testvargroupfile' in kwargs: - var_groups_test_file = kwargs.pop('testvargroupfile') - if var_groups_test_file.endswith('.pkl'): - var_groups_te = pd.read_pickle(var_groups_test_file) - else: - var_groups_te = np.loadtxt(var_groups_test_file) + if 'testvargroup' in kwargs: + var_groups_te = kwargs.pop('testvargroup') else: - var_groups_te = None + if 'testvargroupfile' in kwargs: + var_groups_test_file = kwargs.pop('testvargroupfile') + if var_groups_test_file.endswith('.pkl'): + var_groups_te = pd.read_pickle(var_groups_test_file) + else: + var_groups_te = np.loadtxt(var_groups_test_file) + else: + var_groups_te = None # process test variance covariates - if 'testvarcovfile' in kwargs: - var_cov_test_file = kwargs.get('testvarcovfile') - if var_cov_test_file.endswith('.pkl'): - var_cov_te = pd.read_pickle(var_cov_test_file) - else: - var_cov_te = np.loadtxt(var_cov_test_file) + if 'testvarcov' in kwargs: + var_cov_te = kwargs.pop('testvarcov') else: - var_cov_te = None + if 'testvarcovfile' in kwargs: + var_cov_test_file = kwargs.get('testvarcovfile') + if var_cov_test_file.endswith('.pkl'): + var_cov_te = pd.read_pickle(var_cov_test_file) + else: + var_cov_te = np.loadtxt(var_cov_test_file) + else: + var_cov_te = None # do we want to adjust the responses? - if 'adaptrespfile' in kwargs: - y_adapt = fileio.load(kwargs.pop('adaptrespfile')) - if len(y_adapt.shape) == 1: - y_adapt = y_adapt[:, np.newaxis] + if 'adaptresp' in kwargs: + y_adapt = kwargs.pop('adaptresp') else: - y_adapt = None + if 'adaptrespfile' in kwargs: + y_adapt = fileio.load(kwargs.pop('adaptrespfile')) + if len(y_adapt.shape) == 1: + y_adapt = y_adapt[:, np.newaxis] + else: + y_adapt = None - if 'adaptcovfile' in kwargs: - X_adapt = fileio.load(kwargs.pop('adaptcovfile')) + if 'adaptcov' in kwargs: + X_adapt = kwargs.pop('adaptcov') Phi_adapt = create_poly_basis(X_adapt, self._model_order) else: - Phi_adapt = None - - if 'adaptvargroupfile' in kwargs: - var_groups_adapt_file = kwargs.pop('adaptvargroupfile') - if var_groups_adapt_file.endswith('.pkl'): - var_groups_ad = pd.read_pickle(var_groups_adapt_file) + if 'adaptcovfile' in kwargs: + X_adapt = fileio.load(kwargs.pop('adaptcovfile')) + Phi_adapt = create_poly_basis(X_adapt, self._model_order) else: - var_groups_ad = np.loadtxt(var_groups_adapt_file) + Phi_adapt = None + + if 'adaptvargroup' in kwargs: + var_groups_ad = kwargs.pop('adaptvargroup') else: - var_groups_ad = None - + if 'adaptvargroupfile' in kwargs: + var_groups_adapt_file = kwargs.pop('adaptvargroupfile') + if var_groups_adapt_file.endswith('.pkl'): + var_groups_ad = pd.read_pickle(var_groups_adapt_file) + else: + var_groups_ad = np.loadtxt(var_groups_adapt_file) + else: + var_groups_ad = None if y_adapt is None: yhat, s2 = self.blr.predict(theta, Phi, y, Phis, From 93ea62a73cd5020b9f7f7d7be0664a66678d638e Mon Sep 17 00:00:00 2001 From: Andre Marquand Date: Mon, 7 Nov 2022 15:22:26 +0100 Subject: [PATCH 2/2] clean up --- pcntoolkit/normative.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pcntoolkit/normative.py b/pcntoolkit/normative.py index 8a8fbb70..bfd50553 100755 --- a/pcntoolkit/normative.py +++ b/pcntoolkit/normative.py @@ -1020,12 +1020,6 @@ def transfer(covfile, respfile, testcov=None, testresp=None, maskfile=None, adaptvargroup = batch_effects_train, testvargroup = batch_effects_test, **kwargs) - # yhat, s2 = nm.predict(Xte, X, Y[:, i], - # adaptcovfile = covfile, - # adaptrespfile = respfile, - # adaptvargroupfile = trbefile, - # testvargroupfile = tsbefile, - # **kwargs) if testcov is not None: if outscaler == 'standardize': @@ -1066,7 +1060,6 @@ def transfer(covfile, respfile, testcov=None, testresp=None, maskfile=None, Z = (Yte - Yhat) / np.sqrt(S2) print("Evaluating the model ...") - #results = evaluate(Yte, Yhat, S2=S2, mY=mY, sY=sY) if meta_data and not warp: results = evaluate(Yte, Yhat, S2=S2, mY=mY, sY=sY) else: