From 86ef5aa0c2ee050520f69978b5969a8c995cbb7f Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 25 Jul 2016 17:30:50 -0400 Subject: [PATCH 01/39] Added fit file, moved amares fitting functions to singlet, converted a few values in denoising and singlet to int --- suspect/fitting/fit.py | 443 ++++++++++++++++++++++++++++++++ suspect/fitting/singlet.py | 187 ++++++++------ suspect/processing/denoising.py | 6 +- 3 files changed, 555 insertions(+), 81 deletions(-) create mode 100644 suspect/fitting/fit.py diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py new file mode 100644 index 0000000..e622757 --- /dev/null +++ b/suspect/fitting/fit.py @@ -0,0 +1,443 @@ +""" +File Name: fit.py +Author: Sam Jiang +Purpose: fit_31p() function fits single 31P fid. +Interpreter: Anaconda 3.5.1 + +fit(fid, model) +Input: fid and model in hierarchical dictionary format +Output: dictionary with 'model', 'fit', and 'error' keys and corresponding values +""" + +import os +import numpy +import json +import lmfit +import array +import suspect + + +# Load model in. +def load(fin): + if not os.path.isfile(fin): + raise FileNotFoundError("{} not found.".format(fin)) + with open(fin, 'r+') as f: + model = json.load(f) + return model + + +# Save model or fit out. +def save(output, fout): + # Check if file and directory exists. + directory, name = os.path.split(fout) + if directory == '': + directory = os.getcwd() + if not os.path.isdir(directory): + os.makedirs(directory) + if name == '': + raise IsADirectoryError("{} is not a file name.".format(name)) + ftype = os.path.splitext(name)[1] + + # If model, save as json. + if type(output) is dict: + if ftype != ".json": + raise TypeError("Output file must be a JSON (.json) file.") + + with open(fout, 'w+') as f: + json.dump(output, f) + + # If fit, save as csv or txt. + elif type(output) is array.ArrayType: + if ftype != ".csv" or ftype != ".txt": + raise TypeError("Output file must be a CSV (.csv) or Text (.txt) file.") + + with open(fout, 'w+') as f: + first = True + for x in output.tolist(): + if first: + f.write("{}".format(x)) + first = False + else: + f.write(", {}".format(x)) + + +# Fit fid +# Input: fid, model +# Output: dictionary containing optimized model, fit data, standard errors +def fit(fid, model): + # MODEL + # Get list of metabolite names + # def get_metabolites(model_input): + # metabolites = [] + # for name, value in model_input.items(): + # if type(value) is dict: + # metabolites.append(name) + # return metabolites + + # Get standard errors from lmfit MinimizerResult object. + def get_errors(result): + errors = {} + for name, param in result.params.items(): + errors[name] = param.stderr + return errors + + # Convert lmfit parameters to model format + def parameters_to_model(parameters_obj, param_weights): + metabolite_name_list = [] + for param in parameters_obj.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + # Create dictionary for new model. + new_model = {} + for param_name, param in parameters_obj.items(): + name = param_name.split("_") + name1 = name[0] + if len(name) == 1: # i.e. phase + new_model[name1] = param.value + else: + name2 = name[1] + if name1 not in new_model: + new_model[name1] = {name2: param.value} + else: + new_model[name1][name2] = param.value + + for i, metabolite_name in enumerate(metabolite_name_list): + new_model[metabolite_name]["amplitude"] = param_weights[i] + + return new_model + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + # Check if all model input types are correct. + def check_errors(check_model): + # Allowed names and keys in the model. + allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] + allowed_keys = ["min", "max", "value", "phase", "amplitude"] + + # Scan model. + for name1, value1 in check_model.items(): + if type(value1) is not int and type(value1) is not float and type(value1) is not dict: + raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) + elif name1 not in allowed_names: + raise NameError("{} is not an allowed name.".format(name1)) + elif type(value1) is dict: # i.e. type(value) is not int + for name2, value2 in value1.items(): + if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ + type(value2) is not str: + raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." + .format(name1, name2)) + if type(value2) is dict: + for key in value2: + # Dictionary must have 'value' key. + if "value" not in value2: + raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) + # Dictionary can only have 'min,' 'max,' and 'value'. + if key not in allowed_keys: + raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) + + return + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # MAIN + def main(): + # Minimize and fit 31P data. + # Check for errors in model formatting. + check_errors(model) + + # Set list of metabolite names. + # nonlocal metabolite_name_list + # metabolite_name_list = get_metabolites(model) + + # Convert model to lmfit Parameters object. + parameters = model_to_parameters(model) + + # Fit data. + fitted_weights, fitted_data, fitted_results = suspect.fitting.singlet.fit_data(fid, parameters) + + # Convert fit parameters to model format. + final_model = parameters_to_model(fitted_results.params, fitted_weights) + # Get stderr values for each parameter. + stderr = get_errors(fitted_results) + + # Compile output into a dictionary. + return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} + + return return_dict + + if __name__ == "__main__": + return main() + + +# Test fit function +def test_fit(_plot): + # Test file and model + testfile = "test_timecourse.csv" + modelfile = "model.json" + model = load(modelfile) + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + def fake_spectrum(params, time_axis, weights): + basis = suspect.fitting.singlet.make_basis(params, time_axis) + real_data = numpy.array(numpy.dot(weights, basis)).squeeze() + complex_data = suspect.fitting.singlet.real_to_complex(real_data) + return suspect.MRSData(complex_data, dt, f0, te=0, ppm0=0) + + # Generate fake noise (Testing only) + def generate_noise(n, snr): + # Added (n * 0) to fix unused variable warning. + noise = (n * 0) + (numpy.random.randn(4096) + 1j * numpy.random.randn(4096)) * 6.4e-6 / snr + return noise + + # Generate fake fid (Testing only) + def generate_fid(ground_truth, snr, initial_params): + metabolite_name_list = [] + for param in initial_params.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + + peaks = {"pcr": 1, + "atpc": 0.3, + "atpb": 0.3, + "atpa": 0.3, + "pi": 1, + "pme": 0.05, + "pde": 0.05} + amps = [] + for metabolite in metabolite_name_list: + amps.append(peaks[metabolite]) + + noisy_sequence = numpy.zeros((len(ground_truth), np), 'complex') + noisy_sequence = suspect.MRSData(noisy_sequence, dt, f0, te=0, ppm0=0) + + # Populate the noisy_sequence with data. + generated_fid = fake_spectrum(initial_params, noisy_sequence.time_axis(), amps) + # [1, 0.3, 0.3, 0.3, 1, 0.05, 0.05] + noisy_fid = generated_fid + generate_noise(np, snr) + + return generated_fid, noisy_fid + + # Generate fake FID. + parameters = model_to_parameters(model) + sw = 6e3 + dt = 1.0 / sw + np = 4096 + f0 = 49.885802 + fake_fid, noisy_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid + + # Test fit(). + fit_results = fit(fake_fid, model) + + # Plot fit, noisy_fid, and fake_fid. + if _plot: + from matplotlib import pyplot + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noisy_fid))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) + pyplot.show() + + +test = False +plot = True +if test: + print("Running test.") + test_fit(plot) + print("Test complete.") + diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 39a15e7..6e19dd1 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -1,5 +1,6 @@ import lmfit import numpy +import scipy.optimize import numbers import suspect @@ -32,7 +33,7 @@ def real_to_complex(real_fid): :param real_fid: the real FID to be converted to complex. :return: the complex version of the FID """ - np = real_fid.shape[0] / 2 + np = int(real_fid.shape[0] / 2) complex_fid = numpy.zeros(np, 'complex') complex_fid[:] = real_fid[:np] @@ -42,83 +43,113 @@ def real_to_complex(real_fid): return complex_fid -def gaussian_fid(x, amplitude=1, frequency=0.0, phase=0.0, fwhm=1.0): +# metabolite_name_list = [] + + +def phase_fid(fid_in, phase0, phase1): + """ + This function performs a Fourier Transform on the FID to shift it into phase. + + :param fid_in: FID to be fitted. + :param phase1: phase1 value. + :param phase0: phase0 value. + :return: + """ + spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) + np = fid_in.np + phase_shift = phase0 + phase1 * numpy.linspace(-np / 2, np / 2, np, endpoint=False) + phased_spectrum = spectrum * numpy.exp(1j * phase_shift) + return fid_in.inherit(numpy.fft.ifft(numpy.fft.ifftshift(phased_spectrum))) + + +def make_basis(params, time_axis): + """ + This function generates a basis set. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :return: a matrix containing the generated basis set. + """ + metabolite_name_list = [] + for param in params.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + + basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) + for i, metabolite_name in enumerate(metabolite_name_list): + gaussian = suspect.basis.gaussian(time_axis, + params["{}_frequency".format(metabolite_name)], + params["{}_phase".format(metabolite_name)].value, + params["{}_width".format(metabolite_name)]) + real_gaussian = complex_to_real(gaussian) + basis_matrix[i, :] = real_gaussian + return basis_matrix + + +def do_fit(params, time_axis, real_unphased_data): + """ + This function performs the fitting. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param real_unphased_data: + :return: List of fitted data points. """ - Generates a Gaussian FID for use with the lmfit GaussianFidModel class. The - helper function complex_to_real is used to convert the FID to a real form. - - :param x: the time axis for the fid data - :param amplitude: the amplitude of the Gaussian - :param frequency: the frequency of the Gaussian in Hz - :param phase: the phase in radians - :param fwhm: the full width at half maximum of the Gaussian - :return: a real FID describing a Gaussian relaxation + baseline_points = 16 + basis = make_basis(params, time_axis) + + weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() + return fitted_data + + +def residual(params, time_axis, data): + """ + This function calculates the residual to be minimized by the least squares means method. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param data: FID to be fitted. + :return: residual values of baseline points. """ + baseline_points = 16 + # unphase the data to make it pure absorptive + unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) + real_unphased_data = complex_to_real(unphased_data) - complex_fid = amplitude * suspect.basis.gaussian(x, frequency, phase, fwhm) - return complex_to_real(complex_fid) - - -class GaussianFidModel(lmfit.models.Model): - def __init__(self, *args, **kwargs): - super(GaussianFidModel, self).__init__(gaussian_fid, *args, **kwargs) - self.set_param_hint('fwhm', min=0) - self.set_param_hint('amplitude', min=0) - self.set_param_hint('phase', min=-numpy.pi, max=numpy.pi) - - def guess(self, data=None, **kwargs): - return self.make_params() - - def copy(self, **kwargs): - raise NotImplementedError - - -class Model: - def __init__(self, peaks): - self.model = None - self.params = lmfit.Parameters() - for peak_name, peak_params in peaks.items(): - - current_peak_model = GaussianFidModel(prefix="{}".format(peak_name)) - self.params.update(current_peak_model.make_params()) - - for param_name, param_data in peak_params.items(): - # the - full_name = "{0}{1}".format(peak_name, param_name) - - if full_name in self.params: - if type(param_data) is str: - self.params[full_name].set(expr=param_data) - elif type(param_data) is dict: - if "value" in param_data: - self.params[full_name].set(value=param_data["value"]) - if "min" in param_data: - self.params[full_name].set(min=param_data["min"]) - if "max" in param_data: - self.params[full_name].set(max=param_data["max"]) - if "expr" in param_data: - self.params[full_name].set(expr=param_data) - elif type(param_data) is numbers.Number: - self.params[full_name].set(param_data) - - if self.model is None: - self.model = current_peak_model - else: - self.model += current_peak_model - - def fit(self, data): - fit_result = self.model.fit(complex_to_real(data), x=data.time_axis(), params=self.params) - result_params = {} - for component in self.model.components: - component_name = component.prefix - result_params[component.prefix] = {} - for param in component.make_params(): - param_name = str(param).replace(component_name, "") - result_params[component.prefix][param_name] = fit_result.params[param].value - fit_curve = real_to_complex(fit_result.best_fit) - fit_components = {k: real_to_complex(v) for k, v in fit_result.eval_components().items()} - return { - "params": result_params, - "fit": fit_curve, - "fit_components": fit_components - } + fitted_data = do_fit(params, time_axis, real_unphased_data) + res = fitted_data - real_unphased_data + + return res[baseline_points:-baseline_points] + + +def fit_data(data, initial_params): + """ + This function takes an FID and a set of parameters contained in an lmfit Parameters object, + and fits the data using the least squares means method. + + :param data: FID to be fitted. + :param initial_params: lmfit Parameters object containing fitting parameters. + :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. + """ + baseline_points = 16 + fitting_result = lmfit.minimize(residual, + initial_params, + args=(data.time_axis(), data), + xtol=5e-3) + + unphased_data = phase_fid(data, + -fitting_result.params['phase0'], + -fitting_result.params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) + fitted_data = real_to_complex(real_fitted_data) + fitting_basis = make_basis(fitting_result.params, data.time_axis()) + fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + return fitting_weights, fitted_data, fitting_result diff --git a/suspect/processing/denoising.py b/suspect/processing/denoising.py index 60b0484..7b85dee 100644 --- a/suspect/processing/denoising.py +++ b/suspect/processing/denoising.py @@ -14,7 +14,7 @@ def _pad(input_signal, length, average=10): :return: """ padded_input_signal = numpy.zeros(length, input_signal.dtype) - start_offset = (len(padded_input_signal) - len(input_signal)) / 2. + start_offset = int((len(padded_input_signal) - len(input_signal)) / 2) padded_input_signal[:start_offset] = numpy.average(input_signal[0:average]) padded_input_signal[start_offset:(start_offset + len(input_signal))] = input_signal[:] padded_input_signal[(start_offset + len(input_signal)):] = numpy.average(input_signal[-average:]) @@ -38,7 +38,7 @@ def sliding_gaussian(input_signal, window_width): window /= numpy.sum(window) # pad the signal to cover half the window width on each side padded_input = _pad(input_signal, len(input_signal) + window_width - 1) - result = numpy.zeros(len(input_signal)) + result = numpy.zeros_like(input_signal) for i in range(len(input_signal)): result[i] = numpy.dot(window, padded_input[i:(i + window_width)]) return result @@ -63,7 +63,7 @@ def svd(input_signal, rank): s[rank:] = 0.0 recon = U * numpy.diag(s) * V - result = numpy.zeros(len(input_signal)) + result = numpy.zeros_like(input_signal) for i in range(len(input_signal)): count = 0 for j in range(matrix_height): From 438d51e4e69de8a608a8a90aeae200650242dd75 Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 14:22:23 -0400 Subject: [PATCH 02/39] Added fit file, moved amares fitting functions to singlet, converted a few values in denoising and singlet to int --- suspect/fitting/fit.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py index e622757..fabce0b 100644 --- a/suspect/fitting/fit.py +++ b/suspect/fitting/fit.py @@ -62,9 +62,14 @@ def save(output, fout): # Fit fid -# Input: fid, model -# Output: dictionary containing optimized model, fit data, standard errors def fit(fid, model): + """ + Fit fid with model parameters. + + :param fid: MRSData object of FID to be fit + :param model: dictionary model of fit parameters + :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] + """ # MODEL # Get list of metabolite names # def get_metabolites(model_input): @@ -267,10 +272,9 @@ def main(): # Test fit function -def test_fit(_plot): +def test_fit(_plot, testfile, modelfile): # Test file and model - testfile = "test_timecourse.csv" - modelfile = "model.json" + model = load(modelfile) # Calculate references to determine order for Parameters. @@ -420,24 +424,25 @@ def generate_fid(ground_truth, snr, initial_params): dt = 1.0 / sw np = 4096 f0 = 49.885802 - fake_fid, noisy_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid + fake_fid, noise_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid # Test fit(). fit_results = fit(fake_fid, model) - # Plot fit, noisy_fid, and fake_fid. + # Plot fit, noise_fid, and fake_fid. if _plot: from matplotlib import pyplot pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noisy_fid))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noise_fid))) pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) pyplot.show() test = False -plot = True +plot = False if test: print("Running test.") - test_fit(plot) + test_file = "/home/sam/Desktop/amares/test_timecourse.csv" + test_model = "/home/sam/Desktop/amares/model.json" + test_fit(plot, test_file, test_model) print("Test complete.") - From 82644f8572d9d66dd4ba6a7db2b8c889dc3ad9bc Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 15:22:02 -0400 Subject: [PATCH 03/39] Consolidated fit function into singlet, removed fit.py --- suspect/fitting/fit.py | 448 ------------------------------------- suspect/fitting/singlet.py | 415 +++++++++++++++++++++++++--------- 2 files changed, 311 insertions(+), 552 deletions(-) delete mode 100644 suspect/fitting/fit.py diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py deleted file mode 100644 index fabce0b..0000000 --- a/suspect/fitting/fit.py +++ /dev/null @@ -1,448 +0,0 @@ -""" -File Name: fit.py -Author: Sam Jiang -Purpose: fit_31p() function fits single 31P fid. -Interpreter: Anaconda 3.5.1 - -fit(fid, model) -Input: fid and model in hierarchical dictionary format -Output: dictionary with 'model', 'fit', and 'error' keys and corresponding values -""" - -import os -import numpy -import json -import lmfit -import array -import suspect - - -# Load model in. -def load(fin): - if not os.path.isfile(fin): - raise FileNotFoundError("{} not found.".format(fin)) - with open(fin, 'r+') as f: - model = json.load(f) - return model - - -# Save model or fit out. -def save(output, fout): - # Check if file and directory exists. - directory, name = os.path.split(fout) - if directory == '': - directory = os.getcwd() - if not os.path.isdir(directory): - os.makedirs(directory) - if name == '': - raise IsADirectoryError("{} is not a file name.".format(name)) - ftype = os.path.splitext(name)[1] - - # If model, save as json. - if type(output) is dict: - if ftype != ".json": - raise TypeError("Output file must be a JSON (.json) file.") - - with open(fout, 'w+') as f: - json.dump(output, f) - - # If fit, save as csv or txt. - elif type(output) is array.ArrayType: - if ftype != ".csv" or ftype != ".txt": - raise TypeError("Output file must be a CSV (.csv) or Text (.txt) file.") - - with open(fout, 'w+') as f: - first = True - for x in output.tolist(): - if first: - f.write("{}".format(x)) - first = False - else: - f.write(", {}".format(x)) - - -# Fit fid -def fit(fid, model): - """ - Fit fid with model parameters. - - :param fid: MRSData object of FID to be fit - :param model: dictionary model of fit parameters - :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] - """ - # MODEL - # Get list of metabolite names - # def get_metabolites(model_input): - # metabolites = [] - # for name, value in model_input.items(): - # if type(value) is dict: - # metabolites.append(name) - # return metabolites - - # Get standard errors from lmfit MinimizerResult object. - def get_errors(result): - errors = {} - for name, param in result.params.items(): - errors[name] = param.stderr - return errors - - # Convert lmfit parameters to model format - def parameters_to_model(parameters_obj, param_weights): - metabolite_name_list = [] - for param in parameters_obj.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - # Create dictionary for new model. - new_model = {} - for param_name, param in parameters_obj.items(): - name = param_name.split("_") - name1 = name[0] - if len(name) == 1: # i.e. phase - new_model[name1] = param.value - else: - name2 = name[1] - if name1 not in new_model: - new_model[name1] = {name2: param.value} - else: - new_model[name1][name2] = param.value - - for i, metabolite_name in enumerate(metabolite_name_list): - new_model[metabolite_name]["amplitude"] = param_weights[i] - - return new_model - - # Convert initial model to lmfit parameters. - def model_to_parameters(model_dict): - lmfit_parameters = lmfit.Parameters() - params = [] - ordered_params = [] - # Calculate dependencies/references for each parameter. - depend_dict = calculate_dependencies(model_dict) - - # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: - # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): - # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) - value = None - vary = None - lmfit_min = None - lmfit_max = None - expr = None - if type(value2) is int: - value = value2 - elif type(value2) is str: - expr = value2 - if type(value2) is dict: - if "value" in value2: - value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] - # Add parameter object with defined parameters. - params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) - - # Order parameters based on dependencies. - in_oparams = [] - while len(params) > 0: - front = params.pop(0) - name = front[0] - # If no dependencies, add parameter to list and mark parameter as added. - if name not in depend_dict or depend_dict[name] is None: - ordered_params.append(front) - in_oparams.append(name) - else: - dependencies_present = True - for dependency in depend_dict[name]: - # If dependency not yet added, mark parameter to move to back of queue. - if dependency not in in_oparams: - dependencies_present = False - # If all dependencies present, add parameter to list and mark parameter as added. - if dependencies_present: - ordered_params.append(front) - in_oparams.append(name) - # If dependencies missing, move parameter to back of queue. - else: - params.append(front) - - # Convert all parameters to lmfit Parameter objects. - lmfit_parameters.add_many(*ordered_params) - - return lmfit_parameters - - # Check if all model input types are correct. - def check_errors(check_model): - # Allowed names and keys in the model. - allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] - allowed_keys = ["min", "max", "value", "phase", "amplitude"] - - # Scan model. - for name1, value1 in check_model.items(): - if type(value1) is not int and type(value1) is not float and type(value1) is not dict: - raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) - elif name1 not in allowed_names: - raise NameError("{} is not an allowed name.".format(name1)) - elif type(value1) is dict: # i.e. type(value) is not int - for name2, value2 in value1.items(): - if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ - type(value2) is not str: - raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." - .format(name1, name2)) - if type(value2) is dict: - for key in value2: - # Dictionary must have 'value' key. - if "value" not in value2: - raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) - # Dictionary can only have 'min,' 'max,' and 'value'. - if key not in allowed_keys: - raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) - - return - - # Calculate references to determine order for Parameters. - def calculate_dependencies(unordered_model): - dependencies = {} # (name, [dependencies]) - - # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None - - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) - dependencies[lmfit_name] = [] - for depend in dependencies: - if depend in value2: - dependencies[lmfit_name].append(depend) - - # Check for circular dependencies. - for name, dependents in dependencies.items(): - if type(dependents) is list: - for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." - .format(name, dependent)) - - return dependencies - - # MAIN - def main(): - # Minimize and fit 31P data. - # Check for errors in model formatting. - check_errors(model) - - # Set list of metabolite names. - # nonlocal metabolite_name_list - # metabolite_name_list = get_metabolites(model) - - # Convert model to lmfit Parameters object. - parameters = model_to_parameters(model) - - # Fit data. - fitted_weights, fitted_data, fitted_results = suspect.fitting.singlet.fit_data(fid, parameters) - - # Convert fit parameters to model format. - final_model = parameters_to_model(fitted_results.params, fitted_weights) - # Get stderr values for each parameter. - stderr = get_errors(fitted_results) - - # Compile output into a dictionary. - return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} - - return return_dict - - if __name__ == "__main__": - return main() - - -# Test fit function -def test_fit(_plot, testfile, modelfile): - # Test file and model - - model = load(modelfile) - - # Calculate references to determine order for Parameters. - def calculate_dependencies(unordered_model): - dependencies = {} # (name, [dependencies]) - - # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None - - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) - dependencies[lmfit_name] = [] - for depend in dependencies: - if depend in value2: - dependencies[lmfit_name].append(depend) - - # Check for circular dependencies. - for name, dependents in dependencies.items(): - if type(dependents) is list: - for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." - .format(name, dependent)) - - return dependencies - - # Convert initial model to lmfit parameters. - def model_to_parameters(model_dict): - lmfit_parameters = lmfit.Parameters() - params = [] - ordered_params = [] - # Calculate dependencies/references for each parameter. - depend_dict = calculate_dependencies(model_dict) - - # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: - # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): - # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) - value = None - vary = None - lmfit_min = None - lmfit_max = None - expr = None - if type(value2) is int: - value = value2 - elif type(value2) is str: - expr = value2 - if type(value2) is dict: - if "value" in value2: - value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] - # Add parameter object with defined parameters. - params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) - - # Order parameters based on dependencies. - in_oparams = [] - while len(params) > 0: - front = params.pop(0) - name = front[0] - # If no dependencies, add parameter to list and mark parameter as added. - if name not in depend_dict or depend_dict[name] is None: - ordered_params.append(front) - in_oparams.append(name) - else: - dependencies_present = True - for dependency in depend_dict[name]: - # If dependency not yet added, mark parameter to move to back of queue. - if dependency not in in_oparams: - dependencies_present = False - # If all dependencies present, add parameter to list and mark parameter as added. - if dependencies_present: - ordered_params.append(front) - in_oparams.append(name) - # If dependencies missing, move parameter to back of queue. - else: - params.append(front) - - # Convert all parameters to lmfit Parameter objects. - lmfit_parameters.add_many(*ordered_params) - - return lmfit_parameters - - def fake_spectrum(params, time_axis, weights): - basis = suspect.fitting.singlet.make_basis(params, time_axis) - real_data = numpy.array(numpy.dot(weights, basis)).squeeze() - complex_data = suspect.fitting.singlet.real_to_complex(real_data) - return suspect.MRSData(complex_data, dt, f0, te=0, ppm0=0) - - # Generate fake noise (Testing only) - def generate_noise(n, snr): - # Added (n * 0) to fix unused variable warning. - noise = (n * 0) + (numpy.random.randn(4096) + 1j * numpy.random.randn(4096)) * 6.4e-6 / snr - return noise - - # Generate fake fid (Testing only) - def generate_fid(ground_truth, snr, initial_params): - metabolite_name_list = [] - for param in initial_params.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - - peaks = {"pcr": 1, - "atpc": 0.3, - "atpb": 0.3, - "atpa": 0.3, - "pi": 1, - "pme": 0.05, - "pde": 0.05} - amps = [] - for metabolite in metabolite_name_list: - amps.append(peaks[metabolite]) - - noisy_sequence = numpy.zeros((len(ground_truth), np), 'complex') - noisy_sequence = suspect.MRSData(noisy_sequence, dt, f0, te=0, ppm0=0) - - # Populate the noisy_sequence with data. - generated_fid = fake_spectrum(initial_params, noisy_sequence.time_axis(), amps) - # [1, 0.3, 0.3, 0.3, 1, 0.05, 0.05] - noisy_fid = generated_fid + generate_noise(np, snr) - - return generated_fid, noisy_fid - - # Generate fake FID. - parameters = model_to_parameters(model) - sw = 6e3 - dt = 1.0 / sw - np = 4096 - f0 = 49.885802 - fake_fid, noise_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid - - # Test fit(). - fit_results = fit(fake_fid, model) - - # Plot fit, noise_fid, and fake_fid. - if _plot: - from matplotlib import pyplot - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noise_fid))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) - pyplot.show() - - -test = False -plot = False -if test: - print("Running test.") - test_file = "/home/sam/Desktop/amares/test_timecourse.csv" - test_model = "/home/sam/Desktop/amares/model.json" - test_fit(plot, test_file, test_model) - print("Test complete.") diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 6e19dd1..f659b3d 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -43,113 +43,320 @@ def real_to_complex(real_fid): return complex_fid -# metabolite_name_list = [] - - -def phase_fid(fid_in, phase0, phase1): - """ - This function performs a Fourier Transform on the FID to shift it into phase. - - :param fid_in: FID to be fitted. - :param phase1: phase1 value. - :param phase0: phase0 value. - :return: +def fit(fid, model): """ - spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) - np = fid_in.np - phase_shift = phase0 + phase1 * numpy.linspace(-np / 2, np / 2, np, endpoint=False) - phased_spectrum = spectrum * numpy.exp(1j * phase_shift) - return fid_in.inherit(numpy.fft.ifft(numpy.fft.ifftshift(phased_spectrum))) - + Fit fid with model parameters. -def make_basis(params, time_axis): + :param fid: MRSData object of FID to be fit + :param model: dictionary model of fit parameters + :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] """ - This function generates a basis set. - :param params: lmfit Parameters object containing fitting parameters. - :param time_axis: the time axis. - :return: a matrix containing the generated basis set. - """ + # List of metabolite names metabolite_name_list = [] - for param in params.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - - basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) - for i, metabolite_name in enumerate(metabolite_name_list): - gaussian = suspect.basis.gaussian(time_axis, - params["{}_frequency".format(metabolite_name)], - params["{}_phase".format(metabolite_name)].value, - params["{}_width".format(metabolite_name)]) - real_gaussian = complex_to_real(gaussian) - basis_matrix[i, :] = real_gaussian - return basis_matrix - - -def do_fit(params, time_axis, real_unphased_data): - """ - This function performs the fitting. - - :param params: lmfit Parameters object containing fitting parameters. - :param time_axis: the time axis. - :param real_unphased_data: - :return: List of fitted data points. - """ - baseline_points = 16 - basis = make_basis(params, time_axis) - - weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, - real_unphased_data[baseline_points:-baseline_points])[0] - - fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() - return fitted_data - -def residual(params, time_axis, data): - """ - This function calculates the residual to be minimized by the least squares means method. - - :param params: lmfit Parameters object containing fitting parameters. - :param time_axis: the time axis. - :param data: FID to be fitted. - :return: residual values of baseline points. - """ - baseline_points = 16 - # unphase the data to make it pure absorptive - unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) - real_unphased_data = complex_to_real(unphased_data) - - fitted_data = do_fit(params, time_axis, real_unphased_data) - res = fitted_data - real_unphased_data - - return res[baseline_points:-baseline_points] - - -def fit_data(data, initial_params): - """ - This function takes an FID and a set of parameters contained in an lmfit Parameters object, - and fits the data using the least squares means method. - - :param data: FID to be fitted. - :param initial_params: lmfit Parameters object containing fitting parameters. - :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. - """ - baseline_points = 16 - fitting_result = lmfit.minimize(residual, - initial_params, - args=(data.time_axis(), data), - xtol=5e-3) - - unphased_data = phase_fid(data, - -fitting_result.params['phase0'], - -fitting_result.params['phase1']) - real_unphased_data = complex_to_real(unphased_data) - real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) - fitted_data = real_to_complex(real_fitted_data) - fitting_basis = make_basis(fitting_result.params, data.time_axis()) - fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, - real_unphased_data[baseline_points:-baseline_points])[0] - - return fitting_weights, fitted_data, fitting_result + # Get list of metabolite names. + def get_metabolites(model_input): + metabolites = [] + for name, value in model_input.items(): + if type(value) is dict: + metabolites.append(name) + return metabolites + + # Get standard errors from lmfit MinimizerResult object. + def get_errors(result): + errors = {} + for name, param in result.params.items(): + errors[name] = param.stderr + return errors + + def phase_fid(fid_in, phase0, phase1): + """ + This function performs a Fourier Transform on the FID to shift it into phase. + + :param fid_in: FID to be fitted. + :param phase1: phase1 value. + :param phase0: phase0 value. + :return: + """ + spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) + np = fid_in.np + phase_shift = phase0 + phase1 * numpy.linspace(-np / 2, np / 2, np, endpoint=False) + phased_spectrum = spectrum * numpy.exp(1j * phase_shift) + return fid_in.inherit(numpy.fft.ifft(numpy.fft.ifftshift(phased_spectrum))) + + def make_basis(params, time_axis): + """ + This function generates a basis set. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :return: a matrix containing the generated basis set. + """ + # metabolite_name_list = [] + # for param in params.keys(): + # split = param.split('_') + # if len(split) == 2: + # if split[0] not in metabolite_name_list: + # metabolite_name_list.append(split[0]) + + basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) + for i, metabolite_name in enumerate(metabolite_name_list): + gaussian = suspect.basis.gaussian(time_axis, + params["{}_frequency".format(metabolite_name)], + params["{}_phase".format(metabolite_name)].value, + params["{}_width".format(metabolite_name)]) + real_gaussian = complex_to_real(gaussian) + basis_matrix[i, :] = real_gaussian + return basis_matrix + + def do_fit(params, time_axis, real_unphased_data): + """ + This function performs the fitting. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param real_unphased_data: + :return: List of fitted data points. + """ + baseline_points = 16 + basis = make_basis(params, time_axis) + + weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() + return fitted_data + + def residual(params, time_axis, data): + """ + This function calculates the residual to be minimized by the least squares means method. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param data: FID to be fitted. + :return: residual values of baseline points. + """ + baseline_points = 16 + # unphase the data to make it pure absorptive + unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + + fitted_data = do_fit(params, time_axis, real_unphased_data) + res = fitted_data - real_unphased_data + + return res[baseline_points:-baseline_points] + + def fit_data(data, initial_params): + """ + This function takes an FID and a set of parameters contained in an lmfit Parameters object, + and fits the data using the least squares means method. + + :param data: FID to be fitted. + :param initial_params: lmfit Parameters object containing fitting parameters. + :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. + """ + baseline_points = 16 + fitting_result = lmfit.minimize(residual, + initial_params, + args=(data.time_axis(), data), + xtol=5e-3) + + unphased_data = phase_fid(data, + -fitting_result.params['phase0'], + -fitting_result.params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) + fitted_data = real_to_complex(real_fitted_data) + fitting_basis = make_basis(fitting_result.params, data.time_axis()) + fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + return fitting_weights, fitted_data, fitting_result + + # Convert lmfit parameters to model format + def parameters_to_model(parameters_obj, param_weights): + # metabolite_name_list = [] + # for param in parameters_obj.keys(): + # split = param.split('_') + # if len(split) == 2: + # if split[0] not in metabolite_name_list: + # metabolite_name_list.append(split[0]) + # Create dictionary for new model. + new_model = {} + for param_name, param in parameters_obj.items(): + name = param_name.split("_") + name1 = name[0] + if len(name) == 1: # i.e. phase + new_model[name1] = param.value + else: + name2 = name[1] + if name1 not in new_model: + new_model[name1] = {name2: param.value} + else: + new_model[name1][name2] = param.value + + nonlocal metabolite_name_list + for i, metabolite_name in enumerate(metabolite_name_list): + new_model[metabolite_name]["amplitude"] = param_weights[i] + + return new_model + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + # Check if all model input types are correct. + def check_errors(check_model): + # Allowed names and keys in the model. + allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] + allowed_keys = ["min", "max", "value", "phase", "amplitude"] + + # Scan model. + for name1, value1 in check_model.items(): + if type(value1) is not int and type(value1) is not float and type(value1) is not dict: + raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) + elif name1 not in allowed_names: + raise NameError("{} is not an allowed name.".format(name1)) + elif type(value1) is dict: # i.e. type(value) is not int + for name2, value2 in value1.items(): + if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ + type(value2) is not str: + raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." + .format(name1, name2)) + if type(value2) is dict: + for key in value2: + # Dictionary must have 'value' key. + if "value" not in value2: + raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) + # Dictionary can only have 'min,' 'max,' and 'value'. + if key not in allowed_keys: + raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) + + return + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # Do singlet fitting + def main(): + # Minimize and fit 31P data. + # Check for errors in model formatting. + check_errors(model) + + # Set list of metabolite names. + # nonlocal metabolite_name_list + # metabolite_name_list = get_metabolites(model) + + # Convert model to lmfit Parameters object. + parameters = model_to_parameters(model) + + # Set metabolite name list + nonlocal metabolite_name_list + metabolite_name_list = get_metabolites(model) + + # Fit data. + fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) + + # Convert fit parameters to model format. + final_model = parameters_to_model(fitted_results.params, fitted_weights) + # Get stderr values for each parameter. + stderr = get_errors(fitted_results) + + # Compile output into a dictionary. + return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} + return return_dict + + return main() From a9680051eaaf11b00eaea10c75680e1aa58f627f Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 15:28:37 -0400 Subject: [PATCH 04/39] Consolidated fit function into singlet, removed fit.py --- suspect/fitting/singlet.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index f659b3d..e72109f 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -337,16 +337,12 @@ def main(): check_errors(model) # Set list of metabolite names. - # nonlocal metabolite_name_list - # metabolite_name_list = get_metabolites(model) + nonlocal metabolite_name_list + metabolite_name_list = get_metabolites(model) # Convert model to lmfit Parameters object. parameters = model_to_parameters(model) - # Set metabolite name list - nonlocal metabolite_name_list - metabolite_name_list = get_metabolites(model) - # Fit data. fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) From 398987d1525da82fef7f6200267bf59d3350e181 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 25 Jul 2016 17:30:50 -0400 Subject: [PATCH 05/39] Added fit file, moved amares fitting functions to singlet, converted a few values in denoising and singlet to int --- suspect/fitting/fit.py | 443 ++++++++++++++++++++++++++++++++ suspect/fitting/singlet.py | 187 ++++++++------ suspect/processing/denoising.py | 6 +- 3 files changed, 555 insertions(+), 81 deletions(-) create mode 100644 suspect/fitting/fit.py diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py new file mode 100644 index 0000000..e622757 --- /dev/null +++ b/suspect/fitting/fit.py @@ -0,0 +1,443 @@ +""" +File Name: fit.py +Author: Sam Jiang +Purpose: fit_31p() function fits single 31P fid. +Interpreter: Anaconda 3.5.1 + +fit(fid, model) +Input: fid and model in hierarchical dictionary format +Output: dictionary with 'model', 'fit', and 'error' keys and corresponding values +""" + +import os +import numpy +import json +import lmfit +import array +import suspect + + +# Load model in. +def load(fin): + if not os.path.isfile(fin): + raise FileNotFoundError("{} not found.".format(fin)) + with open(fin, 'r+') as f: + model = json.load(f) + return model + + +# Save model or fit out. +def save(output, fout): + # Check if file and directory exists. + directory, name = os.path.split(fout) + if directory == '': + directory = os.getcwd() + if not os.path.isdir(directory): + os.makedirs(directory) + if name == '': + raise IsADirectoryError("{} is not a file name.".format(name)) + ftype = os.path.splitext(name)[1] + + # If model, save as json. + if type(output) is dict: + if ftype != ".json": + raise TypeError("Output file must be a JSON (.json) file.") + + with open(fout, 'w+') as f: + json.dump(output, f) + + # If fit, save as csv or txt. + elif type(output) is array.ArrayType: + if ftype != ".csv" or ftype != ".txt": + raise TypeError("Output file must be a CSV (.csv) or Text (.txt) file.") + + with open(fout, 'w+') as f: + first = True + for x in output.tolist(): + if first: + f.write("{}".format(x)) + first = False + else: + f.write(", {}".format(x)) + + +# Fit fid +# Input: fid, model +# Output: dictionary containing optimized model, fit data, standard errors +def fit(fid, model): + # MODEL + # Get list of metabolite names + # def get_metabolites(model_input): + # metabolites = [] + # for name, value in model_input.items(): + # if type(value) is dict: + # metabolites.append(name) + # return metabolites + + # Get standard errors from lmfit MinimizerResult object. + def get_errors(result): + errors = {} + for name, param in result.params.items(): + errors[name] = param.stderr + return errors + + # Convert lmfit parameters to model format + def parameters_to_model(parameters_obj, param_weights): + metabolite_name_list = [] + for param in parameters_obj.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + # Create dictionary for new model. + new_model = {} + for param_name, param in parameters_obj.items(): + name = param_name.split("_") + name1 = name[0] + if len(name) == 1: # i.e. phase + new_model[name1] = param.value + else: + name2 = name[1] + if name1 not in new_model: + new_model[name1] = {name2: param.value} + else: + new_model[name1][name2] = param.value + + for i, metabolite_name in enumerate(metabolite_name_list): + new_model[metabolite_name]["amplitude"] = param_weights[i] + + return new_model + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + # Check if all model input types are correct. + def check_errors(check_model): + # Allowed names and keys in the model. + allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] + allowed_keys = ["min", "max", "value", "phase", "amplitude"] + + # Scan model. + for name1, value1 in check_model.items(): + if type(value1) is not int and type(value1) is not float and type(value1) is not dict: + raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) + elif name1 not in allowed_names: + raise NameError("{} is not an allowed name.".format(name1)) + elif type(value1) is dict: # i.e. type(value) is not int + for name2, value2 in value1.items(): + if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ + type(value2) is not str: + raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." + .format(name1, name2)) + if type(value2) is dict: + for key in value2: + # Dictionary must have 'value' key. + if "value" not in value2: + raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) + # Dictionary can only have 'min,' 'max,' and 'value'. + if key not in allowed_keys: + raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) + + return + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # MAIN + def main(): + # Minimize and fit 31P data. + # Check for errors in model formatting. + check_errors(model) + + # Set list of metabolite names. + # nonlocal metabolite_name_list + # metabolite_name_list = get_metabolites(model) + + # Convert model to lmfit Parameters object. + parameters = model_to_parameters(model) + + # Fit data. + fitted_weights, fitted_data, fitted_results = suspect.fitting.singlet.fit_data(fid, parameters) + + # Convert fit parameters to model format. + final_model = parameters_to_model(fitted_results.params, fitted_weights) + # Get stderr values for each parameter. + stderr = get_errors(fitted_results) + + # Compile output into a dictionary. + return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} + + return return_dict + + if __name__ == "__main__": + return main() + + +# Test fit function +def test_fit(_plot): + # Test file and model + testfile = "test_timecourse.csv" + modelfile = "model.json" + model = load(modelfile) + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + def fake_spectrum(params, time_axis, weights): + basis = suspect.fitting.singlet.make_basis(params, time_axis) + real_data = numpy.array(numpy.dot(weights, basis)).squeeze() + complex_data = suspect.fitting.singlet.real_to_complex(real_data) + return suspect.MRSData(complex_data, dt, f0, te=0, ppm0=0) + + # Generate fake noise (Testing only) + def generate_noise(n, snr): + # Added (n * 0) to fix unused variable warning. + noise = (n * 0) + (numpy.random.randn(4096) + 1j * numpy.random.randn(4096)) * 6.4e-6 / snr + return noise + + # Generate fake fid (Testing only) + def generate_fid(ground_truth, snr, initial_params): + metabolite_name_list = [] + for param in initial_params.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + + peaks = {"pcr": 1, + "atpc": 0.3, + "atpb": 0.3, + "atpa": 0.3, + "pi": 1, + "pme": 0.05, + "pde": 0.05} + amps = [] + for metabolite in metabolite_name_list: + amps.append(peaks[metabolite]) + + noisy_sequence = numpy.zeros((len(ground_truth), np), 'complex') + noisy_sequence = suspect.MRSData(noisy_sequence, dt, f0, te=0, ppm0=0) + + # Populate the noisy_sequence with data. + generated_fid = fake_spectrum(initial_params, noisy_sequence.time_axis(), amps) + # [1, 0.3, 0.3, 0.3, 1, 0.05, 0.05] + noisy_fid = generated_fid + generate_noise(np, snr) + + return generated_fid, noisy_fid + + # Generate fake FID. + parameters = model_to_parameters(model) + sw = 6e3 + dt = 1.0 / sw + np = 4096 + f0 = 49.885802 + fake_fid, noisy_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid + + # Test fit(). + fit_results = fit(fake_fid, model) + + # Plot fit, noisy_fid, and fake_fid. + if _plot: + from matplotlib import pyplot + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noisy_fid))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) + pyplot.show() + + +test = False +plot = True +if test: + print("Running test.") + test_fit(plot) + print("Test complete.") + diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 39a15e7..6e19dd1 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -1,5 +1,6 @@ import lmfit import numpy +import scipy.optimize import numbers import suspect @@ -32,7 +33,7 @@ def real_to_complex(real_fid): :param real_fid: the real FID to be converted to complex. :return: the complex version of the FID """ - np = real_fid.shape[0] / 2 + np = int(real_fid.shape[0] / 2) complex_fid = numpy.zeros(np, 'complex') complex_fid[:] = real_fid[:np] @@ -42,83 +43,113 @@ def real_to_complex(real_fid): return complex_fid -def gaussian_fid(x, amplitude=1, frequency=0.0, phase=0.0, fwhm=1.0): +# metabolite_name_list = [] + + +def phase_fid(fid_in, phase0, phase1): + """ + This function performs a Fourier Transform on the FID to shift it into phase. + + :param fid_in: FID to be fitted. + :param phase1: phase1 value. + :param phase0: phase0 value. + :return: + """ + spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) + np = fid_in.np + phase_shift = phase0 + phase1 * numpy.linspace(-np / 2, np / 2, np, endpoint=False) + phased_spectrum = spectrum * numpy.exp(1j * phase_shift) + return fid_in.inherit(numpy.fft.ifft(numpy.fft.ifftshift(phased_spectrum))) + + +def make_basis(params, time_axis): + """ + This function generates a basis set. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :return: a matrix containing the generated basis set. + """ + metabolite_name_list = [] + for param in params.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + + basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) + for i, metabolite_name in enumerate(metabolite_name_list): + gaussian = suspect.basis.gaussian(time_axis, + params["{}_frequency".format(metabolite_name)], + params["{}_phase".format(metabolite_name)].value, + params["{}_width".format(metabolite_name)]) + real_gaussian = complex_to_real(gaussian) + basis_matrix[i, :] = real_gaussian + return basis_matrix + + +def do_fit(params, time_axis, real_unphased_data): + """ + This function performs the fitting. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param real_unphased_data: + :return: List of fitted data points. """ - Generates a Gaussian FID for use with the lmfit GaussianFidModel class. The - helper function complex_to_real is used to convert the FID to a real form. - - :param x: the time axis for the fid data - :param amplitude: the amplitude of the Gaussian - :param frequency: the frequency of the Gaussian in Hz - :param phase: the phase in radians - :param fwhm: the full width at half maximum of the Gaussian - :return: a real FID describing a Gaussian relaxation + baseline_points = 16 + basis = make_basis(params, time_axis) + + weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() + return fitted_data + + +def residual(params, time_axis, data): + """ + This function calculates the residual to be minimized by the least squares means method. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param data: FID to be fitted. + :return: residual values of baseline points. """ + baseline_points = 16 + # unphase the data to make it pure absorptive + unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) + real_unphased_data = complex_to_real(unphased_data) - complex_fid = amplitude * suspect.basis.gaussian(x, frequency, phase, fwhm) - return complex_to_real(complex_fid) - - -class GaussianFidModel(lmfit.models.Model): - def __init__(self, *args, **kwargs): - super(GaussianFidModel, self).__init__(gaussian_fid, *args, **kwargs) - self.set_param_hint('fwhm', min=0) - self.set_param_hint('amplitude', min=0) - self.set_param_hint('phase', min=-numpy.pi, max=numpy.pi) - - def guess(self, data=None, **kwargs): - return self.make_params() - - def copy(self, **kwargs): - raise NotImplementedError - - -class Model: - def __init__(self, peaks): - self.model = None - self.params = lmfit.Parameters() - for peak_name, peak_params in peaks.items(): - - current_peak_model = GaussianFidModel(prefix="{}".format(peak_name)) - self.params.update(current_peak_model.make_params()) - - for param_name, param_data in peak_params.items(): - # the - full_name = "{0}{1}".format(peak_name, param_name) - - if full_name in self.params: - if type(param_data) is str: - self.params[full_name].set(expr=param_data) - elif type(param_data) is dict: - if "value" in param_data: - self.params[full_name].set(value=param_data["value"]) - if "min" in param_data: - self.params[full_name].set(min=param_data["min"]) - if "max" in param_data: - self.params[full_name].set(max=param_data["max"]) - if "expr" in param_data: - self.params[full_name].set(expr=param_data) - elif type(param_data) is numbers.Number: - self.params[full_name].set(param_data) - - if self.model is None: - self.model = current_peak_model - else: - self.model += current_peak_model - - def fit(self, data): - fit_result = self.model.fit(complex_to_real(data), x=data.time_axis(), params=self.params) - result_params = {} - for component in self.model.components: - component_name = component.prefix - result_params[component.prefix] = {} - for param in component.make_params(): - param_name = str(param).replace(component_name, "") - result_params[component.prefix][param_name] = fit_result.params[param].value - fit_curve = real_to_complex(fit_result.best_fit) - fit_components = {k: real_to_complex(v) for k, v in fit_result.eval_components().items()} - return { - "params": result_params, - "fit": fit_curve, - "fit_components": fit_components - } + fitted_data = do_fit(params, time_axis, real_unphased_data) + res = fitted_data - real_unphased_data + + return res[baseline_points:-baseline_points] + + +def fit_data(data, initial_params): + """ + This function takes an FID and a set of parameters contained in an lmfit Parameters object, + and fits the data using the least squares means method. + + :param data: FID to be fitted. + :param initial_params: lmfit Parameters object containing fitting parameters. + :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. + """ + baseline_points = 16 + fitting_result = lmfit.minimize(residual, + initial_params, + args=(data.time_axis(), data), + xtol=5e-3) + + unphased_data = phase_fid(data, + -fitting_result.params['phase0'], + -fitting_result.params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) + fitted_data = real_to_complex(real_fitted_data) + fitting_basis = make_basis(fitting_result.params, data.time_axis()) + fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + return fitting_weights, fitted_data, fitting_result diff --git a/suspect/processing/denoising.py b/suspect/processing/denoising.py index 60b0484..7b85dee 100644 --- a/suspect/processing/denoising.py +++ b/suspect/processing/denoising.py @@ -14,7 +14,7 @@ def _pad(input_signal, length, average=10): :return: """ padded_input_signal = numpy.zeros(length, input_signal.dtype) - start_offset = (len(padded_input_signal) - len(input_signal)) / 2. + start_offset = int((len(padded_input_signal) - len(input_signal)) / 2) padded_input_signal[:start_offset] = numpy.average(input_signal[0:average]) padded_input_signal[start_offset:(start_offset + len(input_signal))] = input_signal[:] padded_input_signal[(start_offset + len(input_signal)):] = numpy.average(input_signal[-average:]) @@ -38,7 +38,7 @@ def sliding_gaussian(input_signal, window_width): window /= numpy.sum(window) # pad the signal to cover half the window width on each side padded_input = _pad(input_signal, len(input_signal) + window_width - 1) - result = numpy.zeros(len(input_signal)) + result = numpy.zeros_like(input_signal) for i in range(len(input_signal)): result[i] = numpy.dot(window, padded_input[i:(i + window_width)]) return result @@ -63,7 +63,7 @@ def svd(input_signal, rank): s[rank:] = 0.0 recon = U * numpy.diag(s) * V - result = numpy.zeros(len(input_signal)) + result = numpy.zeros_like(input_signal) for i in range(len(input_signal)): count = 0 for j in range(matrix_height): From e6a8e36bfcde5575f07c692c76466b9cdf73944d Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 14:22:23 -0400 Subject: [PATCH 06/39] Added fit file, moved amares fitting functions to singlet, converted a few values in denoising and singlet to int --- suspect/fitting/fit.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py index e622757..fabce0b 100644 --- a/suspect/fitting/fit.py +++ b/suspect/fitting/fit.py @@ -62,9 +62,14 @@ def save(output, fout): # Fit fid -# Input: fid, model -# Output: dictionary containing optimized model, fit data, standard errors def fit(fid, model): + """ + Fit fid with model parameters. + + :param fid: MRSData object of FID to be fit + :param model: dictionary model of fit parameters + :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] + """ # MODEL # Get list of metabolite names # def get_metabolites(model_input): @@ -267,10 +272,9 @@ def main(): # Test fit function -def test_fit(_plot): +def test_fit(_plot, testfile, modelfile): # Test file and model - testfile = "test_timecourse.csv" - modelfile = "model.json" + model = load(modelfile) # Calculate references to determine order for Parameters. @@ -420,24 +424,25 @@ def generate_fid(ground_truth, snr, initial_params): dt = 1.0 / sw np = 4096 f0 = 49.885802 - fake_fid, noisy_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid + fake_fid, noise_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid # Test fit(). fit_results = fit(fake_fid, model) - # Plot fit, noisy_fid, and fake_fid. + # Plot fit, noise_fid, and fake_fid. if _plot: from matplotlib import pyplot pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noisy_fid))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noise_fid))) pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) pyplot.show() test = False -plot = True +plot = False if test: print("Running test.") - test_fit(plot) + test_file = "/home/sam/Desktop/amares/test_timecourse.csv" + test_model = "/home/sam/Desktop/amares/model.json" + test_fit(plot, test_file, test_model) print("Test complete.") - From 989a0232ecce008873a7c3d81ed239eef86b8a77 Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 15:22:02 -0400 Subject: [PATCH 07/39] Consolidated fit function into singlet, removed fit.py --- suspect/fitting/fit.py | 448 ------------------------------------- suspect/fitting/singlet.py | 415 +++++++++++++++++++++++++--------- 2 files changed, 311 insertions(+), 552 deletions(-) delete mode 100644 suspect/fitting/fit.py diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py deleted file mode 100644 index fabce0b..0000000 --- a/suspect/fitting/fit.py +++ /dev/null @@ -1,448 +0,0 @@ -""" -File Name: fit.py -Author: Sam Jiang -Purpose: fit_31p() function fits single 31P fid. -Interpreter: Anaconda 3.5.1 - -fit(fid, model) -Input: fid and model in hierarchical dictionary format -Output: dictionary with 'model', 'fit', and 'error' keys and corresponding values -""" - -import os -import numpy -import json -import lmfit -import array -import suspect - - -# Load model in. -def load(fin): - if not os.path.isfile(fin): - raise FileNotFoundError("{} not found.".format(fin)) - with open(fin, 'r+') as f: - model = json.load(f) - return model - - -# Save model or fit out. -def save(output, fout): - # Check if file and directory exists. - directory, name = os.path.split(fout) - if directory == '': - directory = os.getcwd() - if not os.path.isdir(directory): - os.makedirs(directory) - if name == '': - raise IsADirectoryError("{} is not a file name.".format(name)) - ftype = os.path.splitext(name)[1] - - # If model, save as json. - if type(output) is dict: - if ftype != ".json": - raise TypeError("Output file must be a JSON (.json) file.") - - with open(fout, 'w+') as f: - json.dump(output, f) - - # If fit, save as csv or txt. - elif type(output) is array.ArrayType: - if ftype != ".csv" or ftype != ".txt": - raise TypeError("Output file must be a CSV (.csv) or Text (.txt) file.") - - with open(fout, 'w+') as f: - first = True - for x in output.tolist(): - if first: - f.write("{}".format(x)) - first = False - else: - f.write(", {}".format(x)) - - -# Fit fid -def fit(fid, model): - """ - Fit fid with model parameters. - - :param fid: MRSData object of FID to be fit - :param model: dictionary model of fit parameters - :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] - """ - # MODEL - # Get list of metabolite names - # def get_metabolites(model_input): - # metabolites = [] - # for name, value in model_input.items(): - # if type(value) is dict: - # metabolites.append(name) - # return metabolites - - # Get standard errors from lmfit MinimizerResult object. - def get_errors(result): - errors = {} - for name, param in result.params.items(): - errors[name] = param.stderr - return errors - - # Convert lmfit parameters to model format - def parameters_to_model(parameters_obj, param_weights): - metabolite_name_list = [] - for param in parameters_obj.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - # Create dictionary for new model. - new_model = {} - for param_name, param in parameters_obj.items(): - name = param_name.split("_") - name1 = name[0] - if len(name) == 1: # i.e. phase - new_model[name1] = param.value - else: - name2 = name[1] - if name1 not in new_model: - new_model[name1] = {name2: param.value} - else: - new_model[name1][name2] = param.value - - for i, metabolite_name in enumerate(metabolite_name_list): - new_model[metabolite_name]["amplitude"] = param_weights[i] - - return new_model - - # Convert initial model to lmfit parameters. - def model_to_parameters(model_dict): - lmfit_parameters = lmfit.Parameters() - params = [] - ordered_params = [] - # Calculate dependencies/references for each parameter. - depend_dict = calculate_dependencies(model_dict) - - # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: - # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): - # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) - value = None - vary = None - lmfit_min = None - lmfit_max = None - expr = None - if type(value2) is int: - value = value2 - elif type(value2) is str: - expr = value2 - if type(value2) is dict: - if "value" in value2: - value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] - # Add parameter object with defined parameters. - params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) - - # Order parameters based on dependencies. - in_oparams = [] - while len(params) > 0: - front = params.pop(0) - name = front[0] - # If no dependencies, add parameter to list and mark parameter as added. - if name not in depend_dict or depend_dict[name] is None: - ordered_params.append(front) - in_oparams.append(name) - else: - dependencies_present = True - for dependency in depend_dict[name]: - # If dependency not yet added, mark parameter to move to back of queue. - if dependency not in in_oparams: - dependencies_present = False - # If all dependencies present, add parameter to list and mark parameter as added. - if dependencies_present: - ordered_params.append(front) - in_oparams.append(name) - # If dependencies missing, move parameter to back of queue. - else: - params.append(front) - - # Convert all parameters to lmfit Parameter objects. - lmfit_parameters.add_many(*ordered_params) - - return lmfit_parameters - - # Check if all model input types are correct. - def check_errors(check_model): - # Allowed names and keys in the model. - allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] - allowed_keys = ["min", "max", "value", "phase", "amplitude"] - - # Scan model. - for name1, value1 in check_model.items(): - if type(value1) is not int and type(value1) is not float and type(value1) is not dict: - raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) - elif name1 not in allowed_names: - raise NameError("{} is not an allowed name.".format(name1)) - elif type(value1) is dict: # i.e. type(value) is not int - for name2, value2 in value1.items(): - if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ - type(value2) is not str: - raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." - .format(name1, name2)) - if type(value2) is dict: - for key in value2: - # Dictionary must have 'value' key. - if "value" not in value2: - raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) - # Dictionary can only have 'min,' 'max,' and 'value'. - if key not in allowed_keys: - raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) - - return - - # Calculate references to determine order for Parameters. - def calculate_dependencies(unordered_model): - dependencies = {} # (name, [dependencies]) - - # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None - - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) - dependencies[lmfit_name] = [] - for depend in dependencies: - if depend in value2: - dependencies[lmfit_name].append(depend) - - # Check for circular dependencies. - for name, dependents in dependencies.items(): - if type(dependents) is list: - for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." - .format(name, dependent)) - - return dependencies - - # MAIN - def main(): - # Minimize and fit 31P data. - # Check for errors in model formatting. - check_errors(model) - - # Set list of metabolite names. - # nonlocal metabolite_name_list - # metabolite_name_list = get_metabolites(model) - - # Convert model to lmfit Parameters object. - parameters = model_to_parameters(model) - - # Fit data. - fitted_weights, fitted_data, fitted_results = suspect.fitting.singlet.fit_data(fid, parameters) - - # Convert fit parameters to model format. - final_model = parameters_to_model(fitted_results.params, fitted_weights) - # Get stderr values for each parameter. - stderr = get_errors(fitted_results) - - # Compile output into a dictionary. - return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} - - return return_dict - - if __name__ == "__main__": - return main() - - -# Test fit function -def test_fit(_plot, testfile, modelfile): - # Test file and model - - model = load(modelfile) - - # Calculate references to determine order for Parameters. - def calculate_dependencies(unordered_model): - dependencies = {} # (name, [dependencies]) - - # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None - - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) - dependencies[lmfit_name] = [] - for depend in dependencies: - if depend in value2: - dependencies[lmfit_name].append(depend) - - # Check for circular dependencies. - for name, dependents in dependencies.items(): - if type(dependents) is list: - for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." - .format(name, dependent)) - - return dependencies - - # Convert initial model to lmfit parameters. - def model_to_parameters(model_dict): - lmfit_parameters = lmfit.Parameters() - params = [] - ordered_params = [] - # Calculate dependencies/references for each parameter. - depend_dict = calculate_dependencies(model_dict) - - # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: - # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): - # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) - value = None - vary = None - lmfit_min = None - lmfit_max = None - expr = None - if type(value2) is int: - value = value2 - elif type(value2) is str: - expr = value2 - if type(value2) is dict: - if "value" in value2: - value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] - # Add parameter object with defined parameters. - params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) - - # Order parameters based on dependencies. - in_oparams = [] - while len(params) > 0: - front = params.pop(0) - name = front[0] - # If no dependencies, add parameter to list and mark parameter as added. - if name not in depend_dict or depend_dict[name] is None: - ordered_params.append(front) - in_oparams.append(name) - else: - dependencies_present = True - for dependency in depend_dict[name]: - # If dependency not yet added, mark parameter to move to back of queue. - if dependency not in in_oparams: - dependencies_present = False - # If all dependencies present, add parameter to list and mark parameter as added. - if dependencies_present: - ordered_params.append(front) - in_oparams.append(name) - # If dependencies missing, move parameter to back of queue. - else: - params.append(front) - - # Convert all parameters to lmfit Parameter objects. - lmfit_parameters.add_many(*ordered_params) - - return lmfit_parameters - - def fake_spectrum(params, time_axis, weights): - basis = suspect.fitting.singlet.make_basis(params, time_axis) - real_data = numpy.array(numpy.dot(weights, basis)).squeeze() - complex_data = suspect.fitting.singlet.real_to_complex(real_data) - return suspect.MRSData(complex_data, dt, f0, te=0, ppm0=0) - - # Generate fake noise (Testing only) - def generate_noise(n, snr): - # Added (n * 0) to fix unused variable warning. - noise = (n * 0) + (numpy.random.randn(4096) + 1j * numpy.random.randn(4096)) * 6.4e-6 / snr - return noise - - # Generate fake fid (Testing only) - def generate_fid(ground_truth, snr, initial_params): - metabolite_name_list = [] - for param in initial_params.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - - peaks = {"pcr": 1, - "atpc": 0.3, - "atpb": 0.3, - "atpa": 0.3, - "pi": 1, - "pme": 0.05, - "pde": 0.05} - amps = [] - for metabolite in metabolite_name_list: - amps.append(peaks[metabolite]) - - noisy_sequence = numpy.zeros((len(ground_truth), np), 'complex') - noisy_sequence = suspect.MRSData(noisy_sequence, dt, f0, te=0, ppm0=0) - - # Populate the noisy_sequence with data. - generated_fid = fake_spectrum(initial_params, noisy_sequence.time_axis(), amps) - # [1, 0.3, 0.3, 0.3, 1, 0.05, 0.05] - noisy_fid = generated_fid + generate_noise(np, snr) - - return generated_fid, noisy_fid - - # Generate fake FID. - parameters = model_to_parameters(model) - sw = 6e3 - dt = 1.0 / sw - np = 4096 - f0 = 49.885802 - fake_fid, noise_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid - - # Test fit(). - fit_results = fit(fake_fid, model) - - # Plot fit, noise_fid, and fake_fid. - if _plot: - from matplotlib import pyplot - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noise_fid))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) - pyplot.show() - - -test = False -plot = False -if test: - print("Running test.") - test_file = "/home/sam/Desktop/amares/test_timecourse.csv" - test_model = "/home/sam/Desktop/amares/model.json" - test_fit(plot, test_file, test_model) - print("Test complete.") diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 6e19dd1..f659b3d 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -43,113 +43,320 @@ def real_to_complex(real_fid): return complex_fid -# metabolite_name_list = [] - - -def phase_fid(fid_in, phase0, phase1): - """ - This function performs a Fourier Transform on the FID to shift it into phase. - - :param fid_in: FID to be fitted. - :param phase1: phase1 value. - :param phase0: phase0 value. - :return: +def fit(fid, model): """ - spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) - np = fid_in.np - phase_shift = phase0 + phase1 * numpy.linspace(-np / 2, np / 2, np, endpoint=False) - phased_spectrum = spectrum * numpy.exp(1j * phase_shift) - return fid_in.inherit(numpy.fft.ifft(numpy.fft.ifftshift(phased_spectrum))) - + Fit fid with model parameters. -def make_basis(params, time_axis): + :param fid: MRSData object of FID to be fit + :param model: dictionary model of fit parameters + :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] """ - This function generates a basis set. - :param params: lmfit Parameters object containing fitting parameters. - :param time_axis: the time axis. - :return: a matrix containing the generated basis set. - """ + # List of metabolite names metabolite_name_list = [] - for param in params.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - - basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) - for i, metabolite_name in enumerate(metabolite_name_list): - gaussian = suspect.basis.gaussian(time_axis, - params["{}_frequency".format(metabolite_name)], - params["{}_phase".format(metabolite_name)].value, - params["{}_width".format(metabolite_name)]) - real_gaussian = complex_to_real(gaussian) - basis_matrix[i, :] = real_gaussian - return basis_matrix - - -def do_fit(params, time_axis, real_unphased_data): - """ - This function performs the fitting. - - :param params: lmfit Parameters object containing fitting parameters. - :param time_axis: the time axis. - :param real_unphased_data: - :return: List of fitted data points. - """ - baseline_points = 16 - basis = make_basis(params, time_axis) - - weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, - real_unphased_data[baseline_points:-baseline_points])[0] - - fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() - return fitted_data - -def residual(params, time_axis, data): - """ - This function calculates the residual to be minimized by the least squares means method. - - :param params: lmfit Parameters object containing fitting parameters. - :param time_axis: the time axis. - :param data: FID to be fitted. - :return: residual values of baseline points. - """ - baseline_points = 16 - # unphase the data to make it pure absorptive - unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) - real_unphased_data = complex_to_real(unphased_data) - - fitted_data = do_fit(params, time_axis, real_unphased_data) - res = fitted_data - real_unphased_data - - return res[baseline_points:-baseline_points] - - -def fit_data(data, initial_params): - """ - This function takes an FID and a set of parameters contained in an lmfit Parameters object, - and fits the data using the least squares means method. - - :param data: FID to be fitted. - :param initial_params: lmfit Parameters object containing fitting parameters. - :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. - """ - baseline_points = 16 - fitting_result = lmfit.minimize(residual, - initial_params, - args=(data.time_axis(), data), - xtol=5e-3) - - unphased_data = phase_fid(data, - -fitting_result.params['phase0'], - -fitting_result.params['phase1']) - real_unphased_data = complex_to_real(unphased_data) - real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) - fitted_data = real_to_complex(real_fitted_data) - fitting_basis = make_basis(fitting_result.params, data.time_axis()) - fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, - real_unphased_data[baseline_points:-baseline_points])[0] - - return fitting_weights, fitted_data, fitting_result + # Get list of metabolite names. + def get_metabolites(model_input): + metabolites = [] + for name, value in model_input.items(): + if type(value) is dict: + metabolites.append(name) + return metabolites + + # Get standard errors from lmfit MinimizerResult object. + def get_errors(result): + errors = {} + for name, param in result.params.items(): + errors[name] = param.stderr + return errors + + def phase_fid(fid_in, phase0, phase1): + """ + This function performs a Fourier Transform on the FID to shift it into phase. + + :param fid_in: FID to be fitted. + :param phase1: phase1 value. + :param phase0: phase0 value. + :return: + """ + spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) + np = fid_in.np + phase_shift = phase0 + phase1 * numpy.linspace(-np / 2, np / 2, np, endpoint=False) + phased_spectrum = spectrum * numpy.exp(1j * phase_shift) + return fid_in.inherit(numpy.fft.ifft(numpy.fft.ifftshift(phased_spectrum))) + + def make_basis(params, time_axis): + """ + This function generates a basis set. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :return: a matrix containing the generated basis set. + """ + # metabolite_name_list = [] + # for param in params.keys(): + # split = param.split('_') + # if len(split) == 2: + # if split[0] not in metabolite_name_list: + # metabolite_name_list.append(split[0]) + + basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) + for i, metabolite_name in enumerate(metabolite_name_list): + gaussian = suspect.basis.gaussian(time_axis, + params["{}_frequency".format(metabolite_name)], + params["{}_phase".format(metabolite_name)].value, + params["{}_width".format(metabolite_name)]) + real_gaussian = complex_to_real(gaussian) + basis_matrix[i, :] = real_gaussian + return basis_matrix + + def do_fit(params, time_axis, real_unphased_data): + """ + This function performs the fitting. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param real_unphased_data: + :return: List of fitted data points. + """ + baseline_points = 16 + basis = make_basis(params, time_axis) + + weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() + return fitted_data + + def residual(params, time_axis, data): + """ + This function calculates the residual to be minimized by the least squares means method. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param data: FID to be fitted. + :return: residual values of baseline points. + """ + baseline_points = 16 + # unphase the data to make it pure absorptive + unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + + fitted_data = do_fit(params, time_axis, real_unphased_data) + res = fitted_data - real_unphased_data + + return res[baseline_points:-baseline_points] + + def fit_data(data, initial_params): + """ + This function takes an FID and a set of parameters contained in an lmfit Parameters object, + and fits the data using the least squares means method. + + :param data: FID to be fitted. + :param initial_params: lmfit Parameters object containing fitting parameters. + :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. + """ + baseline_points = 16 + fitting_result = lmfit.minimize(residual, + initial_params, + args=(data.time_axis(), data), + xtol=5e-3) + + unphased_data = phase_fid(data, + -fitting_result.params['phase0'], + -fitting_result.params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) + fitted_data = real_to_complex(real_fitted_data) + fitting_basis = make_basis(fitting_result.params, data.time_axis()) + fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + return fitting_weights, fitted_data, fitting_result + + # Convert lmfit parameters to model format + def parameters_to_model(parameters_obj, param_weights): + # metabolite_name_list = [] + # for param in parameters_obj.keys(): + # split = param.split('_') + # if len(split) == 2: + # if split[0] not in metabolite_name_list: + # metabolite_name_list.append(split[0]) + # Create dictionary for new model. + new_model = {} + for param_name, param in parameters_obj.items(): + name = param_name.split("_") + name1 = name[0] + if len(name) == 1: # i.e. phase + new_model[name1] = param.value + else: + name2 = name[1] + if name1 not in new_model: + new_model[name1] = {name2: param.value} + else: + new_model[name1][name2] = param.value + + nonlocal metabolite_name_list + for i, metabolite_name in enumerate(metabolite_name_list): + new_model[metabolite_name]["amplitude"] = param_weights[i] + + return new_model + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + # Check if all model input types are correct. + def check_errors(check_model): + # Allowed names and keys in the model. + allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] + allowed_keys = ["min", "max", "value", "phase", "amplitude"] + + # Scan model. + for name1, value1 in check_model.items(): + if type(value1) is not int and type(value1) is not float and type(value1) is not dict: + raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) + elif name1 not in allowed_names: + raise NameError("{} is not an allowed name.".format(name1)) + elif type(value1) is dict: # i.e. type(value) is not int + for name2, value2 in value1.items(): + if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ + type(value2) is not str: + raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." + .format(name1, name2)) + if type(value2) is dict: + for key in value2: + # Dictionary must have 'value' key. + if "value" not in value2: + raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) + # Dictionary can only have 'min,' 'max,' and 'value'. + if key not in allowed_keys: + raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) + + return + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # Do singlet fitting + def main(): + # Minimize and fit 31P data. + # Check for errors in model formatting. + check_errors(model) + + # Set list of metabolite names. + # nonlocal metabolite_name_list + # metabolite_name_list = get_metabolites(model) + + # Convert model to lmfit Parameters object. + parameters = model_to_parameters(model) + + # Set metabolite name list + nonlocal metabolite_name_list + metabolite_name_list = get_metabolites(model) + + # Fit data. + fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) + + # Convert fit parameters to model format. + final_model = parameters_to_model(fitted_results.params, fitted_weights) + # Get stderr values for each parameter. + stderr = get_errors(fitted_results) + + # Compile output into a dictionary. + return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} + return return_dict + + return main() From 5344abfab8cfa7e2f77d7c029fe75f7ea45b26c1 Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 15:28:37 -0400 Subject: [PATCH 08/39] Consolidated fit function into singlet, removed fit.py --- suspect/fitting/singlet.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index f659b3d..e72109f 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -337,16 +337,12 @@ def main(): check_errors(model) # Set list of metabolite names. - # nonlocal metabolite_name_list - # metabolite_name_list = get_metabolites(model) + nonlocal metabolite_name_list + metabolite_name_list = get_metabolites(model) # Convert model to lmfit Parameters object. parameters = model_to_parameters(model) - # Set metabolite name list - nonlocal metabolite_name_list - metabolite_name_list = get_metabolites(model) - # Fit data. fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) From 122c4e06c67ace372dd88a01a387c5c459ef201f Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 25 Jul 2016 17:30:50 -0400 Subject: [PATCH 09/39] Added fit file, moved amares fitting functions to singlet, converted a few values in denoising and singlet to int --- suspect/fitting/fit.py | 443 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 443 insertions(+) create mode 100644 suspect/fitting/fit.py diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py new file mode 100644 index 0000000..e622757 --- /dev/null +++ b/suspect/fitting/fit.py @@ -0,0 +1,443 @@ +""" +File Name: fit.py +Author: Sam Jiang +Purpose: fit_31p() function fits single 31P fid. +Interpreter: Anaconda 3.5.1 + +fit(fid, model) +Input: fid and model in hierarchical dictionary format +Output: dictionary with 'model', 'fit', and 'error' keys and corresponding values +""" + +import os +import numpy +import json +import lmfit +import array +import suspect + + +# Load model in. +def load(fin): + if not os.path.isfile(fin): + raise FileNotFoundError("{} not found.".format(fin)) + with open(fin, 'r+') as f: + model = json.load(f) + return model + + +# Save model or fit out. +def save(output, fout): + # Check if file and directory exists. + directory, name = os.path.split(fout) + if directory == '': + directory = os.getcwd() + if not os.path.isdir(directory): + os.makedirs(directory) + if name == '': + raise IsADirectoryError("{} is not a file name.".format(name)) + ftype = os.path.splitext(name)[1] + + # If model, save as json. + if type(output) is dict: + if ftype != ".json": + raise TypeError("Output file must be a JSON (.json) file.") + + with open(fout, 'w+') as f: + json.dump(output, f) + + # If fit, save as csv or txt. + elif type(output) is array.ArrayType: + if ftype != ".csv" or ftype != ".txt": + raise TypeError("Output file must be a CSV (.csv) or Text (.txt) file.") + + with open(fout, 'w+') as f: + first = True + for x in output.tolist(): + if first: + f.write("{}".format(x)) + first = False + else: + f.write(", {}".format(x)) + + +# Fit fid +# Input: fid, model +# Output: dictionary containing optimized model, fit data, standard errors +def fit(fid, model): + # MODEL + # Get list of metabolite names + # def get_metabolites(model_input): + # metabolites = [] + # for name, value in model_input.items(): + # if type(value) is dict: + # metabolites.append(name) + # return metabolites + + # Get standard errors from lmfit MinimizerResult object. + def get_errors(result): + errors = {} + for name, param in result.params.items(): + errors[name] = param.stderr + return errors + + # Convert lmfit parameters to model format + def parameters_to_model(parameters_obj, param_weights): + metabolite_name_list = [] + for param in parameters_obj.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + # Create dictionary for new model. + new_model = {} + for param_name, param in parameters_obj.items(): + name = param_name.split("_") + name1 = name[0] + if len(name) == 1: # i.e. phase + new_model[name1] = param.value + else: + name2 = name[1] + if name1 not in new_model: + new_model[name1] = {name2: param.value} + else: + new_model[name1][name2] = param.value + + for i, metabolite_name in enumerate(metabolite_name_list): + new_model[metabolite_name]["amplitude"] = param_weights[i] + + return new_model + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + # Check if all model input types are correct. + def check_errors(check_model): + # Allowed names and keys in the model. + allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] + allowed_keys = ["min", "max", "value", "phase", "amplitude"] + + # Scan model. + for name1, value1 in check_model.items(): + if type(value1) is not int and type(value1) is not float and type(value1) is not dict: + raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) + elif name1 not in allowed_names: + raise NameError("{} is not an allowed name.".format(name1)) + elif type(value1) is dict: # i.e. type(value) is not int + for name2, value2 in value1.items(): + if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ + type(value2) is not str: + raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." + .format(name1, name2)) + if type(value2) is dict: + for key in value2: + # Dictionary must have 'value' key. + if "value" not in value2: + raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) + # Dictionary can only have 'min,' 'max,' and 'value'. + if key not in allowed_keys: + raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) + + return + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # MAIN + def main(): + # Minimize and fit 31P data. + # Check for errors in model formatting. + check_errors(model) + + # Set list of metabolite names. + # nonlocal metabolite_name_list + # metabolite_name_list = get_metabolites(model) + + # Convert model to lmfit Parameters object. + parameters = model_to_parameters(model) + + # Fit data. + fitted_weights, fitted_data, fitted_results = suspect.fitting.singlet.fit_data(fid, parameters) + + # Convert fit parameters to model format. + final_model = parameters_to_model(fitted_results.params, fitted_weights) + # Get stderr values for each parameter. + stderr = get_errors(fitted_results) + + # Compile output into a dictionary. + return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} + + return return_dict + + if __name__ == "__main__": + return main() + + +# Test fit function +def test_fit(_plot): + # Test file and model + testfile = "test_timecourse.csv" + modelfile = "model.json" + model = load(modelfile) + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + def fake_spectrum(params, time_axis, weights): + basis = suspect.fitting.singlet.make_basis(params, time_axis) + real_data = numpy.array(numpy.dot(weights, basis)).squeeze() + complex_data = suspect.fitting.singlet.real_to_complex(real_data) + return suspect.MRSData(complex_data, dt, f0, te=0, ppm0=0) + + # Generate fake noise (Testing only) + def generate_noise(n, snr): + # Added (n * 0) to fix unused variable warning. + noise = (n * 0) + (numpy.random.randn(4096) + 1j * numpy.random.randn(4096)) * 6.4e-6 / snr + return noise + + # Generate fake fid (Testing only) + def generate_fid(ground_truth, snr, initial_params): + metabolite_name_list = [] + for param in initial_params.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + + peaks = {"pcr": 1, + "atpc": 0.3, + "atpb": 0.3, + "atpa": 0.3, + "pi": 1, + "pme": 0.05, + "pde": 0.05} + amps = [] + for metabolite in metabolite_name_list: + amps.append(peaks[metabolite]) + + noisy_sequence = numpy.zeros((len(ground_truth), np), 'complex') + noisy_sequence = suspect.MRSData(noisy_sequence, dt, f0, te=0, ppm0=0) + + # Populate the noisy_sequence with data. + generated_fid = fake_spectrum(initial_params, noisy_sequence.time_axis(), amps) + # [1, 0.3, 0.3, 0.3, 1, 0.05, 0.05] + noisy_fid = generated_fid + generate_noise(np, snr) + + return generated_fid, noisy_fid + + # Generate fake FID. + parameters = model_to_parameters(model) + sw = 6e3 + dt = 1.0 / sw + np = 4096 + f0 = 49.885802 + fake_fid, noisy_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid + + # Test fit(). + fit_results = fit(fake_fid, model) + + # Plot fit, noisy_fid, and fake_fid. + if _plot: + from matplotlib import pyplot + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noisy_fid))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) + pyplot.show() + + +test = False +plot = True +if test: + print("Running test.") + test_fit(plot) + print("Test complete.") + From 799f7e4147adc115c3ff4514ed2c0cc9bf0a511d Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 14:22:23 -0400 Subject: [PATCH 10/39] Added fit file, moved amares fitting functions to singlet, converted a few values in denoising and singlet to int --- suspect/fitting/fit.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py index e622757..fabce0b 100644 --- a/suspect/fitting/fit.py +++ b/suspect/fitting/fit.py @@ -62,9 +62,14 @@ def save(output, fout): # Fit fid -# Input: fid, model -# Output: dictionary containing optimized model, fit data, standard errors def fit(fid, model): + """ + Fit fid with model parameters. + + :param fid: MRSData object of FID to be fit + :param model: dictionary model of fit parameters + :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] + """ # MODEL # Get list of metabolite names # def get_metabolites(model_input): @@ -267,10 +272,9 @@ def main(): # Test fit function -def test_fit(_plot): +def test_fit(_plot, testfile, modelfile): # Test file and model - testfile = "test_timecourse.csv" - modelfile = "model.json" + model = load(modelfile) # Calculate references to determine order for Parameters. @@ -420,24 +424,25 @@ def generate_fid(ground_truth, snr, initial_params): dt = 1.0 / sw np = 4096 f0 = 49.885802 - fake_fid, noisy_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid + fake_fid, noise_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid # Test fit(). fit_results = fit(fake_fid, model) - # Plot fit, noisy_fid, and fake_fid. + # Plot fit, noise_fid, and fake_fid. if _plot: from matplotlib import pyplot pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noisy_fid))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noise_fid))) pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) pyplot.show() test = False -plot = True +plot = False if test: print("Running test.") - test_fit(plot) + test_file = "/home/sam/Desktop/amares/test_timecourse.csv" + test_model = "/home/sam/Desktop/amares/model.json" + test_fit(plot, test_file, test_model) print("Test complete.") - From 3db6696fc458d20bd4d14b6f3430433d80afc48d Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 15:22:02 -0400 Subject: [PATCH 11/39] Consolidated fit function into singlet, removed fit.py --- suspect/fitting/fit.py | 448 ------------------------------------- suspect/fitting/singlet.py | 8 +- 2 files changed, 6 insertions(+), 450 deletions(-) delete mode 100644 suspect/fitting/fit.py diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py deleted file mode 100644 index fabce0b..0000000 --- a/suspect/fitting/fit.py +++ /dev/null @@ -1,448 +0,0 @@ -""" -File Name: fit.py -Author: Sam Jiang -Purpose: fit_31p() function fits single 31P fid. -Interpreter: Anaconda 3.5.1 - -fit(fid, model) -Input: fid and model in hierarchical dictionary format -Output: dictionary with 'model', 'fit', and 'error' keys and corresponding values -""" - -import os -import numpy -import json -import lmfit -import array -import suspect - - -# Load model in. -def load(fin): - if not os.path.isfile(fin): - raise FileNotFoundError("{} not found.".format(fin)) - with open(fin, 'r+') as f: - model = json.load(f) - return model - - -# Save model or fit out. -def save(output, fout): - # Check if file and directory exists. - directory, name = os.path.split(fout) - if directory == '': - directory = os.getcwd() - if not os.path.isdir(directory): - os.makedirs(directory) - if name == '': - raise IsADirectoryError("{} is not a file name.".format(name)) - ftype = os.path.splitext(name)[1] - - # If model, save as json. - if type(output) is dict: - if ftype != ".json": - raise TypeError("Output file must be a JSON (.json) file.") - - with open(fout, 'w+') as f: - json.dump(output, f) - - # If fit, save as csv or txt. - elif type(output) is array.ArrayType: - if ftype != ".csv" or ftype != ".txt": - raise TypeError("Output file must be a CSV (.csv) or Text (.txt) file.") - - with open(fout, 'w+') as f: - first = True - for x in output.tolist(): - if first: - f.write("{}".format(x)) - first = False - else: - f.write(", {}".format(x)) - - -# Fit fid -def fit(fid, model): - """ - Fit fid with model parameters. - - :param fid: MRSData object of FID to be fit - :param model: dictionary model of fit parameters - :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] - """ - # MODEL - # Get list of metabolite names - # def get_metabolites(model_input): - # metabolites = [] - # for name, value in model_input.items(): - # if type(value) is dict: - # metabolites.append(name) - # return metabolites - - # Get standard errors from lmfit MinimizerResult object. - def get_errors(result): - errors = {} - for name, param in result.params.items(): - errors[name] = param.stderr - return errors - - # Convert lmfit parameters to model format - def parameters_to_model(parameters_obj, param_weights): - metabolite_name_list = [] - for param in parameters_obj.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - # Create dictionary for new model. - new_model = {} - for param_name, param in parameters_obj.items(): - name = param_name.split("_") - name1 = name[0] - if len(name) == 1: # i.e. phase - new_model[name1] = param.value - else: - name2 = name[1] - if name1 not in new_model: - new_model[name1] = {name2: param.value} - else: - new_model[name1][name2] = param.value - - for i, metabolite_name in enumerate(metabolite_name_list): - new_model[metabolite_name]["amplitude"] = param_weights[i] - - return new_model - - # Convert initial model to lmfit parameters. - def model_to_parameters(model_dict): - lmfit_parameters = lmfit.Parameters() - params = [] - ordered_params = [] - # Calculate dependencies/references for each parameter. - depend_dict = calculate_dependencies(model_dict) - - # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: - # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): - # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) - value = None - vary = None - lmfit_min = None - lmfit_max = None - expr = None - if type(value2) is int: - value = value2 - elif type(value2) is str: - expr = value2 - if type(value2) is dict: - if "value" in value2: - value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] - # Add parameter object with defined parameters. - params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) - - # Order parameters based on dependencies. - in_oparams = [] - while len(params) > 0: - front = params.pop(0) - name = front[0] - # If no dependencies, add parameter to list and mark parameter as added. - if name not in depend_dict or depend_dict[name] is None: - ordered_params.append(front) - in_oparams.append(name) - else: - dependencies_present = True - for dependency in depend_dict[name]: - # If dependency not yet added, mark parameter to move to back of queue. - if dependency not in in_oparams: - dependencies_present = False - # If all dependencies present, add parameter to list and mark parameter as added. - if dependencies_present: - ordered_params.append(front) - in_oparams.append(name) - # If dependencies missing, move parameter to back of queue. - else: - params.append(front) - - # Convert all parameters to lmfit Parameter objects. - lmfit_parameters.add_many(*ordered_params) - - return lmfit_parameters - - # Check if all model input types are correct. - def check_errors(check_model): - # Allowed names and keys in the model. - allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] - allowed_keys = ["min", "max", "value", "phase", "amplitude"] - - # Scan model. - for name1, value1 in check_model.items(): - if type(value1) is not int and type(value1) is not float and type(value1) is not dict: - raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) - elif name1 not in allowed_names: - raise NameError("{} is not an allowed name.".format(name1)) - elif type(value1) is dict: # i.e. type(value) is not int - for name2, value2 in value1.items(): - if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ - type(value2) is not str: - raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." - .format(name1, name2)) - if type(value2) is dict: - for key in value2: - # Dictionary must have 'value' key. - if "value" not in value2: - raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) - # Dictionary can only have 'min,' 'max,' and 'value'. - if key not in allowed_keys: - raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) - - return - - # Calculate references to determine order for Parameters. - def calculate_dependencies(unordered_model): - dependencies = {} # (name, [dependencies]) - - # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None - - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) - dependencies[lmfit_name] = [] - for depend in dependencies: - if depend in value2: - dependencies[lmfit_name].append(depend) - - # Check for circular dependencies. - for name, dependents in dependencies.items(): - if type(dependents) is list: - for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." - .format(name, dependent)) - - return dependencies - - # MAIN - def main(): - # Minimize and fit 31P data. - # Check for errors in model formatting. - check_errors(model) - - # Set list of metabolite names. - # nonlocal metabolite_name_list - # metabolite_name_list = get_metabolites(model) - - # Convert model to lmfit Parameters object. - parameters = model_to_parameters(model) - - # Fit data. - fitted_weights, fitted_data, fitted_results = suspect.fitting.singlet.fit_data(fid, parameters) - - # Convert fit parameters to model format. - final_model = parameters_to_model(fitted_results.params, fitted_weights) - # Get stderr values for each parameter. - stderr = get_errors(fitted_results) - - # Compile output into a dictionary. - return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} - - return return_dict - - if __name__ == "__main__": - return main() - - -# Test fit function -def test_fit(_plot, testfile, modelfile): - # Test file and model - - model = load(modelfile) - - # Calculate references to determine order for Parameters. - def calculate_dependencies(unordered_model): - dependencies = {} # (name, [dependencies]) - - # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None - - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) - dependencies[lmfit_name] = [] - for depend in dependencies: - if depend in value2: - dependencies[lmfit_name].append(depend) - - # Check for circular dependencies. - for name, dependents in dependencies.items(): - if type(dependents) is list: - for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." - .format(name, dependent)) - - return dependencies - - # Convert initial model to lmfit parameters. - def model_to_parameters(model_dict): - lmfit_parameters = lmfit.Parameters() - params = [] - ordered_params = [] - # Calculate dependencies/references for each parameter. - depend_dict = calculate_dependencies(model_dict) - - # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: - # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): - # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) - value = None - vary = None - lmfit_min = None - lmfit_max = None - expr = None - if type(value2) is int: - value = value2 - elif type(value2) is str: - expr = value2 - if type(value2) is dict: - if "value" in value2: - value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] - # Add parameter object with defined parameters. - params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) - - # Order parameters based on dependencies. - in_oparams = [] - while len(params) > 0: - front = params.pop(0) - name = front[0] - # If no dependencies, add parameter to list and mark parameter as added. - if name not in depend_dict or depend_dict[name] is None: - ordered_params.append(front) - in_oparams.append(name) - else: - dependencies_present = True - for dependency in depend_dict[name]: - # If dependency not yet added, mark parameter to move to back of queue. - if dependency not in in_oparams: - dependencies_present = False - # If all dependencies present, add parameter to list and mark parameter as added. - if dependencies_present: - ordered_params.append(front) - in_oparams.append(name) - # If dependencies missing, move parameter to back of queue. - else: - params.append(front) - - # Convert all parameters to lmfit Parameter objects. - lmfit_parameters.add_many(*ordered_params) - - return lmfit_parameters - - def fake_spectrum(params, time_axis, weights): - basis = suspect.fitting.singlet.make_basis(params, time_axis) - real_data = numpy.array(numpy.dot(weights, basis)).squeeze() - complex_data = suspect.fitting.singlet.real_to_complex(real_data) - return suspect.MRSData(complex_data, dt, f0, te=0, ppm0=0) - - # Generate fake noise (Testing only) - def generate_noise(n, snr): - # Added (n * 0) to fix unused variable warning. - noise = (n * 0) + (numpy.random.randn(4096) + 1j * numpy.random.randn(4096)) * 6.4e-6 / snr - return noise - - # Generate fake fid (Testing only) - def generate_fid(ground_truth, snr, initial_params): - metabolite_name_list = [] - for param in initial_params.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - - peaks = {"pcr": 1, - "atpc": 0.3, - "atpb": 0.3, - "atpa": 0.3, - "pi": 1, - "pme": 0.05, - "pde": 0.05} - amps = [] - for metabolite in metabolite_name_list: - amps.append(peaks[metabolite]) - - noisy_sequence = numpy.zeros((len(ground_truth), np), 'complex') - noisy_sequence = suspect.MRSData(noisy_sequence, dt, f0, te=0, ppm0=0) - - # Populate the noisy_sequence with data. - generated_fid = fake_spectrum(initial_params, noisy_sequence.time_axis(), amps) - # [1, 0.3, 0.3, 0.3, 1, 0.05, 0.05] - noisy_fid = generated_fid + generate_noise(np, snr) - - return generated_fid, noisy_fid - - # Generate fake FID. - parameters = model_to_parameters(model) - sw = 6e3 - dt = 1.0 / sw - np = 4096 - f0 = 49.885802 - fake_fid, noise_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid - - # Test fit(). - fit_results = fit(fake_fid, model) - - # Plot fit, noise_fid, and fake_fid. - if _plot: - from matplotlib import pyplot - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noise_fid))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) - pyplot.show() - - -test = False -plot = False -if test: - print("Running test.") - test_file = "/home/sam/Desktop/amares/test_timecourse.csv" - test_model = "/home/sam/Desktop/amares/model.json" - test_fit(plot, test_file, test_model) - print("Test complete.") diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index e72109f..f659b3d 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -337,12 +337,16 @@ def main(): check_errors(model) # Set list of metabolite names. - nonlocal metabolite_name_list - metabolite_name_list = get_metabolites(model) + # nonlocal metabolite_name_list + # metabolite_name_list = get_metabolites(model) # Convert model to lmfit Parameters object. parameters = model_to_parameters(model) + # Set metabolite name list + nonlocal metabolite_name_list + metabolite_name_list = get_metabolites(model) + # Fit data. fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) From d1e4c4c5e9c5c62a26eec6107ce22703ad01a986 Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 15:28:37 -0400 Subject: [PATCH 12/39] Consolidated fit function into singlet, removed fit.py --- suspect/fitting/singlet.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index f659b3d..e72109f 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -337,16 +337,12 @@ def main(): check_errors(model) # Set list of metabolite names. - # nonlocal metabolite_name_list - # metabolite_name_list = get_metabolites(model) + nonlocal metabolite_name_list + metabolite_name_list = get_metabolites(model) # Convert model to lmfit Parameters object. parameters = model_to_parameters(model) - # Set metabolite name list - nonlocal metabolite_name_list - metabolite_name_list = get_metabolites(model) - # Fit data. fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) From 16fce6322313071d620bdf9fa00a63582d468bf5 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 25 Jul 2016 17:30:50 -0400 Subject: [PATCH 13/39] Added fit file, moved amares fitting functions to singlet, converted a few values in denoising and singlet to int --- suspect/fitting/fit.py | 443 ++++++++++++++++++++++++++++++++ suspect/fitting/singlet.py | 187 ++++++++------ suspect/processing/denoising.py | 6 +- 3 files changed, 555 insertions(+), 81 deletions(-) create mode 100644 suspect/fitting/fit.py diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py new file mode 100644 index 0000000..e622757 --- /dev/null +++ b/suspect/fitting/fit.py @@ -0,0 +1,443 @@ +""" +File Name: fit.py +Author: Sam Jiang +Purpose: fit_31p() function fits single 31P fid. +Interpreter: Anaconda 3.5.1 + +fit(fid, model) +Input: fid and model in hierarchical dictionary format +Output: dictionary with 'model', 'fit', and 'error' keys and corresponding values +""" + +import os +import numpy +import json +import lmfit +import array +import suspect + + +# Load model in. +def load(fin): + if not os.path.isfile(fin): + raise FileNotFoundError("{} not found.".format(fin)) + with open(fin, 'r+') as f: + model = json.load(f) + return model + + +# Save model or fit out. +def save(output, fout): + # Check if file and directory exists. + directory, name = os.path.split(fout) + if directory == '': + directory = os.getcwd() + if not os.path.isdir(directory): + os.makedirs(directory) + if name == '': + raise IsADirectoryError("{} is not a file name.".format(name)) + ftype = os.path.splitext(name)[1] + + # If model, save as json. + if type(output) is dict: + if ftype != ".json": + raise TypeError("Output file must be a JSON (.json) file.") + + with open(fout, 'w+') as f: + json.dump(output, f) + + # If fit, save as csv or txt. + elif type(output) is array.ArrayType: + if ftype != ".csv" or ftype != ".txt": + raise TypeError("Output file must be a CSV (.csv) or Text (.txt) file.") + + with open(fout, 'w+') as f: + first = True + for x in output.tolist(): + if first: + f.write("{}".format(x)) + first = False + else: + f.write(", {}".format(x)) + + +# Fit fid +# Input: fid, model +# Output: dictionary containing optimized model, fit data, standard errors +def fit(fid, model): + # MODEL + # Get list of metabolite names + # def get_metabolites(model_input): + # metabolites = [] + # for name, value in model_input.items(): + # if type(value) is dict: + # metabolites.append(name) + # return metabolites + + # Get standard errors from lmfit MinimizerResult object. + def get_errors(result): + errors = {} + for name, param in result.params.items(): + errors[name] = param.stderr + return errors + + # Convert lmfit parameters to model format + def parameters_to_model(parameters_obj, param_weights): + metabolite_name_list = [] + for param in parameters_obj.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + # Create dictionary for new model. + new_model = {} + for param_name, param in parameters_obj.items(): + name = param_name.split("_") + name1 = name[0] + if len(name) == 1: # i.e. phase + new_model[name1] = param.value + else: + name2 = name[1] + if name1 not in new_model: + new_model[name1] = {name2: param.value} + else: + new_model[name1][name2] = param.value + + for i, metabolite_name in enumerate(metabolite_name_list): + new_model[metabolite_name]["amplitude"] = param_weights[i] + + return new_model + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + # Check if all model input types are correct. + def check_errors(check_model): + # Allowed names and keys in the model. + allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] + allowed_keys = ["min", "max", "value", "phase", "amplitude"] + + # Scan model. + for name1, value1 in check_model.items(): + if type(value1) is not int and type(value1) is not float and type(value1) is not dict: + raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) + elif name1 not in allowed_names: + raise NameError("{} is not an allowed name.".format(name1)) + elif type(value1) is dict: # i.e. type(value) is not int + for name2, value2 in value1.items(): + if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ + type(value2) is not str: + raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." + .format(name1, name2)) + if type(value2) is dict: + for key in value2: + # Dictionary must have 'value' key. + if "value" not in value2: + raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) + # Dictionary can only have 'min,' 'max,' and 'value'. + if key not in allowed_keys: + raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) + + return + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # MAIN + def main(): + # Minimize and fit 31P data. + # Check for errors in model formatting. + check_errors(model) + + # Set list of metabolite names. + # nonlocal metabolite_name_list + # metabolite_name_list = get_metabolites(model) + + # Convert model to lmfit Parameters object. + parameters = model_to_parameters(model) + + # Fit data. + fitted_weights, fitted_data, fitted_results = suspect.fitting.singlet.fit_data(fid, parameters) + + # Convert fit parameters to model format. + final_model = parameters_to_model(fitted_results.params, fitted_weights) + # Get stderr values for each parameter. + stderr = get_errors(fitted_results) + + # Compile output into a dictionary. + return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} + + return return_dict + + if __name__ == "__main__": + return main() + + +# Test fit function +def test_fit(_plot): + # Test file and model + testfile = "test_timecourse.csv" + modelfile = "model.json" + model = load(modelfile) + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + def fake_spectrum(params, time_axis, weights): + basis = suspect.fitting.singlet.make_basis(params, time_axis) + real_data = numpy.array(numpy.dot(weights, basis)).squeeze() + complex_data = suspect.fitting.singlet.real_to_complex(real_data) + return suspect.MRSData(complex_data, dt, f0, te=0, ppm0=0) + + # Generate fake noise (Testing only) + def generate_noise(n, snr): + # Added (n * 0) to fix unused variable warning. + noise = (n * 0) + (numpy.random.randn(4096) + 1j * numpy.random.randn(4096)) * 6.4e-6 / snr + return noise + + # Generate fake fid (Testing only) + def generate_fid(ground_truth, snr, initial_params): + metabolite_name_list = [] + for param in initial_params.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + + peaks = {"pcr": 1, + "atpc": 0.3, + "atpb": 0.3, + "atpa": 0.3, + "pi": 1, + "pme": 0.05, + "pde": 0.05} + amps = [] + for metabolite in metabolite_name_list: + amps.append(peaks[metabolite]) + + noisy_sequence = numpy.zeros((len(ground_truth), np), 'complex') + noisy_sequence = suspect.MRSData(noisy_sequence, dt, f0, te=0, ppm0=0) + + # Populate the noisy_sequence with data. + generated_fid = fake_spectrum(initial_params, noisy_sequence.time_axis(), amps) + # [1, 0.3, 0.3, 0.3, 1, 0.05, 0.05] + noisy_fid = generated_fid + generate_noise(np, snr) + + return generated_fid, noisy_fid + + # Generate fake FID. + parameters = model_to_parameters(model) + sw = 6e3 + dt = 1.0 / sw + np = 4096 + f0 = 49.885802 + fake_fid, noisy_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid + + # Test fit(). + fit_results = fit(fake_fid, model) + + # Plot fit, noisy_fid, and fake_fid. + if _plot: + from matplotlib import pyplot + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noisy_fid))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) + pyplot.show() + + +test = False +plot = True +if test: + print("Running test.") + test_fit(plot) + print("Test complete.") + diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 39a15e7..6e19dd1 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -1,5 +1,6 @@ import lmfit import numpy +import scipy.optimize import numbers import suspect @@ -32,7 +33,7 @@ def real_to_complex(real_fid): :param real_fid: the real FID to be converted to complex. :return: the complex version of the FID """ - np = real_fid.shape[0] / 2 + np = int(real_fid.shape[0] / 2) complex_fid = numpy.zeros(np, 'complex') complex_fid[:] = real_fid[:np] @@ -42,83 +43,113 @@ def real_to_complex(real_fid): return complex_fid -def gaussian_fid(x, amplitude=1, frequency=0.0, phase=0.0, fwhm=1.0): +# metabolite_name_list = [] + + +def phase_fid(fid_in, phase0, phase1): + """ + This function performs a Fourier Transform on the FID to shift it into phase. + + :param fid_in: FID to be fitted. + :param phase1: phase1 value. + :param phase0: phase0 value. + :return: + """ + spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) + np = fid_in.np + phase_shift = phase0 + phase1 * numpy.linspace(-np / 2, np / 2, np, endpoint=False) + phased_spectrum = spectrum * numpy.exp(1j * phase_shift) + return fid_in.inherit(numpy.fft.ifft(numpy.fft.ifftshift(phased_spectrum))) + + +def make_basis(params, time_axis): + """ + This function generates a basis set. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :return: a matrix containing the generated basis set. + """ + metabolite_name_list = [] + for param in params.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + + basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) + for i, metabolite_name in enumerate(metabolite_name_list): + gaussian = suspect.basis.gaussian(time_axis, + params["{}_frequency".format(metabolite_name)], + params["{}_phase".format(metabolite_name)].value, + params["{}_width".format(metabolite_name)]) + real_gaussian = complex_to_real(gaussian) + basis_matrix[i, :] = real_gaussian + return basis_matrix + + +def do_fit(params, time_axis, real_unphased_data): + """ + This function performs the fitting. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param real_unphased_data: + :return: List of fitted data points. """ - Generates a Gaussian FID for use with the lmfit GaussianFidModel class. The - helper function complex_to_real is used to convert the FID to a real form. - - :param x: the time axis for the fid data - :param amplitude: the amplitude of the Gaussian - :param frequency: the frequency of the Gaussian in Hz - :param phase: the phase in radians - :param fwhm: the full width at half maximum of the Gaussian - :return: a real FID describing a Gaussian relaxation + baseline_points = 16 + basis = make_basis(params, time_axis) + + weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() + return fitted_data + + +def residual(params, time_axis, data): + """ + This function calculates the residual to be minimized by the least squares means method. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param data: FID to be fitted. + :return: residual values of baseline points. """ + baseline_points = 16 + # unphase the data to make it pure absorptive + unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) + real_unphased_data = complex_to_real(unphased_data) - complex_fid = amplitude * suspect.basis.gaussian(x, frequency, phase, fwhm) - return complex_to_real(complex_fid) - - -class GaussianFidModel(lmfit.models.Model): - def __init__(self, *args, **kwargs): - super(GaussianFidModel, self).__init__(gaussian_fid, *args, **kwargs) - self.set_param_hint('fwhm', min=0) - self.set_param_hint('amplitude', min=0) - self.set_param_hint('phase', min=-numpy.pi, max=numpy.pi) - - def guess(self, data=None, **kwargs): - return self.make_params() - - def copy(self, **kwargs): - raise NotImplementedError - - -class Model: - def __init__(self, peaks): - self.model = None - self.params = lmfit.Parameters() - for peak_name, peak_params in peaks.items(): - - current_peak_model = GaussianFidModel(prefix="{}".format(peak_name)) - self.params.update(current_peak_model.make_params()) - - for param_name, param_data in peak_params.items(): - # the - full_name = "{0}{1}".format(peak_name, param_name) - - if full_name in self.params: - if type(param_data) is str: - self.params[full_name].set(expr=param_data) - elif type(param_data) is dict: - if "value" in param_data: - self.params[full_name].set(value=param_data["value"]) - if "min" in param_data: - self.params[full_name].set(min=param_data["min"]) - if "max" in param_data: - self.params[full_name].set(max=param_data["max"]) - if "expr" in param_data: - self.params[full_name].set(expr=param_data) - elif type(param_data) is numbers.Number: - self.params[full_name].set(param_data) - - if self.model is None: - self.model = current_peak_model - else: - self.model += current_peak_model - - def fit(self, data): - fit_result = self.model.fit(complex_to_real(data), x=data.time_axis(), params=self.params) - result_params = {} - for component in self.model.components: - component_name = component.prefix - result_params[component.prefix] = {} - for param in component.make_params(): - param_name = str(param).replace(component_name, "") - result_params[component.prefix][param_name] = fit_result.params[param].value - fit_curve = real_to_complex(fit_result.best_fit) - fit_components = {k: real_to_complex(v) for k, v in fit_result.eval_components().items()} - return { - "params": result_params, - "fit": fit_curve, - "fit_components": fit_components - } + fitted_data = do_fit(params, time_axis, real_unphased_data) + res = fitted_data - real_unphased_data + + return res[baseline_points:-baseline_points] + + +def fit_data(data, initial_params): + """ + This function takes an FID and a set of parameters contained in an lmfit Parameters object, + and fits the data using the least squares means method. + + :param data: FID to be fitted. + :param initial_params: lmfit Parameters object containing fitting parameters. + :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. + """ + baseline_points = 16 + fitting_result = lmfit.minimize(residual, + initial_params, + args=(data.time_axis(), data), + xtol=5e-3) + + unphased_data = phase_fid(data, + -fitting_result.params['phase0'], + -fitting_result.params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) + fitted_data = real_to_complex(real_fitted_data) + fitting_basis = make_basis(fitting_result.params, data.time_axis()) + fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + return fitting_weights, fitted_data, fitting_result diff --git a/suspect/processing/denoising.py b/suspect/processing/denoising.py index 60b0484..7b85dee 100644 --- a/suspect/processing/denoising.py +++ b/suspect/processing/denoising.py @@ -14,7 +14,7 @@ def _pad(input_signal, length, average=10): :return: """ padded_input_signal = numpy.zeros(length, input_signal.dtype) - start_offset = (len(padded_input_signal) - len(input_signal)) / 2. + start_offset = int((len(padded_input_signal) - len(input_signal)) / 2) padded_input_signal[:start_offset] = numpy.average(input_signal[0:average]) padded_input_signal[start_offset:(start_offset + len(input_signal))] = input_signal[:] padded_input_signal[(start_offset + len(input_signal)):] = numpy.average(input_signal[-average:]) @@ -38,7 +38,7 @@ def sliding_gaussian(input_signal, window_width): window /= numpy.sum(window) # pad the signal to cover half the window width on each side padded_input = _pad(input_signal, len(input_signal) + window_width - 1) - result = numpy.zeros(len(input_signal)) + result = numpy.zeros_like(input_signal) for i in range(len(input_signal)): result[i] = numpy.dot(window, padded_input[i:(i + window_width)]) return result @@ -63,7 +63,7 @@ def svd(input_signal, rank): s[rank:] = 0.0 recon = U * numpy.diag(s) * V - result = numpy.zeros(len(input_signal)) + result = numpy.zeros_like(input_signal) for i in range(len(input_signal)): count = 0 for j in range(matrix_height): From ae7cf807aff9cfb232a565abc5a13871fee703dc Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 14:22:23 -0400 Subject: [PATCH 14/39] Added fit file, moved amares fitting functions to singlet, converted a few values in denoising and singlet to int --- suspect/fitting/fit.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py index e622757..fabce0b 100644 --- a/suspect/fitting/fit.py +++ b/suspect/fitting/fit.py @@ -62,9 +62,14 @@ def save(output, fout): # Fit fid -# Input: fid, model -# Output: dictionary containing optimized model, fit data, standard errors def fit(fid, model): + """ + Fit fid with model parameters. + + :param fid: MRSData object of FID to be fit + :param model: dictionary model of fit parameters + :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] + """ # MODEL # Get list of metabolite names # def get_metabolites(model_input): @@ -267,10 +272,9 @@ def main(): # Test fit function -def test_fit(_plot): +def test_fit(_plot, testfile, modelfile): # Test file and model - testfile = "test_timecourse.csv" - modelfile = "model.json" + model = load(modelfile) # Calculate references to determine order for Parameters. @@ -420,24 +424,25 @@ def generate_fid(ground_truth, snr, initial_params): dt = 1.0 / sw np = 4096 f0 = 49.885802 - fake_fid, noisy_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid + fake_fid, noise_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid # Test fit(). fit_results = fit(fake_fid, model) - # Plot fit, noisy_fid, and fake_fid. + # Plot fit, noise_fid, and fake_fid. if _plot: from matplotlib import pyplot pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noisy_fid))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noise_fid))) pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) pyplot.show() test = False -plot = True +plot = False if test: print("Running test.") - test_fit(plot) + test_file = "/home/sam/Desktop/amares/test_timecourse.csv" + test_model = "/home/sam/Desktop/amares/model.json" + test_fit(plot, test_file, test_model) print("Test complete.") - From cdbd33b8bb507d57d0d3c6dc623ed1d7ee56cedd Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 15:22:02 -0400 Subject: [PATCH 15/39] Consolidated fit function into singlet, removed fit.py --- suspect/fitting/fit.py | 448 ------------------------------------- suspect/fitting/singlet.py | 415 +++++++++++++++++++++++++--------- 2 files changed, 311 insertions(+), 552 deletions(-) delete mode 100644 suspect/fitting/fit.py diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py deleted file mode 100644 index fabce0b..0000000 --- a/suspect/fitting/fit.py +++ /dev/null @@ -1,448 +0,0 @@ -""" -File Name: fit.py -Author: Sam Jiang -Purpose: fit_31p() function fits single 31P fid. -Interpreter: Anaconda 3.5.1 - -fit(fid, model) -Input: fid and model in hierarchical dictionary format -Output: dictionary with 'model', 'fit', and 'error' keys and corresponding values -""" - -import os -import numpy -import json -import lmfit -import array -import suspect - - -# Load model in. -def load(fin): - if not os.path.isfile(fin): - raise FileNotFoundError("{} not found.".format(fin)) - with open(fin, 'r+') as f: - model = json.load(f) - return model - - -# Save model or fit out. -def save(output, fout): - # Check if file and directory exists. - directory, name = os.path.split(fout) - if directory == '': - directory = os.getcwd() - if not os.path.isdir(directory): - os.makedirs(directory) - if name == '': - raise IsADirectoryError("{} is not a file name.".format(name)) - ftype = os.path.splitext(name)[1] - - # If model, save as json. - if type(output) is dict: - if ftype != ".json": - raise TypeError("Output file must be a JSON (.json) file.") - - with open(fout, 'w+') as f: - json.dump(output, f) - - # If fit, save as csv or txt. - elif type(output) is array.ArrayType: - if ftype != ".csv" or ftype != ".txt": - raise TypeError("Output file must be a CSV (.csv) or Text (.txt) file.") - - with open(fout, 'w+') as f: - first = True - for x in output.tolist(): - if first: - f.write("{}".format(x)) - first = False - else: - f.write(", {}".format(x)) - - -# Fit fid -def fit(fid, model): - """ - Fit fid with model parameters. - - :param fid: MRSData object of FID to be fit - :param model: dictionary model of fit parameters - :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] - """ - # MODEL - # Get list of metabolite names - # def get_metabolites(model_input): - # metabolites = [] - # for name, value in model_input.items(): - # if type(value) is dict: - # metabolites.append(name) - # return metabolites - - # Get standard errors from lmfit MinimizerResult object. - def get_errors(result): - errors = {} - for name, param in result.params.items(): - errors[name] = param.stderr - return errors - - # Convert lmfit parameters to model format - def parameters_to_model(parameters_obj, param_weights): - metabolite_name_list = [] - for param in parameters_obj.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - # Create dictionary for new model. - new_model = {} - for param_name, param in parameters_obj.items(): - name = param_name.split("_") - name1 = name[0] - if len(name) == 1: # i.e. phase - new_model[name1] = param.value - else: - name2 = name[1] - if name1 not in new_model: - new_model[name1] = {name2: param.value} - else: - new_model[name1][name2] = param.value - - for i, metabolite_name in enumerate(metabolite_name_list): - new_model[metabolite_name]["amplitude"] = param_weights[i] - - return new_model - - # Convert initial model to lmfit parameters. - def model_to_parameters(model_dict): - lmfit_parameters = lmfit.Parameters() - params = [] - ordered_params = [] - # Calculate dependencies/references for each parameter. - depend_dict = calculate_dependencies(model_dict) - - # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: - # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): - # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) - value = None - vary = None - lmfit_min = None - lmfit_max = None - expr = None - if type(value2) is int: - value = value2 - elif type(value2) is str: - expr = value2 - if type(value2) is dict: - if "value" in value2: - value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] - # Add parameter object with defined parameters. - params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) - - # Order parameters based on dependencies. - in_oparams = [] - while len(params) > 0: - front = params.pop(0) - name = front[0] - # If no dependencies, add parameter to list and mark parameter as added. - if name not in depend_dict or depend_dict[name] is None: - ordered_params.append(front) - in_oparams.append(name) - else: - dependencies_present = True - for dependency in depend_dict[name]: - # If dependency not yet added, mark parameter to move to back of queue. - if dependency not in in_oparams: - dependencies_present = False - # If all dependencies present, add parameter to list and mark parameter as added. - if dependencies_present: - ordered_params.append(front) - in_oparams.append(name) - # If dependencies missing, move parameter to back of queue. - else: - params.append(front) - - # Convert all parameters to lmfit Parameter objects. - lmfit_parameters.add_many(*ordered_params) - - return lmfit_parameters - - # Check if all model input types are correct. - def check_errors(check_model): - # Allowed names and keys in the model. - allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] - allowed_keys = ["min", "max", "value", "phase", "amplitude"] - - # Scan model. - for name1, value1 in check_model.items(): - if type(value1) is not int and type(value1) is not float and type(value1) is not dict: - raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) - elif name1 not in allowed_names: - raise NameError("{} is not an allowed name.".format(name1)) - elif type(value1) is dict: # i.e. type(value) is not int - for name2, value2 in value1.items(): - if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ - type(value2) is not str: - raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." - .format(name1, name2)) - if type(value2) is dict: - for key in value2: - # Dictionary must have 'value' key. - if "value" not in value2: - raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) - # Dictionary can only have 'min,' 'max,' and 'value'. - if key not in allowed_keys: - raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) - - return - - # Calculate references to determine order for Parameters. - def calculate_dependencies(unordered_model): - dependencies = {} # (name, [dependencies]) - - # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None - - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) - dependencies[lmfit_name] = [] - for depend in dependencies: - if depend in value2: - dependencies[lmfit_name].append(depend) - - # Check for circular dependencies. - for name, dependents in dependencies.items(): - if type(dependents) is list: - for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." - .format(name, dependent)) - - return dependencies - - # MAIN - def main(): - # Minimize and fit 31P data. - # Check for errors in model formatting. - check_errors(model) - - # Set list of metabolite names. - # nonlocal metabolite_name_list - # metabolite_name_list = get_metabolites(model) - - # Convert model to lmfit Parameters object. - parameters = model_to_parameters(model) - - # Fit data. - fitted_weights, fitted_data, fitted_results = suspect.fitting.singlet.fit_data(fid, parameters) - - # Convert fit parameters to model format. - final_model = parameters_to_model(fitted_results.params, fitted_weights) - # Get stderr values for each parameter. - stderr = get_errors(fitted_results) - - # Compile output into a dictionary. - return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} - - return return_dict - - if __name__ == "__main__": - return main() - - -# Test fit function -def test_fit(_plot, testfile, modelfile): - # Test file and model - - model = load(modelfile) - - # Calculate references to determine order for Parameters. - def calculate_dependencies(unordered_model): - dependencies = {} # (name, [dependencies]) - - # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None - - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) - dependencies[lmfit_name] = [] - for depend in dependencies: - if depend in value2: - dependencies[lmfit_name].append(depend) - - # Check for circular dependencies. - for name, dependents in dependencies.items(): - if type(dependents) is list: - for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." - .format(name, dependent)) - - return dependencies - - # Convert initial model to lmfit parameters. - def model_to_parameters(model_dict): - lmfit_parameters = lmfit.Parameters() - params = [] - ordered_params = [] - # Calculate dependencies/references for each parameter. - depend_dict = calculate_dependencies(model_dict) - - # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: - # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): - # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) - value = None - vary = None - lmfit_min = None - lmfit_max = None - expr = None - if type(value2) is int: - value = value2 - elif type(value2) is str: - expr = value2 - if type(value2) is dict: - if "value" in value2: - value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] - # Add parameter object with defined parameters. - params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) - - # Order parameters based on dependencies. - in_oparams = [] - while len(params) > 0: - front = params.pop(0) - name = front[0] - # If no dependencies, add parameter to list and mark parameter as added. - if name not in depend_dict or depend_dict[name] is None: - ordered_params.append(front) - in_oparams.append(name) - else: - dependencies_present = True - for dependency in depend_dict[name]: - # If dependency not yet added, mark parameter to move to back of queue. - if dependency not in in_oparams: - dependencies_present = False - # If all dependencies present, add parameter to list and mark parameter as added. - if dependencies_present: - ordered_params.append(front) - in_oparams.append(name) - # If dependencies missing, move parameter to back of queue. - else: - params.append(front) - - # Convert all parameters to lmfit Parameter objects. - lmfit_parameters.add_many(*ordered_params) - - return lmfit_parameters - - def fake_spectrum(params, time_axis, weights): - basis = suspect.fitting.singlet.make_basis(params, time_axis) - real_data = numpy.array(numpy.dot(weights, basis)).squeeze() - complex_data = suspect.fitting.singlet.real_to_complex(real_data) - return suspect.MRSData(complex_data, dt, f0, te=0, ppm0=0) - - # Generate fake noise (Testing only) - def generate_noise(n, snr): - # Added (n * 0) to fix unused variable warning. - noise = (n * 0) + (numpy.random.randn(4096) + 1j * numpy.random.randn(4096)) * 6.4e-6 / snr - return noise - - # Generate fake fid (Testing only) - def generate_fid(ground_truth, snr, initial_params): - metabolite_name_list = [] - for param in initial_params.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - - peaks = {"pcr": 1, - "atpc": 0.3, - "atpb": 0.3, - "atpa": 0.3, - "pi": 1, - "pme": 0.05, - "pde": 0.05} - amps = [] - for metabolite in metabolite_name_list: - amps.append(peaks[metabolite]) - - noisy_sequence = numpy.zeros((len(ground_truth), np), 'complex') - noisy_sequence = suspect.MRSData(noisy_sequence, dt, f0, te=0, ppm0=0) - - # Populate the noisy_sequence with data. - generated_fid = fake_spectrum(initial_params, noisy_sequence.time_axis(), amps) - # [1, 0.3, 0.3, 0.3, 1, 0.05, 0.05] - noisy_fid = generated_fid + generate_noise(np, snr) - - return generated_fid, noisy_fid - - # Generate fake FID. - parameters = model_to_parameters(model) - sw = 6e3 - dt = 1.0 / sw - np = 4096 - f0 = 49.885802 - fake_fid, noise_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid - - # Test fit(). - fit_results = fit(fake_fid, model) - - # Plot fit, noise_fid, and fake_fid. - if _plot: - from matplotlib import pyplot - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noise_fid))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) - pyplot.show() - - -test = False -plot = False -if test: - print("Running test.") - test_file = "/home/sam/Desktop/amares/test_timecourse.csv" - test_model = "/home/sam/Desktop/amares/model.json" - test_fit(plot, test_file, test_model) - print("Test complete.") diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 6e19dd1..f659b3d 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -43,113 +43,320 @@ def real_to_complex(real_fid): return complex_fid -# metabolite_name_list = [] - - -def phase_fid(fid_in, phase0, phase1): - """ - This function performs a Fourier Transform on the FID to shift it into phase. - - :param fid_in: FID to be fitted. - :param phase1: phase1 value. - :param phase0: phase0 value. - :return: +def fit(fid, model): """ - spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) - np = fid_in.np - phase_shift = phase0 + phase1 * numpy.linspace(-np / 2, np / 2, np, endpoint=False) - phased_spectrum = spectrum * numpy.exp(1j * phase_shift) - return fid_in.inherit(numpy.fft.ifft(numpy.fft.ifftshift(phased_spectrum))) - + Fit fid with model parameters. -def make_basis(params, time_axis): + :param fid: MRSData object of FID to be fit + :param model: dictionary model of fit parameters + :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] """ - This function generates a basis set. - :param params: lmfit Parameters object containing fitting parameters. - :param time_axis: the time axis. - :return: a matrix containing the generated basis set. - """ + # List of metabolite names metabolite_name_list = [] - for param in params.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - - basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) - for i, metabolite_name in enumerate(metabolite_name_list): - gaussian = suspect.basis.gaussian(time_axis, - params["{}_frequency".format(metabolite_name)], - params["{}_phase".format(metabolite_name)].value, - params["{}_width".format(metabolite_name)]) - real_gaussian = complex_to_real(gaussian) - basis_matrix[i, :] = real_gaussian - return basis_matrix - - -def do_fit(params, time_axis, real_unphased_data): - """ - This function performs the fitting. - - :param params: lmfit Parameters object containing fitting parameters. - :param time_axis: the time axis. - :param real_unphased_data: - :return: List of fitted data points. - """ - baseline_points = 16 - basis = make_basis(params, time_axis) - - weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, - real_unphased_data[baseline_points:-baseline_points])[0] - - fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() - return fitted_data - -def residual(params, time_axis, data): - """ - This function calculates the residual to be minimized by the least squares means method. - - :param params: lmfit Parameters object containing fitting parameters. - :param time_axis: the time axis. - :param data: FID to be fitted. - :return: residual values of baseline points. - """ - baseline_points = 16 - # unphase the data to make it pure absorptive - unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) - real_unphased_data = complex_to_real(unphased_data) - - fitted_data = do_fit(params, time_axis, real_unphased_data) - res = fitted_data - real_unphased_data - - return res[baseline_points:-baseline_points] - - -def fit_data(data, initial_params): - """ - This function takes an FID and a set of parameters contained in an lmfit Parameters object, - and fits the data using the least squares means method. - - :param data: FID to be fitted. - :param initial_params: lmfit Parameters object containing fitting parameters. - :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. - """ - baseline_points = 16 - fitting_result = lmfit.minimize(residual, - initial_params, - args=(data.time_axis(), data), - xtol=5e-3) - - unphased_data = phase_fid(data, - -fitting_result.params['phase0'], - -fitting_result.params['phase1']) - real_unphased_data = complex_to_real(unphased_data) - real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) - fitted_data = real_to_complex(real_fitted_data) - fitting_basis = make_basis(fitting_result.params, data.time_axis()) - fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, - real_unphased_data[baseline_points:-baseline_points])[0] - - return fitting_weights, fitted_data, fitting_result + # Get list of metabolite names. + def get_metabolites(model_input): + metabolites = [] + for name, value in model_input.items(): + if type(value) is dict: + metabolites.append(name) + return metabolites + + # Get standard errors from lmfit MinimizerResult object. + def get_errors(result): + errors = {} + for name, param in result.params.items(): + errors[name] = param.stderr + return errors + + def phase_fid(fid_in, phase0, phase1): + """ + This function performs a Fourier Transform on the FID to shift it into phase. + + :param fid_in: FID to be fitted. + :param phase1: phase1 value. + :param phase0: phase0 value. + :return: + """ + spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) + np = fid_in.np + phase_shift = phase0 + phase1 * numpy.linspace(-np / 2, np / 2, np, endpoint=False) + phased_spectrum = spectrum * numpy.exp(1j * phase_shift) + return fid_in.inherit(numpy.fft.ifft(numpy.fft.ifftshift(phased_spectrum))) + + def make_basis(params, time_axis): + """ + This function generates a basis set. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :return: a matrix containing the generated basis set. + """ + # metabolite_name_list = [] + # for param in params.keys(): + # split = param.split('_') + # if len(split) == 2: + # if split[0] not in metabolite_name_list: + # metabolite_name_list.append(split[0]) + + basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) + for i, metabolite_name in enumerate(metabolite_name_list): + gaussian = suspect.basis.gaussian(time_axis, + params["{}_frequency".format(metabolite_name)], + params["{}_phase".format(metabolite_name)].value, + params["{}_width".format(metabolite_name)]) + real_gaussian = complex_to_real(gaussian) + basis_matrix[i, :] = real_gaussian + return basis_matrix + + def do_fit(params, time_axis, real_unphased_data): + """ + This function performs the fitting. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param real_unphased_data: + :return: List of fitted data points. + """ + baseline_points = 16 + basis = make_basis(params, time_axis) + + weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() + return fitted_data + + def residual(params, time_axis, data): + """ + This function calculates the residual to be minimized by the least squares means method. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param data: FID to be fitted. + :return: residual values of baseline points. + """ + baseline_points = 16 + # unphase the data to make it pure absorptive + unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + + fitted_data = do_fit(params, time_axis, real_unphased_data) + res = fitted_data - real_unphased_data + + return res[baseline_points:-baseline_points] + + def fit_data(data, initial_params): + """ + This function takes an FID and a set of parameters contained in an lmfit Parameters object, + and fits the data using the least squares means method. + + :param data: FID to be fitted. + :param initial_params: lmfit Parameters object containing fitting parameters. + :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. + """ + baseline_points = 16 + fitting_result = lmfit.minimize(residual, + initial_params, + args=(data.time_axis(), data), + xtol=5e-3) + + unphased_data = phase_fid(data, + -fitting_result.params['phase0'], + -fitting_result.params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) + fitted_data = real_to_complex(real_fitted_data) + fitting_basis = make_basis(fitting_result.params, data.time_axis()) + fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + return fitting_weights, fitted_data, fitting_result + + # Convert lmfit parameters to model format + def parameters_to_model(parameters_obj, param_weights): + # metabolite_name_list = [] + # for param in parameters_obj.keys(): + # split = param.split('_') + # if len(split) == 2: + # if split[0] not in metabolite_name_list: + # metabolite_name_list.append(split[0]) + # Create dictionary for new model. + new_model = {} + for param_name, param in parameters_obj.items(): + name = param_name.split("_") + name1 = name[0] + if len(name) == 1: # i.e. phase + new_model[name1] = param.value + else: + name2 = name[1] + if name1 not in new_model: + new_model[name1] = {name2: param.value} + else: + new_model[name1][name2] = param.value + + nonlocal metabolite_name_list + for i, metabolite_name in enumerate(metabolite_name_list): + new_model[metabolite_name]["amplitude"] = param_weights[i] + + return new_model + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + # Check if all model input types are correct. + def check_errors(check_model): + # Allowed names and keys in the model. + allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] + allowed_keys = ["min", "max", "value", "phase", "amplitude"] + + # Scan model. + for name1, value1 in check_model.items(): + if type(value1) is not int and type(value1) is not float and type(value1) is not dict: + raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) + elif name1 not in allowed_names: + raise NameError("{} is not an allowed name.".format(name1)) + elif type(value1) is dict: # i.e. type(value) is not int + for name2, value2 in value1.items(): + if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ + type(value2) is not str: + raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." + .format(name1, name2)) + if type(value2) is dict: + for key in value2: + # Dictionary must have 'value' key. + if "value" not in value2: + raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) + # Dictionary can only have 'min,' 'max,' and 'value'. + if key not in allowed_keys: + raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) + + return + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # Do singlet fitting + def main(): + # Minimize and fit 31P data. + # Check for errors in model formatting. + check_errors(model) + + # Set list of metabolite names. + # nonlocal metabolite_name_list + # metabolite_name_list = get_metabolites(model) + + # Convert model to lmfit Parameters object. + parameters = model_to_parameters(model) + + # Set metabolite name list + nonlocal metabolite_name_list + metabolite_name_list = get_metabolites(model) + + # Fit data. + fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) + + # Convert fit parameters to model format. + final_model = parameters_to_model(fitted_results.params, fitted_weights) + # Get stderr values for each parameter. + stderr = get_errors(fitted_results) + + # Compile output into a dictionary. + return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} + return return_dict + + return main() From 7adca0d9997d63523ee5f693ec5cb5e970ebcf22 Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 15:28:37 -0400 Subject: [PATCH 16/39] Consolidated fit function into singlet, removed fit.py --- suspect/fitting/singlet.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index f659b3d..e72109f 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -337,16 +337,12 @@ def main(): check_errors(model) # Set list of metabolite names. - # nonlocal metabolite_name_list - # metabolite_name_list = get_metabolites(model) + nonlocal metabolite_name_list + metabolite_name_list = get_metabolites(model) # Convert model to lmfit Parameters object. parameters = model_to_parameters(model) - # Set metabolite name list - nonlocal metabolite_name_list - metabolite_name_list = get_metabolites(model) - # Fit data. fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) From 055241501c4f3f234d12b261f91349f6a725b30d Mon Sep 17 00:00:00 2001 From: bennyrowland Date: Fri, 9 Sep 2016 19:26:00 -0400 Subject: [PATCH 17/39] adjust Gaussian generation to take phase in radians --- suspect/basis/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suspect/basis/__init__.py b/suspect/basis/__init__.py index 1bb6424..c8d4eb3 100644 --- a/suspect/basis/__init__.py +++ b/suspect/basis/__init__.py @@ -2,7 +2,7 @@ def gaussian(time_axis, frequency, phase, fwhm): - oscillatory_term = numpy.exp(2j * numpy.pi * (frequency * time_axis + phase)) + oscillatory_term = numpy.exp(2j * numpy.pi * (frequency * time_axis) + 1j * phase) damping = numpy.exp(-time_axis ** 2 / 4 * numpy.pi ** 2 / numpy.log(2) * fwhm ** 2) fid = oscillatory_term * damping fid[0] /= 2.0 From 13cee9a8231456d205af05b5192d337503074b1c Mon Sep 17 00:00:00 2001 From: bennyrowland Date: Fri, 9 Sep 2016 19:26:54 -0400 Subject: [PATCH 18/39] fixed errors in singlet fitting and added first unit test --- suspect/fitting/singlet.py | 21 +++++------ tests/test_mrs/test_fitting.py | 66 ++++++++++------------------------ 2 files changed, 26 insertions(+), 61 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index e72109f..345f898 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -3,7 +3,7 @@ import scipy.optimize import numbers -import suspect +import suspect.basis def complex_to_real(complex_fid): @@ -43,12 +43,13 @@ def real_to_complex(real_fid): return complex_fid -def fit(fid, model): +def fit(fid, model, baseline_points=16): """ Fit fid with model parameters. :param fid: MRSData object of FID to be fit :param model: dictionary model of fit parameters + :param baseline_points: the number of points at the start of the FID to ignore :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] """ @@ -117,16 +118,15 @@ def do_fit(params, time_axis, real_unphased_data): :param params: lmfit Parameters object containing fitting parameters. :param time_axis: the time axis. :param real_unphased_data: - :return: List of fitted data points. + :return: List of fitted data points and amplitudes of each singlet. """ - baseline_points = 16 basis = make_basis(params, time_axis) weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, real_unphased_data[baseline_points:-baseline_points])[0] fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() - return fitted_data + return fitted_data, weights def residual(params, time_axis, data): """ @@ -137,12 +137,11 @@ def residual(params, time_axis, data): :param data: FID to be fitted. :return: residual values of baseline points. """ - baseline_points = 16 # unphase the data to make it pure absorptive unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) real_unphased_data = complex_to_real(unphased_data) - fitted_data = do_fit(params, time_axis, real_unphased_data) + fitted_data, _ = do_fit(params, time_axis, real_unphased_data) res = fitted_data - real_unphased_data return res[baseline_points:-baseline_points] @@ -156,7 +155,6 @@ def fit_data(data, initial_params): :param initial_params: lmfit Parameters object containing fitting parameters. :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. """ - baseline_points = 16 fitting_result = lmfit.minimize(residual, initial_params, args=(data.time_axis(), data), @@ -166,11 +164,8 @@ def fit_data(data, initial_params): -fitting_result.params['phase0'], -fitting_result.params['phase1']) real_unphased_data = complex_to_real(unphased_data) - real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) + real_fitted_data, fitting_weights = do_fit(fitting_result.params, data.time_axis(), real_unphased_data) fitted_data = real_to_complex(real_fitted_data) - fitting_basis = make_basis(fitting_result.params, data.time_axis()) - fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, - real_unphased_data[baseline_points:-baseline_points])[0] return fitting_weights, fitted_data, fitting_result @@ -222,7 +217,7 @@ def model_to_parameters(model_dict): # Initialize lmfit parameter arguments. name = "{}_{}".format(name1, name2) value = None - vary = None + vary = True lmfit_min = None lmfit_max = None expr = None diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index 20b3fce..71af908 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -6,56 +6,26 @@ def test_gaussian(): time_axis = numpy.arange(0, 0.512, 5e-4) - fid = basis.gaussian(time_axis, 0, 0, 50.0) + fid = basis.gaussian(time_axis, 0, 0, 50.0) + 0.00001 * (numpy.random.rand(1024) - 0.5) data = MRSData(fid, 5e-4, 123) - model = singlet.Model({ - "g1": { - "amplitude": 1.0, - "fwhm": 50.0, + model = { + "phase0": 0, + "phase1": 0, + "pcr": { + "amplitude": 1, + "width": { + "value": 45, + "min": 42, + "max": 55 + }, "phase": "0", - "frequency": 0.0 + "frequency": 0 } - }) - fitting_result = model.fit(data) - numpy.testing.assert_allclose(fitting_result["params"]["g1"]["fwhm"], 50.0) - numpy.testing.assert_allclose(fitting_result["params"]["g1"]["amplitude"], 1.0) - numpy.testing.assert_allclose(fitting_result["params"]["g1"]["frequency"], 0.0, atol=1e-7) + } + fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["fit"], fid) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=1e-2) - -# def test_multiple_gaussians(): -# time_axis = numpy.arange(0, 0.512, 5e-4) -# g1 = basis.gaussian(time_axis, 0, 0, 50.0) -# g2 = basis.gaussian(time_axis, 250, 0, 40.0) -# fid = g1 + g2 -# data = MRSData(fid, 5e-4, 123) -# model = singlet.Model({ -# "g1": { -# "amplitude": 1.0, -# "fwhm": 50.0, -# "phase": "0", -# "frequency": {"value": 0.0, "min": -50, "max": 50} -# }, -# "g2": { -# "amplitude": "g1amplitude", -# "fwhm": 45, -# "phase": "0", -# "frequency": 230 -# } -# }) -# -# fitting_result = model.fit(data) -# params = fitting_result["params"] -# -# print(params["g1"]) -# numpy.testing.assert_allclose(params["g1"]["fwhm"], 50.0) -# numpy.testing.assert_allclose(params["g1"]["amplitude"], 1.0) -# numpy.testing.assert_allclose(params["g1"]["frequency"], 0.0, atol=1e-7) -# numpy.testing.assert_allclose(params["g2"]["fwhm"], 40.0) -# numpy.testing.assert_allclose(params["g2"]["amplitude"], 1.0) -# numpy.testing.assert_allclose(params["g2"]["frequency"], 250.0) -# -# numpy.testing.assert_allclose(fitting_result["fit"], fid) -# numpy.testing.assert_allclose(fitting_result["fit_components"]["g1"], g1) -# numpy.testing.assert_allclose(fitting_result["fit_components"]["g2"], g2) + numpy.testing.assert_allclose(fitting_result["fit"], fid, atol=0.001) From 56bb1d8c043d64353b0b81f1610229a7ce6f8cb9 Mon Sep 17 00:00:00 2001 From: lasyasreepada Date: Wed, 14 Sep 2016 10:51:02 -0400 Subject: [PATCH 19/39] Modified the files suspect/fitting/singlet.py and tests/test_mrs/test_fitting in order to improve existing code and write a more comprehensive test --- suspect/fitting/singlet.py | 87 +++++++++++---------------- tests/test_mrs/test_fitting.py | 105 ++++++++++++++++++++++++++++++--- 2 files changed, 131 insertions(+), 61 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 345f898..511bbd0 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -5,7 +5,6 @@ import suspect.basis - def complex_to_real(complex_fid): """ Standard optimization routines as used in lmfit require real data. This @@ -50,12 +49,9 @@ def fit(fid, model, baseline_points=16): :param fid: MRSData object of FID to be fit :param model: dictionary model of fit parameters :param baseline_points: the number of points at the start of the FID to ignore - :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] + :return: Dictionary containing ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] """ - # List of metabolite names - metabolite_name_list = [] - # Get list of metabolite names. def get_metabolites(model_input): metabolites = [] @@ -78,7 +74,7 @@ def phase_fid(fid_in, phase0, phase1): :param fid_in: FID to be fitted. :param phase1: phase1 value. :param phase0: phase0 value. - :return: + :return: FID that has been shifted into phase by FFT """ spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) np = fid_in.np @@ -111,6 +107,13 @@ def make_basis(params, time_axis): basis_matrix[i, :] = real_gaussian return basis_matrix + def unphase(data, params): + + unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + + return real_unphased_data + def do_fit(params, time_axis, real_unphased_data): """ This function performs the fitting. @@ -137,11 +140,9 @@ def residual(params, time_axis, data): :param data: FID to be fitted. :return: residual values of baseline points. """ - # unphase the data to make it pure absorptive - unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) - real_unphased_data = complex_to_real(unphased_data) - fitted_data, _ = do_fit(params, time_axis, real_unphased_data) + real_unphased_data = unphase(data, params) + fitted_data, weights = do_fit(params, time_axis, real_unphased_data) res = fitted_data - real_unphased_data return res[baseline_points:-baseline_points] @@ -160,11 +161,7 @@ def fit_data(data, initial_params): args=(data.time_axis(), data), xtol=5e-3) - unphased_data = phase_fid(data, - -fitting_result.params['phase0'], - -fitting_result.params['phase1']) - real_unphased_data = complex_to_real(unphased_data) - real_fitted_data, fitting_weights = do_fit(fitting_result.params, data.time_axis(), real_unphased_data) + real_fitted_data, fitting_weights = do_fit(fitting_result.params, data.time_axis(), unphase(data, fitting_result.params)) fitted_data = real_to_complex(real_fitted_data) return fitting_weights, fitted_data, fitting_result @@ -191,7 +188,6 @@ def parameters_to_model(parameters_obj, param_weights): else: new_model[name1][name2] = param.value - nonlocal metabolite_name_list for i, metabolite_name in enumerate(metabolite_name_list): new_model[metabolite_name]["amplitude"] = param_weights[i] @@ -207,7 +203,7 @@ def model_to_parameters(model_dict): # Construct lmfit Parameter input for each parameter. for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) + if isinstance(value1, numbers.Number): # (e.g. phase0) params.append((name1, value1)) if type(value1) is dict: # Fix phase value to 0 by default. @@ -221,15 +217,13 @@ def model_to_parameters(model_dict): lmfit_min = None lmfit_max = None expr = None - if type(value2) is int: + if isinstance(value2, numbers.Number): value = value2 - elif type(value2) is str: + elif isinstance(value2, str): expr = value2 - if type(value2) is dict: + elif isinstance(value2, dict): if "value" in value2: value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] if "min" in value2: lmfit_min = value2["min"] if "max" in value2: @@ -267,20 +261,16 @@ def model_to_parameters(model_dict): # Check if all model input types are correct. def check_errors(check_model): - # Allowed names and keys in the model. - allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] + # Allowed keys in the model. allowed_keys = ["min", "max", "value", "phase", "amplitude"] # Scan model. for name1, value1 in check_model.items(): - if type(value1) is not int and type(value1) is not float and type(value1) is not dict: + if not isinstance(value1, (numbers.Number, dict)): raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) - elif name1 not in allowed_names: - raise NameError("{} is not an allowed name.".format(name1)) elif type(value1) is dict: # i.e. type(value) is not int for name2, value2 in value1.items(): - if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ - type(value2) is not str: + if not isinstance(value2,(numbers.Number,dict,str)): raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." .format(name1, name2)) if type(value2) is dict: @@ -292,21 +282,18 @@ def check_errors(check_model): if key not in allowed_keys: raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) - return - # Calculate references to determine order for Parameters. def calculate_dependencies(unordered_model): dependencies = {} # (name, [dependencies]) - # Compile dictionary of effective names. for name1, value1 in unordered_model.items(): if type(value1) is dict: # i.e. not phase + + # Compile dictionary of effective names. for name2 in value1: dependencies["{}_{}".format(name1, name2)] = None - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase + # Find dependencies for each effective name. for name2, value2 in value1.items(): if type(value2) is str: lmfit_name = "{}_{}".format(name1, name2) @@ -326,28 +313,20 @@ def calculate_dependencies(unordered_model): return dependencies # Do singlet fitting - def main(): - # Minimize and fit 31P data. - # Check for errors in model formatting. - check_errors(model) + # Minimize and fit 31P data. + + check_errors(model) # Check for errors in model formatting. + + metabolite_name_list = get_metabolites(model) # Set list of metabolite names. - # Set list of metabolite names. - nonlocal metabolite_name_list - metabolite_name_list = get_metabolites(model) + parameters = model_to_parameters(model) # Convert model to lmfit Parameters object. - # Convert model to lmfit Parameters object. - parameters = model_to_parameters(model) + fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) # Fit data. - # Fit data. - fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) + final_model = parameters_to_model(fitted_results.params, fitted_weights) # Convert fit parameters to model format. - # Convert fit parameters to model format. - final_model = parameters_to_model(fitted_results.params, fitted_weights) - # Get stderr values for each parameter. - stderr = get_errors(fitted_results) + stderr = get_errors(fitted_results) # Get stderr values for each parameter. - # Compile output into a dictionary. - return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} - return return_dict + return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} # Compile output into a dictionary. + return return_dict - return main() diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index 71af908..12e57e1 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -3,12 +3,13 @@ import numpy - def test_gaussian(): time_axis = numpy.arange(0, 0.512, 5e-4) fid = basis.gaussian(time_axis, 0, 0, 50.0) + 0.00001 * (numpy.random.rand(1024) - 0.5) data = MRSData(fid, 5e-4, 123) - model = { + + #Original test with all parameters passed in; correct data types; integer values + model1 = { "phase0": 0, "phase1": 0, "pcr": { @@ -22,10 +23,100 @@ def test_gaussian(): "frequency": 0 } } - fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=1e-2) + #Floating point values; invalid key added to width dict, to test whether KeyError is raised + model2 = { + "phase0": 0.0, + "phase1": 0.0, + "pcr": { + "amplitude": 1.0, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + "avg": 47 + }, + "phase": "0", + "frequency": 0.0 + } + } + + #No width value passed in, to test whether KeyError is raised + model3 = { + "phase0": 0, + "phase1": 0, + "pcr": { + "amplitude": 1, + "width": { + #"value": 45, + "min": 42, + "max": 55, + + }, + "phase": "0", + "frequency": 0 + } + } + + #No phase value passed in, to test whether phase is fixed to 0 by default + model4 = { + "phase0": 0, + "phase1": 0, + "pcr": { + "amplitude": 1, + "width": { + "value": 45, + "min": 42, + "max": 55, + }, + # "phase": "0", + "frequency": 0 + } + } + + #Str value supplied for phase0 and phase1, to test whether TypeError is raised + model5 = { + "phase0": str, + "phase1": str, + "pcr": { + "amplitude": 1.0, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + }, + "phase": "0", + "frequency": 0.0 + } + } + + # Str value supplied for amplitude, to test whether TypeError is raised + model6 = { + "phase0": 0.0, + "phase1": 0.0, + "pcr": { + "amplitude": str, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + }, + "phase": "0", + "frequency": 0.0 + } + } + + set_of_models = [model1, model2, model3, model4, model5, model6] + + for model in set_of_models: + try: + fitting_result = singlet.fit(data, model) + + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=1e-2) + + numpy.testing.assert_allclose(fitting_result["fit"], fid, atol=0.001) - numpy.testing.assert_allclose(fitting_result["fit"], fid, atol=0.001) + except(TypeError, KeyError, ReferenceError): + pass From fe88364a4e7d0f4fa17f4901277feac8b7d6bdf3 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 25 Jul 2016 17:30:50 -0400 Subject: [PATCH 20/39] Added fit file, moved amares fitting functions to singlet, converted a few values in denoising and singlet to int --- suspect/fitting/fit.py | 443 ++++++++++++++++++++++++++++++++ suspect/fitting/singlet.py | 187 ++++++++------ suspect/processing/denoising.py | 6 +- 3 files changed, 555 insertions(+), 81 deletions(-) create mode 100644 suspect/fitting/fit.py diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py new file mode 100644 index 0000000..e622757 --- /dev/null +++ b/suspect/fitting/fit.py @@ -0,0 +1,443 @@ +""" +File Name: fit.py +Author: Sam Jiang +Purpose: fit_31p() function fits single 31P fid. +Interpreter: Anaconda 3.5.1 + +fit(fid, model) +Input: fid and model in hierarchical dictionary format +Output: dictionary with 'model', 'fit', and 'error' keys and corresponding values +""" + +import os +import numpy +import json +import lmfit +import array +import suspect + + +# Load model in. +def load(fin): + if not os.path.isfile(fin): + raise FileNotFoundError("{} not found.".format(fin)) + with open(fin, 'r+') as f: + model = json.load(f) + return model + + +# Save model or fit out. +def save(output, fout): + # Check if file and directory exists. + directory, name = os.path.split(fout) + if directory == '': + directory = os.getcwd() + if not os.path.isdir(directory): + os.makedirs(directory) + if name == '': + raise IsADirectoryError("{} is not a file name.".format(name)) + ftype = os.path.splitext(name)[1] + + # If model, save as json. + if type(output) is dict: + if ftype != ".json": + raise TypeError("Output file must be a JSON (.json) file.") + + with open(fout, 'w+') as f: + json.dump(output, f) + + # If fit, save as csv or txt. + elif type(output) is array.ArrayType: + if ftype != ".csv" or ftype != ".txt": + raise TypeError("Output file must be a CSV (.csv) or Text (.txt) file.") + + with open(fout, 'w+') as f: + first = True + for x in output.tolist(): + if first: + f.write("{}".format(x)) + first = False + else: + f.write(", {}".format(x)) + + +# Fit fid +# Input: fid, model +# Output: dictionary containing optimized model, fit data, standard errors +def fit(fid, model): + # MODEL + # Get list of metabolite names + # def get_metabolites(model_input): + # metabolites = [] + # for name, value in model_input.items(): + # if type(value) is dict: + # metabolites.append(name) + # return metabolites + + # Get standard errors from lmfit MinimizerResult object. + def get_errors(result): + errors = {} + for name, param in result.params.items(): + errors[name] = param.stderr + return errors + + # Convert lmfit parameters to model format + def parameters_to_model(parameters_obj, param_weights): + metabolite_name_list = [] + for param in parameters_obj.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + # Create dictionary for new model. + new_model = {} + for param_name, param in parameters_obj.items(): + name = param_name.split("_") + name1 = name[0] + if len(name) == 1: # i.e. phase + new_model[name1] = param.value + else: + name2 = name[1] + if name1 not in new_model: + new_model[name1] = {name2: param.value} + else: + new_model[name1][name2] = param.value + + for i, metabolite_name in enumerate(metabolite_name_list): + new_model[metabolite_name]["amplitude"] = param_weights[i] + + return new_model + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + # Check if all model input types are correct. + def check_errors(check_model): + # Allowed names and keys in the model. + allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] + allowed_keys = ["min", "max", "value", "phase", "amplitude"] + + # Scan model. + for name1, value1 in check_model.items(): + if type(value1) is not int and type(value1) is not float and type(value1) is not dict: + raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) + elif name1 not in allowed_names: + raise NameError("{} is not an allowed name.".format(name1)) + elif type(value1) is dict: # i.e. type(value) is not int + for name2, value2 in value1.items(): + if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ + type(value2) is not str: + raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." + .format(name1, name2)) + if type(value2) is dict: + for key in value2: + # Dictionary must have 'value' key. + if "value" not in value2: + raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) + # Dictionary can only have 'min,' 'max,' and 'value'. + if key not in allowed_keys: + raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) + + return + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # MAIN + def main(): + # Minimize and fit 31P data. + # Check for errors in model formatting. + check_errors(model) + + # Set list of metabolite names. + # nonlocal metabolite_name_list + # metabolite_name_list = get_metabolites(model) + + # Convert model to lmfit Parameters object. + parameters = model_to_parameters(model) + + # Fit data. + fitted_weights, fitted_data, fitted_results = suspect.fitting.singlet.fit_data(fid, parameters) + + # Convert fit parameters to model format. + final_model = parameters_to_model(fitted_results.params, fitted_weights) + # Get stderr values for each parameter. + stderr = get_errors(fitted_results) + + # Compile output into a dictionary. + return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} + + return return_dict + + if __name__ == "__main__": + return main() + + +# Test fit function +def test_fit(_plot): + # Test file and model + testfile = "test_timecourse.csv" + modelfile = "model.json" + model = load(modelfile) + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + def fake_spectrum(params, time_axis, weights): + basis = suspect.fitting.singlet.make_basis(params, time_axis) + real_data = numpy.array(numpy.dot(weights, basis)).squeeze() + complex_data = suspect.fitting.singlet.real_to_complex(real_data) + return suspect.MRSData(complex_data, dt, f0, te=0, ppm0=0) + + # Generate fake noise (Testing only) + def generate_noise(n, snr): + # Added (n * 0) to fix unused variable warning. + noise = (n * 0) + (numpy.random.randn(4096) + 1j * numpy.random.randn(4096)) * 6.4e-6 / snr + return noise + + # Generate fake fid (Testing only) + def generate_fid(ground_truth, snr, initial_params): + metabolite_name_list = [] + for param in initial_params.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + + peaks = {"pcr": 1, + "atpc": 0.3, + "atpb": 0.3, + "atpa": 0.3, + "pi": 1, + "pme": 0.05, + "pde": 0.05} + amps = [] + for metabolite in metabolite_name_list: + amps.append(peaks[metabolite]) + + noisy_sequence = numpy.zeros((len(ground_truth), np), 'complex') + noisy_sequence = suspect.MRSData(noisy_sequence, dt, f0, te=0, ppm0=0) + + # Populate the noisy_sequence with data. + generated_fid = fake_spectrum(initial_params, noisy_sequence.time_axis(), amps) + # [1, 0.3, 0.3, 0.3, 1, 0.05, 0.05] + noisy_fid = generated_fid + generate_noise(np, snr) + + return generated_fid, noisy_fid + + # Generate fake FID. + parameters = model_to_parameters(model) + sw = 6e3 + dt = 1.0 / sw + np = 4096 + f0 = 49.885802 + fake_fid, noisy_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid + + # Test fit(). + fit_results = fit(fake_fid, model) + + # Plot fit, noisy_fid, and fake_fid. + if _plot: + from matplotlib import pyplot + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noisy_fid))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) + pyplot.show() + + +test = False +plot = True +if test: + print("Running test.") + test_fit(plot) + print("Test complete.") + diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 39a15e7..6e19dd1 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -1,5 +1,6 @@ import lmfit import numpy +import scipy.optimize import numbers import suspect @@ -32,7 +33,7 @@ def real_to_complex(real_fid): :param real_fid: the real FID to be converted to complex. :return: the complex version of the FID """ - np = real_fid.shape[0] / 2 + np = int(real_fid.shape[0] / 2) complex_fid = numpy.zeros(np, 'complex') complex_fid[:] = real_fid[:np] @@ -42,83 +43,113 @@ def real_to_complex(real_fid): return complex_fid -def gaussian_fid(x, amplitude=1, frequency=0.0, phase=0.0, fwhm=1.0): +# metabolite_name_list = [] + + +def phase_fid(fid_in, phase0, phase1): + """ + This function performs a Fourier Transform on the FID to shift it into phase. + + :param fid_in: FID to be fitted. + :param phase1: phase1 value. + :param phase0: phase0 value. + :return: + """ + spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) + np = fid_in.np + phase_shift = phase0 + phase1 * numpy.linspace(-np / 2, np / 2, np, endpoint=False) + phased_spectrum = spectrum * numpy.exp(1j * phase_shift) + return fid_in.inherit(numpy.fft.ifft(numpy.fft.ifftshift(phased_spectrum))) + + +def make_basis(params, time_axis): + """ + This function generates a basis set. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :return: a matrix containing the generated basis set. + """ + metabolite_name_list = [] + for param in params.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + + basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) + for i, metabolite_name in enumerate(metabolite_name_list): + gaussian = suspect.basis.gaussian(time_axis, + params["{}_frequency".format(metabolite_name)], + params["{}_phase".format(metabolite_name)].value, + params["{}_width".format(metabolite_name)]) + real_gaussian = complex_to_real(gaussian) + basis_matrix[i, :] = real_gaussian + return basis_matrix + + +def do_fit(params, time_axis, real_unphased_data): + """ + This function performs the fitting. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param real_unphased_data: + :return: List of fitted data points. """ - Generates a Gaussian FID for use with the lmfit GaussianFidModel class. The - helper function complex_to_real is used to convert the FID to a real form. - - :param x: the time axis for the fid data - :param amplitude: the amplitude of the Gaussian - :param frequency: the frequency of the Gaussian in Hz - :param phase: the phase in radians - :param fwhm: the full width at half maximum of the Gaussian - :return: a real FID describing a Gaussian relaxation + baseline_points = 16 + basis = make_basis(params, time_axis) + + weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() + return fitted_data + + +def residual(params, time_axis, data): + """ + This function calculates the residual to be minimized by the least squares means method. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param data: FID to be fitted. + :return: residual values of baseline points. """ + baseline_points = 16 + # unphase the data to make it pure absorptive + unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) + real_unphased_data = complex_to_real(unphased_data) - complex_fid = amplitude * suspect.basis.gaussian(x, frequency, phase, fwhm) - return complex_to_real(complex_fid) - - -class GaussianFidModel(lmfit.models.Model): - def __init__(self, *args, **kwargs): - super(GaussianFidModel, self).__init__(gaussian_fid, *args, **kwargs) - self.set_param_hint('fwhm', min=0) - self.set_param_hint('amplitude', min=0) - self.set_param_hint('phase', min=-numpy.pi, max=numpy.pi) - - def guess(self, data=None, **kwargs): - return self.make_params() - - def copy(self, **kwargs): - raise NotImplementedError - - -class Model: - def __init__(self, peaks): - self.model = None - self.params = lmfit.Parameters() - for peak_name, peak_params in peaks.items(): - - current_peak_model = GaussianFidModel(prefix="{}".format(peak_name)) - self.params.update(current_peak_model.make_params()) - - for param_name, param_data in peak_params.items(): - # the - full_name = "{0}{1}".format(peak_name, param_name) - - if full_name in self.params: - if type(param_data) is str: - self.params[full_name].set(expr=param_data) - elif type(param_data) is dict: - if "value" in param_data: - self.params[full_name].set(value=param_data["value"]) - if "min" in param_data: - self.params[full_name].set(min=param_data["min"]) - if "max" in param_data: - self.params[full_name].set(max=param_data["max"]) - if "expr" in param_data: - self.params[full_name].set(expr=param_data) - elif type(param_data) is numbers.Number: - self.params[full_name].set(param_data) - - if self.model is None: - self.model = current_peak_model - else: - self.model += current_peak_model - - def fit(self, data): - fit_result = self.model.fit(complex_to_real(data), x=data.time_axis(), params=self.params) - result_params = {} - for component in self.model.components: - component_name = component.prefix - result_params[component.prefix] = {} - for param in component.make_params(): - param_name = str(param).replace(component_name, "") - result_params[component.prefix][param_name] = fit_result.params[param].value - fit_curve = real_to_complex(fit_result.best_fit) - fit_components = {k: real_to_complex(v) for k, v in fit_result.eval_components().items()} - return { - "params": result_params, - "fit": fit_curve, - "fit_components": fit_components - } + fitted_data = do_fit(params, time_axis, real_unphased_data) + res = fitted_data - real_unphased_data + + return res[baseline_points:-baseline_points] + + +def fit_data(data, initial_params): + """ + This function takes an FID and a set of parameters contained in an lmfit Parameters object, + and fits the data using the least squares means method. + + :param data: FID to be fitted. + :param initial_params: lmfit Parameters object containing fitting parameters. + :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. + """ + baseline_points = 16 + fitting_result = lmfit.minimize(residual, + initial_params, + args=(data.time_axis(), data), + xtol=5e-3) + + unphased_data = phase_fid(data, + -fitting_result.params['phase0'], + -fitting_result.params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) + fitted_data = real_to_complex(real_fitted_data) + fitting_basis = make_basis(fitting_result.params, data.time_axis()) + fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + return fitting_weights, fitted_data, fitting_result diff --git a/suspect/processing/denoising.py b/suspect/processing/denoising.py index 60b0484..7b85dee 100644 --- a/suspect/processing/denoising.py +++ b/suspect/processing/denoising.py @@ -14,7 +14,7 @@ def _pad(input_signal, length, average=10): :return: """ padded_input_signal = numpy.zeros(length, input_signal.dtype) - start_offset = (len(padded_input_signal) - len(input_signal)) / 2. + start_offset = int((len(padded_input_signal) - len(input_signal)) / 2) padded_input_signal[:start_offset] = numpy.average(input_signal[0:average]) padded_input_signal[start_offset:(start_offset + len(input_signal))] = input_signal[:] padded_input_signal[(start_offset + len(input_signal)):] = numpy.average(input_signal[-average:]) @@ -38,7 +38,7 @@ def sliding_gaussian(input_signal, window_width): window /= numpy.sum(window) # pad the signal to cover half the window width on each side padded_input = _pad(input_signal, len(input_signal) + window_width - 1) - result = numpy.zeros(len(input_signal)) + result = numpy.zeros_like(input_signal) for i in range(len(input_signal)): result[i] = numpy.dot(window, padded_input[i:(i + window_width)]) return result @@ -63,7 +63,7 @@ def svd(input_signal, rank): s[rank:] = 0.0 recon = U * numpy.diag(s) * V - result = numpy.zeros(len(input_signal)) + result = numpy.zeros_like(input_signal) for i in range(len(input_signal)): count = 0 for j in range(matrix_height): From 7d63c90530a034a3ee5a7b8b5c34c3aaeed3cdfe Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 14:22:23 -0400 Subject: [PATCH 21/39] Added fit file, moved amares fitting functions to singlet, converted a few values in denoising and singlet to int --- suspect/fitting/fit.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py index e622757..fabce0b 100644 --- a/suspect/fitting/fit.py +++ b/suspect/fitting/fit.py @@ -62,9 +62,14 @@ def save(output, fout): # Fit fid -# Input: fid, model -# Output: dictionary containing optimized model, fit data, standard errors def fit(fid, model): + """ + Fit fid with model parameters. + + :param fid: MRSData object of FID to be fit + :param model: dictionary model of fit parameters + :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] + """ # MODEL # Get list of metabolite names # def get_metabolites(model_input): @@ -267,10 +272,9 @@ def main(): # Test fit function -def test_fit(_plot): +def test_fit(_plot, testfile, modelfile): # Test file and model - testfile = "test_timecourse.csv" - modelfile = "model.json" + model = load(modelfile) # Calculate references to determine order for Parameters. @@ -420,24 +424,25 @@ def generate_fid(ground_truth, snr, initial_params): dt = 1.0 / sw np = 4096 f0 = 49.885802 - fake_fid, noisy_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid + fake_fid, noise_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid # Test fit(). fit_results = fit(fake_fid, model) - # Plot fit, noisy_fid, and fake_fid. + # Plot fit, noise_fid, and fake_fid. if _plot: from matplotlib import pyplot pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noisy_fid))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noise_fid))) pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) pyplot.show() test = False -plot = True +plot = False if test: print("Running test.") - test_fit(plot) + test_file = "/home/sam/Desktop/amares/test_timecourse.csv" + test_model = "/home/sam/Desktop/amares/model.json" + test_fit(plot, test_file, test_model) print("Test complete.") - From cd62a6260dedf591a962d61dacec15b1b14fb60d Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 15:22:02 -0400 Subject: [PATCH 22/39] Consolidated fit function into singlet, removed fit.py --- suspect/fitting/fit.py | 448 ------------------------------------- suspect/fitting/singlet.py | 415 +++++++++++++++++++++++++--------- 2 files changed, 311 insertions(+), 552 deletions(-) delete mode 100644 suspect/fitting/fit.py diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py deleted file mode 100644 index fabce0b..0000000 --- a/suspect/fitting/fit.py +++ /dev/null @@ -1,448 +0,0 @@ -""" -File Name: fit.py -Author: Sam Jiang -Purpose: fit_31p() function fits single 31P fid. -Interpreter: Anaconda 3.5.1 - -fit(fid, model) -Input: fid and model in hierarchical dictionary format -Output: dictionary with 'model', 'fit', and 'error' keys and corresponding values -""" - -import os -import numpy -import json -import lmfit -import array -import suspect - - -# Load model in. -def load(fin): - if not os.path.isfile(fin): - raise FileNotFoundError("{} not found.".format(fin)) - with open(fin, 'r+') as f: - model = json.load(f) - return model - - -# Save model or fit out. -def save(output, fout): - # Check if file and directory exists. - directory, name = os.path.split(fout) - if directory == '': - directory = os.getcwd() - if not os.path.isdir(directory): - os.makedirs(directory) - if name == '': - raise IsADirectoryError("{} is not a file name.".format(name)) - ftype = os.path.splitext(name)[1] - - # If model, save as json. - if type(output) is dict: - if ftype != ".json": - raise TypeError("Output file must be a JSON (.json) file.") - - with open(fout, 'w+') as f: - json.dump(output, f) - - # If fit, save as csv or txt. - elif type(output) is array.ArrayType: - if ftype != ".csv" or ftype != ".txt": - raise TypeError("Output file must be a CSV (.csv) or Text (.txt) file.") - - with open(fout, 'w+') as f: - first = True - for x in output.tolist(): - if first: - f.write("{}".format(x)) - first = False - else: - f.write(", {}".format(x)) - - -# Fit fid -def fit(fid, model): - """ - Fit fid with model parameters. - - :param fid: MRSData object of FID to be fit - :param model: dictionary model of fit parameters - :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] - """ - # MODEL - # Get list of metabolite names - # def get_metabolites(model_input): - # metabolites = [] - # for name, value in model_input.items(): - # if type(value) is dict: - # metabolites.append(name) - # return metabolites - - # Get standard errors from lmfit MinimizerResult object. - def get_errors(result): - errors = {} - for name, param in result.params.items(): - errors[name] = param.stderr - return errors - - # Convert lmfit parameters to model format - def parameters_to_model(parameters_obj, param_weights): - metabolite_name_list = [] - for param in parameters_obj.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - # Create dictionary for new model. - new_model = {} - for param_name, param in parameters_obj.items(): - name = param_name.split("_") - name1 = name[0] - if len(name) == 1: # i.e. phase - new_model[name1] = param.value - else: - name2 = name[1] - if name1 not in new_model: - new_model[name1] = {name2: param.value} - else: - new_model[name1][name2] = param.value - - for i, metabolite_name in enumerate(metabolite_name_list): - new_model[metabolite_name]["amplitude"] = param_weights[i] - - return new_model - - # Convert initial model to lmfit parameters. - def model_to_parameters(model_dict): - lmfit_parameters = lmfit.Parameters() - params = [] - ordered_params = [] - # Calculate dependencies/references for each parameter. - depend_dict = calculate_dependencies(model_dict) - - # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: - # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): - # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) - value = None - vary = None - lmfit_min = None - lmfit_max = None - expr = None - if type(value2) is int: - value = value2 - elif type(value2) is str: - expr = value2 - if type(value2) is dict: - if "value" in value2: - value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] - # Add parameter object with defined parameters. - params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) - - # Order parameters based on dependencies. - in_oparams = [] - while len(params) > 0: - front = params.pop(0) - name = front[0] - # If no dependencies, add parameter to list and mark parameter as added. - if name not in depend_dict or depend_dict[name] is None: - ordered_params.append(front) - in_oparams.append(name) - else: - dependencies_present = True - for dependency in depend_dict[name]: - # If dependency not yet added, mark parameter to move to back of queue. - if dependency not in in_oparams: - dependencies_present = False - # If all dependencies present, add parameter to list and mark parameter as added. - if dependencies_present: - ordered_params.append(front) - in_oparams.append(name) - # If dependencies missing, move parameter to back of queue. - else: - params.append(front) - - # Convert all parameters to lmfit Parameter objects. - lmfit_parameters.add_many(*ordered_params) - - return lmfit_parameters - - # Check if all model input types are correct. - def check_errors(check_model): - # Allowed names and keys in the model. - allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] - allowed_keys = ["min", "max", "value", "phase", "amplitude"] - - # Scan model. - for name1, value1 in check_model.items(): - if type(value1) is not int and type(value1) is not float and type(value1) is not dict: - raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) - elif name1 not in allowed_names: - raise NameError("{} is not an allowed name.".format(name1)) - elif type(value1) is dict: # i.e. type(value) is not int - for name2, value2 in value1.items(): - if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ - type(value2) is not str: - raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." - .format(name1, name2)) - if type(value2) is dict: - for key in value2: - # Dictionary must have 'value' key. - if "value" not in value2: - raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) - # Dictionary can only have 'min,' 'max,' and 'value'. - if key not in allowed_keys: - raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) - - return - - # Calculate references to determine order for Parameters. - def calculate_dependencies(unordered_model): - dependencies = {} # (name, [dependencies]) - - # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None - - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) - dependencies[lmfit_name] = [] - for depend in dependencies: - if depend in value2: - dependencies[lmfit_name].append(depend) - - # Check for circular dependencies. - for name, dependents in dependencies.items(): - if type(dependents) is list: - for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." - .format(name, dependent)) - - return dependencies - - # MAIN - def main(): - # Minimize and fit 31P data. - # Check for errors in model formatting. - check_errors(model) - - # Set list of metabolite names. - # nonlocal metabolite_name_list - # metabolite_name_list = get_metabolites(model) - - # Convert model to lmfit Parameters object. - parameters = model_to_parameters(model) - - # Fit data. - fitted_weights, fitted_data, fitted_results = suspect.fitting.singlet.fit_data(fid, parameters) - - # Convert fit parameters to model format. - final_model = parameters_to_model(fitted_results.params, fitted_weights) - # Get stderr values for each parameter. - stderr = get_errors(fitted_results) - - # Compile output into a dictionary. - return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} - - return return_dict - - if __name__ == "__main__": - return main() - - -# Test fit function -def test_fit(_plot, testfile, modelfile): - # Test file and model - - model = load(modelfile) - - # Calculate references to determine order for Parameters. - def calculate_dependencies(unordered_model): - dependencies = {} # (name, [dependencies]) - - # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None - - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) - dependencies[lmfit_name] = [] - for depend in dependencies: - if depend in value2: - dependencies[lmfit_name].append(depend) - - # Check for circular dependencies. - for name, dependents in dependencies.items(): - if type(dependents) is list: - for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." - .format(name, dependent)) - - return dependencies - - # Convert initial model to lmfit parameters. - def model_to_parameters(model_dict): - lmfit_parameters = lmfit.Parameters() - params = [] - ordered_params = [] - # Calculate dependencies/references for each parameter. - depend_dict = calculate_dependencies(model_dict) - - # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: - # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): - # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) - value = None - vary = None - lmfit_min = None - lmfit_max = None - expr = None - if type(value2) is int: - value = value2 - elif type(value2) is str: - expr = value2 - if type(value2) is dict: - if "value" in value2: - value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] - # Add parameter object with defined parameters. - params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) - - # Order parameters based on dependencies. - in_oparams = [] - while len(params) > 0: - front = params.pop(0) - name = front[0] - # If no dependencies, add parameter to list and mark parameter as added. - if name not in depend_dict or depend_dict[name] is None: - ordered_params.append(front) - in_oparams.append(name) - else: - dependencies_present = True - for dependency in depend_dict[name]: - # If dependency not yet added, mark parameter to move to back of queue. - if dependency not in in_oparams: - dependencies_present = False - # If all dependencies present, add parameter to list and mark parameter as added. - if dependencies_present: - ordered_params.append(front) - in_oparams.append(name) - # If dependencies missing, move parameter to back of queue. - else: - params.append(front) - - # Convert all parameters to lmfit Parameter objects. - lmfit_parameters.add_many(*ordered_params) - - return lmfit_parameters - - def fake_spectrum(params, time_axis, weights): - basis = suspect.fitting.singlet.make_basis(params, time_axis) - real_data = numpy.array(numpy.dot(weights, basis)).squeeze() - complex_data = suspect.fitting.singlet.real_to_complex(real_data) - return suspect.MRSData(complex_data, dt, f0, te=0, ppm0=0) - - # Generate fake noise (Testing only) - def generate_noise(n, snr): - # Added (n * 0) to fix unused variable warning. - noise = (n * 0) + (numpy.random.randn(4096) + 1j * numpy.random.randn(4096)) * 6.4e-6 / snr - return noise - - # Generate fake fid (Testing only) - def generate_fid(ground_truth, snr, initial_params): - metabolite_name_list = [] - for param in initial_params.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - - peaks = {"pcr": 1, - "atpc": 0.3, - "atpb": 0.3, - "atpa": 0.3, - "pi": 1, - "pme": 0.05, - "pde": 0.05} - amps = [] - for metabolite in metabolite_name_list: - amps.append(peaks[metabolite]) - - noisy_sequence = numpy.zeros((len(ground_truth), np), 'complex') - noisy_sequence = suspect.MRSData(noisy_sequence, dt, f0, te=0, ppm0=0) - - # Populate the noisy_sequence with data. - generated_fid = fake_spectrum(initial_params, noisy_sequence.time_axis(), amps) - # [1, 0.3, 0.3, 0.3, 1, 0.05, 0.05] - noisy_fid = generated_fid + generate_noise(np, snr) - - return generated_fid, noisy_fid - - # Generate fake FID. - parameters = model_to_parameters(model) - sw = 6e3 - dt = 1.0 / sw - np = 4096 - f0 = 49.885802 - fake_fid, noise_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid - - # Test fit(). - fit_results = fit(fake_fid, model) - - # Plot fit, noise_fid, and fake_fid. - if _plot: - from matplotlib import pyplot - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noise_fid))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) - pyplot.show() - - -test = False -plot = False -if test: - print("Running test.") - test_file = "/home/sam/Desktop/amares/test_timecourse.csv" - test_model = "/home/sam/Desktop/amares/model.json" - test_fit(plot, test_file, test_model) - print("Test complete.") diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 6e19dd1..f659b3d 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -43,113 +43,320 @@ def real_to_complex(real_fid): return complex_fid -# metabolite_name_list = [] - - -def phase_fid(fid_in, phase0, phase1): - """ - This function performs a Fourier Transform on the FID to shift it into phase. - - :param fid_in: FID to be fitted. - :param phase1: phase1 value. - :param phase0: phase0 value. - :return: +def fit(fid, model): """ - spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) - np = fid_in.np - phase_shift = phase0 + phase1 * numpy.linspace(-np / 2, np / 2, np, endpoint=False) - phased_spectrum = spectrum * numpy.exp(1j * phase_shift) - return fid_in.inherit(numpy.fft.ifft(numpy.fft.ifftshift(phased_spectrum))) - + Fit fid with model parameters. -def make_basis(params, time_axis): + :param fid: MRSData object of FID to be fit + :param model: dictionary model of fit parameters + :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] """ - This function generates a basis set. - :param params: lmfit Parameters object containing fitting parameters. - :param time_axis: the time axis. - :return: a matrix containing the generated basis set. - """ + # List of metabolite names metabolite_name_list = [] - for param in params.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - - basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) - for i, metabolite_name in enumerate(metabolite_name_list): - gaussian = suspect.basis.gaussian(time_axis, - params["{}_frequency".format(metabolite_name)], - params["{}_phase".format(metabolite_name)].value, - params["{}_width".format(metabolite_name)]) - real_gaussian = complex_to_real(gaussian) - basis_matrix[i, :] = real_gaussian - return basis_matrix - - -def do_fit(params, time_axis, real_unphased_data): - """ - This function performs the fitting. - - :param params: lmfit Parameters object containing fitting parameters. - :param time_axis: the time axis. - :param real_unphased_data: - :return: List of fitted data points. - """ - baseline_points = 16 - basis = make_basis(params, time_axis) - - weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, - real_unphased_data[baseline_points:-baseline_points])[0] - - fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() - return fitted_data - -def residual(params, time_axis, data): - """ - This function calculates the residual to be minimized by the least squares means method. - - :param params: lmfit Parameters object containing fitting parameters. - :param time_axis: the time axis. - :param data: FID to be fitted. - :return: residual values of baseline points. - """ - baseline_points = 16 - # unphase the data to make it pure absorptive - unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) - real_unphased_data = complex_to_real(unphased_data) - - fitted_data = do_fit(params, time_axis, real_unphased_data) - res = fitted_data - real_unphased_data - - return res[baseline_points:-baseline_points] - - -def fit_data(data, initial_params): - """ - This function takes an FID and a set of parameters contained in an lmfit Parameters object, - and fits the data using the least squares means method. - - :param data: FID to be fitted. - :param initial_params: lmfit Parameters object containing fitting parameters. - :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. - """ - baseline_points = 16 - fitting_result = lmfit.minimize(residual, - initial_params, - args=(data.time_axis(), data), - xtol=5e-3) - - unphased_data = phase_fid(data, - -fitting_result.params['phase0'], - -fitting_result.params['phase1']) - real_unphased_data = complex_to_real(unphased_data) - real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) - fitted_data = real_to_complex(real_fitted_data) - fitting_basis = make_basis(fitting_result.params, data.time_axis()) - fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, - real_unphased_data[baseline_points:-baseline_points])[0] - - return fitting_weights, fitted_data, fitting_result + # Get list of metabolite names. + def get_metabolites(model_input): + metabolites = [] + for name, value in model_input.items(): + if type(value) is dict: + metabolites.append(name) + return metabolites + + # Get standard errors from lmfit MinimizerResult object. + def get_errors(result): + errors = {} + for name, param in result.params.items(): + errors[name] = param.stderr + return errors + + def phase_fid(fid_in, phase0, phase1): + """ + This function performs a Fourier Transform on the FID to shift it into phase. + + :param fid_in: FID to be fitted. + :param phase1: phase1 value. + :param phase0: phase0 value. + :return: + """ + spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) + np = fid_in.np + phase_shift = phase0 + phase1 * numpy.linspace(-np / 2, np / 2, np, endpoint=False) + phased_spectrum = spectrum * numpy.exp(1j * phase_shift) + return fid_in.inherit(numpy.fft.ifft(numpy.fft.ifftshift(phased_spectrum))) + + def make_basis(params, time_axis): + """ + This function generates a basis set. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :return: a matrix containing the generated basis set. + """ + # metabolite_name_list = [] + # for param in params.keys(): + # split = param.split('_') + # if len(split) == 2: + # if split[0] not in metabolite_name_list: + # metabolite_name_list.append(split[0]) + + basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) + for i, metabolite_name in enumerate(metabolite_name_list): + gaussian = suspect.basis.gaussian(time_axis, + params["{}_frequency".format(metabolite_name)], + params["{}_phase".format(metabolite_name)].value, + params["{}_width".format(metabolite_name)]) + real_gaussian = complex_to_real(gaussian) + basis_matrix[i, :] = real_gaussian + return basis_matrix + + def do_fit(params, time_axis, real_unphased_data): + """ + This function performs the fitting. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param real_unphased_data: + :return: List of fitted data points. + """ + baseline_points = 16 + basis = make_basis(params, time_axis) + + weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() + return fitted_data + + def residual(params, time_axis, data): + """ + This function calculates the residual to be minimized by the least squares means method. + + :param params: lmfit Parameters object containing fitting parameters. + :param time_axis: the time axis. + :param data: FID to be fitted. + :return: residual values of baseline points. + """ + baseline_points = 16 + # unphase the data to make it pure absorptive + unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + + fitted_data = do_fit(params, time_axis, real_unphased_data) + res = fitted_data - real_unphased_data + + return res[baseline_points:-baseline_points] + + def fit_data(data, initial_params): + """ + This function takes an FID and a set of parameters contained in an lmfit Parameters object, + and fits the data using the least squares means method. + + :param data: FID to be fitted. + :param initial_params: lmfit Parameters object containing fitting parameters. + :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. + """ + baseline_points = 16 + fitting_result = lmfit.minimize(residual, + initial_params, + args=(data.time_axis(), data), + xtol=5e-3) + + unphased_data = phase_fid(data, + -fitting_result.params['phase0'], + -fitting_result.params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) + fitted_data = real_to_complex(real_fitted_data) + fitting_basis = make_basis(fitting_result.params, data.time_axis()) + fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, + real_unphased_data[baseline_points:-baseline_points])[0] + + return fitting_weights, fitted_data, fitting_result + + # Convert lmfit parameters to model format + def parameters_to_model(parameters_obj, param_weights): + # metabolite_name_list = [] + # for param in parameters_obj.keys(): + # split = param.split('_') + # if len(split) == 2: + # if split[0] not in metabolite_name_list: + # metabolite_name_list.append(split[0]) + # Create dictionary for new model. + new_model = {} + for param_name, param in parameters_obj.items(): + name = param_name.split("_") + name1 = name[0] + if len(name) == 1: # i.e. phase + new_model[name1] = param.value + else: + name2 = name[1] + if name1 not in new_model: + new_model[name1] = {name2: param.value} + else: + new_model[name1][name2] = param.value + + nonlocal metabolite_name_list + for i, metabolite_name in enumerate(metabolite_name_list): + new_model[metabolite_name]["amplitude"] = param_weights[i] + + return new_model + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + # Check if all model input types are correct. + def check_errors(check_model): + # Allowed names and keys in the model. + allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] + allowed_keys = ["min", "max", "value", "phase", "amplitude"] + + # Scan model. + for name1, value1 in check_model.items(): + if type(value1) is not int and type(value1) is not float and type(value1) is not dict: + raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) + elif name1 not in allowed_names: + raise NameError("{} is not an allowed name.".format(name1)) + elif type(value1) is dict: # i.e. type(value) is not int + for name2, value2 in value1.items(): + if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ + type(value2) is not str: + raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." + .format(name1, name2)) + if type(value2) is dict: + for key in value2: + # Dictionary must have 'value' key. + if "value" not in value2: + raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) + # Dictionary can only have 'min,' 'max,' and 'value'. + if key not in allowed_keys: + raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) + + return + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # Do singlet fitting + def main(): + # Minimize and fit 31P data. + # Check for errors in model formatting. + check_errors(model) + + # Set list of metabolite names. + # nonlocal metabolite_name_list + # metabolite_name_list = get_metabolites(model) + + # Convert model to lmfit Parameters object. + parameters = model_to_parameters(model) + + # Set metabolite name list + nonlocal metabolite_name_list + metabolite_name_list = get_metabolites(model) + + # Fit data. + fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) + + # Convert fit parameters to model format. + final_model = parameters_to_model(fitted_results.params, fitted_weights) + # Get stderr values for each parameter. + stderr = get_errors(fitted_results) + + # Compile output into a dictionary. + return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} + return return_dict + + return main() From a05fd42a28cd7175699f2226e1437038053d6530 Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 15:28:37 -0400 Subject: [PATCH 23/39] Consolidated fit function into singlet, removed fit.py --- suspect/fitting/singlet.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index f659b3d..e72109f 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -337,16 +337,12 @@ def main(): check_errors(model) # Set list of metabolite names. - # nonlocal metabolite_name_list - # metabolite_name_list = get_metabolites(model) + nonlocal metabolite_name_list + metabolite_name_list = get_metabolites(model) # Convert model to lmfit Parameters object. parameters = model_to_parameters(model) - # Set metabolite name list - nonlocal metabolite_name_list - metabolite_name_list = get_metabolites(model) - # Fit data. fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) From e0d7fb2ff4c69cbcc345f4634592ddefccfded12 Mon Sep 17 00:00:00 2001 From: bennyrowland Date: Fri, 9 Sep 2016 19:26:00 -0400 Subject: [PATCH 24/39] adjust Gaussian generation to take phase in radians --- suspect/basis/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suspect/basis/__init__.py b/suspect/basis/__init__.py index 1bb6424..c8d4eb3 100644 --- a/suspect/basis/__init__.py +++ b/suspect/basis/__init__.py @@ -2,7 +2,7 @@ def gaussian(time_axis, frequency, phase, fwhm): - oscillatory_term = numpy.exp(2j * numpy.pi * (frequency * time_axis + phase)) + oscillatory_term = numpy.exp(2j * numpy.pi * (frequency * time_axis) + 1j * phase) damping = numpy.exp(-time_axis ** 2 / 4 * numpy.pi ** 2 / numpy.log(2) * fwhm ** 2) fid = oscillatory_term * damping fid[0] /= 2.0 From 1f9a8d6437d927470a5670737741308271b0dbb3 Mon Sep 17 00:00:00 2001 From: bennyrowland Date: Fri, 9 Sep 2016 19:26:54 -0400 Subject: [PATCH 25/39] fixed errors in singlet fitting and added first unit test --- suspect/fitting/singlet.py | 21 +++++------ tests/test_mrs/test_fitting.py | 66 ++++++++++------------------------ 2 files changed, 26 insertions(+), 61 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index e72109f..345f898 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -3,7 +3,7 @@ import scipy.optimize import numbers -import suspect +import suspect.basis def complex_to_real(complex_fid): @@ -43,12 +43,13 @@ def real_to_complex(real_fid): return complex_fid -def fit(fid, model): +def fit(fid, model, baseline_points=16): """ Fit fid with model parameters. :param fid: MRSData object of FID to be fit :param model: dictionary model of fit parameters + :param baseline_points: the number of points at the start of the FID to ignore :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] """ @@ -117,16 +118,15 @@ def do_fit(params, time_axis, real_unphased_data): :param params: lmfit Parameters object containing fitting parameters. :param time_axis: the time axis. :param real_unphased_data: - :return: List of fitted data points. + :return: List of fitted data points and amplitudes of each singlet. """ - baseline_points = 16 basis = make_basis(params, time_axis) weights = scipy.optimize.nnls(basis[:, baseline_points:-baseline_points].T, real_unphased_data[baseline_points:-baseline_points])[0] fitted_data = numpy.array(numpy.dot(weights, basis)).squeeze() - return fitted_data + return fitted_data, weights def residual(params, time_axis, data): """ @@ -137,12 +137,11 @@ def residual(params, time_axis, data): :param data: FID to be fitted. :return: residual values of baseline points. """ - baseline_points = 16 # unphase the data to make it pure absorptive unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) real_unphased_data = complex_to_real(unphased_data) - fitted_data = do_fit(params, time_axis, real_unphased_data) + fitted_data, _ = do_fit(params, time_axis, real_unphased_data) res = fitted_data - real_unphased_data return res[baseline_points:-baseline_points] @@ -156,7 +155,6 @@ def fit_data(data, initial_params): :param initial_params: lmfit Parameters object containing fitting parameters. :return: tuple of weights as a list, data as a list, and result as an lmift MinimizerResult object. """ - baseline_points = 16 fitting_result = lmfit.minimize(residual, initial_params, args=(data.time_axis(), data), @@ -166,11 +164,8 @@ def fit_data(data, initial_params): -fitting_result.params['phase0'], -fitting_result.params['phase1']) real_unphased_data = complex_to_real(unphased_data) - real_fitted_data = do_fit(initial_params, data.time_axis(), real_unphased_data) + real_fitted_data, fitting_weights = do_fit(fitting_result.params, data.time_axis(), real_unphased_data) fitted_data = real_to_complex(real_fitted_data) - fitting_basis = make_basis(fitting_result.params, data.time_axis()) - fitting_weights = scipy.optimize.nnls(fitting_basis[:, baseline_points:-baseline_points].T, - real_unphased_data[baseline_points:-baseline_points])[0] return fitting_weights, fitted_data, fitting_result @@ -222,7 +217,7 @@ def model_to_parameters(model_dict): # Initialize lmfit parameter arguments. name = "{}_{}".format(name1, name2) value = None - vary = None + vary = True lmfit_min = None lmfit_max = None expr = None diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index 20b3fce..71af908 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -6,56 +6,26 @@ def test_gaussian(): time_axis = numpy.arange(0, 0.512, 5e-4) - fid = basis.gaussian(time_axis, 0, 0, 50.0) + fid = basis.gaussian(time_axis, 0, 0, 50.0) + 0.00001 * (numpy.random.rand(1024) - 0.5) data = MRSData(fid, 5e-4, 123) - model = singlet.Model({ - "g1": { - "amplitude": 1.0, - "fwhm": 50.0, + model = { + "phase0": 0, + "phase1": 0, + "pcr": { + "amplitude": 1, + "width": { + "value": 45, + "min": 42, + "max": 55 + }, "phase": "0", - "frequency": 0.0 + "frequency": 0 } - }) - fitting_result = model.fit(data) - numpy.testing.assert_allclose(fitting_result["params"]["g1"]["fwhm"], 50.0) - numpy.testing.assert_allclose(fitting_result["params"]["g1"]["amplitude"], 1.0) - numpy.testing.assert_allclose(fitting_result["params"]["g1"]["frequency"], 0.0, atol=1e-7) + } + fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["fit"], fid) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=1e-2) - -# def test_multiple_gaussians(): -# time_axis = numpy.arange(0, 0.512, 5e-4) -# g1 = basis.gaussian(time_axis, 0, 0, 50.0) -# g2 = basis.gaussian(time_axis, 250, 0, 40.0) -# fid = g1 + g2 -# data = MRSData(fid, 5e-4, 123) -# model = singlet.Model({ -# "g1": { -# "amplitude": 1.0, -# "fwhm": 50.0, -# "phase": "0", -# "frequency": {"value": 0.0, "min": -50, "max": 50} -# }, -# "g2": { -# "amplitude": "g1amplitude", -# "fwhm": 45, -# "phase": "0", -# "frequency": 230 -# } -# }) -# -# fitting_result = model.fit(data) -# params = fitting_result["params"] -# -# print(params["g1"]) -# numpy.testing.assert_allclose(params["g1"]["fwhm"], 50.0) -# numpy.testing.assert_allclose(params["g1"]["amplitude"], 1.0) -# numpy.testing.assert_allclose(params["g1"]["frequency"], 0.0, atol=1e-7) -# numpy.testing.assert_allclose(params["g2"]["fwhm"], 40.0) -# numpy.testing.assert_allclose(params["g2"]["amplitude"], 1.0) -# numpy.testing.assert_allclose(params["g2"]["frequency"], 250.0) -# -# numpy.testing.assert_allclose(fitting_result["fit"], fid) -# numpy.testing.assert_allclose(fitting_result["fit_components"]["g1"], g1) -# numpy.testing.assert_allclose(fitting_result["fit_components"]["g2"], g2) + numpy.testing.assert_allclose(fitting_result["fit"], fid, atol=0.001) From 2a6e32c9f5e96f75d09fd2e0367a00a7b1ac86bc Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 25 Jul 2016 17:30:50 -0400 Subject: [PATCH 26/39] Added fit file, moved amares fitting functions to singlet, converted a few values in denoising and singlet to int --- suspect/fitting/fit.py | 443 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 443 insertions(+) create mode 100644 suspect/fitting/fit.py diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py new file mode 100644 index 0000000..e622757 --- /dev/null +++ b/suspect/fitting/fit.py @@ -0,0 +1,443 @@ +""" +File Name: fit.py +Author: Sam Jiang +Purpose: fit_31p() function fits single 31P fid. +Interpreter: Anaconda 3.5.1 + +fit(fid, model) +Input: fid and model in hierarchical dictionary format +Output: dictionary with 'model', 'fit', and 'error' keys and corresponding values +""" + +import os +import numpy +import json +import lmfit +import array +import suspect + + +# Load model in. +def load(fin): + if not os.path.isfile(fin): + raise FileNotFoundError("{} not found.".format(fin)) + with open(fin, 'r+') as f: + model = json.load(f) + return model + + +# Save model or fit out. +def save(output, fout): + # Check if file and directory exists. + directory, name = os.path.split(fout) + if directory == '': + directory = os.getcwd() + if not os.path.isdir(directory): + os.makedirs(directory) + if name == '': + raise IsADirectoryError("{} is not a file name.".format(name)) + ftype = os.path.splitext(name)[1] + + # If model, save as json. + if type(output) is dict: + if ftype != ".json": + raise TypeError("Output file must be a JSON (.json) file.") + + with open(fout, 'w+') as f: + json.dump(output, f) + + # If fit, save as csv or txt. + elif type(output) is array.ArrayType: + if ftype != ".csv" or ftype != ".txt": + raise TypeError("Output file must be a CSV (.csv) or Text (.txt) file.") + + with open(fout, 'w+') as f: + first = True + for x in output.tolist(): + if first: + f.write("{}".format(x)) + first = False + else: + f.write(", {}".format(x)) + + +# Fit fid +# Input: fid, model +# Output: dictionary containing optimized model, fit data, standard errors +def fit(fid, model): + # MODEL + # Get list of metabolite names + # def get_metabolites(model_input): + # metabolites = [] + # for name, value in model_input.items(): + # if type(value) is dict: + # metabolites.append(name) + # return metabolites + + # Get standard errors from lmfit MinimizerResult object. + def get_errors(result): + errors = {} + for name, param in result.params.items(): + errors[name] = param.stderr + return errors + + # Convert lmfit parameters to model format + def parameters_to_model(parameters_obj, param_weights): + metabolite_name_list = [] + for param in parameters_obj.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + # Create dictionary for new model. + new_model = {} + for param_name, param in parameters_obj.items(): + name = param_name.split("_") + name1 = name[0] + if len(name) == 1: # i.e. phase + new_model[name1] = param.value + else: + name2 = name[1] + if name1 not in new_model: + new_model[name1] = {name2: param.value} + else: + new_model[name1][name2] = param.value + + for i, metabolite_name in enumerate(metabolite_name_list): + new_model[metabolite_name]["amplitude"] = param_weights[i] + + return new_model + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + # Check if all model input types are correct. + def check_errors(check_model): + # Allowed names and keys in the model. + allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] + allowed_keys = ["min", "max", "value", "phase", "amplitude"] + + # Scan model. + for name1, value1 in check_model.items(): + if type(value1) is not int and type(value1) is not float and type(value1) is not dict: + raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) + elif name1 not in allowed_names: + raise NameError("{} is not an allowed name.".format(name1)) + elif type(value1) is dict: # i.e. type(value) is not int + for name2, value2 in value1.items(): + if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ + type(value2) is not str: + raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." + .format(name1, name2)) + if type(value2) is dict: + for key in value2: + # Dictionary must have 'value' key. + if "value" not in value2: + raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) + # Dictionary can only have 'min,' 'max,' and 'value'. + if key not in allowed_keys: + raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) + + return + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # MAIN + def main(): + # Minimize and fit 31P data. + # Check for errors in model formatting. + check_errors(model) + + # Set list of metabolite names. + # nonlocal metabolite_name_list + # metabolite_name_list = get_metabolites(model) + + # Convert model to lmfit Parameters object. + parameters = model_to_parameters(model) + + # Fit data. + fitted_weights, fitted_data, fitted_results = suspect.fitting.singlet.fit_data(fid, parameters) + + # Convert fit parameters to model format. + final_model = parameters_to_model(fitted_results.params, fitted_weights) + # Get stderr values for each parameter. + stderr = get_errors(fitted_results) + + # Compile output into a dictionary. + return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} + + return return_dict + + if __name__ == "__main__": + return main() + + +# Test fit function +def test_fit(_plot): + # Test file and model + testfile = "test_timecourse.csv" + modelfile = "model.json" + model = load(modelfile) + + # Calculate references to determine order for Parameters. + def calculate_dependencies(unordered_model): + dependencies = {} # (name, [dependencies]) + + # Compile dictionary of effective names. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2 in value1: + dependencies["{}_{}".format(name1, name2)] = None + + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase + for name2, value2 in value1.items(): + if type(value2) is str: + lmfit_name = "{}_{}".format(name1, name2) + dependencies[lmfit_name] = [] + for depend in dependencies: + if depend in value2: + dependencies[lmfit_name].append(depend) + + # Check for circular dependencies. + for name, dependents in dependencies.items(): + if type(dependents) is list: + for dependent in dependents: + if name in dependencies[dependent]: + raise ReferenceError("{} and {} reference each other, creating a circular reference." + .format(name, dependent)) + + return dependencies + + # Convert initial model to lmfit parameters. + def model_to_parameters(model_dict): + lmfit_parameters = lmfit.Parameters() + params = [] + ordered_params = [] + # Calculate dependencies/references for each parameter. + depend_dict = calculate_dependencies(model_dict) + + # Construct lmfit Parameter input for each parameter. + for name1, value1 in model_dict.items(): + if type(value1) is int: # (e.g. phase0) + params.append((name1, value1)) + if type(value1) is dict: + # Fix phase value to 0 by default. + if "phase" not in value1: + params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + for name2, value2 in value1.items(): + # Initialize lmfit parameter arguments. + name = "{}_{}".format(name1, name2) + value = None + vary = None + lmfit_min = None + lmfit_max = None + expr = None + if type(value2) is int: + value = value2 + elif type(value2) is str: + expr = value2 + if type(value2) is dict: + if "value" in value2: + value = value2["value"] + # if "vary" in value2: + # vary = value2["vary"] + if "min" in value2: + lmfit_min = value2["min"] + if "max" in value2: + lmfit_max = value2["max"] + # Add parameter object with defined parameters. + params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) + + # Order parameters based on dependencies. + in_oparams = [] + while len(params) > 0: + front = params.pop(0) + name = front[0] + # If no dependencies, add parameter to list and mark parameter as added. + if name not in depend_dict or depend_dict[name] is None: + ordered_params.append(front) + in_oparams.append(name) + else: + dependencies_present = True + for dependency in depend_dict[name]: + # If dependency not yet added, mark parameter to move to back of queue. + if dependency not in in_oparams: + dependencies_present = False + # If all dependencies present, add parameter to list and mark parameter as added. + if dependencies_present: + ordered_params.append(front) + in_oparams.append(name) + # If dependencies missing, move parameter to back of queue. + else: + params.append(front) + + # Convert all parameters to lmfit Parameter objects. + lmfit_parameters.add_many(*ordered_params) + + return lmfit_parameters + + def fake_spectrum(params, time_axis, weights): + basis = suspect.fitting.singlet.make_basis(params, time_axis) + real_data = numpy.array(numpy.dot(weights, basis)).squeeze() + complex_data = suspect.fitting.singlet.real_to_complex(real_data) + return suspect.MRSData(complex_data, dt, f0, te=0, ppm0=0) + + # Generate fake noise (Testing only) + def generate_noise(n, snr): + # Added (n * 0) to fix unused variable warning. + noise = (n * 0) + (numpy.random.randn(4096) + 1j * numpy.random.randn(4096)) * 6.4e-6 / snr + return noise + + # Generate fake fid (Testing only) + def generate_fid(ground_truth, snr, initial_params): + metabolite_name_list = [] + for param in initial_params.keys(): + split = param.split('_') + if len(split) == 2: + if split[0] not in metabolite_name_list: + metabolite_name_list.append(split[0]) + + peaks = {"pcr": 1, + "atpc": 0.3, + "atpb": 0.3, + "atpa": 0.3, + "pi": 1, + "pme": 0.05, + "pde": 0.05} + amps = [] + for metabolite in metabolite_name_list: + amps.append(peaks[metabolite]) + + noisy_sequence = numpy.zeros((len(ground_truth), np), 'complex') + noisy_sequence = suspect.MRSData(noisy_sequence, dt, f0, te=0, ppm0=0) + + # Populate the noisy_sequence with data. + generated_fid = fake_spectrum(initial_params, noisy_sequence.time_axis(), amps) + # [1, 0.3, 0.3, 0.3, 1, 0.05, 0.05] + noisy_fid = generated_fid + generate_noise(np, snr) + + return generated_fid, noisy_fid + + # Generate fake FID. + parameters = model_to_parameters(model) + sw = 6e3 + dt = 1.0 / sw + np = 4096 + f0 = 49.885802 + fake_fid, noisy_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid + + # Test fit(). + fit_results = fit(fake_fid, model) + + # Plot fit, noisy_fid, and fake_fid. + if _plot: + from matplotlib import pyplot + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noisy_fid))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) + pyplot.show() + + +test = False +plot = True +if test: + print("Running test.") + test_fit(plot) + print("Test complete.") + From b80da66a4733f2df5fe188bb3291b5a0c0958fc9 Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 14:22:23 -0400 Subject: [PATCH 27/39] Added fit file, moved amares fitting functions to singlet, converted a few values in denoising and singlet to int --- suspect/fitting/fit.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py index e622757..fabce0b 100644 --- a/suspect/fitting/fit.py +++ b/suspect/fitting/fit.py @@ -62,9 +62,14 @@ def save(output, fout): # Fit fid -# Input: fid, model -# Output: dictionary containing optimized model, fit data, standard errors def fit(fid, model): + """ + Fit fid with model parameters. + + :param fid: MRSData object of FID to be fit + :param model: dictionary model of fit parameters + :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] + """ # MODEL # Get list of metabolite names # def get_metabolites(model_input): @@ -267,10 +272,9 @@ def main(): # Test fit function -def test_fit(_plot): +def test_fit(_plot, testfile, modelfile): # Test file and model - testfile = "test_timecourse.csv" - modelfile = "model.json" + model = load(modelfile) # Calculate references to determine order for Parameters. @@ -420,24 +424,25 @@ def generate_fid(ground_truth, snr, initial_params): dt = 1.0 / sw np = 4096 f0 = 49.885802 - fake_fid, noisy_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid + fake_fid, noise_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid # Test fit(). fit_results = fit(fake_fid, model) - # Plot fit, noisy_fid, and fake_fid. + # Plot fit, noise_fid, and fake_fid. if _plot: from matplotlib import pyplot pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noisy_fid))) + pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noise_fid))) pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) pyplot.show() test = False -plot = True +plot = False if test: print("Running test.") - test_fit(plot) + test_file = "/home/sam/Desktop/amares/test_timecourse.csv" + test_model = "/home/sam/Desktop/amares/model.json" + test_fit(plot, test_file, test_model) print("Test complete.") - From e2eae9a095e364ce40118e599466ff97c4a5d68c Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 4 Aug 2016 15:22:02 -0400 Subject: [PATCH 28/39] Consolidated fit function into singlet, removed fit.py --- suspect/fitting/fit.py | 448 ----------------------------------------- 1 file changed, 448 deletions(-) delete mode 100644 suspect/fitting/fit.py diff --git a/suspect/fitting/fit.py b/suspect/fitting/fit.py deleted file mode 100644 index fabce0b..0000000 --- a/suspect/fitting/fit.py +++ /dev/null @@ -1,448 +0,0 @@ -""" -File Name: fit.py -Author: Sam Jiang -Purpose: fit_31p() function fits single 31P fid. -Interpreter: Anaconda 3.5.1 - -fit(fid, model) -Input: fid and model in hierarchical dictionary format -Output: dictionary with 'model', 'fit', and 'error' keys and corresponding values -""" - -import os -import numpy -import json -import lmfit -import array -import suspect - - -# Load model in. -def load(fin): - if not os.path.isfile(fin): - raise FileNotFoundError("{} not found.".format(fin)) - with open(fin, 'r+') as f: - model = json.load(f) - return model - - -# Save model or fit out. -def save(output, fout): - # Check if file and directory exists. - directory, name = os.path.split(fout) - if directory == '': - directory = os.getcwd() - if not os.path.isdir(directory): - os.makedirs(directory) - if name == '': - raise IsADirectoryError("{} is not a file name.".format(name)) - ftype = os.path.splitext(name)[1] - - # If model, save as json. - if type(output) is dict: - if ftype != ".json": - raise TypeError("Output file must be a JSON (.json) file.") - - with open(fout, 'w+') as f: - json.dump(output, f) - - # If fit, save as csv or txt. - elif type(output) is array.ArrayType: - if ftype != ".csv" or ftype != ".txt": - raise TypeError("Output file must be a CSV (.csv) or Text (.txt) file.") - - with open(fout, 'w+') as f: - first = True - for x in output.tolist(): - if first: - f.write("{}".format(x)) - first = False - else: - f.write(", {}".format(x)) - - -# Fit fid -def fit(fid, model): - """ - Fit fid with model parameters. - - :param fid: MRSData object of FID to be fit - :param model: dictionary model of fit parameters - :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] - """ - # MODEL - # Get list of metabolite names - # def get_metabolites(model_input): - # metabolites = [] - # for name, value in model_input.items(): - # if type(value) is dict: - # metabolites.append(name) - # return metabolites - - # Get standard errors from lmfit MinimizerResult object. - def get_errors(result): - errors = {} - for name, param in result.params.items(): - errors[name] = param.stderr - return errors - - # Convert lmfit parameters to model format - def parameters_to_model(parameters_obj, param_weights): - metabolite_name_list = [] - for param in parameters_obj.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - # Create dictionary for new model. - new_model = {} - for param_name, param in parameters_obj.items(): - name = param_name.split("_") - name1 = name[0] - if len(name) == 1: # i.e. phase - new_model[name1] = param.value - else: - name2 = name[1] - if name1 not in new_model: - new_model[name1] = {name2: param.value} - else: - new_model[name1][name2] = param.value - - for i, metabolite_name in enumerate(metabolite_name_list): - new_model[metabolite_name]["amplitude"] = param_weights[i] - - return new_model - - # Convert initial model to lmfit parameters. - def model_to_parameters(model_dict): - lmfit_parameters = lmfit.Parameters() - params = [] - ordered_params = [] - # Calculate dependencies/references for each parameter. - depend_dict = calculate_dependencies(model_dict) - - # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: - # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): - # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) - value = None - vary = None - lmfit_min = None - lmfit_max = None - expr = None - if type(value2) is int: - value = value2 - elif type(value2) is str: - expr = value2 - if type(value2) is dict: - if "value" in value2: - value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] - # Add parameter object with defined parameters. - params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) - - # Order parameters based on dependencies. - in_oparams = [] - while len(params) > 0: - front = params.pop(0) - name = front[0] - # If no dependencies, add parameter to list and mark parameter as added. - if name not in depend_dict or depend_dict[name] is None: - ordered_params.append(front) - in_oparams.append(name) - else: - dependencies_present = True - for dependency in depend_dict[name]: - # If dependency not yet added, mark parameter to move to back of queue. - if dependency not in in_oparams: - dependencies_present = False - # If all dependencies present, add parameter to list and mark parameter as added. - if dependencies_present: - ordered_params.append(front) - in_oparams.append(name) - # If dependencies missing, move parameter to back of queue. - else: - params.append(front) - - # Convert all parameters to lmfit Parameter objects. - lmfit_parameters.add_many(*ordered_params) - - return lmfit_parameters - - # Check if all model input types are correct. - def check_errors(check_model): - # Allowed names and keys in the model. - allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] - allowed_keys = ["min", "max", "value", "phase", "amplitude"] - - # Scan model. - for name1, value1 in check_model.items(): - if type(value1) is not int and type(value1) is not float and type(value1) is not dict: - raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) - elif name1 not in allowed_names: - raise NameError("{} is not an allowed name.".format(name1)) - elif type(value1) is dict: # i.e. type(value) is not int - for name2, value2 in value1.items(): - if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ - type(value2) is not str: - raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." - .format(name1, name2)) - if type(value2) is dict: - for key in value2: - # Dictionary must have 'value' key. - if "value" not in value2: - raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) - # Dictionary can only have 'min,' 'max,' and 'value'. - if key not in allowed_keys: - raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) - - return - - # Calculate references to determine order for Parameters. - def calculate_dependencies(unordered_model): - dependencies = {} # (name, [dependencies]) - - # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None - - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) - dependencies[lmfit_name] = [] - for depend in dependencies: - if depend in value2: - dependencies[lmfit_name].append(depend) - - # Check for circular dependencies. - for name, dependents in dependencies.items(): - if type(dependents) is list: - for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." - .format(name, dependent)) - - return dependencies - - # MAIN - def main(): - # Minimize and fit 31P data. - # Check for errors in model formatting. - check_errors(model) - - # Set list of metabolite names. - # nonlocal metabolite_name_list - # metabolite_name_list = get_metabolites(model) - - # Convert model to lmfit Parameters object. - parameters = model_to_parameters(model) - - # Fit data. - fitted_weights, fitted_data, fitted_results = suspect.fitting.singlet.fit_data(fid, parameters) - - # Convert fit parameters to model format. - final_model = parameters_to_model(fitted_results.params, fitted_weights) - # Get stderr values for each parameter. - stderr = get_errors(fitted_results) - - # Compile output into a dictionary. - return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} - - return return_dict - - if __name__ == "__main__": - return main() - - -# Test fit function -def test_fit(_plot, testfile, modelfile): - # Test file and model - - model = load(modelfile) - - # Calculate references to determine order for Parameters. - def calculate_dependencies(unordered_model): - dependencies = {} # (name, [dependencies]) - - # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None - - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) - dependencies[lmfit_name] = [] - for depend in dependencies: - if depend in value2: - dependencies[lmfit_name].append(depend) - - # Check for circular dependencies. - for name, dependents in dependencies.items(): - if type(dependents) is list: - for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." - .format(name, dependent)) - - return dependencies - - # Convert initial model to lmfit parameters. - def model_to_parameters(model_dict): - lmfit_parameters = lmfit.Parameters() - params = [] - ordered_params = [] - # Calculate dependencies/references for each parameter. - depend_dict = calculate_dependencies(model_dict) - - # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: - # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): - # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) - value = None - vary = None - lmfit_min = None - lmfit_max = None - expr = None - if type(value2) is int: - value = value2 - elif type(value2) is str: - expr = value2 - if type(value2) is dict: - if "value" in value2: - value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] - # Add parameter object with defined parameters. - params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) - - # Order parameters based on dependencies. - in_oparams = [] - while len(params) > 0: - front = params.pop(0) - name = front[0] - # If no dependencies, add parameter to list and mark parameter as added. - if name not in depend_dict or depend_dict[name] is None: - ordered_params.append(front) - in_oparams.append(name) - else: - dependencies_present = True - for dependency in depend_dict[name]: - # If dependency not yet added, mark parameter to move to back of queue. - if dependency not in in_oparams: - dependencies_present = False - # If all dependencies present, add parameter to list and mark parameter as added. - if dependencies_present: - ordered_params.append(front) - in_oparams.append(name) - # If dependencies missing, move parameter to back of queue. - else: - params.append(front) - - # Convert all parameters to lmfit Parameter objects. - lmfit_parameters.add_many(*ordered_params) - - return lmfit_parameters - - def fake_spectrum(params, time_axis, weights): - basis = suspect.fitting.singlet.make_basis(params, time_axis) - real_data = numpy.array(numpy.dot(weights, basis)).squeeze() - complex_data = suspect.fitting.singlet.real_to_complex(real_data) - return suspect.MRSData(complex_data, dt, f0, te=0, ppm0=0) - - # Generate fake noise (Testing only) - def generate_noise(n, snr): - # Added (n * 0) to fix unused variable warning. - noise = (n * 0) + (numpy.random.randn(4096) + 1j * numpy.random.randn(4096)) * 6.4e-6 / snr - return noise - - # Generate fake fid (Testing only) - def generate_fid(ground_truth, snr, initial_params): - metabolite_name_list = [] - for param in initial_params.keys(): - split = param.split('_') - if len(split) == 2: - if split[0] not in metabolite_name_list: - metabolite_name_list.append(split[0]) - - peaks = {"pcr": 1, - "atpc": 0.3, - "atpb": 0.3, - "atpa": 0.3, - "pi": 1, - "pme": 0.05, - "pde": 0.05} - amps = [] - for metabolite in metabolite_name_list: - amps.append(peaks[metabolite]) - - noisy_sequence = numpy.zeros((len(ground_truth), np), 'complex') - noisy_sequence = suspect.MRSData(noisy_sequence, dt, f0, te=0, ppm0=0) - - # Populate the noisy_sequence with data. - generated_fid = fake_spectrum(initial_params, noisy_sequence.time_axis(), amps) - # [1, 0.3, 0.3, 0.3, 1, 0.05, 0.05] - noisy_fid = generated_fid + generate_noise(np, snr) - - return generated_fid, noisy_fid - - # Generate fake FID. - parameters = model_to_parameters(model) - sw = 6e3 - dt = 1.0 / sw - np = 4096 - f0 = 49.885802 - fake_fid, noise_fid = generate_fid(numpy.loadtxt(testfile), 4, parameters) # using filename, not actual fid - - # Test fit(). - fit_results = fit(fake_fid, model) - - # Plot fit, noise_fid, and fake_fid. - if _plot: - from matplotlib import pyplot - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fit_results["fit"]))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(noise_fid))) - pyplot.plot(numpy.fft.fftshift(numpy.fft.fft(fake_fid))) - pyplot.show() - - -test = False -plot = False -if test: - print("Running test.") - test_file = "/home/sam/Desktop/amares/test_timecourse.csv" - test_model = "/home/sam/Desktop/amares/model.json" - test_fit(plot, test_file, test_model) - print("Test complete.") From 99658841b2186dc4571f3b42bfde95edaef1dd30 Mon Sep 17 00:00:00 2001 From: bennyrowland Date: Wed, 14 Sep 2016 11:11:27 -0400 Subject: [PATCH 29/39] fixed Gaussian fitting accuracy --- tests/test_mrs/test_fitting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index 71af908..2cd5703 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -24,7 +24,7 @@ def test_gaussian(): } fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=1e-2) From d156fca97a268477701af3fd7fcfe108becefffb Mon Sep 17 00:00:00 2001 From: lasyasreepada Date: Wed, 14 Sep 2016 10:51:02 -0400 Subject: [PATCH 30/39] Modified the files suspect/fitting/singlet.py and tests/test_mrs/test_fitting in order to improve existing code and write a more comprehensive test --- suspect/fitting/singlet.py | 87 +++++++++++---------------- tests/test_mrs/test_fitting.py | 105 ++++++++++++++++++++++++++++++--- 2 files changed, 131 insertions(+), 61 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 345f898..511bbd0 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -5,7 +5,6 @@ import suspect.basis - def complex_to_real(complex_fid): """ Standard optimization routines as used in lmfit require real data. This @@ -50,12 +49,9 @@ def fit(fid, model, baseline_points=16): :param fid: MRSData object of FID to be fit :param model: dictionary model of fit parameters :param baseline_points: the number of points at the start of the FID to ignore - :return: ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] + :return: Dictionary containing ["model": optimized model, "fit": fitting data, "err": dictionary of standard errors] """ - # List of metabolite names - metabolite_name_list = [] - # Get list of metabolite names. def get_metabolites(model_input): metabolites = [] @@ -78,7 +74,7 @@ def phase_fid(fid_in, phase0, phase1): :param fid_in: FID to be fitted. :param phase1: phase1 value. :param phase0: phase0 value. - :return: + :return: FID that has been shifted into phase by FFT """ spectrum = numpy.fft.fftshift(numpy.fft.fft(fid_in)) np = fid_in.np @@ -111,6 +107,13 @@ def make_basis(params, time_axis): basis_matrix[i, :] = real_gaussian return basis_matrix + def unphase(data, params): + + unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) + real_unphased_data = complex_to_real(unphased_data) + + return real_unphased_data + def do_fit(params, time_axis, real_unphased_data): """ This function performs the fitting. @@ -137,11 +140,9 @@ def residual(params, time_axis, data): :param data: FID to be fitted. :return: residual values of baseline points. """ - # unphase the data to make it pure absorptive - unphased_data = phase_fid(data, -params['phase0'], -params['phase1']) - real_unphased_data = complex_to_real(unphased_data) - fitted_data, _ = do_fit(params, time_axis, real_unphased_data) + real_unphased_data = unphase(data, params) + fitted_data, weights = do_fit(params, time_axis, real_unphased_data) res = fitted_data - real_unphased_data return res[baseline_points:-baseline_points] @@ -160,11 +161,7 @@ def fit_data(data, initial_params): args=(data.time_axis(), data), xtol=5e-3) - unphased_data = phase_fid(data, - -fitting_result.params['phase0'], - -fitting_result.params['phase1']) - real_unphased_data = complex_to_real(unphased_data) - real_fitted_data, fitting_weights = do_fit(fitting_result.params, data.time_axis(), real_unphased_data) + real_fitted_data, fitting_weights = do_fit(fitting_result.params, data.time_axis(), unphase(data, fitting_result.params)) fitted_data = real_to_complex(real_fitted_data) return fitting_weights, fitted_data, fitting_result @@ -191,7 +188,6 @@ def parameters_to_model(parameters_obj, param_weights): else: new_model[name1][name2] = param.value - nonlocal metabolite_name_list for i, metabolite_name in enumerate(metabolite_name_list): new_model[metabolite_name]["amplitude"] = param_weights[i] @@ -207,7 +203,7 @@ def model_to_parameters(model_dict): # Construct lmfit Parameter input for each parameter. for name1, value1 in model_dict.items(): - if type(value1) is int: # (e.g. phase0) + if isinstance(value1, numbers.Number): # (e.g. phase0) params.append((name1, value1)) if type(value1) is dict: # Fix phase value to 0 by default. @@ -221,15 +217,13 @@ def model_to_parameters(model_dict): lmfit_min = None lmfit_max = None expr = None - if type(value2) is int: + if isinstance(value2, numbers.Number): value = value2 - elif type(value2) is str: + elif isinstance(value2, str): expr = value2 - if type(value2) is dict: + elif isinstance(value2, dict): if "value" in value2: value = value2["value"] - # if "vary" in value2: - # vary = value2["vary"] if "min" in value2: lmfit_min = value2["min"] if "max" in value2: @@ -267,20 +261,16 @@ def model_to_parameters(model_dict): # Check if all model input types are correct. def check_errors(check_model): - # Allowed names and keys in the model. - allowed_names = ["pcr", "atpc", "atpb", "atpa", "pi", "pme", "pde", "phase0", "phase1"] + # Allowed keys in the model. allowed_keys = ["min", "max", "value", "phase", "amplitude"] # Scan model. for name1, value1 in check_model.items(): - if type(value1) is not int and type(value1) is not float and type(value1) is not dict: + if not isinstance(value1, (numbers.Number, dict)): raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) - elif name1 not in allowed_names: - raise NameError("{} is not an allowed name.".format(name1)) elif type(value1) is dict: # i.e. type(value) is not int for name2, value2 in value1.items(): - if type(value2) is not int and type(value2) is not float and type(value2) is not dict and \ - type(value2) is not str: + if not isinstance(value2,(numbers.Number,dict,str)): raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." .format(name1, name2)) if type(value2) is dict: @@ -292,21 +282,18 @@ def check_errors(check_model): if key not in allowed_keys: raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) - return - # Calculate references to determine order for Parameters. def calculate_dependencies(unordered_model): dependencies = {} # (name, [dependencies]) - # Compile dictionary of effective names. for name1, value1 in unordered_model.items(): if type(value1) is dict: # i.e. not phase + + # Compile dictionary of effective names. for name2 in value1: dependencies["{}_{}".format(name1, name2)] = None - # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase + # Find dependencies for each effective name. for name2, value2 in value1.items(): if type(value2) is str: lmfit_name = "{}_{}".format(name1, name2) @@ -326,28 +313,20 @@ def calculate_dependencies(unordered_model): return dependencies # Do singlet fitting - def main(): - # Minimize and fit 31P data. - # Check for errors in model formatting. - check_errors(model) + # Minimize and fit 31P data. + + check_errors(model) # Check for errors in model formatting. + + metabolite_name_list = get_metabolites(model) # Set list of metabolite names. - # Set list of metabolite names. - nonlocal metabolite_name_list - metabolite_name_list = get_metabolites(model) + parameters = model_to_parameters(model) # Convert model to lmfit Parameters object. - # Convert model to lmfit Parameters object. - parameters = model_to_parameters(model) + fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) # Fit data. - # Fit data. - fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) + final_model = parameters_to_model(fitted_results.params, fitted_weights) # Convert fit parameters to model format. - # Convert fit parameters to model format. - final_model = parameters_to_model(fitted_results.params, fitted_weights) - # Get stderr values for each parameter. - stderr = get_errors(fitted_results) + stderr = get_errors(fitted_results) # Get stderr values for each parameter. - # Compile output into a dictionary. - return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} - return return_dict + return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} # Compile output into a dictionary. + return return_dict - return main() diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index 2cd5703..12e57e1 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -3,12 +3,13 @@ import numpy - def test_gaussian(): time_axis = numpy.arange(0, 0.512, 5e-4) fid = basis.gaussian(time_axis, 0, 0, 50.0) + 0.00001 * (numpy.random.rand(1024) - 0.5) data = MRSData(fid, 5e-4, 123) - model = { + + #Original test with all parameters passed in; correct data types; integer values + model1 = { "phase0": 0, "phase1": 0, "pcr": { @@ -22,10 +23,100 @@ def test_gaussian(): "frequency": 0 } } - fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=1e-2) + #Floating point values; invalid key added to width dict, to test whether KeyError is raised + model2 = { + "phase0": 0.0, + "phase1": 0.0, + "pcr": { + "amplitude": 1.0, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + "avg": 47 + }, + "phase": "0", + "frequency": 0.0 + } + } + + #No width value passed in, to test whether KeyError is raised + model3 = { + "phase0": 0, + "phase1": 0, + "pcr": { + "amplitude": 1, + "width": { + #"value": 45, + "min": 42, + "max": 55, + + }, + "phase": "0", + "frequency": 0 + } + } + + #No phase value passed in, to test whether phase is fixed to 0 by default + model4 = { + "phase0": 0, + "phase1": 0, + "pcr": { + "amplitude": 1, + "width": { + "value": 45, + "min": 42, + "max": 55, + }, + # "phase": "0", + "frequency": 0 + } + } + + #Str value supplied for phase0 and phase1, to test whether TypeError is raised + model5 = { + "phase0": str, + "phase1": str, + "pcr": { + "amplitude": 1.0, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + }, + "phase": "0", + "frequency": 0.0 + } + } + + # Str value supplied for amplitude, to test whether TypeError is raised + model6 = { + "phase0": 0.0, + "phase1": 0.0, + "pcr": { + "amplitude": str, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + }, + "phase": "0", + "frequency": 0.0 + } + } + + set_of_models = [model1, model2, model3, model4, model5, model6] + + for model in set_of_models: + try: + fitting_result = singlet.fit(data, model) + + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=1e-2) + + numpy.testing.assert_allclose(fitting_result["fit"], fid, atol=0.001) - numpy.testing.assert_allclose(fitting_result["fit"], fid, atol=0.001) + except(TypeError, KeyError, ReferenceError): + pass From 9310fd50f0660667bb7afc5322133c0fad223505 Mon Sep 17 00:00:00 2001 From: lasyasreepada Date: Thu, 15 Sep 2016 15:00:46 -0400 Subject: [PATCH 31/39] Fixed a bug in calculate_dependencies function in singlet.py. Added dependency tests to test_fitting. Seeded the random number generator and created fixtures for fid generation in order to ensure testing consistency. --- suspect/fitting/singlet.py | 43 +++---- tests/test_mrs/test_fitting.py | 210 ++++++++++++++++++++++++++++----- 2 files changed, 205 insertions(+), 48 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 511bbd0..14c4ec8 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -208,10 +208,10 @@ def model_to_parameters(model_dict): if type(value1) is dict: # Fix phase value to 0 by default. if "phase" not in value1: - params.append(("{}_{}".format(name1, "phase"), None, None, None, None, "0")) + params.append(("{0}_{1}".format(name1, "phase"), None, None, None, None, "0")) for name2, value2 in value1.items(): # Initialize lmfit parameter arguments. - name = "{}_{}".format(name1, name2) + name = "{0}_{1}".format(name1, name2) value = None vary = True lmfit_min = None @@ -267,65 +267,68 @@ def check_errors(check_model): # Scan model. for name1, value1 in check_model.items(): if not isinstance(value1, (numbers.Number, dict)): - raise TypeError("Value of {} must be a number (for phases), or a dictionary.".format(name1)) + raise TypeError("Value of {0} must be a number (for phases), or a dictionary.".format(name1)) elif type(value1) is dict: # i.e. type(value) is not int for name2, value2 in value1.items(): if not isinstance(value2,(numbers.Number,dict,str)): - raise TypeError("Value of {}_{} must be a value, an expression, or a dictionary." + raise TypeError("Value of {0}_{1} must be a value, an expression, or a dictionary." .format(name1, name2)) if type(value2) is dict: for key in value2: # Dictionary must have 'value' key. if "value" not in value2: - raise KeyError("Dictionary {}_{} is missing 'value' key.".format(name1, name2)) + raise KeyError("Dictionary {0}_{1} is missing 'value' key.".format(name1, name2)) # Dictionary can only have 'min,' 'max,' and 'value'. if key not in allowed_keys: - raise KeyError("In {}_{}, '{}' is not an allowed key.".format(name1, name2, key)) + raise KeyError("In {0}_{1}, '{2}' is not an allowed key.".format(name1, name2, key)) # Calculate references to determine order for Parameters. def calculate_dependencies(unordered_model): dependencies = {} # (name, [dependencies]) + # Compile dictionary of effective names. for name1, value1 in unordered_model.items(): if type(value1) is dict: # i.e. not phase - - # Compile dictionary of effective names. for name2 in value1: - dependencies["{}_{}".format(name1, name2)] = None + dependencies["{0}_{1}".format(name1, name2)] = None - # Find dependencies for each effective name. + # Find dependencies for each effective name. + for name1, value1 in unordered_model.items(): + if type(value1) is dict: # i.e. not phase for name2, value2 in value1.items(): if type(value2) is str: - lmfit_name = "{}_{}".format(name1, name2) + lmfit_name = "{0}_{1}".format(name1, name2) dependencies[lmfit_name] = [] for depend in dependencies: if depend in value2: dependencies[lmfit_name].append(depend) + print(dependencies) # Check for circular dependencies. for name, dependents in dependencies.items(): if type(dependents) is list: for dependent in dependents: - if name in dependencies[dependent]: - raise ReferenceError("{} and {} reference each other, creating a circular reference." + #print(dependent) + #print(dependencies) + if dependencies[dependent] is not None and name in dependencies[dependent]: + raise ReferenceError("{0} and {1} reference each other, creating a circular reference." .format(name, dependent)) - return dependencies # Do singlet fitting # Minimize and fit 31P data. - check_errors(model) # Check for errors in model formatting. + check_errors(model) # Check for errors in model formatting. - metabolite_name_list = get_metabolites(model) # Set list of metabolite names. + metabolite_name_list = get_metabolites(model) # Set list of metabolite names. - parameters = model_to_parameters(model) # Convert model to lmfit Parameters object. + parameters = model_to_parameters(model) # Convert model to lmfit Parameters object. - fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) # Fit data. + fitted_weights, fitted_data, fitted_results = fit_data(fid, parameters) # Fit data. - final_model = parameters_to_model(fitted_results.params, fitted_weights) # Convert fit parameters to model format. + final_model = parameters_to_model(fitted_results.params, fitted_weights) # Convert fit parameters to model format. - stderr = get_errors(fitted_results) # Get stderr values for each parameter. + stderr = get_errors(fitted_results) # Get stderr values for each parameter. return_dict = {"model": final_model, "fit": fitted_data, "errors": stderr} # Compile output into a dictionary. return return_dict diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index 12e57e1..ac467a8 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -2,14 +2,34 @@ from suspect import basis, MRSData import numpy +import pytest +import random + +random.seed() + +@pytest.fixture +def fixed_fid(): -def test_gaussian(): time_axis = numpy.arange(0, 0.512, 5e-4) fid = basis.gaussian(time_axis, 0, 0, 50.0) + 0.00001 * (numpy.random.rand(1024) - 0.5) - data = MRSData(fid, 5e-4, 123) + return fid + - #Original test with all parameters passed in; correct data types; integer values - model1 = { +@pytest.fixture +def fixed_fid_sum(): + + time_axis = numpy.arange(0, 0.512, 5e-4) + fid = basis.gaussian(time_axis, 0, 0, 50.0) + 0.00001 * (numpy.random.rand(1024) - 0.5) + fid2 = basis.gaussian(time_axis, 200, 0, 50.0) + return fid + fid2 + + +def test_gaussian(fixed_fid): + + data = MRSData(fixed_fid, 5e-4, 123) + + # Original test with all parameters passed in; correct data types; integer values + model = { "phase0": 0, "phase1": 0, "pcr": { @@ -23,9 +43,21 @@ def test_gaussian(): "frequency": 0 } } + fitting_result = singlet.fit(data, model) + + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) - #Floating point values; invalid key added to width dict, to test whether KeyError is raised - model2 = { + numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) + + +def test_gaussian_2(fixed_fid): + + data = MRSData(fixed_fid, 5e-4, 123) + + # Floating point values; invalid key added to width dict, to test whether KeyError is raised + model = { "phase0": 0.0, "phase1": 0.0, "pcr": { @@ -40,15 +72,28 @@ def test_gaussian(): "frequency": 0.0 } } + with pytest.raises(KeyError): + fitting_result = singlet.fit(data, model) + + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) + + numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) - #No width value passed in, to test whether KeyError is raised - model3 = { + +def test_gaussian_3(fixed_fid): + + data = MRSData(fixed_fid, 5e-4, 123) + + # No width value passed in, to test whether KeyError is raised + model = { "phase0": 0, "phase1": 0, "pcr": { "amplitude": 1, "width": { - #"value": 45, + # "value": 45, "min": 42, "max": 55, @@ -57,9 +102,21 @@ def test_gaussian(): "frequency": 0 } } + with pytest.raises(KeyError): + fitting_result = singlet.fit(data, model) + + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) + + numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) - #No phase value passed in, to test whether phase is fixed to 0 by default - model4 = { +def test_gaussian_4(fixed_fid): + + data = MRSData(fixed_fid, 5e-4, 123) + + # No phase value passed in, to test whether phase is fixed to 0 by default + model = { "phase0": 0, "phase1": 0, "pcr": { @@ -74,10 +131,23 @@ def test_gaussian(): } } - #Str value supplied for phase0 and phase1, to test whether TypeError is raised - model5 = { - "phase0": str, - "phase1": str, + fitting_result = singlet.fit(data, model) + + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) + + numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) + + +def test_gaussian_5(fixed_fid): + + data = MRSData(fixed_fid, 5e-4, 123) + + # None value supplied for phase0 and phase1, to test whether TypeError is raised + model = { + "phase0": None, + "phase1": None, "pcr": { "amplitude": 1.0, "width": { @@ -89,13 +159,26 @@ def test_gaussian(): "frequency": 0.0 } } + with pytest.raises(TypeError): + fitting_result = singlet.fit(data, model) + + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) + + numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) + - # Str value supplied for amplitude, to test whether TypeError is raised - model6 = { +def test_gaussian_6(fixed_fid): + + data = MRSData(fixed_fid, 5e-4, 123) + + # None value supplied for amplitude, to test whether TypeError is raised + model = { "phase0": 0.0, "phase1": 0.0, "pcr": { - "amplitude": str, + "amplitude": None, "width": { "value": 45.0, "min": 42.0, @@ -106,17 +189,88 @@ def test_gaussian(): } } - set_of_models = [model1, model2, model3, model4, model5, model6] + with pytest.raises(TypeError): + fitting_result = singlet.fit(data, model) + + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) + + numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) + + +def test_gaussian_dependencies(fixed_fid): + + data = MRSData(fixed_fid, 5e-4, 123) + + model = { + "phase0": 0.0, + "phase1": 0.0, + "pcr": { + "amplitude": 1.0, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + }, + "phase": "0", + "frequency": "pcr2_frequency+200" + }, + "pcr2": { + "amplitude": 1.0, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + }, + "phase": "0", + "frequency": "pcr_frequency-200" + } + } + + with pytest.raises(ReferenceError): + fitting_result = singlet.fit(data, model) + + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) + + numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) + + +def test_gaussian_dependencies2(fixed_fid_sum): - for model in set_of_models: - try: - fitting_result = singlet.fit(data, model) + data = MRSData(fixed_fid_sum, 5e-4, 123) + + model = { + "phase0": 0.0, + "phase1": 0.0, + "pcr": { + "amplitude": 1.0, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + }, + "phase": "0", + "frequency": 0 + }, + "pcr2": { + "amplitude": 1.0, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + }, + "phase": "0", + "frequency": "pcr_frequency+200" + } + } - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=1e-2) + fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["fit"], fid, atol=0.001) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) - except(TypeError, KeyError, ReferenceError): - pass + numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid_sum, atol=0.001) \ No newline at end of file From e7996be9daac4252fb1468731a4a0e4f1a817d04 Mon Sep 17 00:00:00 2001 From: lasyasreepada Date: Thu, 15 Sep 2016 15:05:47 -0400 Subject: [PATCH 32/39] Changed tolerances to lower values. --- tests/test_mrs/test_fitting.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index ac467a8..726c971 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -45,8 +45,8 @@ def test_gaussian(fixed_fid): } fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) @@ -75,8 +75,8 @@ def test_gaussian_2(fixed_fid): with pytest.raises(KeyError): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) @@ -105,8 +105,8 @@ def test_gaussian_3(fixed_fid): with pytest.raises(KeyError): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) @@ -133,8 +133,8 @@ def test_gaussian_4(fixed_fid): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) @@ -162,8 +162,8 @@ def test_gaussian_5(fixed_fid): with pytest.raises(TypeError): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) @@ -192,8 +192,8 @@ def test_gaussian_6(fixed_fid): with pytest.raises(TypeError): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) @@ -231,8 +231,8 @@ def test_gaussian_dependencies(fixed_fid): with pytest.raises(ReferenceError): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) @@ -269,8 +269,8 @@ def test_gaussian_dependencies2(fixed_fid_sum): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid_sum, atol=0.001) \ No newline at end of file From d598448588c35e78ad0f9f6c99bb6cc0fd682d0b Mon Sep 17 00:00:00 2001 From: lasyasreepada Date: Fri, 16 Sep 2016 14:35:32 -0400 Subject: [PATCH 33/39] Fixed some coding style issues and modified variable names in fit.model_to_parameters function of singlet.py to be more descriptive. Changed the loop constructing lmfit Parameter input in said function to iterate only over peak dictionaries by handling phase parameters outside the loop. Modified some tolerance values in test_fitting --- suspect/fitting/singlet.py | 111 +++++++++++++++------------------ tests/test_mrs/test_fitting.py | 6 +- 2 files changed, 53 insertions(+), 64 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 14c4ec8..9110c88 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -2,9 +2,11 @@ import numpy import scipy.optimize import numbers +import copy import suspect.basis + def complex_to_real(complex_fid): """ Standard optimization routines as used in lmfit require real data. This @@ -55,9 +57,9 @@ def fit(fid, model, baseline_points=16): # Get list of metabolite names. def get_metabolites(model_input): metabolites = [] - for name, value in model_input.items(): - if type(value) is dict: - metabolites.append(name) + for fid_property_name, fid_property_value in model_input.items(): + if type(fid_property_value) is dict: + metabolites.append(fid_property_name) return metabolites # Get standard errors from lmfit MinimizerResult object. @@ -90,12 +92,6 @@ def make_basis(params, time_axis): :param time_axis: the time axis. :return: a matrix containing the generated basis set. """ - # metabolite_name_list = [] - # for param in params.keys(): - # split = param.split('_') - # if len(split) == 2: - # if split[0] not in metabolite_name_list: - # metabolite_name_list.append(split[0]) basis_matrix = numpy.matrix(numpy.zeros((len(metabolite_name_list), len(time_axis) * 2))) for i, metabolite_name in enumerate(metabolite_name_list): @@ -168,13 +164,6 @@ def fit_data(data, initial_params): # Convert lmfit parameters to model format def parameters_to_model(parameters_obj, param_weights): - # metabolite_name_list = [] - # for param in parameters_obj.keys(): - # split = param.split('_') - # if len(split) == 2: - # if split[0] not in metabolite_name_list: - # metabolite_name_list.append(split[0]) - # Create dictionary for new model. new_model = {} for param_name, param in parameters_obj.items(): name = param_name.split("_") @@ -201,33 +190,34 @@ def model_to_parameters(model_dict): # Calculate dependencies/references for each parameter. depend_dict = calculate_dependencies(model_dict) + model_dict_copy = copy.deepcopy(model_dict) + params.append(("phase0", model_dict_copy.pop("phase0"))) + params.append(("phase1", model_dict_copy.pop("phase1"))) + # Construct lmfit Parameter input for each parameter. - for name1, value1 in model_dict.items(): - if isinstance(value1, numbers.Number): # (e.g. phase0) - params.append((name1, value1)) - if type(value1) is dict: + for peak_dictionary, peak_property in model_dict_copy.items(): # Fix phase value to 0 by default. - if "phase" not in value1: - params.append(("{0}_{1}".format(name1, "phase"), None, None, None, None, "0")) - for name2, value2 in value1.items(): + if "phase" not in peak_property: + params.append(("{0}_{1}".format(peak_dictionary, "phase"), None, None, None, None, "0")) + for property_name, property_value in peak_property.items(): # Initialize lmfit parameter arguments. - name = "{0}_{1}".format(name1, name2) + name = "{0}_{1}".format(peak_dictionary, property_name) value = None vary = True lmfit_min = None lmfit_max = None expr = None - if isinstance(value2, numbers.Number): - value = value2 - elif isinstance(value2, str): - expr = value2 - elif isinstance(value2, dict): - if "value" in value2: - value = value2["value"] - if "min" in value2: - lmfit_min = value2["min"] - if "max" in value2: - lmfit_max = value2["max"] + if isinstance(property_value, numbers.Number): + value = property_value + elif isinstance(property_value, str): + expr = property_value + elif isinstance(property_value, dict): + if "value" in property_value: + value = property_value["value"] + if "min" in property_value: + lmfit_min = property_value["min"] + if "max" in property_value: + lmfit_max = property_value["max"] # Add parameter object with defined parameters. params.append((name, value, vary, lmfit_min, lmfit_max, expr)) # (lmfit Parameter input format) @@ -265,51 +255,50 @@ def check_errors(check_model): allowed_keys = ["min", "max", "value", "phase", "amplitude"] # Scan model. - for name1, value1 in check_model.items(): - if not isinstance(value1, (numbers.Number, dict)): - raise TypeError("Value of {0} must be a number (for phases), or a dictionary.".format(name1)) - elif type(value1) is dict: # i.e. type(value) is not int - for name2, value2 in value1.items(): - if not isinstance(value2,(numbers.Number,dict,str)): + for model_property, model_values in check_model.items(): + if not isinstance(model_values, (numbers.Number, dict)): + raise TypeError("Value of {0} must be a number (for phases), or a dictionary.".format(model_property)) + elif type(model_values) is dict: # i.e. type(value) is not int + for peak_property, peak_value in model_values.items(): + if not isinstance(peak_value,(numbers.Number,dict,str)): raise TypeError("Value of {0}_{1} must be a value, an expression, or a dictionary." - .format(name1, name2)) - if type(value2) is dict: - for key in value2: + .format(model_property, peak_property)) + if type(peak_value) is dict: + for width_param in peak_value: # Dictionary must have 'value' key. - if "value" not in value2: - raise KeyError("Dictionary {0}_{1} is missing 'value' key.".format(name1, name2)) + if "value" not in peak_value: + raise KeyError("Dictionary {0}_{1} is missing 'value' key." + .format(model_property, peak_property)) # Dictionary can only have 'min,' 'max,' and 'value'. - if key not in allowed_keys: - raise KeyError("In {0}_{1}, '{2}' is not an allowed key.".format(name1, name2, key)) + if width_param not in allowed_keys: + raise KeyError("In {0}_{1}, '{2}' is not an allowed key." + .format(model_property, peak_property, width_param)) # Calculate references to determine order for Parameters. def calculate_dependencies(unordered_model): dependencies = {} # (name, [dependencies]) # Compile dictionary of effective names. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2 in value1: - dependencies["{0}_{1}".format(name1, name2)] = None + for model_property, model_values in unordered_model.items(): + if type(model_values) is dict: # i.e. pcr, not phase + for peak_property in model_values: + dependencies["{0}_{1}".format(model_property, peak_property)] = None # Find dependencies for each effective name. - for name1, value1 in unordered_model.items(): - if type(value1) is dict: # i.e. not phase - for name2, value2 in value1.items(): - if type(value2) is str: - lmfit_name = "{0}_{1}".format(name1, name2) + for model_property, model_values in unordered_model.items(): + if type(model_values) is dict: # i.e. not phase + for peak_property, peak_value in model_values.items(): + if type(peak_value) is str: + lmfit_name = "{0}_{1}".format(model_property, peak_property) dependencies[lmfit_name] = [] for depend in dependencies: - if depend in value2: + if depend in peak_value: dependencies[lmfit_name].append(depend) - print(dependencies) # Check for circular dependencies. for name, dependents in dependencies.items(): if type(dependents) is list: for dependent in dependents: - #print(dependent) - #print(dependencies) if dependencies[dependent] is not None and name in dependencies[dependent]: raise ReferenceError("{0} and {1} reference each other, creating a circular reference." .format(name, dependent)) diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index 726c971..11e22ac 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -5,7 +5,7 @@ import pytest import random -random.seed() +numpy.random.seed(1024) @pytest.fixture def fixed_fid(): @@ -133,8 +133,8 @@ def test_gaussian_4(fixed_fid): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) From 1bcc8891bffd50b8793a68395b735bf010f1088d Mon Sep 17 00:00:00 2001 From: lasyasreepada Date: Fri, 16 Sep 2016 15:09:34 -0400 Subject: [PATCH 34/39] Changed variable names to more accurately reflect what they contain --- suspect/fitting/singlet.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index 9110c88..f14831d 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -195,13 +195,13 @@ def model_to_parameters(model_dict): params.append(("phase1", model_dict_copy.pop("phase1"))) # Construct lmfit Parameter input for each parameter. - for peak_dictionary, peak_property in model_dict_copy.items(): + for peak_name, peak_properties in model_dict_copy.items(): # Fix phase value to 0 by default. - if "phase" not in peak_property: - params.append(("{0}_{1}".format(peak_dictionary, "phase"), None, None, None, None, "0")) - for property_name, property_value in peak_property.items(): + if "phase" not in peak_properties: + params.append(("{0}_{1}".format(peak_name, "phase"), None, None, None, None, "0")) + for property_name, property_value in peak_properties.items(): # Initialize lmfit parameter arguments. - name = "{0}_{1}".format(peak_dictionary, property_name) + name = "{0}_{1}".format(peak_name, property_name) value = None vary = True lmfit_min = None From b94ad39b75cbe50489c78300ca0d67fdd40aa67e Mon Sep 17 00:00:00 2001 From: lasyasreepada Date: Tue, 20 Sep 2016 14:30:18 -0400 Subject: [PATCH 35/39] The file test_fitting has a new test for dependencies, not working as of yet. --- tests/test_mrs/test_fitting.py | 43 +++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index 11e22ac..a7504f7 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -273,4 +273,45 @@ def test_gaussian_dependencies2(fixed_fid_sum): numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) - numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid_sum, atol=0.001) \ No newline at end of file + numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid_sum, atol=0.001) + + +def test_gaussian_dependencies3(fixed_fid_sum): + + data = MRSData(fixed_fid_sum, 5e-4, 123) + + model = { + "phase0": 0.0, + "phase1": 0.0, + + "pcr2": { + + "amplitude": 1.0, + "frequency": "pcr3_frequency+200", + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + }, + "phase": "0", + + }, + "pcr": { + "amplitude": 1.0, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + }, + "phase": "0", + "frequency": 0 + } + } + + fitting_result = singlet.fit(data, model) + + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) + + numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid_sum, atol=0.001) From 680483bc8539f30dacd59991ec2d5dafb03e3b7f Mon Sep 17 00:00:00 2001 From: bennyrowland Date: Tue, 20 Sep 2016 15:21:14 -0400 Subject: [PATCH 36/39] TST: removed unnecessary assertion tests When testing with pytest.raises, the test passes as soon as the error appears so we don't need to keep the assertions about the quality of the fit, as the fitting is not performed. --- tests/test_mrs/test_fitting.py | 35 ---------------------------------- 1 file changed, 35 deletions(-) diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index a7504f7..4acbc00 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -75,12 +75,6 @@ def test_gaussian_2(fixed_fid): with pytest.raises(KeyError): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) - - numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) - def test_gaussian_3(fixed_fid): @@ -105,11 +99,6 @@ def test_gaussian_3(fixed_fid): with pytest.raises(KeyError): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) - - numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) def test_gaussian_4(fixed_fid): @@ -162,12 +151,6 @@ def test_gaussian_5(fixed_fid): with pytest.raises(TypeError): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) - - numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) - def test_gaussian_6(fixed_fid): @@ -192,12 +175,6 @@ def test_gaussian_6(fixed_fid): with pytest.raises(TypeError): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) - - numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) - def test_gaussian_dependencies(fixed_fid): @@ -231,12 +208,6 @@ def test_gaussian_dependencies(fixed_fid): with pytest.raises(ReferenceError): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) - - numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) - def test_gaussian_dependencies2(fixed_fid_sum): @@ -309,9 +280,3 @@ def test_gaussian_dependencies3(fixed_fid_sum): } fitting_result = singlet.fit(data, model) - - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) - - numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid_sum, atol=0.001) From fef31d2b70bed97dd8f38db36be7451514f6ab41 Mon Sep 17 00:00:00 2001 From: bennyrowland Date: Tue, 20 Sep 2016 15:29:00 -0400 Subject: [PATCH 37/39] TST: Added fitting test with reordered dependencies lmfit parameters must be added in the correct order to prevent NameErrors, so we have to sort them by dependency to ensure all dependencies have already been added. --- tests/test_mrs/test_fitting.py | 41 +++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index 4acbc00..be78192 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -247,6 +247,44 @@ def test_gaussian_dependencies2(fixed_fid_sum): numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid_sum, atol=0.001) +def test_reordered_dependencies2(fixed_fid_sum): + + data = MRSData(fixed_fid_sum, 5e-4, 123) + + model = { + "phase0": 0.0, + "phase1": 0.0, + "pcr": { + "amplitude": 1.0, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + }, + "phase": "0", + "frequency": "pcr2_frequency+200" + }, + "pcr2": { + "amplitude": 1.0, + "width": { + "value": 45.0, + "min": 42.0, + "max": 55.0, + }, + "phase": "0", + "frequency": 0 + } + } + + fitting_result = singlet.fit(data, model) + + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 200.0, atol=1e-1) + + numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid_sum, atol=0.001) + + def test_gaussian_dependencies3(fixed_fid_sum): data = MRSData(fixed_fid_sum, 5e-4, 123) @@ -279,4 +317,5 @@ def test_gaussian_dependencies3(fixed_fid_sum): } } - fitting_result = singlet.fit(data, model) + with pytest.raises(NameError): + fitting_result = singlet.fit(data, model) From 46b3cec6bf94d64c7d6a05c99fd5163b18b6f724 Mon Sep 17 00:00:00 2001 From: bennyrowland Date: Tue, 20 Sep 2016 15:32:55 -0400 Subject: [PATCH 38/39] TST: Renamed tests with more descriptive versions --- tests/test_mrs/test_fitting.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index be78192..dc6c99e 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -36,11 +36,11 @@ def test_gaussian(fixed_fid): "amplitude": 1, "width": { "value": 45, - "min": 42, + "min": 42.0, "max": 55 }, "phase": "0", - "frequency": 0 + "frequency": 0.0 } } fitting_result = singlet.fit(data, model) @@ -52,11 +52,11 @@ def test_gaussian(fixed_fid): numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) -def test_gaussian_2(fixed_fid): +def test_bad_param(fixed_fid): data = MRSData(fixed_fid, 5e-4, 123) - # Floating point values; invalid key added to width dict, to test whether KeyError is raised + # invalid key added to width dict, to test whether KeyError is raised model = { "phase0": 0.0, "phase1": 0.0, @@ -66,7 +66,7 @@ def test_gaussian_2(fixed_fid): "value": 45.0, "min": 42.0, "max": 55.0, - "avg": 47 + "avg": 47 # this is the bad key }, "phase": "0", "frequency": 0.0 @@ -76,7 +76,7 @@ def test_gaussian_2(fixed_fid): fitting_result = singlet.fit(data, model) -def test_gaussian_3(fixed_fid): +def test_missing_param(fixed_fid): data = MRSData(fixed_fid, 5e-4, 123) @@ -90,7 +90,6 @@ def test_gaussian_3(fixed_fid): # "value": 45, "min": 42, "max": 55, - }, "phase": "0", "frequency": 0 @@ -100,7 +99,7 @@ def test_gaussian_3(fixed_fid): fitting_result = singlet.fit(data, model) -def test_gaussian_4(fixed_fid): +def test_missing_peak_phase(fixed_fid): data = MRSData(fixed_fid, 5e-4, 123) @@ -129,7 +128,7 @@ def test_gaussian_4(fixed_fid): numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid, atol=0.001) -def test_gaussian_5(fixed_fid): +def test_missing_global_phase(fixed_fid): data = MRSData(fixed_fid, 5e-4, 123) @@ -152,7 +151,7 @@ def test_gaussian_5(fixed_fid): fitting_result = singlet.fit(data, model) -def test_gaussian_6(fixed_fid): +def test_bad_param_value(fixed_fid): data = MRSData(fixed_fid, 5e-4, 123) @@ -176,7 +175,7 @@ def test_gaussian_6(fixed_fid): fitting_result = singlet.fit(data, model) -def test_gaussian_dependencies(fixed_fid): +def test_circular_dependencies(fixed_fid): data = MRSData(fixed_fid, 5e-4, 123) @@ -209,7 +208,7 @@ def test_gaussian_dependencies(fixed_fid): fitting_result = singlet.fit(data, model) -def test_gaussian_dependencies2(fixed_fid_sum): +def test_dependencies(fixed_fid_sum): data = MRSData(fixed_fid_sum, 5e-4, 123) @@ -247,7 +246,7 @@ def test_gaussian_dependencies2(fixed_fid_sum): numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid_sum, atol=0.001) -def test_reordered_dependencies2(fixed_fid_sum): +def test_reordered_dependencies(fixed_fid_sum): data = MRSData(fixed_fid_sum, 5e-4, 123) @@ -285,14 +284,13 @@ def test_reordered_dependencies2(fixed_fid_sum): numpy.testing.assert_allclose(fitting_result["fit"], fixed_fid_sum, atol=0.001) -def test_gaussian_dependencies3(fixed_fid_sum): +def test_missing_dependencies(fixed_fid_sum): data = MRSData(fixed_fid_sum, 5e-4, 123) model = { "phase0": 0.0, "phase1": 0.0, - "pcr2": { "amplitude": 1.0, @@ -303,7 +301,6 @@ def test_gaussian_dependencies3(fixed_fid_sum): "max": 55.0, }, "phase": "0", - }, "pcr": { "amplitude": 1.0, From 6afc09bede728ec1b05dd802f72b240e1ba7dc82 Mon Sep 17 00:00:00 2001 From: bennyrowland Date: Tue, 20 Sep 2016 15:38:45 -0400 Subject: [PATCH 39/39] API: Renamed fitting parameter "width" to "fwhm" --- suspect/fitting/singlet.py | 2 +- tests/test_mrs/test_fitting.py | 36 +++++++++++++++++----------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/suspect/fitting/singlet.py b/suspect/fitting/singlet.py index f14831d..3ea7b86 100644 --- a/suspect/fitting/singlet.py +++ b/suspect/fitting/singlet.py @@ -98,7 +98,7 @@ def make_basis(params, time_axis): gaussian = suspect.basis.gaussian(time_axis, params["{}_frequency".format(metabolite_name)], params["{}_phase".format(metabolite_name)].value, - params["{}_width".format(metabolite_name)]) + params["{}_fwhm".format(metabolite_name)]) real_gaussian = complex_to_real(gaussian) basis_matrix[i, :] = real_gaussian return basis_matrix diff --git a/tests/test_mrs/test_fitting.py b/tests/test_mrs/test_fitting.py index dc6c99e..76287ad 100644 --- a/tests/test_mrs/test_fitting.py +++ b/tests/test_mrs/test_fitting.py @@ -34,7 +34,7 @@ def test_gaussian(fixed_fid): "phase1": 0, "pcr": { "amplitude": 1, - "width": { + "fwhm": { "value": 45, "min": 42.0, "max": 55 @@ -45,7 +45,7 @@ def test_gaussian(fixed_fid): } fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["fwhm"], 50.0, rtol=1e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) @@ -62,7 +62,7 @@ def test_bad_param(fixed_fid): "phase1": 0.0, "pcr": { "amplitude": 1.0, - "width": { + "fwhm": { "value": 45.0, "min": 42.0, "max": 55.0, @@ -86,7 +86,7 @@ def test_missing_param(fixed_fid): "phase1": 0, "pcr": { "amplitude": 1, - "width": { + "fwhm": { # "value": 45, "min": 42, "max": 55, @@ -109,7 +109,7 @@ def test_missing_peak_phase(fixed_fid): "phase1": 0, "pcr": { "amplitude": 1, - "width": { + "fwhm": { "value": 45, "min": 42, "max": 55, @@ -121,7 +121,7 @@ def test_missing_peak_phase(fixed_fid): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=5e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["fwhm"], 50.0, rtol=5e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=5e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) @@ -138,7 +138,7 @@ def test_missing_global_phase(fixed_fid): "phase1": None, "pcr": { "amplitude": 1.0, - "width": { + "fwhm": { "value": 45.0, "min": 42.0, "max": 55.0, @@ -161,7 +161,7 @@ def test_bad_param_value(fixed_fid): "phase1": 0.0, "pcr": { "amplitude": None, - "width": { + "fwhm": { "value": 45.0, "min": 42.0, "max": 55.0, @@ -184,7 +184,7 @@ def test_circular_dependencies(fixed_fid): "phase1": 0.0, "pcr": { "amplitude": 1.0, - "width": { + "fwhm": { "value": 45.0, "min": 42.0, "max": 55.0, @@ -194,7 +194,7 @@ def test_circular_dependencies(fixed_fid): }, "pcr2": { "amplitude": 1.0, - "width": { + "fwhm": { "value": 45.0, "min": 42.0, "max": 55.0, @@ -217,7 +217,7 @@ def test_dependencies(fixed_fid_sum): "phase1": 0.0, "pcr": { "amplitude": 1.0, - "width": { + "fwhm": { "value": 45.0, "min": 42.0, "max": 55.0, @@ -227,7 +227,7 @@ def test_dependencies(fixed_fid_sum): }, "pcr2": { "amplitude": 1.0, - "width": { + "fwhm": { "value": 45.0, "min": 42.0, "max": 55.0, @@ -239,7 +239,7 @@ def test_dependencies(fixed_fid_sum): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["fwhm"], 50.0, rtol=1e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 0.0, atol=5e-2) @@ -255,7 +255,7 @@ def test_reordered_dependencies(fixed_fid_sum): "phase1": 0.0, "pcr": { "amplitude": 1.0, - "width": { + "fwhm": { "value": 45.0, "min": 42.0, "max": 55.0, @@ -265,7 +265,7 @@ def test_reordered_dependencies(fixed_fid_sum): }, "pcr2": { "amplitude": 1.0, - "width": { + "fwhm": { "value": 45.0, "min": 42.0, "max": 55.0, @@ -277,7 +277,7 @@ def test_reordered_dependencies(fixed_fid_sum): fitting_result = singlet.fit(data, model) - numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["width"], 50.0, rtol=1e-2) + numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["fwhm"], 50.0, rtol=1e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["amplitude"], 1.0, rtol=2e-2) numpy.testing.assert_allclose(fitting_result["model"]["pcr"]["frequency"], 200.0, atol=1e-1) @@ -295,7 +295,7 @@ def test_missing_dependencies(fixed_fid_sum): "amplitude": 1.0, "frequency": "pcr3_frequency+200", - "width": { + "fwhm": { "value": 45.0, "min": 42.0, "max": 55.0, @@ -304,7 +304,7 @@ def test_missing_dependencies(fixed_fid_sum): }, "pcr": { "amplitude": 1.0, - "width": { + "fwhm": { "value": 45.0, "min": 42.0, "max": 55.0,