From 847b8cd453688944cacce34efa97fdc07579f2db Mon Sep 17 00:00:00 2001 From: Aaronearlerichardson Date: Fri, 24 May 2024 15:11:50 -0400 Subject: [PATCH 1/5] added 'bad' option to BIDS_coding --- BIDS_converter/utils/organize.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/BIDS_converter/utils/organize.py b/BIDS_converter/utils/organize.py index 6a636e9..4f02841 100644 --- a/BIDS_converter/utils/organize.py +++ b/BIDS_converter/utils/organize.py @@ -55,8 +55,8 @@ def frame2bids(df_in: pd.DataFrame, event_format: Dict[str, str], def reframe_events(df_in: pd.DataFrame, events: Union[list, dict], - stim_dir: PathLike, mat_sample_rate, - audio_correction) -> pd.DataFrame: + stim_dir: PathLike, mat_sample_rate: int, + audio_correction=None) -> pd.DataFrame: df = df_in.copy() new_df = None if isinstance(events, dict): @@ -103,6 +103,13 @@ def reframe_events(df_in: pd.DataFrame, events: Union[list, dict], float) / mat_sample_rate else: temp_df[key] = eval_df(df, value) + + if "bad" in temp_df.columns: + temp_df["bad"].fillna(False) + func = lambda r: "bad " + r["trial_type"] if r["bad"] and not r["trial_type"].startswith("bad") else r["trial_type"] + temp_df["trial_type"] = temp_df.apply(func, axis=1) + temp_df.drop("bad", axis=1, inplace=True) + if "trial_num" not in temp_df.columns: temp_df["trial_num"] = [1 + i for i in list(range(temp_df.shape[0]))] temp_df.dropna(axis=0, how="all", subset=["onset", "duration"], inplace=True) @@ -397,7 +404,7 @@ def eval_df(df: pd.DataFrame, exp: str) -> pd.Series: """ if exp in df.columns: return df[exp].squeeze() - fields = [i for i in re.split(r"[ +\-/*%]", exp) if i != ''] + fields = [i for i in re.split(r"[ +\-/*%(==)><(>=)(<=)]", exp) if i != ''] df["SPLITCHAR"] = pd.Series(["/"] * df.shape[0], dtype="string") for name in fields: From 98b296dba9733948a83714fff53c567bbb334057 Mon Sep 17 00:00:00 2001 From: Aaronearlerichardson Date: Fri, 24 May 2024 15:12:10 -0400 Subject: [PATCH 2/5] fixed by adding 'headerdata' to config usage --- BIDS_converter/data2bids.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/BIDS_converter/data2bids.py b/BIDS_converter/data2bids.py index 9b66de0..291417b 100644 --- a/BIDS_converter/data2bids.py +++ b/BIDS_converter/data2bids.py @@ -217,7 +217,7 @@ def chan_walk(self, root: PathLike, files: List[PathLike], for i, file in enumerate(files): src = op.join(root, file) if any(f in op.basename(src) for f in - ieeg_conf["channels"].keys()): + ieeg_conf["headerData"]["channels"].keys()): self._channels_file[part_match] = src for name, var in ieeg_conf["headerData"].items(): @@ -231,7 +231,7 @@ def scan_chans(self, src: PathLike, var: str, part_match: str): self.channels[part_match] = self.channels[part_match] + org.mat2df( src, var).tolist() self.sample_rate[part_match] = int( - org.mat2df(src, self._config['ieeg']['sampleRate']).iloc[0]) + org.mat2df(src, self._config['ieeg']["headerData"]['sampleRate']).iloc[0]) self._ignore.append(src) elif name.endswith((".txt", ".csv", ".tsv")): f = open(name, 'r') @@ -699,7 +699,7 @@ def read_edf(self, file_name: PathLike, print("Reading " + file_name + "...") [array, signal_headers, _] = highlevel.read_edf( file_name, ch_nrs=chn_nums, - digital=self._config["ieeg"]["digital"], verbose=True) + digital=self._config["ieeg"]["headerData"]["digital"], verbose=True) print("read it") if extra_arrays: array = array + extra_arrays From eadcadc357cf0d5fbc1f1733e56578dce6d3c4a9 Mon Sep 17 00:00:00 2001 From: Aaronearlerichardson Date: Fri, 24 May 2024 15:12:10 -0400 Subject: [PATCH 3/5] fixed by adding 'headerdata' to config usage --- BIDS_converter/data2bids.py | 6 +++--- BIDS_converter/utils/organize.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/BIDS_converter/data2bids.py b/BIDS_converter/data2bids.py index 9b66de0..291417b 100644 --- a/BIDS_converter/data2bids.py +++ b/BIDS_converter/data2bids.py @@ -217,7 +217,7 @@ def chan_walk(self, root: PathLike, files: List[PathLike], for i, file in enumerate(files): src = op.join(root, file) if any(f in op.basename(src) for f in - ieeg_conf["channels"].keys()): + ieeg_conf["headerData"]["channels"].keys()): self._channels_file[part_match] = src for name, var in ieeg_conf["headerData"].items(): @@ -231,7 +231,7 @@ def scan_chans(self, src: PathLike, var: str, part_match: str): self.channels[part_match] = self.channels[part_match] + org.mat2df( src, var).tolist() self.sample_rate[part_match] = int( - org.mat2df(src, self._config['ieeg']['sampleRate']).iloc[0]) + org.mat2df(src, self._config['ieeg']["headerData"]['sampleRate']).iloc[0]) self._ignore.append(src) elif name.endswith((".txt", ".csv", ".tsv")): f = open(name, 'r') @@ -699,7 +699,7 @@ def read_edf(self, file_name: PathLike, print("Reading " + file_name + "...") [array, signal_headers, _] = highlevel.read_edf( file_name, ch_nrs=chn_nums, - digital=self._config["ieeg"]["digital"], verbose=True) + digital=self._config["ieeg"]["headerData"]["digital"], verbose=True) print("read it") if extra_arrays: array = array + extra_arrays diff --git a/BIDS_converter/utils/organize.py b/BIDS_converter/utils/organize.py index 4f02841..2a62cd4 100644 --- a/BIDS_converter/utils/organize.py +++ b/BIDS_converter/utils/organize.py @@ -333,7 +333,7 @@ def sort_by_list(df: pd.DataFrame, ord: list[str], col: str) -> pd.DataFrame: def prep_tsv(file_path: PathLike, task: str, pmatchz: str, ieeg_config: dict, bids_dir: PathLike) -> (str, pd.DataFrame): df = None - for name, var in ieeg_config["channels"].items(): + for name, var in ieeg_config["headerData"]["channels"].items(): if name in file_path: df = mat2df(file_path, var) if "highpass_cutoff" in df.columns.to_list(): From 4db7f3142d456d361ae079fdd8235494e8db4839 Mon Sep 17 00:00:00 2001 From: Aaronearlerichardson Date: Fri, 24 May 2024 16:44:30 -0400 Subject: [PATCH 4/5] fixed by adding 'headerdata' to config usage again and fixed configs --- BIDS_converter/BIDS_convert_wsl.sh | 11 +++++------ BIDS_converter/config.json | 19 +++++-------------- BIDS_converter/data2bids.py | 4 ++-- BIDS_converter/utils/organize.py | 2 +- 4 files changed, 13 insertions(+), 23 deletions(-) diff --git a/BIDS_converter/BIDS_convert_wsl.sh b/BIDS_converter/BIDS_convert_wsl.sh index 53fbc3b..e22436e 100755 --- a/BIDS_converter/BIDS_convert_wsl.sh +++ b/BIDS_converter/BIDS_convert_wsl.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash -ORIG_DATA_DIR="$HOME/Library/CloudStorage/Box-Box/CoganLab" +ORIG_DATA_DIR="$HOME/Box/CoganLab" # OUTPUT_DIR="$ORIG_DATA_DIR/BIDS-1.1_GlobalLocal" -OUTPUT_DIR="$ORIG_DATA_DIR/BIDS-1.2_GlobalLocal" -TASKS=("GlobalLocal") +OUTPUT_DIR="$HOME/Workspace/SentenceRep" +TASKS=("SentenceRep") #(D3 D5 D6 D7 D8 D9 D12 D14 D15 D16 D17 D18 D20 D22 D23 D24 D26 D27 D28 D29 D30 D31 D32 D53 # D57 D59 D60 D61 D65 D66 D69 D70 D71 D72 D73) #D3 D5 D6 D7 D8 D9 D18 D20 D22 D23 D24 D26 D27 D28 D29 D30 D31 D32 D53 @@ -16,7 +16,7 @@ TASKS=("GlobalLocal") for TASK in "${TASKS[@]}" do mapfile -t SUB_IDS < <(find "$ORIG_DATA_DIR/D_Data/$TASK" -maxdepth 1 -type d -name "D*" -exec basename {} \;) - SUB_IDS=(D57) +# SUB_IDS=(D30) # OUTPUT_DIR="$ORIG_DATA_DIR/$TASK" BIDS_DIR="$OUTPUT_DIR/BIDS" ZIP=false @@ -34,8 +34,7 @@ for TASK in "${TASKS[@]}" # mkdir -p $BIDS_DIR # mkdir -p "$OUTPUT_DIR/stimuli" # shellcheck disable=SC2038 - # find "$ORIG_DATA_DIR/task_stimuli" -iname "sentence_rep" -type d -exec echo "{}/." \; | xargs -I{} cp -afv {} "$OUTPUT_DIR/stimuli/" - TASKLOWER=$(echo $TASK | tr '[:upper:]' '[:lower:]') + find "$ORIG_DATA_DIR/task_stimuli" -iname $TASK -type d -exec echo "{}/." \; | xargs -I{} cp -afv {} "$OUTPUT_DIR/stimuli/" #echo "$ORIG_DATA_DIR/task_stimuli/$TASKLOWER/." #cp -av "$ORIG_DATA_DIR/task_stimuli/$TASKLOWER/." "$BIDS_DIR/stimuli/" diff --git a/BIDS_converter/config.json b/BIDS_converter/config.json index 41ebae4..6923f1a 100644 --- a/BIDS_converter/config.json +++ b/BIDS_converter/config.json @@ -26,7 +26,7 @@ "binaryEncoding": "float32", "headerData": { "experiment.mat": "channels.name", - "default": "~/Library/CloudStorage/Box-Box/CoganLab/ECoG_Task_Data/Timestamps (MASTER).xlsx", + "default": "~/Box/CoganLab/ECoG_Task_Data/Timestamps (MASTER).xlsx", "channels": { "experiment.mat": "channels" }, @@ -137,19 +137,10 @@ "SampleRate": 30000, "AudioCorrection": "n/a", "Events": [ - { - "onset": "stimulus", - "duration": "0.5 * 30", - "trial_num": "trial", - "trial_type": "Stimulus + SPLITCHAR + congruency + percIncongruent + SPLITCHAR + switchType + percSwitch + SPLITCHAR + globalStim + SPLITCHAR + localStim + SPLITCHAR + task + SPLITCHAR + target + SPLITCHAR + partResponded + SPLITCHAR + partResp + SPLITCHAR + corrResp + SPLITCHAR + trialCount + SPLITCHAR + blockTrialCount" - }, - { - "onset": "response", - "duration": "RT", - "trial_num": "trial", - "trial_type": "Response + SPLITCHAR + congruency + percIncongruent + SPLITCHAR + switchType + percSwitch + SPLITCHAR + globalStim + SPLITCHAR + localStim + SPLITCHAR + task + SPLITCHAR + target + SPLITCHAR + partResponded + SPLITCHAR + partResp + SPLITCHAR + corrResp + SPLITCHAR + trialCount + SPLITCHAR + blockTrialCount" - } - ] + {"onset": "Start","duration": "(cueEnd - cueStart) * 30", "trial_num": "Trial","trial_type": "cue", "bad": "Noisy == 1"}, + {"onset": "Auditory", "stim_file": "sound", "trial_num": "Trial", "trial_type": "Audio", "bad": "Noisy == 1"}, + {"onset": "Go", "duration": "(goEnd - goStart) * 30", "trial_num": "Trial", "trial_type": "go", "bad": "Noisy == 1"}, + {"onset": "ResponseStart", "duration": "ResponseEnd - ResponseStart", "trial_num": "Trial","trial_type": "Response", "bad": "Noisy == 1"}] }, "split": { "Sep": "all", diff --git a/BIDS_converter/data2bids.py b/BIDS_converter/data2bids.py index 291417b..853928f 100644 --- a/BIDS_converter/data2bids.py +++ b/BIDS_converter/data2bids.py @@ -898,7 +898,7 @@ def write_edf(self, array: np.ndarray, signal_headers: List[dict], exist_ok=True) highlevel.write_edf(practice, np.split(array, [ 0, start_nums[0][0]], axis=1)[1], signal_headers, - header, digital=self._config["ieeg"][ + header, digital=self._config["ieeg"]["headerData"][ "digital"]) self.bidsignore("*practice*") else: @@ -916,7 +916,7 @@ def write_edf(self, array: np.ndarray, signal_headers: List[dict], print(full_name + "(Samples[" + str(start) + ":" + str( end) + "]) ---> " + edf_name) highlevel.write_edf(edf_name, new_array, signal_headers, header, - digital=self._config["ieeg"]["digital"]) + digital=self._config["ieeg"]["headerData"]["digital"]) # zero the timing so that each file starts at t=0 if i > 0: org.reset_zero(tsv_name, start_nums[i - 1][1], diff --git a/BIDS_converter/utils/organize.py b/BIDS_converter/utils/organize.py index 2a62cd4..e367f48 100644 --- a/BIDS_converter/utils/organize.py +++ b/BIDS_converter/utils/organize.py @@ -341,7 +341,7 @@ def prep_tsv(file_path: PathLike, task: str, pmatchz: str, ieeg_config: dict, if "lowpass_cutoff" in df.columns.to_list(): df = df.rename(columns={"lowpass_cutoff": "low_cutoff"}) df["type"] = ieeg_config["type"] - df["units"] = ieeg_config["units"] + df["units"] = ieeg_config["headerData"]["units"] new_row = pd.DataFrame( {"name": ["Trigger"], "high_cutoff": [1], "low_cutoff": [1000], "type": ["TRIG"], "units": ["uV"]}) From c7023ec91ac8ddba948aacdb506985db89872391 Mon Sep 17 00:00:00 2001 From: Aaronearlerichardson Date: Tue, 28 May 2024 16:16:04 -0400 Subject: [PATCH 5/5] sentencerep gocue fixed --- BIDS_converter/BIDS_convert_wsl.sh | 12 ++++++------ BIDS_converter/config.json | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/BIDS_converter/BIDS_convert_wsl.sh b/BIDS_converter/BIDS_convert_wsl.sh index e22436e..63857e0 100755 --- a/BIDS_converter/BIDS_convert_wsl.sh +++ b/BIDS_converter/BIDS_convert_wsl.sh @@ -16,7 +16,7 @@ TASKS=("SentenceRep") for TASK in "${TASKS[@]}" do mapfile -t SUB_IDS < <(find "$ORIG_DATA_DIR/D_Data/$TASK" -maxdepth 1 -type d -name "D*" -exec basename {} \;) -# SUB_IDS=(D30) + SUB_IDS=(D5 D7 D8 D9 D15 D16 D17 D18 D20 D22 D23 D24 D26 D27 D28 D29 D30 D31) # OUTPUT_DIR="$ORIG_DATA_DIR/$TASK" BIDS_DIR="$OUTPUT_DIR/BIDS" ZIP=false @@ -67,15 +67,15 @@ for TASK in "${TASKS[@]}" rsync -P "$ORIG_DATA_DIR/ECoG_Task_Data/response_coding/PhonemeSequencingStimStarts.txt" "$OUTPUT_DIR/$SUB_ID/${SUB_ID}_PhonemeSequencingStimStarts.txt" fi #eeg files - if ! find "$ORIG_DATA_DIR/D_Data/$TASK/" -iregex ".*$SUB_ID.*\.edf" -exec false {} + ; then #search for edf files - find "$ORIG_DATA_DIR/D_Data/$TASK/" -iregex ".*$SUB_ID.*\.edf" -exec cp -v -t "$OUTPUT_DIR/$SUB_ID/" {} + + if ! find "$ORIG_DATA_DIR/D_Data/$TASK/" -iregex ".*$SUB_ID .*\.edf" -exec false {} + ; then #search for edf files + find "$ORIG_DATA_DIR/D_Data/$TASK/" -iregex ".*$SUB_ID .*\.edf" -exec cp -v -t "$OUTPUT_DIR/$SUB_ID/" {} + find "$OUTPUT_DIR/$SUB_ID/" -regex ".*\.EDF" | xargs -I{} basename -s ".EDF" {} | xargs -I{} mv -v "$OUTPUT_DIR/$SUB_ID/{}.EDF" "$OUTPUT_DIR/$SUB_ID/{}.edf" if $ZIP ; then - find "$ORIG_DATA_DIR/D_Data/$TASK/$SUB_ID" -regex ".*\.\(EDF\)\|\(edf\)" | xargs -I{} basename {} | xargs -I{} cut -f 1 -d '.' {} | xargs -I{} echo "$OUTPUT_DIR/$SUB_ID/{}.edf" | xargs -I{} gzip -6 -v {} + find "$ORIG_DATA_DIR/D_Data/$TASK/$SUB_ID " -regex ".*\.\(EDF\)\|\(edf\)" | xargs -I{} basename {} | xargs -I{} cut -f 1 -d '.' {} | xargs -I{} echo "$OUTPUT_DIR/$SUB_ID/{}.edf" | xargs -I{} gzip -6 -v {} fi else #if no edf files find binary files - x=( $(find "$ORIG_DATA_DIR/D_Data/$TASK/$SUB_ID" -regex ".*$SUB_ID.*\.ieeg.dat" -type f | rev | sed -r "s|/|_|" | sed -r "s|/|noisseS/|" | rev | xargs -n 1 basename | sed -r "s|${SUB_ID}_||") ) - y=( $(find "$ORIG_DATA_DIR/D_Data/$TASK/$SUB_ID" -regex ".*$SUB_ID.*\.ieeg.dat") ) + x=( $(find "$ORIG_DATA_DIR/D_Data/$TASK/$SUB_ID " -regex ".*$SUB_ID.*\.ieeg.dat" -type f | rev | sed -r "s|/|_|" | sed -r "s|/|noisseS/|" | rev | xargs -n 1 basename | sed -r "s|${SUB_ID}_||") ) + y=( $(find "$ORIG_DATA_DIR/D_Data/$TASK/$SUB_ID " -regex ".*$SUB_ID.*\.ieeg.dat") ) z=${#x[@]} if $ZIP ; then for((i=0;i<=$z-1;i+=1)); do gzip -c -6 -v ${y[$i]} > "$OUTPUT_DIR/$SUB_ID/${SUB_ID}_${x[$i]}.gz" ; done diff --git a/BIDS_converter/config.json b/BIDS_converter/config.json index 6923f1a..0df1bf8 100644 --- a/BIDS_converter/config.json +++ b/BIDS_converter/config.json @@ -113,8 +113,8 @@ "content": ["[0-9]{2}"] }, "acq": { - "left": "((Session)|(Part)|(part))", - "right": "[_ \\.]", + "left": "((Session)|(Part)|(part)|(_))", + "right": "[_ \\.(\\.edf)]", "content": ["[0-9]{1,3}"], "fill": 2 }, @@ -132,14 +132,14 @@ }, "Timing": { "start": "stimulus", - "end": "response" + "end": "go" }, "SampleRate": 30000, "AudioCorrection": "n/a", "Events": [ {"onset": "Start","duration": "(cueEnd - cueStart) * 30", "trial_num": "Trial","trial_type": "cue", "bad": "Noisy == 1"}, {"onset": "Auditory", "stim_file": "sound", "trial_num": "Trial", "trial_type": "Audio", "bad": "Noisy == 1"}, - {"onset": "Go", "duration": "(goEnd - goStart) * 30", "trial_num": "Trial", "trial_type": "go", "bad": "Noisy == 1"}, + {"onset": "Go", "duration": "(goEnd - goStart) * 30", "trial_num": "Trial", "trial_type": "GoCue + SPLITCHAR + go", "bad": "Noisy == 1"}, {"onset": "ResponseStart", "duration": "ResponseEnd - ResponseStart", "trial_num": "Trial","trial_type": "Response", "bad": "Noisy == 1"}] }, "split": {