Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add filtering for center packet loss #123

Merged
merged 5 commits into from
Feb 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
# SSINS Change Log

## Unreleased
- Added handling for negative `sig_thresh` such that when set, the associated shape will only trigger
on a negative sig smaller than `sig_thresh`. Behavior for positive `sig_thresh` is unchanged;
this will continue to use the absolute value of sig. Setting a negative `sig_thresh` for the
`narrow` shape is not supported and will trigger a warning.
- Now requires pyuvdata 2.4.1, which introduces several new required parameters for
UVData (thus SS) and UVFlag (thus INS). This required refactoring the INS constructor,
though the API has been preserved. However, if reading SSINS h5 files written with an earlier
Expand Down
3 changes: 3 additions & 0 deletions SSINS/data/MWA_EoR_Highband_shape_dict.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@ TV8:
TV9:
- 195000000.0
- 202000000.0
center_packet_loss:
- 177000000.0
- 187000000.0
20 changes: 17 additions & 3 deletions SSINS/match_filter.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,10 @@ def match_test(self, INS):
shape_max = None
for shape in self.slice_dict:
if shape == 'narrow':
if self.sig_thresh[shape] < 0:
warnings.warn("negative sig_thresh not supported for shape: 'narrow'. The absolute "
" value of sig_thresh will be used.")
self.sig_thresh[shape] = np.abs(self.sig_thresh[shape])
t, f, p = np.unravel_index(np.absolute(INS.metric_ms).argmax(),
INS.metric_ms.shape)
sig = np.absolute(INS.metric_ms[t, f, p])
Expand All @@ -146,14 +150,24 @@ def match_test(self, INS):
else:
N = np.count_nonzero(np.logical_not(INS.metric_ms[:, self.slice_dict[shape]].mask),
axis=1)
sliced_arr = np.absolute(INS.metric_ms[:, self.slice_dict[shape]].mean(axis=1)) * np.sqrt(N)
t, p = np.unravel_index((sliced_arr / self.sig_thresh[shape]).argmax(),
if self.sig_thresh[shape] < 0:
sliced_arr = (INS.metric_ms[:, self.slice_dict[shape]].mean(axis=1)) * np.sqrt(N)
t, p = np.unravel_index((sliced_arr / np.abs(self.sig_thresh[shape])).argmin(),
sliced_arr.shape)
else:
sliced_arr = np.absolute(INS.metric_ms[:, self.slice_dict[shape]].mean(axis=1)) * np.sqrt(N)
t, p = np.unravel_index((sliced_arr / self.sig_thresh[shape]).argmax(),
sliced_arr.shape)
t = slice(t, t + 1)
f = self.slice_dict[shape]
# Pull out the number instead of a sliced arr
sig = sliced_arr[t, p][0]
if sig > self.sig_thresh[shape]:
if self.sig_thresh[shape] < 0:
if sig < 0:
sig = np.absolute(sig)
else:
continue
if sig > np.absolute(self.sig_thresh[shape]):
if sig > sig_max:
t_max, f_max, shape_max, sig_max = (t, f, shape, sig)

Expand Down
69 changes: 69 additions & 0 deletions SSINS/tests/test_MF.py
Original file line number Diff line number Diff line change
Expand Up @@ -378,3 +378,72 @@ def test_MF_write(tmp_path):

with pytest.raises(ValueError, match="matchfilter file with prefix"):
mf.write(f"{prefix}_test", clobber=False)

def test_negative_sig_thresh():
obs = '1061313128_99bl_1pol_half_time'
insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs)

ins = INS(insfile)

# Mock a simple metric_array and freq_array
ins.metric_array[:] = np.ones_like(ins.metric_array)
ins.weights_array = np.copy(ins.metric_array)
ins.weights_square_array = np.copy(ins.weights_array)

# Make a shape dictionary for a shape that will be injected later
ch_wid = ins.freq_array[1] - ins.freq_array[0]
shape = [ins.freq_array[127] - 0.2 * ch_wid, ins.freq_array[255] + 0.2 * ch_wid]
shape_dict = {'neg_shape': shape, 'pos_shape': shape}
sig_thresh = {'neg_shape': -5, 'pos_shape': 5, 'narrow': 5, 'streak': 5}
mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict)

# Inject a negative shape event and a streak event
ins.metric_array[7, 127:256] = -10
ins.metric_array[13] = 10
ins.metric_ms = ins.mean_subtract()

mf.apply_match_test(ins, event_record=True)

# Check that events are flagged correctly
test_mask = np.zeros(ins.metric_array.shape, dtype=bool)
test_mask[7, 127:256] = 1
test_mask[13, :] = 1

assert np.all(test_mask == ins.metric_array.mask), "Flags are incorrect"

test_match_events_slc = [(slice(13, 14, None), slice(0, 384, None), 'streak'),
(slice(7, 8, None), slice(127, 256, None), 'neg_shape')
]

for i, event in enumerate(test_match_events_slc):
assert ins.match_events[i][:-1] == test_match_events_slc[i], f"{i}th event is wrong"

def test_negative_narrow():
obs = '1061313128_99bl_1pol_half_time'
insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs)

ins = INS(insfile)

# Mock a simple metric_array and freq_array
ins.metric_array[:] = np.ones_like(ins.metric_array)
ins.weights_array = np.copy(ins.metric_array)
ins.weights_square_array = np.copy(ins.weights_array)

# Make a shape dictionary
shape_dict={}
sig_thresh = {'narrow': -5, 'streak': 5}
mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict)

# Inject a positive narrowband rfi
ins.metric_array[7:9, 125] = 10
ins.metric_ms = ins.mean_subtract()

with pytest.warns(UserWarning, match="negative sig_thresh not supported"):
mf.apply_match_test(ins, event_record=True)

# Check that the narrowband rfi was detected
test_match_events_slc = [(slice(7, 8, None), slice(125, 126, None), 'narrow_%.3fMHz' % (ins.freq_array[125] * 10**(-6))),
(slice(8, 9, None), slice(125, 126, None), 'narrow_%.3fMHz' % (ins.freq_array[125] * 10**(-6)))]

for i, event in enumerate(test_match_events_slc):
assert ins.match_events[i][:-1] == test_match_events_slc[i], f"{i}th event is wrong"
Loading