From 4fae57859ba2ee11ffbf0dccaeb74db77b0b5b10 Mon Sep 17 00:00:00 2001 From: flixha Date: Tue, 15 Feb 2022 15:11:16 +0100 Subject: [PATCH 01/61] add function that takes care of NaNs when event cannot be directly correlated --- eqcorrscan/utils/clustering.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/eqcorrscan/utils/clustering.py b/eqcorrscan/utils/clustering.py index d9657ead5..b95aa80e7 100644 --- a/eqcorrscan/utils/clustering.py +++ b/eqcorrscan/utils/clustering.py @@ -131,6 +131,22 @@ def cross_chan_correlation( return _coherances, _positions +def fill_nans_with_rowcol_mean(dist_mat): + """ + # Fill missing correlations (nans) with the mean of column and row + """ + missing_corrs = ~np.isfinite(dist_mat) + col_mean = np.nanmean(dist_mat, 0, keepdims=1) + row_mean = np.nanmean(dist_mat, 1, keepdims=1) + missing_mean = ((np.repeat(col_mean, len(row_mean), 0) + + np.repeat(row_mean, len(col_mean), 1)) / 2) + dist_mat = np.where(missing_corrs, missing_mean, dist_mat) + assert np.allclose(dist_mat, dist_mat.T, atol=0.00001) + # force perfect symmetry + dist_mat = (dist_mat + dist_mat.T) / 2 + return dist_mat + + def distance_matrix(stream_list, shift_len=0.0, allow_individual_trace_shifts=True, cores=1): """ @@ -203,9 +219,7 @@ def distance_matrix(stream_list, shift_len=0.0, trace_shift_dict = dict(zip(master_ids, shift_mat_list)) shift_dict[i] = trace_shift_dict if shift_len == 0: - assert np.allclose(dist_mat, dist_mat.T, atol=0.00001) - # Force perfect symmetry - dist_mat = (dist_mat + dist_mat.T) / 2 + dist_mat = fill_nans_with_rowcol_mean(dist_mat) else: # get the shortest distance for each correlation pair dist_mat_shortest = np.minimum(dist_mat, dist_mat.T) @@ -284,6 +298,7 @@ def cluster(template_list, show=True, corr_thresh=0.3, shift_len=0, if save_corrmat: np.save('dist_mat.npy', dist_mat) Logger.info('Saved the distance matrix as dist_mat.npy') + dist_mat = fill_nans_with_rowcol_mean(dist_mat) dist_vec = squareform(dist_mat) Logger.info('Computing linkage') Z = linkage(dist_vec) From ca0992fa0e36e9996c597ed6867c1f79227f1317 Mon Sep 17 00:00:00 2001 From: flixha Date: Tue, 15 Feb 2022 15:13:45 +0100 Subject: [PATCH 02/61] add test to check handling of NaNs in dist_mat --- eqcorrscan/tests/clustering_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/eqcorrscan/tests/clustering_test.py b/eqcorrscan/tests/clustering_test.py index d4dfa7586..b2988e835 100644 --- a/eqcorrscan/tests/clustering_test.py +++ b/eqcorrscan/tests/clustering_test.py @@ -256,6 +256,21 @@ def test_clustered(self): show=False, corr_thresh=0.3) self.assertEqual(len(groups), 9) + def test_clustered_with_nan_links(self): + """ + Test clustering when some events are not directly linked by matching + traces. + """ + template_list=[(st.copy(), i) for i, st in enumerate(self.stream_list)] + # For the first 2 templates: remove the first / the second half of the + # traces, respectively. + for j, tr in enumerate(template_list[0][0][0:4]): + template_list[0][0].remove(tr) + for j, tr in enumerate(template_list[1][0][4:]): + template_list[1][0].remove(tr) + groups = cluster(template_list, show=False, corr_thresh=0.3) + self.assertEqual(len(groups), 9) + class DistanceClusterTests(unittest.TestCase): @classmethod From 007f67e505edcf434a70c39810da3d8ecd1d04f6 Mon Sep 17 00:00:00 2001 From: flixha Date: Tue, 15 Feb 2022 15:39:15 +0100 Subject: [PATCH 03/61] add changelog --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 7890fe58c..4d356668a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,9 @@ the old parallelization strategy across traces. - Now includes `all_horiz`-option that will correlate all matching horizontal channels no matter to which of these the S-pick is linking. +* utils.clustering + - handle indirect comparison of event-waveforms when (i.e., events without + matching traces which can be compared indirectly via a third event) ## 0.4.3 * core.match_filter From 9cda424976946f002fdc129675d41b19fb3e8854 Mon Sep 17 00:00:00 2001 From: flixha Date: Tue, 15 Feb 2022 18:27:35 +0100 Subject: [PATCH 04/61] add comment and replace depreceated np.int with in --- eqcorrscan/utils/clustering.py | 4 ++++ eqcorrscan/utils/correlate.py | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/eqcorrscan/utils/clustering.py b/eqcorrscan/utils/clustering.py index b95aa80e7..cab83b82b 100644 --- a/eqcorrscan/utils/clustering.py +++ b/eqcorrscan/utils/clustering.py @@ -254,6 +254,10 @@ def cluster(template_list, show=True, corr_thresh=0.3, shift_len=0, similar events. Groups are then created by clustering the distance matrix at distances less than 1 - corr_thresh. + When distance_matrix contains NaNs (event pairs that cannot be directly + compared), then the mean correlation between templates is used instead of + NaN (see https://github.com/eqcorrscan/EQcorrscan/issues/484). + Will compute the distance matrix in parallel, using all available cores :type template_list: list diff --git a/eqcorrscan/utils/correlate.py b/eqcorrscan/utils/correlate.py index 74025f591..4eb9afa94 100644 --- a/eqcorrscan/utils/correlate.py +++ b/eqcorrscan/utils/correlate.py @@ -197,7 +197,7 @@ def _pool_normxcorr(templates, stream, stack, pool, func, *args, **kwargs): cccsums = np.sum(xcorrs, axis=0) else: cccsums = np.asarray(xcorrs).swapaxes(0, 1) - no_chans = np.sum(np.array(tr_chans).astype(np.int), axis=0) + no_chans = np.sum(np.array(tr_chans).astype(int), axis=0) for seed_id, tr_chan in zip(seed_ids, tr_chans): for chan, state in zip(chans, tr_chan): if state: @@ -249,7 +249,7 @@ def stream_xcorr(templates, stream, stack=True, *args, **kwargs): cccsums = np.sum([cccsums, tr_cc], axis=0) else: cccsums[:, chan_no] = tr_cc - no_chans += tr_chans.astype(np.int) + no_chans += tr_chans.astype(int) for chan, state in zip(chans, tr_chans): if state: chan.append(seed_id) @@ -704,7 +704,7 @@ def _time_threaded_normxcorr(templates, stream, stack=True, *args, **kwargs): cccsums = np.sum([cccsums, tr_cc], axis=0) else: cccsums[:, chan_no] = tr_cc - no_chans += tr_chans.astype(np.int) + no_chans += tr_chans.astype(int) for chan, state in zip(chans, tr_chans): if state: chan.append(seed_id) @@ -759,7 +759,7 @@ def _fftw_stream_xcorr(templates, stream, stack=True, *args, **kwargs): template_array=template_dict, stream_array=stream_dict, pad_array=pad_dict, seed_ids=seed_ids, cores_inner=num_cores_inner, stack=stack, *args, **kwargs) - no_chans = np.sum(np.array(tr_chans).astype(np.int), axis=0) + no_chans = np.sum(np.array(tr_chans).astype(int), axis=0) for seed_id, tr_chan in zip(seed_ids, tr_chans): for chan, state in zip(chans, tr_chan): if state: @@ -1062,7 +1062,7 @@ def _fmf_multi_xcorr(templates, stream, *args, **kwargs): tr_chans = np.array([~np.isnan(template_dict[seed_id]).any(axis=1) for seed_id in seed_ids]) - no_chans = np.sum(np.array(tr_chans).astype(np.int), axis=0) + no_chans = np.sum(np.array(tr_chans).astype(int), axis=0) # Note: FMF already returns the zeroed end of correlations - we don't # need to call _get_valid_correlation_sum for seed_id, tr_chan in zip(seed_ids, tr_chans): From e498bf5a327cea18454dad5dc1189e9e56642fb9 Mon Sep 17 00:00:00 2001 From: flixha Date: Wed, 16 Feb 2022 14:06:18 +0100 Subject: [PATCH 05/61] make nan-handling in dist_mat optional, raise proper error --- eqcorrscan/tests/clustering_test.py | 7 ++- eqcorrscan/utils/clustering.py | 74 ++++++++++++++++++++++++----- 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/eqcorrscan/tests/clustering_test.py b/eqcorrscan/tests/clustering_test.py index b2988e835..cef562c7e 100644 --- a/eqcorrscan/tests/clustering_test.py +++ b/eqcorrscan/tests/clustering_test.py @@ -261,14 +261,17 @@ def test_clustered_with_nan_links(self): Test clustering when some events are not directly linked by matching traces. """ - template_list=[(st.copy(), i) for i, st in enumerate(self.stream_list)] + template_list = [ + (st.copy(), i) for i, st in enumerate(self.stream_list)] # For the first 2 templates: remove the first / the second half of the # traces, respectively. for j, tr in enumerate(template_list[0][0][0:4]): template_list[0][0].remove(tr) for j, tr in enumerate(template_list[1][0][4:]): template_list[1][0].remove(tr) - groups = cluster(template_list, show=False, corr_thresh=0.3) + groups = cluster(template_list, show=True, corr_thresh=0.3, + replace_nan_distances_with=1, method='complete', + metric='chebyshev', optimal_ordering=True) self.assertEqual(len(groups), 9) diff --git a/eqcorrscan/utils/clustering.py b/eqcorrscan/utils/clustering.py index cab83b82b..89d75a312 100644 --- a/eqcorrscan/utils/clustering.py +++ b/eqcorrscan/utils/clustering.py @@ -11,6 +11,7 @@ import os import logging from multiprocessing import cpu_count +from re import A import matplotlib.pyplot as plt import numpy as np @@ -20,7 +21,8 @@ from eqcorrscan.utils import stacking from eqcorrscan.utils.archive_read import read_data -from eqcorrscan.utils.correlate import get_array_xcorr, get_stream_xcorr +from eqcorrscan.utils.correlate import ( + get_array_xcorr, get_stream_xcorr, CorrelationError) from eqcorrscan.utils.pre_processing import _prep_data_for_correlation Logger = logging.getLogger(__name__) @@ -131,16 +133,51 @@ def cross_chan_correlation( return _coherances, _positions -def fill_nans_with_rowcol_mean(dist_mat): +def handle_distmat_nans(dist_mat, replace_nan_distances_with=None): """ - # Fill missing correlations (nans) with the mean of column and row + Checks for nans and optionally fills missing correlations (nans) with a + replacement value. + + :type dist_mat: np.ndarray + :param dist_mag: Distance matrix. + :type replace_nan_distances_with: None, 'mean', 'min', or float + :param replace_nan_distances_with: + Controls how the clustering handles nan-distances in the distance + matrix. None/False only performs a check, while other choices (e.g., + 1, 'mean', 'min' or float) replace nans in the distance matrix. + + :returns: + Distance matrix. + :type: np.ndarray """ missing_corrs = ~np.isfinite(dist_mat) - col_mean = np.nanmean(dist_mat, 0, keepdims=1) - row_mean = np.nanmean(dist_mat, 1, keepdims=1) - missing_mean = ((np.repeat(col_mean, len(row_mean), 0) + - np.repeat(row_mean, len(col_mean), 1)) / 2) - dist_mat = np.where(missing_corrs, missing_mean, dist_mat) + if not replace_nan_distances_with: + if np.isnan(dist_mat).any(): + raise CorrelationError( + "The distance matrix contains nans, which may indicate that " + + "some templates have no matching traces and can only be " + + "indirectly linked via other templates. You should check " + + " templates. Then you can try cluster() with " + "replace_nan_distances_with=1") + return dist_mat + elif isinstance(replace_nan_distances_with, (int, float)): + missing_vals = np.full_like( + dist_mat, fill_value=replace_nan_distances_with) + elif replace_nan_distances_with == 'mean': + col_mean = np.nanmean(dist_mat, 0, keepdims=1) + row_mean = np.nanmean(dist_mat, 1, keepdims=1) + missing_vals = ((np.repeat(col_mean, len(row_mean), 0) + + np.repeat(row_mean, len(col_mean), 1)) / 2) + elif replace_nan_distances_with == 'min': + col_min = np.nanmin(dist_mat, 0, keepdims=1) + row_min = np.nanmin(dist_mat, 1, keepdims=1) + missing_vals = np.minimum(((np.repeat(col_min, len(row_min), 0), + np.repeat(row_min, len(col_min), 1)))) + else: + raise NotImplementedError( + 'replace_nan_distances_with={} is not supported'.format( + replace_nan_distances_with)) + dist_mat = np.where(missing_corrs, missing_vals, dist_mat) assert np.allclose(dist_mat, dist_mat.T, atol=0.00001) # force perfect symmetry dist_mat = (dist_mat + dist_mat.T) / 2 @@ -148,6 +185,7 @@ def fill_nans_with_rowcol_mean(dist_mat): def distance_matrix(stream_list, shift_len=0.0, + replace_nan_distances_with=None, allow_individual_trace_shifts=True, cores=1): """ Compute distance matrix for waveforms based on cross-correlations. @@ -168,6 +206,11 @@ def distance_matrix(stream_list, shift_len=0.0, Controls whether templates are shifted by shift_len in relation to the picks as a whole, or whether each trace can be shifted individually. Defaults to True. + :type replace_nan_distances_with: None, 'mean', 'min', or float + :param replace_nan_distances_with: + Controls how the clustering handles nan-distances in the distance + matrix. None/False only performs a check, while other choices (e.g., + 1, 'mean', 'min' or float) replace nans in the distance matrix. :type cores: int :param cores: Number of cores to parallel process using, defaults to 1. @@ -219,7 +262,8 @@ def distance_matrix(stream_list, shift_len=0.0, trace_shift_dict = dict(zip(master_ids, shift_mat_list)) shift_dict[i] = trace_shift_dict if shift_len == 0: - dist_mat = fill_nans_with_rowcol_mean(dist_mat) + dist_mat = handle_distmat_nans( + dist_mat, replace_nan_distances_with=replace_nan_distances_with) else: # get the shortest distance for each correlation pair dist_mat_shortest = np.minimum(dist_mat, dist_mat.T) @@ -241,7 +285,7 @@ def distance_matrix(stream_list, shift_len=0.0, def cluster(template_list, show=True, corr_thresh=0.3, shift_len=0, allow_individual_trace_shifts=True, save_corrmat=False, - cores='all'): + replace_nan_distances_with=None, cores='all', **kwargs): """ Cluster template waveforms based on average correlations. @@ -279,6 +323,11 @@ def cluster(template_list, show=True, corr_thresh=0.3, shift_len=0, :param save_corrmat: If True will save the distance matrix to dist_mat.npy in the local directory. + :type replace_nan_distances_with: None, 'mean', 'min', or float + :param replace_nan_distances_with: + Controls how the clustering handles nan-distances in the distance + matrix. None/False only performs a check, while other choices (e.g., + 1, 'mean', 'min' or float) replace nans in the distance matrix. :type cores: int :param cores: number of cores to use when computing the distance matrix, defaults to @@ -298,14 +347,15 @@ def cluster(template_list, show=True, corr_thresh=0.3, shift_len=0, Logger.info('Computing the distance matrix using %i cores' % num_cores) dist_mat, shift_mat, shift_dict = distance_matrix( stream_list=stream_list, shift_len=shift_len, cores=num_cores, + replace_nan_distances_with=replace_nan_distances_with, allow_individual_trace_shifts=allow_individual_trace_shifts) if save_corrmat: np.save('dist_mat.npy', dist_mat) Logger.info('Saved the distance matrix as dist_mat.npy') - dist_mat = fill_nans_with_rowcol_mean(dist_mat) + dist_mat = handle_distmat_nans(dist_mat, replace_nan_distances_with) dist_vec = squareform(dist_mat) Logger.info('Computing linkage') - Z = linkage(dist_vec) + Z = linkage(dist_vec, **kwargs) if show: Logger.info('Plotting the dendrogram') dendrogram(Z, color_threshold=1 - corr_thresh, From d6c7e34c4a7a876e66a79c1ec6acb106df15a9e0 Mon Sep 17 00:00:00 2001 From: flixha Date: Wed, 16 Feb 2022 14:10:53 +0100 Subject: [PATCH 06/61] extra docs --- CHANGES.md | 6 ++++-- eqcorrscan/utils/clustering.py | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4d356668a..c8809997d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,8 +19,10 @@ - Now includes `all_horiz`-option that will correlate all matching horizontal channels no matter to which of these the S-pick is linking. * utils.clustering - - handle indirect comparison of event-waveforms when (i.e., events without - matching traces which can be compared indirectly via a third event) + - Allow to handle indirect comparison of event-waveforms when (i.e., events + without matching traces which can be compared indirectly via a third event) + - Allows to set clustering method, metric, and sort_order from + scipy.cluster.hierarchy.linkage. ## 0.4.3 * core.match_filter diff --git a/eqcorrscan/utils/clustering.py b/eqcorrscan/utils/clustering.py index 89d75a312..07fcdef5d 100644 --- a/eqcorrscan/utils/clustering.py +++ b/eqcorrscan/utils/clustering.py @@ -302,7 +302,10 @@ def cluster(template_list, show=True, corr_thresh=0.3, shift_len=0, compared), then the mean correlation between templates is used instead of NaN (see https://github.com/eqcorrscan/EQcorrscan/issues/484). - Will compute the distance matrix in parallel, using all available cores + Will compute the distance matrix in parallel, using all available cores. + The method, metric, and order to compute linkage from the distance matrix + can be controled with parameters from scipy.cluster.hierarchy.linkage as + kwargs. :type template_list: list :param template_list: From 580cd5b9fb888371dfdd8d72c65a3929e4ba2b71 Mon Sep 17 00:00:00 2001 From: flixha Date: Wed, 16 Feb 2022 14:25:00 +0100 Subject: [PATCH 07/61] suppress image show --- eqcorrscan/tests/clustering_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eqcorrscan/tests/clustering_test.py b/eqcorrscan/tests/clustering_test.py index cef562c7e..505b6ef9a 100644 --- a/eqcorrscan/tests/clustering_test.py +++ b/eqcorrscan/tests/clustering_test.py @@ -269,7 +269,7 @@ def test_clustered_with_nan_links(self): template_list[0][0].remove(tr) for j, tr in enumerate(template_list[1][0][4:]): template_list[1][0].remove(tr) - groups = cluster(template_list, show=True, corr_thresh=0.3, + groups = cluster(template_list, show=False, corr_thresh=0.3, replace_nan_distances_with=1, method='complete', metric='chebyshev', optimal_ordering=True) self.assertEqual(len(groups), 9) From f074c294e04c488f9955e92f5748a1ffb60276c6 Mon Sep 17 00:00:00 2001 From: flixha Date: Wed, 16 Feb 2022 14:58:49 +0100 Subject: [PATCH 08/61] fix when no nan-replacement --- eqcorrscan/utils/clustering.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eqcorrscan/utils/clustering.py b/eqcorrscan/utils/clustering.py index 07fcdef5d..7dd5a7a3e 100644 --- a/eqcorrscan/utils/clustering.py +++ b/eqcorrscan/utils/clustering.py @@ -151,6 +151,7 @@ def handle_distmat_nans(dist_mat, replace_nan_distances_with=None): :type: np.ndarray """ missing_corrs = ~np.isfinite(dist_mat) + missing_vals = np.full_like(dist_mat, fill_value=1) if not replace_nan_distances_with: if np.isnan(dist_mat).any(): raise CorrelationError( @@ -159,7 +160,6 @@ def handle_distmat_nans(dist_mat, replace_nan_distances_with=None): "indirectly linked via other templates. You should check " + " templates. Then you can try cluster() with " "replace_nan_distances_with=1") - return dist_mat elif isinstance(replace_nan_distances_with, (int, float)): missing_vals = np.full_like( dist_mat, fill_value=replace_nan_distances_with) @@ -355,7 +355,8 @@ def cluster(template_list, show=True, corr_thresh=0.3, shift_len=0, if save_corrmat: np.save('dist_mat.npy', dist_mat) Logger.info('Saved the distance matrix as dist_mat.npy') - dist_mat = handle_distmat_nans(dist_mat, replace_nan_distances_with) + dist_mat = handle_distmat_nans( + dist_mat, replace_nan_distances_with=replace_nan_distances_with) dist_vec = squareform(dist_mat) Logger.info('Computing linkage') Z = linkage(dist_vec, **kwargs) From 45d6543a69282f2e07babaab1319fcb69b908cf2 Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Thu, 17 Feb 2022 09:11:40 +1300 Subject: [PATCH 09/61] Remove unused import --- eqcorrscan/utils/clustering.py | 1 - 1 file changed, 1 deletion(-) diff --git a/eqcorrscan/utils/clustering.py b/eqcorrscan/utils/clustering.py index 7dd5a7a3e..77667f99d 100644 --- a/eqcorrscan/utils/clustering.py +++ b/eqcorrscan/utils/clustering.py @@ -11,7 +11,6 @@ import os import logging from multiprocessing import cpu_count -from re import A import matplotlib.pyplot as plt import numpy as np From 4179ad5ee84e5414a6f24e8d201db0bf88d7ed93 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 15 Mar 2022 09:31:42 +1300 Subject: [PATCH 10/61] Shorten ccsum out-of-range WARNING msg to only 2 lines and terminate with newline. --- CONTRIBUTORS.md | 1 + eqcorrscan/utils/src/multi_corr.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 528bf7dab..79165b4a5 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -8,3 +8,4 @@ * Felix Halpaap * Iman Kahbasi * eQ Halauwet +* Glenn Nelson diff --git a/eqcorrscan/utils/src/multi_corr.c b/eqcorrscan/utils/src/multi_corr.c index 46ecef3e8..426d5707c 100644 --- a/eqcorrscan/utils/src/multi_corr.c +++ b/eqcorrscan/utils/src/multi_corr.c @@ -514,8 +514,8 @@ static inline int set_ncc( } else if (fabsf(value) > 1.01) { // this will raise a warning when we return to Python - printf("WARNING: Correlation out of range at:\n\tncc_index: %ld\n\ttemplate: %ld\n\tchannel: %i\n\tindex: %ld\n\tvalue: %f\nSETTING TO ZERO.", - ncc_index, t, chan, i, value); + printf("WARNING: Correlation value=%f:\tncc_index: %ld\ttemplate: %ld\tchannel: %i\tindex: %ld\nSETTING TO ZERO.\n", + value, ncc_index, t, chan, i); value = 0.0; status = 1; } From 00492b34e47d7e1288310c3d283618171f32aa6a Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Tue, 15 Mar 2022 10:31:25 +1300 Subject: [PATCH 11/61] Merge upstream changes into develop (#489) * Start an FAQ * Remove relative magnitude methods for now, see issue #385 * Add SAC question. * implement magnitude-bias correction as in Schaff & Richards 2014 * add change-comment * stickler * syntax * fix L2-norm SNR * fix signal norm normalization (now RMS amplitude) * add python version to conda environment creation * Update installation.rst Simplify install and let conda work out Python versions. * Update faq.rst * Bump obspy version to 1.3.0 (#486) * Bump obspy version to 1.3.0 * Test on py 3.9, 3.10 * Encapsulate python versions in quotes * Remove future use * Temporarally pin pyfftw version * Use pip for pyfftw * Don't pin pyproj * Try using micromamba * Try using mamba * underscore * update maco env * update slow tests * Force h5py version to avoid dependency conflict * Debug macos omp linking error * Use conda * Add back in condarc * Don't run on 3.10 for now - macos requirements are not there Co-authored-by: flixha Co-authored-by: Tyler Newton --- .github/test_conda_env.yml | 9 ++- .github/test_conda_env_macOS.yml | 11 +-- .github/test_condarc.yml | 2 +- .github/workflows/runtest.yml | 2 +- CHANGES.md | 3 + eqcorrscan/doc/faq.rst | 114 +++++++++++++++++++++++++++++ eqcorrscan/doc/index.rst | 1 + eqcorrscan/doc/installation.rst | 8 +- eqcorrscan/tests/mag_calc_test.py | 21 ++++-- eqcorrscan/utils/clustering.py | 18 ++--- eqcorrscan/utils/correlate.py | 35 +++++---- eqcorrscan/utils/findpeaks.py | 35 +++++---- eqcorrscan/utils/mag_calc.py | 65 ++++++++++------ eqcorrscan/utils/pre_processing.py | 3 +- requirements.txt | 6 +- 15 files changed, 240 insertions(+), 93 deletions(-) create mode 100644 eqcorrscan/doc/faq.rst diff --git a/.github/test_conda_env.yml b/.github/test_conda_env.yml index 5c0210cbb..36ad70d98 100644 --- a/.github/test_conda_env.yml +++ b/.github/test_conda_env.yml @@ -7,17 +7,18 @@ dependencies: - matplotlib>=1.3.0 - scipy>=0.18 - mock - - obspy>=1.0.3 + - obspy>=1.3.0 - h5py - pyyaml - bottleneck - fftw - - pyfftw>=0.12.0 - - pyproj<2 - pytest>=2.0.0 - pytest-cov - pytest-pep8 - pytest-xdist - pytest-rerunfailures - pytest-mpl - - codecov \ No newline at end of file + - codecov + - pip + - pip: + - pyfftw diff --git a/.github/test_conda_env_macOS.yml b/.github/test_conda_env_macOS.yml index f0d964219..4e846445e 100644 --- a/.github/test_conda_env_macOS.yml +++ b/.github/test_conda_env_macOS.yml @@ -16,17 +16,18 @@ dependencies: - matplotlib>=1.3.0 - scipy>=0.18 - mock - - obspy>=1.0.3 - - h5py + - obspy>=1.3.0 + - h5py<3.2 # Issue with dep resolution: https://github.com/conda-forge/h5py-feedstock/issues/92 - pyyaml - bottleneck - fftw - - pyfftw>=0.12.0 - - pyproj<2 - pytest>=2.0.0 - pytest-cov - pytest-pep8 - pytest-xdist - pytest-rerunfailures - pytest-mpl - - codecov \ No newline at end of file + - codecov + - pip + - pip: + - pyfftw diff --git a/.github/test_condarc.yml b/.github/test_condarc.yml index 5d95569c9..0d9409ec4 100644 --- a/.github/test_condarc.yml +++ b/.github/test_condarc.yml @@ -1,4 +1,4 @@ channels: - conda-forge - defaults -auto_activate_base: false \ No newline at end of file +auto_activate_base: false diff --git a/.github/workflows/runtest.yml b/.github/workflows/runtest.yml index fbc39e2db..f31d44c67 100644 --- a/.github/workflows/runtest.yml +++ b/.github/workflows/runtest.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.7, 3.8] + python-version: ['3.7', '3.8', '3.9'] fail-fast: false # continue-on-error: true diff --git a/CHANGES.md b/CHANGES.md index 7890fe58c..ef46454f5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,9 @@ * utils.mag_calc.relative_magnitude - fixed bug where S-picks / traces were used for relative-magnitude calculation against user's choice. + - implemented full magnitude bias-correction for CC and SNR +* utils.mag_calc.relative_amplitude: + - returns dicts for SNR measurements * utils.catalog_to_dd.write_correlations - Fixed bug on execution of parallel execution. - Added parallel-options for catalog-dt measurements and for stream-preparation diff --git a/eqcorrscan/doc/faq.rst b/eqcorrscan/doc/faq.rst new file mode 100644 index 000000000..2f3ccd9ee --- /dev/null +++ b/eqcorrscan/doc/faq.rst @@ -0,0 +1,114 @@ +EQcorrscan FAQs +=============== + +This is a developing list of frequently asked questions. If you find yourself +experiencing a problem similar to one of these, then try the solution here first! + +If your problem/question isn't answered here then check the issues on the github page +and, if there isn't an issue related to your problem/question then open a new one and +we will try to help. + +---------------------------------------------------------------------- + +Usage Questions +--------------- + +No output to terminal +..................... + +EQcorrscan uses `Logging `_ +to handle output. If you are seeing no output to screen you +probably haven't set up your logging settings. A simple way to do +this is to run code like: + +.. code-block:: python + import logging + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s") + +You can set different levels to get more or less output (DEBUG is the +most output, then INFO, then WARNNG, then ERROR, then CRITICAL). You will need to run this before any calls to EQcorrscan's functions to +get logging output. + +--- + +No correlations computed +........................ + +Frequently the cause of no correlations being computed is that the +SEED ID (network.station.location.channel) for your template do not +match your continuous data. Check that they match, and try increasing +the logging output (above) to help you find the issue. + +--- + +Everything is done multiple times! +.................................. + +EQcorrscan uses `multiprocessing `_ +under the hood, which will spawn multiple processes when called. To +ensure that programs do not run multiple times you should always +write your scripts with a form: + +.. code-block:: python + + def main(): + # Do the mahi + + if __name__ == "__main__": + main() + +See the `multiprocessing note on "Safe importing of main module" `_ for more info. + +--- + +Making templates from SAC files +............................... + +While there is support for making templates from SAC, it generally +isn't a good idea, unless you have SAC waveforms of the length that +you want to correlate with. This is for two reasons: +1. Because templates and continuous +data *must* be processed the same, including using the same length +FFT, for correlations to be accurate. If you make your template from +short data you will be forced to correlate with short chunks of data, which is not very efficient +2. The programming team are not SAC users, and do not understand the nuances of where picks can be saved in SAC headers. + +Instead: you might find it better to convert your SAC picks into +another obspy readable pick/event format. You can try EQcorrscan's +basic sac_utils to do this, but not everything will be retained. + +---------------------------------------------------------------------- + +Design Questions +---------------- + +Can I use different lengths for different channels in a template? +................................................................. + +Not yet in EQcorrscan - we want this as well, but haven't had time to implement it. +If you want this then we would really appreciate the contribution! There are two +main issues with this that require some thought: 1) How correlations are +computed, and 2) how correlations are weighted in the correlation sum. + +Why doesn't EQcorrscan have a GUI? +.................................. + +This would be cool, and it would be great if someone wants to contribute this, +however, the developers thus far have been focused on other things and don't have +unlimited time. + +If you want this, and know how to program GUIs then please do contribute, it would +open EQcorrscan up to more users, which would be great! + +Why do you have a functional and object-oriented API? +..................................................... + +Simply legacy, when Calum started writing EQcorrscan he didn't know +anything about classes. The functional API is retained so that old +codes can still be run, but if you are starting from scratch please use the OO API where possible. + + + diff --git a/eqcorrscan/doc/index.rst b/eqcorrscan/doc/index.rst index 33ac8aa3f..d2230746d 100644 --- a/eqcorrscan/doc/index.rst +++ b/eqcorrscan/doc/index.rst @@ -60,6 +60,7 @@ Contents: intro installation + faq updates tutorial api diff --git a/eqcorrscan/doc/installation.rst b/eqcorrscan/doc/installation.rst index bec8f9db0..594d4f13d 100644 --- a/eqcorrscan/doc/installation.rst +++ b/eqcorrscan/doc/installation.rst @@ -20,10 +20,14 @@ with the following: .. code-block:: bash - conda create -n eqcorrscan -c conda-forge colorama numpy scipy matplotlib obspy bottleneck pyproj + conda create -n eqcorrscan -c conda-forge eqcorrscan source activate eqcorrscan + +This will create an environment called eqcorrscan and install eqcorrscan and its +dependancies in that environment. -To then install EQcorrscan you can simply run: +If you already have a conda environment that you want to use then to install EQcorrscan +you can simply run: .. code-block:: bash diff --git a/eqcorrscan/tests/mag_calc_test.py b/eqcorrscan/tests/mag_calc_test.py index 7a6e1bd31..cb03eb74e 100644 --- a/eqcorrscan/tests/mag_calc_test.py +++ b/eqcorrscan/tests/mag_calc_test.py @@ -153,7 +153,7 @@ def test_scaled_event(self): event2 = event1 for tr in st2: tr.data *= scale_factor - relative_amplitudes = relative_amplitude( + relative_amplitudes, _, _ = relative_amplitude( st1=st1, st2=st2, event1=event1, event2=event2) self.assertEqual(len(relative_amplitudes), len(st1)) for value in relative_amplitudes.values(): @@ -171,7 +171,7 @@ def test_no_suitable_picks_event1(self): event2 = event1 for tr in st2: tr.data *= scale_factor - relative_amplitudes = relative_amplitude( + relative_amplitudes, _, _ = relative_amplitude( st1=st1, st2=st2, event1=event1, event2=event2) self.assertEqual(len(relative_amplitudes), 0) @@ -191,7 +191,7 @@ def test_no_suitable_picks_event2(self): pick.phase_hint = "S" for tr in st2: tr.data *= scale_factor - relative_amplitudes = relative_amplitude( + relative_amplitudes, _, _ = relative_amplitude( st1=st1, st2=st2, event1=event1, event2=event2) self.assertEqual(len(relative_amplitudes), 0) @@ -210,7 +210,7 @@ def test_no_picks_event2(self): event2.picks = [] for tr in st2: tr.data *= scale_factor - relative_amplitudes = relative_amplitude( + relative_amplitudes, _, _ = relative_amplitude( st1=st1, st2=st2, event1=event1, event2=event2) self.assertEqual(len(relative_amplitudes), 0) @@ -223,7 +223,7 @@ def test_no_picks_event1(self): event2 = event1 for tr in st2: tr.data *= scale_factor - relative_amplitudes = relative_amplitude( + relative_amplitudes, _, _ = relative_amplitude( st1=st1, st2=st2, event1=event1, event2=event2) self.assertEqual(len(relative_amplitudes), 0) @@ -240,14 +240,14 @@ def test_low_snr(self): event2 = event1 for tr in st2: tr.data *= scale_factor - relative_amplitudes = relative_amplitude( + relative_amplitudes, _, _ = relative_amplitude( st1=st1, st2=st2, event1=event1, event2=event2) self.assertEqual(len(relative_amplitudes), 1) for value in relative_amplitudes.values(): self.assertAlmostEqual(value, scale_factor) def test_real_near_repeat(self): - relative_amplitudes = relative_amplitude( + relative_amplitudes, _, _ = relative_amplitude( st1=self.st1, st2=self.st2, event1=self.event1, event2=self.event2) for seed_id, ratio in relative_amplitudes.items(): self.assertLess(abs(0.8 - ratio), 0.1) @@ -259,6 +259,13 @@ def test_real_near_repeat_magnitudes(self): for seed_id, mag_diff in relative_magnitudes.items(): self.assertLess(abs(mag_diff + 0.15), 0.1) + def test_real_near_repeat_magnitudes_no_bias_correction(self): + relative_magnitudes, correlations = relative_magnitude( + st1=self.st1, st2=self.st2, event1=self.event1, event2=self.event2, + return_correlations=True, correct_mag_bias=False) + for seed_id, mag_diff in relative_magnitudes.items(): + self.assertLess(abs(mag_diff + 0.15), 0.1) + def test_real_near_repeat_magnitudes_corr_provided(self): relative_magnitudes = relative_magnitude( st1=self.st1, st2=self.st2, event1=self.event1, event2=self.event2, diff --git a/eqcorrscan/utils/clustering.py b/eqcorrscan/utils/clustering.py index d9657ead5..2294f71bc 100644 --- a/eqcorrscan/utils/clustering.py +++ b/eqcorrscan/utils/clustering.py @@ -837,21 +837,20 @@ def remove_unclustered(catalog, distance_cutoff, num_threads=None): """ import ctypes from eqcorrscan.utils.libnames import _load_cdll - from future.utils import native_str from math import radians utilslib = _load_cdll('libutils') utilslib.remove_unclustered.argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=np.float32, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=np.float32, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_long, np.ctypeslib.ndpointer(dtype=np.uint8, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_float, ctypes.c_int] utilslib.remove_unclustered.restype = ctypes.c_int @@ -905,20 +904,19 @@ def dist_mat_km(catalog, num_threads=None): """ import ctypes from eqcorrscan.utils.libnames import _load_cdll - from future.utils import native_str utilslib = _load_cdll('libutils') utilslib.distance_matrix.argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=np.float32, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=np.float32, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_long, np.ctypeslib.ndpointer(dtype=np.float32, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_int] utilslib.distance_matrix.restype = ctypes.c_int diff --git a/eqcorrscan/utils/correlate.py b/eqcorrscan/utils/correlate.py index 74025f591..ada573fc1 100644 --- a/eqcorrscan/utils/correlate.py +++ b/eqcorrscan/utils/correlate.py @@ -28,7 +28,6 @@ import numpy as np import math -from future.utils import native_str from packaging import version from eqcorrscan.utils.libnames import _load_cdll @@ -488,13 +487,13 @@ def time_multi_normxcorr(templates, stream, pads, threaded=False, *args, argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, ndim=1, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_int, ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.float32, ndim=1, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.float32, ndim=1, - flags=native_str('C_CONTIGUOUS'))] + flags='C_CONTIGUOUS')] restype = ctypes.c_int if threaded: func = utilslib.multi_normxcorr_time_threaded @@ -563,22 +562,22 @@ def fftw_normxcorr(templates, stream, pads, threaded=False, *args, **kwargs): argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, ndim=1, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_long, ctypes.c_long, np.ctypeslib.ndpointer(dtype=np.float32, ndim=1, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_long, np.ctypeslib.ndpointer(dtype=np.float32, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_long, np.ctypeslib.ndpointer(dtype=np.intc, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=np.intc, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=np.intc, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=np.intc, - flags=native_str('C_CONTIGUOUS'))] + flags='C_CONTIGUOUS')] restype = ctypes.c_int if threaded: @@ -792,23 +791,23 @@ def fftw_multi_normxcorr(template_array, stream_array, pad_array, seed_ids, utilslib.multi_normxcorr_fftw.argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_long, ctypes.c_long, ctypes.c_long, np.ctypeslib.ndpointer(dtype=np.float32, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_long, np.ctypeslib.ndpointer(dtype=np.float32, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_long, np.ctypeslib.ndpointer(dtype=np.intc, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=np.intc, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.intc, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=np.intc, - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_int] utilslib.multi_normxcorr_fftw.restype = ctypes.c_int ''' diff --git a/eqcorrscan/utils/findpeaks.py b/eqcorrscan/utils/findpeaks.py index c5f98e58a..d04d9c990 100644 --- a/eqcorrscan/utils/findpeaks.py +++ b/eqcorrscan/utils/findpeaks.py @@ -14,7 +14,6 @@ from multiprocessing import Pool, cpu_count from scipy import ndimage -from future.utils import native_str from eqcorrscan.utils.correlate import pool_boy from eqcorrscan.utils.libnames import _load_cdll @@ -344,17 +343,17 @@ def _multi_decluster(peaks, indices, trig_int, thresholds, cores): func.argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, shape=(total_length,), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=long_type, shape=(total_length,), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=long_type, shape=(n,), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.float32, shape=(n,), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), long_type, np.ctypeslib.ndpointer(dtype=np.uint32, shape=(total_length,), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_int] func.restype = ctypes.c_int @@ -440,14 +439,14 @@ def decluster_distance_time(peaks, index, trig_int, catalog, func.argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, shape=(length,), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=long_type, shape=(length,), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=np.float32, shape=(length * length,), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), long_type, ctypes.c_float, long_type, ctypes.c_float, np.ctypeslib.ndpointer(dtype=np.uint32, shape=(length,), - flags=native_str('C_CONTIGUOUS'))] + flags='C_CONTIGUOUS')] func.restype = ctypes.c_int sorted_inds = np.abs(peaks).argsort() @@ -505,12 +504,12 @@ def decluster(peaks, index, trig_int, threshold=0): func.argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, shape=(length,), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), np.ctypeslib.ndpointer(dtype=long_type, shape=(length,), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), long_type, ctypes.c_float, long_type, np.ctypeslib.ndpointer(dtype=np.uint32, shape=(length,), - flags=native_str('C_CONTIGUOUS'))] + flags='C_CONTIGUOUS')] func.restype = ctypes.c_int sorted_inds = np.abs(peaks).argsort() @@ -539,10 +538,10 @@ def _find_peaks_c(array, threshold): length = array.shape[0] utilslib.find_peaks.argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, shape=(length, ), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_long, ctypes.c_float, np.ctypeslib.ndpointer(dtype=np.uint32, shape=(length, ), - flags=native_str('C_CONTIGUOUS'))] + flags='C_CONTIGUOUS')] utilslib.find_peaks.restype = ctypes.c_int arr = np.ascontiguousarray(array, np.float32) out = np.ascontiguousarray(np.zeros((length, ), dtype=np.uint32)) @@ -567,13 +566,13 @@ def _multi_find_peaks_c(arrays, thresholds, threads): arr = np.ascontiguousarray(arrays.flatten(), np.float32) utilslib.multi_find_peaks.argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, shape=(n * length,), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_long, ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.float32, shape=(n, ), - flags=native_str('C_CONTIGUOUS')), + flags='C_CONTIGUOUS'), ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.uint32, shape=(n * length, ), - flags=native_str('C_CONTIGUOUS'))] + flags='C_CONTIGUOUS')] utilslib.multi_find_peaks.restype = ctypes.c_int out = np.ascontiguousarray(np.zeros((n * length, ), dtype=np.uint32)) diff --git a/eqcorrscan/utils/mag_calc.py b/eqcorrscan/utils/mag_calc.py index 29165ac27..c91a6259a 100644 --- a/eqcorrscan/utils/mag_calc.py +++ b/eqcorrscan/utils/mag_calc.py @@ -425,11 +425,13 @@ def _snr(tr, noise_window, signal_window): def _get_signal_and_noise(stream, event, seed_id, noise_window, signal_window, use_s_picks): """ - Get noise and signal amplitudes and signal standard deviation for an event - on a specific channel. + Get noise and signal RMS-amplitudes and signal standard deviation for an + event on a specific channel. - Noise amplitude is calculated as the RMS amplitude in the noise window, - signal amplitude is the maximum amplitude in the signal window. + (Until v.0.4.3, this function calculated noise amplitude as the RMS + amplitude of the noise window and signal amplitude as the maximum amplitude + in the signal window. This was changed to only RMS amplitudes to align it + with the methodology in Schaff & Richards 2014-paper.) """ from eqcorrscan.core.template_gen import _rms @@ -444,9 +446,10 @@ def _get_signal_and_noise(stream, event, seed_id, noise_window, if len(tr) == 0: return None, None, None tr = tr[0] - noise_amp = _rms(tr.slice( + noise = tr.slice( starttime=pick.time + noise_window[0], - endtime=pick.time + noise_window[1]).data) + endtime=pick.time + noise_window[1]).data + noise_amp = _rms(noise) if np.isnan(noise_amp): noise_amp = None signal = tr.slice( @@ -457,11 +460,12 @@ def _get_signal_and_noise(stream, event, seed_id, noise_window, pick.time + signal_window[0], pick.time + signal_window[1])) Logger.debug(tr) return noise_amp, None, None - return noise_amp, signal.max(), signal.std() + signal_amp = _rms(signal) + return noise_amp, signal_amp, signal.std() def relative_amplitude(st1, st2, event1, event2, noise_window=(-20, -1), - signal_window=(-.5, 20), min_snr=5.0, + signal_window=(-.5, 20), min_snr=1.5, use_s_picks=False): """ Compute the relative amplitudes between two streams. @@ -477,7 +481,10 @@ def relative_amplitude(st1, st2, event1, event2, noise_window=(-20, -1), from st2. The standard deviation of the amplitudes is computed in the signal window given. If the ratio of amplitudes between the signal window and the noise window is below `min_snr` then no result is returned for that - trace. Windows are computed relative to the first pick for that station. + trace. The SNR here is defined as the ratio of RMS-amplitudes of signal + and noise (equal to ratio of L2-norms of signal and noise, but normalized + for signal length). The Windows are computed relative to the first pick + for that station. If one stream has insufficient data to estimate noise amplitude, the noise amplitude of the other will be used. @@ -504,8 +511,11 @@ def relative_amplitude(st1, st2, event1, event2, noise_window=(-20, -1), Note that noise and signal windows are relative to pick-times, so using an S-pick might result in a noise window including P-energy. - :rtype: dict - :return: Dictionary of relative amplitudes keyed by seed-id + :rtype: dict, dict, dict + :return: + Dictionary of relative amplitudes keyed by seed-id + Dictionary of signal-to-noise ratios for st1 + Dictionary of signal-to-noise ratios for st2 """ # keep input safe event1 = event1.copy() @@ -518,6 +528,8 @@ def relative_amplitude(st1, st2, event1, event2, noise_window=(-20, -1), for p in event1.picks]]) seed_ids = {tr.id for tr in st1}.intersection({tr.id for tr in st2}) amplitudes = {} + snrs_1 = {} + snrs_2 = {} for seed_id in seed_ids: noise1, signal1, std1 = _get_signal_and_noise( stream=st1, event=event1, signal_window=signal_window, @@ -546,7 +558,9 @@ def relative_amplitude(st1, st2, event1, event2, noise_window=(-20, -1), Logger.debug("Channel: {0} Relative amplitude: {1:.2f}".format( seed_id, ratio)) amplitudes.update({seed_id: ratio}) - return amplitudes + snrs_1.update({seed_id: snr1}) + snrs_2.update({seed_id: snr2}) + return amplitudes, snrs_1, snrs_2 # Magnitude estimation functions @@ -554,7 +568,7 @@ def relative_amplitude(st1, st2, event1, event2, noise_window=(-20, -1), def relative_magnitude(st1, st2, event1, event2, noise_window=(-20, -1), signal_window=(-.5, 20), min_snr=5.0, min_cc=0.7, use_s_picks=False, correlations=None, shift=.2, - return_correlations=False, weight_by_correlation=True): + return_correlations=False, correct_mag_bias=True): """ Compute the relative magnitudes between two events. @@ -566,7 +580,8 @@ def relative_magnitude(st1, st2, event1, event2, noise_window=(-20, -1), .. math:: - \\Delta m = \\log{\\frac{std(tr2)}{std(tr1)}} \\times CC + \\Delta m = \\log{\\frac{std(tr2)}{std(tr1)}} + \\log{ + \\frac{(1+\\frac{1}{snr_x^2})}{1+\\frac{1}{snr_y^2}}\\times CC} If you decide to use this function you should definitely read the paper to understand what you can use this for and cite the paper! @@ -609,9 +624,11 @@ def relative_magnitude(st1, st2, event1, event2, noise_window=(-20, -1), :type return_correlations: bool :param return_correlations: If true will also return maximum correlations as a dictionary. - :type weight_by_correlation: bool - :param weight_by_correlation: - Whether to weight the magnitude by the correlation or not. + :type correct_mag_bias: bool + :param correct_mag_bias: + Whether to correct for the magnitude-bias introduced by cc<1 and the + presence of noise (i.e., SNR << ∞). Without bias-correction, the + relative magnitudes are simple L2-norm-ratio relative magnitudes. :rtype: dict :return: Dictionary of relative magnitudes keyed by seed-id @@ -624,7 +641,7 @@ def relative_magnitude(st1, st2, event1, event2, noise_window=(-20, -1), if correlations is None: correlations = {} compute_correlations = True - relative_amplitudes = relative_amplitude( + relative_amplitudes, snrs_1, snrs_2 = relative_amplitude( st1=st1, st2=st2, event1=event1, event2=event2, noise_window=noise_window, signal_window=signal_window, min_snr=min_snr, use_s_picks=use_s_picks) @@ -655,10 +672,14 @@ def relative_magnitude(st1, st2, event1, event2, noise_window=(-20, -1), f"Correlation of {cc} less than {min_cc} for {seed_id}, " "skipping.") continue - if not weight_by_correlation: - cc = 1.0 - # Weight and add to relative_magnitudes - rel_mag = math.log10(amplitude_ratio) * cc + snr_x = snrs_1[seed_id] + snr_y = snrs_2[seed_id] + if not correct_mag_bias: + cc = snr_x = snr_y = 1.0 + # Correct for CC and SNR-bias and add to relative_magnitudes + # This is equation 10 from Schaff & Richards 2014: + rel_mag = math.log10(amplitude_ratio) + math.log10( + math.sqrt((1 + 1 / snr_y**2) / (1 + 1 / snr_x**2)) * cc) Logger.info(f"Channel: {seed_id} Magnitude change {rel_mag:.2f}") relative_magnitudes.update({seed_id: rel_mag}) if return_correlations: diff --git a/eqcorrscan/utils/pre_processing.py b/eqcorrscan/utils/pre_processing.py index 241560309..e0f515ff3 100644 --- a/eqcorrscan/utils/pre_processing.py +++ b/eqcorrscan/utils/pre_processing.py @@ -632,7 +632,6 @@ def _resample(tr, sampling_rate, threads=1): Provide a pyfftw version of obspy's trace resampling. This code is modified from obspy's Trace.resample method. """ - from future.utils import native_str from scipy.signal import get_window from pyfftw.interfaces.scipy_fftpack import rfft, irfft @@ -648,7 +647,7 @@ def _resample(tr, sampling_rate, threads=1): x_i = x[1::2] large_w = np.fft.ifftshift( - get_window(native_str("hanning"), tr.stats.npts)) + get_window("hanning", tr.stats.npts)) x_r *= large_w[:tr.stats.npts // 2 + 1] x_i *= large_w[:tr.stats.npts // 2 + 1] diff --git a/requirements.txt b/requirements.txt index 1726ca487..34adadcb6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,8 @@ numpy>=1.12 matplotlib>=1.3.0 scipy>=0.18 bottleneck -obspy>=1.0.3 -pyfftw +obspy>=1.3.0 # ObsPy <1.3.0 is incompatible with numpy >= 1.22: https://github.com/obspy/obspy/issues/2912 +pyfftw # PyFFTW 0.13 on conda has a build issue: https://github.com/conda-forge/pyfftw-feedstock/issues/51 h5py pytest>=2.0.0 pytest-cov @@ -12,4 +12,4 @@ pytest-xdist pytest-rerunfailures pytest-mpl codecov -pillow>=6.2.3 # not directly required, pinned by Snyk to avoid a vulnerability \ No newline at end of file +pillow>=6.2.3 # not directly required, pinned by Snyk to avoid a vulnerability From ceb0372166c60fd2705af3f76ac753bed4c6ca28 Mon Sep 17 00:00:00 2001 From: flixha Date: Fri, 25 Mar 2022 15:25:29 +0100 Subject: [PATCH 12/61] Trigger tests From cf1bf465aa139f374a453bcba0dd4e24adae8774 Mon Sep 17 00:00:00 2001 From: flixha Date: Fri, 25 Mar 2022 15:30:14 +0100 Subject: [PATCH 13/61] Trigger tests From c46d9273762ab55d19aae547e249d3c43c3e45b4 Mon Sep 17 00:00:00 2001 From: flixha Date: Fri, 25 Mar 2022 16:56:35 +0100 Subject: [PATCH 14/61] fix test so that it succeeds on more systems --- eqcorrscan/tests/clustering_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eqcorrscan/tests/clustering_test.py b/eqcorrscan/tests/clustering_test.py index 505b6ef9a..eaa874dcf 100644 --- a/eqcorrscan/tests/clustering_test.py +++ b/eqcorrscan/tests/clustering_test.py @@ -269,10 +269,10 @@ def test_clustered_with_nan_links(self): template_list[0][0].remove(tr) for j, tr in enumerate(template_list[1][0][4:]): template_list[1][0].remove(tr) - groups = cluster(template_list, show=False, corr_thresh=0.3, + groups = cluster(template_list, show=False, corr_thresh=0.35, replace_nan_distances_with=1, method='complete', metric='chebyshev', optimal_ordering=True) - self.assertEqual(len(groups), 9) + self.assertEqual(len(groups), 11) class DistanceClusterTests(unittest.TestCase): From ddf2c3388d721acbda5d92a420ee8d06bdc1b8a6 Mon Sep 17 00:00:00 2001 From: flixha Date: Fri, 25 Mar 2022 17:21:58 +0100 Subject: [PATCH 15/61] fix test - order of files depends on system --- eqcorrscan/tests/clustering_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eqcorrscan/tests/clustering_test.py b/eqcorrscan/tests/clustering_test.py index eaa874dcf..c93fa29e1 100644 --- a/eqcorrscan/tests/clustering_test.py +++ b/eqcorrscan/tests/clustering_test.py @@ -213,6 +213,7 @@ def setUpClass(cls): os.path.abspath(os.path.dirname(__file__)), 'test_data', 'similar_events_processed') stream_files = glob.glob(os.path.join(testing_path, '*')) + stream_files = sorted(stream_files) stream_list = [read(stream_file) for stream_file in stream_files] for st in stream_list: st.detrend().filter("bandpass", freqmin=5.0, freqmax=15.0) @@ -269,10 +270,10 @@ def test_clustered_with_nan_links(self): template_list[0][0].remove(tr) for j, tr in enumerate(template_list[1][0][4:]): template_list[1][0].remove(tr) - groups = cluster(template_list, show=False, corr_thresh=0.35, + groups = cluster(template_list, show=False, corr_thresh=0.3, replace_nan_distances_with=1, method='complete', metric='chebyshev', optimal_ordering=True) - self.assertEqual(len(groups), 11) + self.assertEqual(len(groups), 9) class DistanceClusterTests(unittest.TestCase): From 2cdc9c0c74941ae6adab53ee81ab08db086d7498 Mon Sep 17 00:00:00 2001 From: flixha Date: Mon, 21 Mar 2022 16:30:19 +0100 Subject: [PATCH 16/61] simplify CC-interpolation through resampling --- eqcorrscan/core/lag_calc.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/eqcorrscan/core/lag_calc.py b/eqcorrscan/core/lag_calc.py index 5ee6286d9..f88929d96 100644 --- a/eqcorrscan/core/lag_calc.py +++ b/eqcorrscan/core/lag_calc.py @@ -12,6 +12,7 @@ import scipy import logging import os +import scipy from collections import Counter, namedtuple @@ -43,7 +44,7 @@ def __str__(self): return 'LagCalcError: ' + self.value -def _xcorr_interp(ccc, dt): +def _xcorr_interp_old(ccc, dt): """ Interpolate around the maximum correlation value for sub-sample precision. @@ -59,6 +60,7 @@ def _xcorr_interp(ccc, dt): cc = ccc[0] else: cc = ccc + # Code borrowed from obspy.signal.cross_correlation.xcorr_pick_correction cc_curvature = np.concatenate((np.zeros(1), np.diff(cc, 2), np.zeros(1))) cc_t = np.arange(0, len(cc) * dt, dt) @@ -103,6 +105,39 @@ def _xcorr_interp(ccc, dt): return shift, coeff +def _xcorr_interp(ccc, dt, resample_factor=10): + """ + Resample correlation-trace and check if there is a better CCC peak for + sub-sample precision. + + :param ccc: Cross-correlation array + :type ccc: numpy.ndarray + :param dt: sample interval + :type dt: float + + :return: Position of interpolated maximum in seconds from start of ccc + :rtype: float + """ + if ccc.shape[0] == 1: + cc = ccc[0] + else: + cc = ccc + + cc_resampled = scipy.signal.resample(cc, len(cc) * resample_factor + 1) + dt_resampled = dt / resample_factor + cc_t = np.arange(0, len(cc_resampled) * dt_resampled, dt_resampled) + peak_index = cc_resampled.argmax() + cc_peak = max(cc_resampled) + + shift = cc_t[peak_index] + if cc_peak < np.amax(cc) or cc_peak > 1.0 or not 0 < shift < len(ccc) * dt: + # Sometimes the interpolation returns a worse result. + Logger.warning("Interpolation did not give an accurate result, " + "returning maximum in data") + return np.argmax(ccc) * dt, np.amax(ccc) + return shift, cc_peak + + def _concatenate_and_correlate(streams, template, cores): """ Concatenate a list of streams into one stream and correlate that with a From 8f8b78a31259e0a9f90e942dd1df4f0844f1f2eb Mon Sep 17 00:00:00 2001 From: flixha Date: Fri, 25 Mar 2022 18:03:11 +0100 Subject: [PATCH 17/61] add kwargs for interpol --- eqcorrscan/core/lag_calc.py | 66 ++----------------------------- eqcorrscan/utils/catalog_to_dd.py | 5 ++- 2 files changed, 6 insertions(+), 65 deletions(-) diff --git a/eqcorrscan/core/lag_calc.py b/eqcorrscan/core/lag_calc.py index f88929d96..2ede990f4 100644 --- a/eqcorrscan/core/lag_calc.py +++ b/eqcorrscan/core/lag_calc.py @@ -44,67 +44,6 @@ def __str__(self): return 'LagCalcError: ' + self.value -def _xcorr_interp_old(ccc, dt): - """ - Interpolate around the maximum correlation value for sub-sample precision. - - :param ccc: Cross-correlation array - :type ccc: numpy.ndarray - :param dt: sample interval - :type dt: float - - :return: Position of interpolated maximum in seconds from start of ccc - :rtype: float - """ - if ccc.shape[0] == 1: - cc = ccc[0] - else: - cc = ccc - - # Code borrowed from obspy.signal.cross_correlation.xcorr_pick_correction - cc_curvature = np.concatenate((np.zeros(1), np.diff(cc, 2), np.zeros(1))) - cc_t = np.arange(0, len(cc) * dt, dt) - peak_index = cc.argmax() - first_sample = peak_index - # XXX this could be improved.. - while first_sample > 0 and cc_curvature[first_sample - 1] <= 0: - first_sample -= 1 - last_sample = peak_index - while last_sample < len(cc) - 1 and cc_curvature[last_sample + 1] <= 0: - last_sample += 1 - num_samples = last_sample - first_sample + 1 - if num_samples < 3: - Logger.warning( - "Fewer than 3 samples selected for fit to cross correlation: " - "{0}, returning maximum in data".format(num_samples)) - return np.argmax(cc) * dt, np.amax(cc) - if num_samples < 5: - Logger.debug( - "Fewer than 5 samples selected for fit to cross correlation: " - "{0}".format(num_samples)) - coeffs, residual = np.polyfit( - cc_t[first_sample:last_sample + 1], - cc[first_sample:last_sample + 1], deg=2, full=True)[:2] - # check results of fit - if coeffs[0] >= 0: - Logger.info("Fitted parabola opens upwards!") - if residual > 0.1: - Logger.info( - "Residual in quadratic fit to cross correlation maximum larger " - "than 0.1: {0}".format(residual)) - # X coordinate of vertex of parabola gives time shift to correct - # differential pick time. Y coordinate gives maximum correlation - # coefficient. - shift = -coeffs[1] / 2.0 / coeffs[0] - coeff = (4 * coeffs[0] * coeffs[2] - coeffs[1] ** 2) / (4 * coeffs[0]) - if coeff < np.amax(ccc) or coeff > 1.0 or not 0 < shift < len(ccc) * dt: - # Sometimes the interpolation returns a worse result. - Logger.warning("Interpolation did not give an accurate result, " - "returning maximum in data") - return np.argmax(ccc) * dt, np.amax(ccc) - return shift, coeff - - def _xcorr_interp(ccc, dt, resample_factor=10): """ Resample correlation-trace and check if there is a better CCC peak for @@ -226,7 +165,8 @@ def xcorr_pick_family(family, stream, shift_len=0.2, min_cc=0.4, min_cc_from_mean_cc_factor=None, horizontal_chans=['E', 'N', '1', '2'], vertical_chans=['Z'], cores=1, interpolate=False, - plot=False, plotdir=None, export_cc=False, cc_dir=None): + plot=False, plotdir=None, export_cc=False, cc_dir=None, + **kwargs): """ Compute cross-correlation picks for detections in a family. @@ -320,7 +260,7 @@ def xcorr_pick_family(family, stream, shift_len=0.2, min_cc=0.4, tr = detect_stream.select( station=stachan.channel[0], channel=stachan.channel[1])[0] if interpolate: - shift, cc_max = _xcorr_interp(correlation, dt=delta) + shift, cc_max = _xcorr_interp(correlation, dt=delta, **kwargs) else: cc_max = np.amax(correlation) shift = np.argmax(correlation) * delta diff --git a/eqcorrscan/utils/catalog_to_dd.py b/eqcorrscan/utils/catalog_to_dd.py index 4f6c2dceb..29b90070b 100644 --- a/eqcorrscan/utils/catalog_to_dd.py +++ b/eqcorrscan/utils/catalog_to_dd.py @@ -221,7 +221,7 @@ def _prepare_stream(stream, event, extract_len, pre_pick, seed_pick_ids=None): def _compute_dt_correlations(catalog, master, min_link, event_id_mapper, stream_dict, min_cc, extract_len, pre_pick, - shift_len, interpolate, max_workers=1): + shift_len, interpolate, max_workers=1, **kwargs): """ Compute cross-correlation delay times. """ max_workers = max_workers or 1 Logger.info( @@ -352,7 +352,8 @@ def _compute_dt_correlations(catalog, master, min_link, event_id_mapper, continue correlation = ccc_out[i][j] if interpolate: - shift, cc_max = _xcorr_interp(correlation, dt=delta) + shift, cc_max = _xcorr_interp(correlation, dt=delta, + **kwargs) else: cc_max = np.amax(correlation) shift = np.argmax(correlation) * delta From 17a6edc5e9b511e6f6d31d117a9f5a7b0770a093 Mon Sep 17 00:00:00 2001 From: flixha Date: Fri, 25 Mar 2022 18:25:09 +0100 Subject: [PATCH 18/61] add documentation --- CHANGES.md | 2 ++ eqcorrscan/core/lag_calc.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ef46454f5..d87ce44ea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,8 @@ ## Current * core.match_filter.tribe - Detect now allows passing of pre-processed data +* core.lag_calc._xcorr_interp + - -CC-interpolation replaced with resampling (more robust) * utils.correlate - Fast Matched Filter now supported natively for version >= 1.4.0 - Only full correlation stacks are returned now (e.g. where fewer than than diff --git a/eqcorrscan/core/lag_calc.py b/eqcorrscan/core/lag_calc.py index 2ede990f4..c985bad8d 100644 --- a/eqcorrscan/core/lag_calc.py +++ b/eqcorrscan/core/lag_calc.py @@ -53,6 +53,8 @@ def _xcorr_interp(ccc, dt, resample_factor=10): :type ccc: numpy.ndarray :param dt: sample interval :type dt: float + :param resample_factor: Factor for upsampling CC-values. + :type resample_factor: int :return: Position of interpolated maximum in seconds from start of ccc :rtype: float From 8baa34501beadf3653dd1d9d320a7218ddcc32eb Mon Sep 17 00:00:00 2001 From: flixha Date: Mon, 28 Mar 2022 10:38:02 +0200 Subject: [PATCH 19/61] make old interpolation method default; add deprecation warning (once); add test --- eqcorrscan/core/lag_calc.py | 79 ++++++++++++++++++++++++++----- eqcorrscan/tests/lag_calc_test.py | 12 ++++- 2 files changed, 79 insertions(+), 12 deletions(-) diff --git a/eqcorrscan/core/lag_calc.py b/eqcorrscan/core/lag_calc.py index c985bad8d..16faf2087 100644 --- a/eqcorrscan/core/lag_calc.py +++ b/eqcorrscan/core/lag_calc.py @@ -11,6 +11,7 @@ import numpy as np import scipy import logging +import warnings import os import scipy @@ -26,6 +27,7 @@ from eqcorrscan.core.match_filter.template import Template from eqcorrscan.utils.plotting import plot_repicked +show_interp_deprec_warning = True Logger = logging.getLogger(__name__) @@ -44,7 +46,7 @@ def __str__(self): return 'LagCalcError: ' + self.value -def _xcorr_interp(ccc, dt, resample_factor=10): +def _xcorr_interp(ccc, dt, resample_factor=10, use_new_resamp_method=False): """ Resample correlation-trace and check if there is a better CCC peak for sub-sample precision. @@ -53,7 +55,8 @@ def _xcorr_interp(ccc, dt, resample_factor=10): :type ccc: numpy.ndarray :param dt: sample interval :type dt: float - :param resample_factor: Factor for upsampling CC-values. + :param resample_factor: + Factor for upsampling CC-values (only for use_new_resamp_method=True) :type resample_factor: int :return: Position of interpolated maximum in seconds from start of ccc @@ -64,19 +67,73 @@ def _xcorr_interp(ccc, dt, resample_factor=10): else: cc = ccc - cc_resampled = scipy.signal.resample(cc, len(cc) * resample_factor + 1) - dt_resampled = dt / resample_factor - cc_t = np.arange(0, len(cc_resampled) * dt_resampled, dt_resampled) - peak_index = cc_resampled.argmax() - cc_peak = max(cc_resampled) - - shift = cc_t[peak_index] - if cc_peak < np.amax(cc) or cc_peak > 1.0 or not 0 < shift < len(ccc) * dt: + # New method with resampling - make this the default in a future version + if use_new_resamp_method: + cc_resampled = scipy.signal.resample(cc, len(cc) * resample_factor + 1) + dt_resampled = dt / resample_factor + cc_t = np.arange(0, len(cc_resampled) * dt_resampled, dt_resampled) + peak_index = cc_resampled.argmax() + cc_peak = max(cc_resampled) + + shift = cc_t[peak_index] + if (cc_peak < np.amax(cc) or cc_peak > 1.0 or + not 0 < shift < len(ccc) * dt): + # Sometimes the interpolation returns a worse result. + Logger.warning("Interpolation did not give an accurate result, " + "returning maximum in data") + return np.argmax(ccc) * dt, np.amax(ccc) + return shift, cc_peak + + # Otherwise use old interpolation method, but warn with deprcation message + # (but show it only once): + global show_interp_deprec_warning + if show_interp_deprec_warning: + Logger.warning( + 'This method for interpolating cross-correlations is deprecated, ' + 'use a more robust method with use_new_resamp_method=True') + show_interp_deprec_warning = False + # Code borrowed from obspy.signal.cross_correlation.xcorr_pick_correction + cc_curvature = np.concatenate((np.zeros(1), np.diff(cc, 2), np.zeros(1))) + cc_t = np.arange(0, len(cc) * dt, dt) + peak_index = cc.argmax() + first_sample = peak_index + # XXX this could be improved.. + while first_sample > 0 and cc_curvature[first_sample - 1] <= 0: + first_sample -= 1 + last_sample = peak_index + while last_sample < len(cc) - 1 and cc_curvature[last_sample + 1] <= 0: + last_sample += 1 + num_samples = last_sample - first_sample + 1 + if num_samples < 3: + Logger.warning( + "Fewer than 3 samples selected for fit to cross correlation: " + "{0}, returning maximum in data".format(num_samples)) + return np.argmax(cc) * dt, np.amax(cc) + if num_samples < 5: + Logger.debug( + "Fewer than 5 samples selected for fit to cross correlation: " + "{0}".format(num_samples)) + coeffs, residual = np.polyfit( + cc_t[first_sample:last_sample + 1], + cc[first_sample:last_sample + 1], deg=2, full=True)[:2] + # check results of fit + if coeffs[0] >= 0: + Logger.info("Fitted parabola opens upwards!") + if residual > 0.1: + Logger.info( + "Residual in quadratic fit to cross correlation maximum larger " + "than 0.1: {0}".format(residual)) + # X coordinate of vertex of parabola gives time shift to correct + # differential pick time. Y coordinate gives maximum correlation + # coefficient. + shift = -coeffs[1] / 2.0 / coeffs[0] + coeff = (4 * coeffs[0] * coeffs[2] - coeffs[1] ** 2) / (4 * coeffs[0]) + if coeff < np.amax(ccc) or coeff > 1.0 or not 0 < shift < len(ccc) * dt: # Sometimes the interpolation returns a worse result. Logger.warning("Interpolation did not give an accurate result, " "returning maximum in data") return np.argmax(ccc) * dt, np.amax(ccc) - return shift, cc_peak + return shift, coeff def _concatenate_and_correlate(streams, template, cores): diff --git a/eqcorrscan/tests/lag_calc_test.py b/eqcorrscan/tests/lag_calc_test.py index c92dc62f7..cd9a35b6f 100644 --- a/eqcorrscan/tests/lag_calc_test.py +++ b/eqcorrscan/tests/lag_calc_test.py @@ -164,6 +164,16 @@ def test_family_picking_with_interpolation(self): self.assertAlmostEqual( float(pick.comments[0].text.split("=")[-1]), 1.0, 1) + def test_family_picking_with_new_interpolation(self): + catalog_dict = xcorr_pick_family( + family=self.party[0], stream=self.data, shift_len=0.2, plot=False, + interpolate=True, use_new_resamp_method=True, export_cc=False) + for event in catalog_dict.values(): + for pick in event.picks: + self.assertTrue("cc_max=" in pick.comments[0].text) + self.assertAlmostEqual( + float(pick.comments[0].text.split("=")[-1]), 1.0, 1) + def test_lag_calc_api(self): detections = [d for f in self.party for d in f] templates = [f.template.st for f in self.party] @@ -258,7 +268,7 @@ def test_bad_interp(self): 0.60129057, -0.71043723, 0.16709118, 0.96839009, 1.58283915, -0.3053663]) - _xcorr_interp(ccc, 0.1) + _xcorr_interp(ccc, 0.1, use_new_resamp_method=True) self.assertEqual(len(self.log_messages['warning']), 1) self.assertTrue( 'not give an accurate result' in self.log_messages['warning'][0]) From 69fda3c20ea2469dc57019302799f90e63596293 Mon Sep 17 00:00:00 2001 From: flixha Date: Mon, 28 Mar 2022 10:41:28 +0200 Subject: [PATCH 20/61] refine change message --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d87ce44ea..382bbd52a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,8 @@ * core.match_filter.tribe - Detect now allows passing of pre-processed data * core.lag_calc._xcorr_interp - - -CC-interpolation replaced with resampling (more robust) + - CC-interpolation replaced with resampling (more robust), old method + deprecated. Use new method with use_new_resamp_method=True as **kwarg. * utils.correlate - Fast Matched Filter now supported natively for version >= 1.4.0 - Only full correlation stacks are returned now (e.g. where fewer than than From 6ed3cf8688c2013ccd58ea51aabac3aa0ae615ff Mon Sep 17 00:00:00 2001 From: flixha Date: Mon, 28 Mar 2022 14:02:18 +0200 Subject: [PATCH 21/61] remove all seishub options and tests --- eqcorrscan/core/match_filter/template.py | 7 ++-- eqcorrscan/core/match_filter/tribe.py | 9 ++--- eqcorrscan/core/template_gen.py | 30 +++++------------ eqcorrscan/tests/template_gen_test.py | 43 ------------------------ eqcorrscan/utils/archive_read.py | 23 +++---------- eqcorrscan/utils/clustering.py | 2 +- 6 files changed, 18 insertions(+), 96 deletions(-) diff --git a/eqcorrscan/core/match_filter/template.py b/eqcorrscan/core/match_filter/template.py index e290bf07b..41faa3dcd 100644 --- a/eqcorrscan/core/match_filter/template.py +++ b/eqcorrscan/core/match_filter/template.py @@ -532,8 +532,8 @@ def construct(self, method, name, lowcut, highcut, samp_rate, filt_order, :param method: Method to make the template, the only available method is: - `from_sac`. For all other methods (`from_seishub`, `from_client` - and `from_meta_file`) use `Tribe.construct()`. + `from_sac`. For all other methods (`from_client` and + `from_meta_file`) use `Tribe.construct()`. :type method: str :type name: str :param name: Name for the template @@ -635,8 +635,7 @@ def construct(self, method, name, lowcut, highcut, samp_rate, filt_order, Tribe.construct instead. """ - if method in ['from_meta_file', 'from_seishub', 'from_client', - 'multi_template_gen']: + if method in ['from_meta_file', 'from_client', 'multi_template_gen']: raise NotImplementedError('Method is not supported, ' 'use Tribe.construct instead.') streams, events, process_lengths = template_gen.template_gen( diff --git a/eqcorrscan/core/match_filter/tribe.py b/eqcorrscan/core/match_filter/tribe.py index 4c9829a84..f03bf77a4 100644 --- a/eqcorrscan/core/match_filter/tribe.py +++ b/eqcorrscan/core/match_filter/tribe.py @@ -924,8 +924,8 @@ def construct(self, method, lowcut, highcut, samp_rate, filt_order, :type method: str :param method: Method of Tribe generation. Possible options are: `from_client`, - `from_seishub`, `from_meta_file`. See below on the additional - required arguments for each method. + `from_meta_file`. See below on the additional required arguments + for each method. :type lowcut: float :param lowcut: Low cut (Hz), if set to None will not apply a lowcut @@ -999,11 +999,6 @@ def construct(self, method, lowcut, highcut, samp_rate, filt_order, :param `obspy.core.event.Catalog` catalog: Catalog of events to generate template for :param float data_pad: Pad length for data-downloads in seconds - - `from_seishub` requires: - :param str url: url to seishub database - :param `obspy.core.event.Catalog` catalog: - Catalog of events to generate template for - :param float data_pad: Pad length for data-downloads in seconds - `from_meta_file` requires: :param str meta_file: Path to obspy-readable event file, or an obspy Catalog diff --git a/eqcorrscan/core/template_gen.py b/eqcorrscan/core/template_gen.py index bcf530c65..f06794da8 100644 --- a/eqcorrscan/core/template_gen.py +++ b/eqcorrscan/core/template_gen.py @@ -24,7 +24,6 @@ from obspy import Stream, read, Trace, UTCDateTime, read_events from obspy.core.event import Catalog from obspy.clients.fdsn import Client as FDSNClient -from obspy.clients.seishub import Client as SeisHubClient from eqcorrscan.utils.sac_util import sactoevent from eqcorrscan.utils import pre_processing @@ -62,9 +61,9 @@ def template_gen(method, lowcut, highcut, samp_rate, filt_order, :type method: str :param method: - Template generation method, must be one of ('from_client', - 'from_seishub', 'from_sac', 'from_meta_file'). - Each method requires - associated arguments, see note below. + Template generation method, must be one of ('from_client', 'from_sac', + 'from_meta_file'). - Each method requires associated arguments, see + note below. :type lowcut: float :param lowcut: Low cut (Hz), if set to None will not apply a lowcut. :type highcut: float @@ -153,11 +152,6 @@ def template_gen(method, lowcut, highcut, samp_rate, filt_order, :param `obspy.core.event.Catalog` catalog: Catalog of events to generate template for :param float data_pad: Pad length for data-downloads in seconds - - `from_seishub` requires: - :param str url: url to seishub database - :param `obspy.core.event.Catalog` catalog: - Catalog of events to generate template for - :param float data_pad: Pad length for data-downloads in seconds - `from_sac` requires: :param list sac_files: osbpy.core.stream.Stream of sac waveforms, or list of paths to @@ -237,13 +231,12 @@ def template_gen(method, lowcut, highcut, samp_rate, filt_order, >>> print(len(templates[0])) 15 """ - client_map = {'from_client': 'fdsn', 'from_seishub': 'seishub'} - assert method in ('from_client', 'from_seishub', 'from_meta_file', - 'from_sac') + client_map = {'from_client': 'fdsn'} + assert method in ('from_client', 'from_meta_file', 'from_sac') if not isinstance(swin, list): swin = [swin] process = True - if method in ['from_client', 'from_seishub']: + if method in ['from_client']: catalog = kwargs.get('catalog', Catalog()) data_pad = kwargs.get('data_pad', 90) # Group catalog into days and only download the data once per day @@ -262,9 +255,6 @@ def template_gen(method, lowcut, highcut, samp_rate, filt_order, "with a get_waveforms method" ) available_stations = [] - else: - client = SeisHubClient(kwargs.get('url', None), timeout=10) - available_stations = client.waveform.get_station_ids() elif method == 'from_meta_file': if isinstance(kwargs.get('meta_file'), Catalog): catalog = kwargs.get('meta_file') @@ -301,7 +291,7 @@ def template_gen(method, lowcut, highcut, samp_rate, filt_order, else: all_channels = False for sub_catalog in sub_catalogs: - if method in ['from_seishub', 'from_client']: + if method in ['from_client']: Logger.info("Downloading data") st = _download_from_client( client=client, client_type=client_map[method], @@ -507,7 +497,7 @@ def extract_from_stack(stack, template, length, pre_pick, pre_pad, def _download_from_client(client, client_type, catalog, data_pad, process_len, available_stations=[], all_channels=False): """ - Internal function to handle downloading from either seishub or fdsn client + Internal function to handle downloading from fdsn client """ st = Stream() catalog = Catalog(sorted(catalog, key=lambda e: e.origins[0].time)) @@ -541,10 +531,6 @@ def _download_from_client(client, client_type, catalog, data_pad, process_len, dropped_pick_stations = 0 for waveform_info in all_waveform_info: net, sta, chan, loc = waveform_info - if client_type == 'seishub' and sta not in available_stations: - Logger.error("Station not found in SeisHub DB") - dropped_pick_stations += 1 - continue Logger.info('Downloading for start-time: {0} end-time: {1}'.format( starttime, endtime)) Logger.debug('.'.join([net, sta, loc, chan])) diff --git a/eqcorrscan/tests/template_gen_test.py b/eqcorrscan/tests/template_gen_test.py index dad3e3506..88b0ebada 100644 --- a/eqcorrscan/tests/template_gen_test.py +++ b/eqcorrscan/tests/template_gen_test.py @@ -256,49 +256,6 @@ def test_all_phase_methods(self): self.assertLess(abs(tr.stats.starttime - (pick.time - 0.2)), tr.stats.delta) - def test_seishub(self): - """Test the seishub method, use obspy default seishub client.""" - import sys - if sys.version_info.major == 2: - from future.backports.urllib.request import URLError - else: - from urllib.request import URLError - t = UTCDateTime(2009, 9, 3) - test_cat = Catalog() - test_cat.append(Event()) - test_cat[0].origins.append(Origin()) - test_cat[0].origins[0].time = t - test_cat[0].origins[0].latitude = 45 - test_cat[0].origins[0].longitude = 45 - test_cat[0].origins[0].depth = 5000 - test_cat[0].picks.append(Pick( - waveform_id=WaveformStreamID( - station_code='MANZ', channel_code='EHZ', network_code='BW'), - phase_hint='PG', time=t + 2000)) - test_cat[0].picks.append(Pick( - waveform_id=WaveformStreamID( - station_code='MANZ', channel_code='EHN', network_code='BW'), - phase_hint='SG', time=t + 2005)) - test_cat[0].picks.append(Pick( - waveform_id=WaveformStreamID( - station_code='MANZ', channel_code='EHE', network_code='BW'), - phase_hint='SG', time=t + 2005.5)) - - test_url = "http://teide.geophysik.uni-muenchen.de:8080" - - if sys.version_info.major == 3: - try: - template = template_gen( - method="from_seishub", catalog=test_cat, url=test_url, - lowcut=1.0, highcut=5.0, samp_rate=20, filt_order=4, - length=3, prepick=0.5, swin='all', process_len=300) - except URLError: - pass - else: - pass - if 'template' in locals(): - self.assertEqual(len(template), 3) - def test_catalog_grouping(self): testing_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'test_data', 'REA', 'TEST_', '*.S??????') diff --git a/eqcorrscan/utils/archive_read.py b/eqcorrscan/utils/archive_read.py index f04ea4260..14d361d64 100644 --- a/eqcorrscan/utils/archive_read.py +++ b/eqcorrscan/utils/archive_read.py @@ -20,7 +20,6 @@ from obspy import read, UTCDateTime, Stream from obspy.clients.fdsn.header import FDSNException -from obspy.clients.seishub import Client as SeishubClient from obspy.clients.fdsn import Client as FDSNClient from obspy.clients.filesystem.sds import Client as SDSClient @@ -34,12 +33,11 @@ def read_data(archive, arc_type, day, stachans, length=86400): :type archive: str :param archive: - The archive source - if arc_type is seishub, this should be a url, - if the arc_type is FDSN then this can be either a url or a known obspy - client. If arc_type is day_vols, then this is the path to the top - directory. + The archive source - if arc_type is FDSN then this can be either a url + or a known obspy client. If arc_type is day_vols, then this is the + path to the top directory. :type arc_type: str - :param arc_type: The type of archive, can be: seishub, FDSN, day_volumes + :param arc_type: The type of archive, can be: FDSN, day_volumes :type day: datetime.date :param day: Date to retrieve data for :type stachans: list @@ -120,12 +118,6 @@ def read_data(archive, arc_type, day, stachans, length=86400): day.strftime('%Y/%m/%d')]) Logger.warning(msg) continue - if arc_type.lower() == 'seishub': - client = SeishubClient(archive) - st += client.get_waveforms( - network='*', station=station_map[0], location='*', - channel=station_map[1], starttime=UTCDateTime(day), - endtime=UTCDateTime(day) + length) elif arc_type.upper() == "FDSN": client = FDSNClient(archive) try: @@ -204,7 +196,6 @@ def _check_available_data(archive, arc_type, day): :returns: list of tuples of (station, channel) as available. - .. note:: Currently the seishub options are untested. """ available_stations = [] @@ -215,12 +206,6 @@ def _check_available_data(archive, arc_type, day): header = read(wavefile, headonly=True) available_stations.append((header[0].stats.station, header[0].stats.channel)) - elif arc_type.lower() == 'seishub': - client = SeishubClient(archive) - st = client.get_previews(starttime=UTCDateTime(day), - endtime=UTCDateTime(day) + 86400) - for tr in st: - available_stations.append((tr.stats.station, tr.stats.channel)) elif arc_type.lower() == 'fdsn': client = FDSNClient(archive) inventory = client.get_stations(starttime=UTCDateTime(day), diff --git a/eqcorrscan/utils/clustering.py b/eqcorrscan/utils/clustering.py index 2294f71bc..b96c6e7ec 100644 --- a/eqcorrscan/utils/clustering.py +++ b/eqcorrscan/utils/clustering.py @@ -638,7 +638,7 @@ def extract_detections(detections, templates, archive, arc_type, Either name of archive or path to continuous data, see :func:`eqcorrscan.utils.archive_read` for details :type arc_type: str - :param arc_type: Type of archive, either seishub, FDSN, day_vols + :param arc_type: Type of archive, either FDSN, day_vols :type extract_len: float :param extract_len: Length to extract around the detection (will be equally cut around From cb8fc0ee245ea521947353b161b3b4d6f06244c0 Mon Sep 17 00:00:00 2001 From: flixha Date: Mon, 28 Mar 2022 14:03:55 +0200 Subject: [PATCH 22/61] add changelog --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ef46454f5..18272c04d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,8 @@ the old parallelization strategy across traces. - Now includes `all_horiz`-option that will correlate all matching horizontal channels no matter to which of these the S-pick is linking. +* tribe, template, template_gen, archive_read, clustering: remove option to read + from seishub (deprecated in obspy). ## 0.4.3 * core.match_filter From f42b469e82ef0dc86b7da736cb02a8cfa4ba153a Mon Sep 17 00:00:00 2001 From: flixha Date: Mon, 28 Mar 2022 14:42:13 +0200 Subject: [PATCH 23/61] fix setup for breaking change from setuptools --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a7ed29553..b5a992e0e 100644 --- a/setup.py +++ b/setup.py @@ -395,7 +395,9 @@ def setup_package(): 'tests_require': ['pytest>=2.0.0', 'pytest-cov', 'pytest-pep8', 'pytest-xdist', 'pytest-rerunfailures', 'obspy>=1.1.0'], - 'cmdclass': {'build_ext': CustomBuildExt} + 'cmdclass': {'build_ext': CustomBuildExt}, + # Declare packages explicitly so setuptools>=61.0.0 does not auto discover + 'packages': [] } if using_setuptools: From 63e1088cd5052c0c4b993a60d744435affccb570 Mon Sep 17 00:00:00 2001 From: flixha Date: Mon, 28 Mar 2022 14:42:13 +0200 Subject: [PATCH 24/61] fix setup for breaking change from setuptools --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a7ed29553..b5a992e0e 100644 --- a/setup.py +++ b/setup.py @@ -395,7 +395,9 @@ def setup_package(): 'tests_require': ['pytest>=2.0.0', 'pytest-cov', 'pytest-pep8', 'pytest-xdist', 'pytest-rerunfailures', 'obspy>=1.1.0'], - 'cmdclass': {'build_ext': CustomBuildExt} + 'cmdclass': {'build_ext': CustomBuildExt}, + # Declare packages explicitly so setuptools>=61.0.0 does not auto discover + 'packages': [] } if using_setuptools: From 76e1ff87ad0c58fc3435272ddc64a4429d348f59 Mon Sep 17 00:00:00 2001 From: flixha Date: Tue, 29 Mar 2022 10:26:37 +0200 Subject: [PATCH 25/61] make compute_dt_correlations send kwargs forward --- eqcorrscan/utils/catalog_to_dd.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eqcorrscan/utils/catalog_to_dd.py b/eqcorrscan/utils/catalog_to_dd.py index 29b90070b..263a9d2ab 100644 --- a/eqcorrscan/utils/catalog_to_dd.py +++ b/eqcorrscan/utils/catalog_to_dd.py @@ -540,6 +540,8 @@ def compute_differential_times(catalog, correlation, stream_dict=None, min_cc=min_cc, stream_dict=stream_dict, extract_len=extract_len, pre_pick=pre_pick, shift_len=shift_len, interpolate=interpolate, max_workers=max_workers) + for key, value in kwargs.items(): + correlation_kwargs.update({key: value}) if correlation: for arg, name in correlation_kwargs.items(): assert arg is not None, "{0} is required for correlation".format( From 7654c84d3a62de6e4e31e9fa8ae8caa9051dff58 Mon Sep 17 00:00:00 2001 From: flixha Date: Fri, 22 Apr 2022 13:04:30 +0200 Subject: [PATCH 26/61] add kwargs to run lag_calc with new resample method --- eqcorrscan/core/lag_calc.py | 7 ++++--- eqcorrscan/core/match_filter/family.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/eqcorrscan/core/lag_calc.py b/eqcorrscan/core/lag_calc.py index 16faf2087..1ab4224c8 100644 --- a/eqcorrscan/core/lag_calc.py +++ b/eqcorrscan/core/lag_calc.py @@ -46,7 +46,8 @@ def __str__(self): return 'LagCalcError: ' + self.value -def _xcorr_interp(ccc, dt, resample_factor=10, use_new_resamp_method=False): +def _xcorr_interp(ccc, dt, resample_factor=10, use_new_resamp_method=False, + **kwargs): """ Resample correlation-trace and check if there is a better CCC peak for sub-sample precision. @@ -453,7 +454,7 @@ def lag_calc(detections, detect_data, template_names, templates, shift_len=0.2, min_cc=0.4, min_cc_from_mean_cc_factor=None, horizontal_chans=['E', 'N', '1', '2'], vertical_chans=['Z'], cores=1, interpolate=False, - plot=False, plotdir=None, export_cc=False, cc_dir=None): + plot=False, plotdir=None, export_cc=False, cc_dir=None, **kwargs): """ Cross-correlation derived picking of seismic events. @@ -600,7 +601,7 @@ def lag_calc(detections, detect_data, template_names, templates, horizontal_chans=horizontal_chans, vertical_chans=vertical_chans, interpolate=interpolate, cores=cores, shift_len=shift_len, plot=plot, plotdir=plotdir, - export_cc=export_cc, cc_dir=cc_dir) + export_cc=export_cc, cc_dir=cc_dir, **kwargs) initial_cat.update(template_dict) # Order the catalogue to match the input output_cat = Catalog() diff --git a/eqcorrscan/core/match_filter/family.py b/eqcorrscan/core/match_filter/family.py index a12f9c665..e9e1978ec 100644 --- a/eqcorrscan/core/match_filter/family.py +++ b/eqcorrscan/core/match_filter/family.py @@ -618,7 +618,7 @@ def lag_calc(self, stream, pre_processed, shift_len=0.2, min_cc=0.4, min_cc_from_mean_cc_factor=min_cc_from_mean_cc_factor, vertical_chans=vertical_chans, cores=cores, interpolate=interpolate, plot=plot, plotdir=plotdir, - export_cc=export_cc, cc_dir=cc_dir) + export_cc=export_cc, cc_dir=cc_dir, **kwargs) catalog_out = Catalog([ev for ev in picked_dict.values()]) for detection_id, event in picked_dict.items(): for pick in event.picks: From c63db250a4e73c5c94a8607c61f4f52d0c01a45f Mon Sep 17 00:00:00 2001 From: flixha Date: Fri, 22 Apr 2022 14:45:35 +0200 Subject: [PATCH 27/61] stickler --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b5a992e0e..1a2f6fd9e 100644 --- a/setup.py +++ b/setup.py @@ -373,7 +373,8 @@ def setup_package(): setup_args = { 'name': 'EQcorrscan', 'version': VERSION, - 'description': 'EQcorrscan - matched-filter earthquake detection and analysis', + 'description': + 'EQcorrscan - matched-filter earthquake detection and analysis', 'long_description': long_description, 'url': 'https://github.com/eqcorrscan/EQcorrscan', 'author': 'Calum Chamberlain', @@ -396,7 +397,8 @@ def setup_package(): 'pytest-xdist', 'pytest-rerunfailures', 'obspy>=1.1.0'], 'cmdclass': {'build_ext': CustomBuildExt}, - # Declare packages explicitly so setuptools>=61.0.0 does not auto discover + # Declare packages explicitly so setuptools>=61.0.0 does not auto + # discover 'packages': [] } From c1f7505cda3c6cc6d6502210272b1091a310d859 Mon Sep 17 00:00:00 2001 From: flixha Date: Fri, 22 Apr 2022 14:49:02 +0200 Subject: [PATCH 28/61] stickler --- eqcorrscan/core/lag_calc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/eqcorrscan/core/lag_calc.py b/eqcorrscan/core/lag_calc.py index 1ab4224c8..2b1e43deb 100644 --- a/eqcorrscan/core/lag_calc.py +++ b/eqcorrscan/core/lag_calc.py @@ -11,9 +11,7 @@ import numpy as np import scipy import logging -import warnings import os -import scipy from collections import Counter, namedtuple From 9238a3690abc8fe2f49383005de821c31e576a7c Mon Sep 17 00:00:00 2001 From: Tyler Newton <54011699+tjnewton@users.noreply.github.com> Date: Sun, 12 Jun 2022 12:19:41 -0700 Subject: [PATCH 29/61] fix broken link to Rubinstein and Ellsworth paper --- eqcorrscan/doc/tutorials/mag-calc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eqcorrscan/doc/tutorials/mag-calc.rst b/eqcorrscan/doc/tutorials/mag-calc.rst index 889bd8460..ecb76b4c9 100644 --- a/eqcorrscan/doc/tutorials/mag-calc.rst +++ b/eqcorrscan/doc/tutorials/mag-calc.rst @@ -11,7 +11,7 @@ This method closely follows the method outlined by |SVD_mag_link|. .. |SVD_mag_link| raw:: html - Rubinstein & Ellsworth 2010 + Rubinstein & Ellsworth 2010 This example requires data downloaded from the eqcorrscan github repository. From 53c16b4399986d60de0762b56b346d5c939ffff8 Mon Sep 17 00:00:00 2001 From: Tyler Newton <54011699+tjnewton@users.noreply.github.com> Date: Sun, 12 Jun 2022 12:21:52 -0700 Subject: [PATCH 30/61] fix broken link to Rubinstein and Ellsworth paper --- eqcorrscan/utils/mag_calc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eqcorrscan/utils/mag_calc.py b/eqcorrscan/utils/mag_calc.py index c91a6259a..cd0a3caa4 100644 --- a/eqcorrscan/utils/mag_calc.py +++ b/eqcorrscan/utils/mag_calc.py @@ -1022,7 +1022,7 @@ def svd_moments(u, s, v, stachans, event_list, n_svs=2): For more information see the paper by `Rubinstein & Ellsworth (2010). - `_ + `_ :type u: list :param u: From 8f52f85d87815314a5fe413ffa6972502ae1fe7c Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Thu, 14 Jul 2022 10:05:35 +1200 Subject: [PATCH 31/61] Include prepick in trimming --- eqcorrscan/core/lag_calc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eqcorrscan/core/lag_calc.py b/eqcorrscan/core/lag_calc.py index 5ee6286d9..b27c3e5be 100644 --- a/eqcorrscan/core/lag_calc.py +++ b/eqcorrscan/core/lag_calc.py @@ -387,7 +387,7 @@ def _prepare_data(family, detect_data, shift_len): length = round(length_samples) / family.template.samp_rate Logger.info("Setting length to {0}s to give an integer number of " "samples".format(length)) - prepick = shift_len + prepick = shift_len + family.template.prepick detect_streams_dict = family.extract_streams( stream=detect_data, length=length, prepick=prepick) for key, detect_stream in detect_streams_dict.items(): @@ -557,7 +557,7 @@ def lag_calc(detections, detect_data, template_names, templates, detections=template_detections, template=Template( name=template_name, st=template, - samp_rate=template[0].stats.sampling_rate)) + samp_rate=template[0].stats.sampling_rate, prepick=0.0)) # Make a sparse template if len(template_detections) > 0: template_dict = xcorr_pick_family( From 8760d60df719fee807c2161ef085a277b2936664 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Thu, 14 Jul 2022 10:05:50 +1200 Subject: [PATCH 32/61] Test extracted length --- eqcorrscan/tests/lag_calc_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eqcorrscan/tests/lag_calc_test.py b/eqcorrscan/tests/lag_calc_test.py index c92dc62f7..c934e30a5 100644 --- a/eqcorrscan/tests/lag_calc_test.py +++ b/eqcorrscan/tests/lag_calc_test.py @@ -78,6 +78,9 @@ def _prepare_data_checks(self, detect_stream_dict, shift_len, family): self.assertAlmostEqual( tr.stats.endtime - tr.stats.starttime, (2 * shift_len) + self.t_length, 1) + pick = [p for p in detection.event.picks if p.waveform_id.get_seed_string() == tr.id][0] + self.assertAlmostEqual( + tr.stats.starttime, pick.time - (family.template.prepick + shift_len), 1) def test_prepare_data(self): shift_len = 0.2 From 98bc877a21b98280892c96563c2bb1f741e2bd62 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Thu, 14 Jul 2022 12:30:23 +1200 Subject: [PATCH 33/61] Try and get eqcorrscan installable on github actions again --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index a7ed29553..b81830c4f 100644 --- a/setup.py +++ b/setup.py @@ -395,7 +395,11 @@ def setup_package(): 'tests_require': ['pytest>=2.0.0', 'pytest-cov', 'pytest-pep8', 'pytest-xdist', 'pytest-rerunfailures', 'obspy>=1.1.0'], - 'cmdclass': {'build_ext': CustomBuildExt} + 'cmdclass': {'build_ext': CustomBuildExt}, + 'packages': [ + 'eqcorrscan', 'eqcorrscan.utils', 'eqcorrscan.core', + 'eqcorrscan.core.match_filter', 'eqcorrscan.utils.lib', + 'eqcorrscan.tutorials', 'eqcorrscan.helpers', 'eqcorrscan.tests'], } if using_setuptools: @@ -413,10 +417,6 @@ def setup_package(): # For these actions, NumPy is not required. pass else: - setup_args['packages'] = [ - 'eqcorrscan', 'eqcorrscan.utils', 'eqcorrscan.core', - 'eqcorrscan.core.match_filter', 'eqcorrscan.utils.lib', - 'eqcorrscan.tutorials', 'eqcorrscan.helpers', 'eqcorrscan.tests'] setup_args['ext_modules'] = get_extensions(no_mkl=no_mkl) setup_args['package_data'] = get_package_data() setup_args['package_dir'] = get_package_dir() From 7d7689ff45a4a514be2d1d5cf8a2b6e7146873db Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Thu, 14 Jul 2022 13:39:03 +1200 Subject: [PATCH 34/61] Skip pytest-mpl 0.16.0 --- .github/test_conda_env.yml | 2 +- .github/test_conda_env_macOS.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/test_conda_env.yml b/.github/test_conda_env.yml index 36ad70d98..0d02dc3d4 100644 --- a/.github/test_conda_env.yml +++ b/.github/test_conda_env.yml @@ -17,7 +17,7 @@ dependencies: - pytest-pep8 - pytest-xdist - pytest-rerunfailures - - pytest-mpl + - pytest-mpl<0.16.0,>0.16.0 - codecov - pip - pip: diff --git a/.github/test_conda_env_macOS.yml b/.github/test_conda_env_macOS.yml index 4e846445e..c68bb5b3a 100644 --- a/.github/test_conda_env_macOS.yml +++ b/.github/test_conda_env_macOS.yml @@ -26,7 +26,7 @@ dependencies: - pytest-pep8 - pytest-xdist - pytest-rerunfailures - - pytest-mpl + - pytest-mpl<0.16.0,>0.16.0 - codecov - pip - pip: From f85d864a83996a4c7b9f439ba45e1346f0a361fa Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Thu, 14 Jul 2022 13:47:49 +1200 Subject: [PATCH 35/61] Logic --- .github/test_conda_env.yml | 2 +- .github/test_conda_env_macOS.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/test_conda_env.yml b/.github/test_conda_env.yml index 0d02dc3d4..83f8ba40d 100644 --- a/.github/test_conda_env.yml +++ b/.github/test_conda_env.yml @@ -17,7 +17,7 @@ dependencies: - pytest-pep8 - pytest-xdist - pytest-rerunfailures - - pytest-mpl<0.16.0,>0.16.0 + - pytest-mpl<0.16.0 - codecov - pip - pip: diff --git a/.github/test_conda_env_macOS.yml b/.github/test_conda_env_macOS.yml index c68bb5b3a..199f7f29a 100644 --- a/.github/test_conda_env_macOS.yml +++ b/.github/test_conda_env_macOS.yml @@ -26,7 +26,7 @@ dependencies: - pytest-pep8 - pytest-xdist - pytest-rerunfailures - - pytest-mpl<0.16.0,>0.16.0 + - pytest-mpl<0.16.0 - codecov - pip - pip: From 78579ec6d14d284f3d994ad4924b8a52a9de2aa3 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Thu, 14 Jul 2022 14:50:25 +1200 Subject: [PATCH 36/61] stickler --- eqcorrscan/tests/lag_calc_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eqcorrscan/tests/lag_calc_test.py b/eqcorrscan/tests/lag_calc_test.py index c934e30a5..aef8a1e57 100644 --- a/eqcorrscan/tests/lag_calc_test.py +++ b/eqcorrscan/tests/lag_calc_test.py @@ -78,9 +78,11 @@ def _prepare_data_checks(self, detect_stream_dict, shift_len, family): self.assertAlmostEqual( tr.stats.endtime - tr.stats.starttime, (2 * shift_len) + self.t_length, 1) - pick = [p for p in detection.event.picks if p.waveform_id.get_seed_string() == tr.id][0] + pick = [p for p in detection.event.picks + if p.waveform_id.get_seed_string() == tr.id][0] self.assertAlmostEqual( - tr.stats.starttime, pick.time - (family.template.prepick + shift_len), 1) + tr.stats.starttime, + pick.time - (family.template.prepick + shift_len), 1) def test_prepare_data(self): shift_len = 0.2 From fcfb927163e957551f23d3171ec2409c8ba5ab8e Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Mon, 18 Jul 2022 12:57:34 +1200 Subject: [PATCH 37/61] Use loadscope for test running to avoid setupcls multiple times: --- .github/workflows/runtest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runtest.yml b/.github/workflows/runtest.yml index f31d44c67..49408e66b 100644 --- a/.github/workflows/runtest.yml +++ b/.github/workflows/runtest.yml @@ -65,7 +65,7 @@ jobs: - name: run main test suite shell: bash -l {0} run: | - py.test -n 2 -m "not serial and not network and not superslow" --cov-report=xml + py.test -n 2 -m "not serial and not network and not superslow" --cov-report=xml --dist loadscope - name: run serial test if: always() From b3fc1f1e5e25715336c3adbfdb4ce101c6614787 Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Mon, 18 Jul 2022 12:57:47 +1200 Subject: [PATCH 38/61] Make tests more robust --- eqcorrscan/tests/lag_calc_test.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/eqcorrscan/tests/lag_calc_test.py b/eqcorrscan/tests/lag_calc_test.py index aef8a1e57..a30f8a04c 100644 --- a/eqcorrscan/tests/lag_calc_test.py +++ b/eqcorrscan/tests/lag_calc_test.py @@ -3,10 +3,12 @@ """ import glob import os +import shutil import unittest import numpy as np import logging +import pytest from obspy import read, UTCDateTime from eqcorrscan.utils.synth_seis import generate_synth_data @@ -18,19 +20,20 @@ from eqcorrscan.core.match_filter import Detection, Family, Party, Template from eqcorrscan.helpers.mock_logger import MockLoggingHandler -np.random.seed(999) +# np.random.seed(999) class SyntheticTests(unittest.TestCase): """ Test lag-calc with synthetic data. """ @classmethod def setUpClass(cls): + print("Setting up class") samp_rate = 50 - cls.t_length = .75 + t_length = .75 # Make some synthetic templates templates, data, seeds = generate_synth_data( nsta=5, ntemplates=5, nseeds=10, samp_rate=samp_rate, - t_length=cls.t_length, max_amp=10, max_lag=15, phaseout="both", + t_length=t_length, max_amp=10, max_lag=15, phaseout="both", jitter=0, noise=False, same_phase=True) # Rename channels channel_mapper = {"SYN_Z": "HHZ", "SYN_H": "HHN"} @@ -39,7 +42,7 @@ def setUpClass(cls): for template in templates: for tr in template: tr.stats.channel = channel_mapper[tr.stats.channel] - cls.party = Party() + party = Party() t = 0 data_start = data[0].stats.starttime for template, template_seeds in zip(templates, seeds): @@ -64,9 +67,12 @@ def setUpClass(cls): samp_rate=samp_rate, filt_order=4, process_length=86400, prepick=10. / samp_rate, event=None) family = Family(template=_template, detections=detections) - cls.party += family + party += family t += 1 + cls.party = party cls.data = data + cls.t_length = t_length + assert len(data) == 10 def _prepare_data_checks(self, detect_stream_dict, shift_len, family): self.assertEqual( @@ -146,8 +152,8 @@ def test_family_picking_missing_data(self): export_cc=False) gap = gappy_data.split().get_gaps() for event in catalog_dict.values(): - if len(event.picks) != len(self.data): - self.assertEqual(len(event.picks), len(self.data) - 1) + if len(event.picks) != len(gappy_data): + self.assertEqual(len(event.picks), len(gappy_data) - 1) # Check that the event happened to be in the gap self.assertTrue(gap[0][4] <= event.picks[0].time <= gap[0][5]) # Check that there isn't a pick on the channel missing data @@ -182,16 +188,22 @@ def test_lag_calc_api(self): self.assertEqual(len(output_cat), len(detections)) for event in output_cat: self.assertEqual(len(event.picks), len(self.data)) + for pick in event.picks: + self.assertTrue("cc_max=" in pick.comments[0].text) + self.assertAlmostEqual( + float(pick.comments[0].text.split("=")[-1]), 1.0, 1) def test_xcorr_pick_family_export_cc(self): - cc_dir = 'cc_exported' + cc_dir = 'lag_calc_cc_exported' xcorr_pick_family( - family=self.party[0], stream=self.data, shift_len=0.2, plot=False, + family=self.party[0], stream=self.data.copy(), shift_len=0.2, plot=False, interpolate=False, export_cc=True, cc_dir=cc_dir) cc_files = glob.glob(os.path.join(cc_dir, '*.npy')) assert len(cc_files) == len(self.party[0]) for fcc in cc_files: np.load(fcc) + if os.path.isdir(cc_dir): + shutil.rmtree(cc_dir) class SimpleRealDataTests(unittest.TestCase): From 412fc02652ac7c3f3c4bfe1acf3e0b5cf20d92e4 Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Mon, 18 Jul 2022 12:58:13 +1200 Subject: [PATCH 39/61] randn was resulting in damage to lag-calc randn tests --- eqcorrscan/tests/find_peaks_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eqcorrscan/tests/find_peaks_test.py b/eqcorrscan/tests/find_peaks_test.py index b8c85cb21..87d5d836a 100644 --- a/eqcorrscan/tests/find_peaks_test.py +++ b/eqcorrscan/tests/find_peaks_test.py @@ -192,7 +192,7 @@ def not_below_threshold(self): spike_locs = np.random.randint(0, self.data_len, size=500) threshold = 0.5 for spike_loc in spike_locs: - arr[spike_loc] *= 100 * np.random.randn() + arr[spike_loc] *= 100 * np.random.random() return arr, spike_locs, threshold @pytest.fixture(scope='class') From 6199d9a221b0e58c7f4daf500a6129defa258755 Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Mon, 18 Jul 2022 13:03:52 +1200 Subject: [PATCH 40/61] stickler --- eqcorrscan/tests/lag_calc_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eqcorrscan/tests/lag_calc_test.py b/eqcorrscan/tests/lag_calc_test.py index a30f8a04c..397e1ce11 100644 --- a/eqcorrscan/tests/lag_calc_test.py +++ b/eqcorrscan/tests/lag_calc_test.py @@ -8,7 +8,6 @@ import numpy as np import logging -import pytest from obspy import read, UTCDateTime from eqcorrscan.utils.synth_seis import generate_synth_data @@ -196,8 +195,8 @@ def test_lag_calc_api(self): def test_xcorr_pick_family_export_cc(self): cc_dir = 'lag_calc_cc_exported' xcorr_pick_family( - family=self.party[0], stream=self.data.copy(), shift_len=0.2, plot=False, - interpolate=False, export_cc=True, cc_dir=cc_dir) + family=self.party[0], stream=self.data.copy(), shift_len=0.2, + plot=False, interpolate=False, export_cc=True, cc_dir=cc_dir) cc_files = glob.glob(os.path.join(cc_dir, '*.npy')) assert len(cc_files) == len(self.party[0]) for fcc in cc_files: From 44fbb6345362e67b14a472a52240b470a2acab51 Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Mon, 18 Jul 2022 13:10:20 +1200 Subject: [PATCH 41/61] Remove lingering future imports --- eqcorrscan/utils/lib/__init__.py | 3 --- setup.py | 1 - 2 files changed, 4 deletions(-) diff --git a/eqcorrscan/utils/lib/__init__.py b/eqcorrscan/utils/lib/__init__.py index 811551b44..e69de29bb 100644 --- a/eqcorrscan/utils/lib/__init__.py +++ b/eqcorrscan/utils/lib/__init__.py @@ -1,3 +0,0 @@ -from __future__ import (absolute_import, division, print_function, - unicode_literals) -from future.builtins import * # NOQA \ No newline at end of file diff --git a/setup.py b/setup.py index 68b3ad56e..afcb8f168 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -from __future__ import print_function try: # use setuptools if we can from setuptools import setup, Command, Extension From 2d79ada69e4bb9c47fc98c717cddeffbc022f505 Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Mon, 18 Jul 2022 14:04:51 +1200 Subject: [PATCH 42/61] Update lag_calc_test.py --- eqcorrscan/tests/lag_calc_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eqcorrscan/tests/lag_calc_test.py b/eqcorrscan/tests/lag_calc_test.py index 010ddfd34..635e57de0 100644 --- a/eqcorrscan/tests/lag_calc_test.py +++ b/eqcorrscan/tests/lag_calc_test.py @@ -19,7 +19,7 @@ from eqcorrscan.core.match_filter import Detection, Family, Party, Template from eqcorrscan.helpers.mock_logger import MockLoggingHandler -# np.random.seed(999) +np.random.seed(999) class SyntheticTests(unittest.TestCase): From ec67b7ac97bc2c27ba93c3d0a07b853b7da7932c Mon Sep 17 00:00:00 2001 From: flixha Date: Fri, 22 Apr 2022 13:50:38 +0200 Subject: [PATCH 43/61] fix bug with min_cc_from_mean_cc_factor when detect_val is negative --- eqcorrscan/core/lag_calc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eqcorrscan/core/lag_calc.py b/eqcorrscan/core/lag_calc.py index 3596ed73a..dd71efc36 100644 --- a/eqcorrscan/core/lag_calc.py +++ b/eqcorrscan/core/lag_calc.py @@ -306,8 +306,9 @@ def xcorr_pick_family(family, stream, shift_len=0.2, min_cc=0.4, checksum, cccsum, used_chans = 0.0, 0.0, 0 event = Event() if min_cc_from_mean_cc_factor is not None: - cc_thresh = min(detection.detect_val / detection.no_chans - * min_cc_from_mean_cc_factor, min_cc) + cc_thresh = min(abs(detection.detect_val / detection.no_chans + * min_cc_from_mean_cc_factor), + min_cc) Logger.info('Setting minimum cc-threshold for detection %s to %s', detection.id, str(cc_thresh)) else: From c85d4490b105521ab9554d7d2a771fed9b4e9834 Mon Sep 17 00:00:00 2001 From: flixha Date: Thu, 21 Jul 2022 16:55:37 +0200 Subject: [PATCH 44/61] changelog entry --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index b93a936d2..93ad3b51c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ * core.lag_calc._xcorr_interp - CC-interpolation replaced with resampling (more robust), old method deprecated. Use new method with use_new_resamp_method=True as **kwarg. +* core.lag_calc: + - Fixed bug where minimum CC defined via min_cc_from_mean_cc_factor was not + set correctly for negative correlation sums. * utils.correlate - Fast Matched Filter now supported natively for version >= 1.4.0 - Only full correlation stacks are returned now (e.g. where fewer than than From e064b5120e75a685b850c0f9b20cf0c0381a42df Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Fri, 5 Aug 2022 16:33:55 +1200 Subject: [PATCH 45/61] Only update peak_cores arg if it is't given explicitly already --- CHANGES.md | 3 +++ eqcorrscan/core/match_filter/matched_filter.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b93a936d2..14dad024b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,7 @@ ## Current +* core.match_filter + - Bug-fix: peak-cores could be defined twice in _group_detect through kwargs. + Fix: only update peak_cores if it isn't there already. * core.match_filter.tribe - Detect now allows passing of pre-processed data * core.lag_calc._xcorr_interp diff --git a/eqcorrscan/core/match_filter/matched_filter.py b/eqcorrscan/core/match_filter/matched_filter.py index d7f680721..b9334c9b1 100644 --- a/eqcorrscan/core/match_filter/matched_filter.py +++ b/eqcorrscan/core/match_filter/matched_filter.py @@ -201,6 +201,7 @@ def _group_detect(templates, stream, threshold, threshold_type, trig_int, n_groups += 1 else: n_groups = 1 + kwargs.update({'peak_cores': kwargs.get('peak_cores', process_cores)}) for st_chunk in streams: chunk_start, chunk_end = (min(tr.stats.starttime for tr in st_chunk), max(tr.stats.endtime for tr in st_chunk)) @@ -226,8 +227,7 @@ def _group_detect(templates, stream, threshold, threshold_type, trig_int, xcorr_func=xcorr_func, concurrency=concurrency, threshold=threshold, threshold_type=threshold_type, trig_int=trig_int, plot=plot, plotdir=plotdir, cores=cores, - full_peaks=full_peaks, peak_cores=process_cores, - **kwargs) + full_peaks=full_peaks, **kwargs) for template in template_group: family = Family(template=template, detections=[]) for detection in detections: From 56d9304c796ea1dc75b3202c01cc1096f3bffbdb Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 10:33:41 +1200 Subject: [PATCH 46/61] Fix #479 - cope with missing phase-hints --- eqcorrscan/core/template_gen.py | 22 ++++++++++++++++++---- eqcorrscan/tests/template_gen_test.py | 18 +++++++++++++++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/eqcorrscan/core/template_gen.py b/eqcorrscan/core/template_gen.py index f06794da8..cfb5d49f1 100644 --- a/eqcorrscan/core/template_gen.py +++ b/eqcorrscan/core/template_gen.py @@ -716,14 +716,20 @@ def _template_gen(picks, st, length, swin='all', prepick=0.05, tr.stats.station] if _swin == 'P_all': p_pick = [pick for pick in station_picks - if pick.phase_hint.upper()[0] == 'P'] + if pick.phase_hint and + pick.phase_hint.upper()[0] == 'P'] if len(p_pick) == 0: + Logger.debug(f"No picks with phase_hint P " + f"found for {tr.stats.station}") continue starttime.update({'picks': p_pick}) elif _swin == 'S_all': s_pick = [pick for pick in station_picks - if pick.phase_hint.upper()[0] == 'S'] + if pick.phase_hint and + pick.phase_hint.upper()[0] == 'S'] if len(s_pick) == 0: + Logger.debug(f"No picks with phase_hint S " + f"found for {tr.stats.station}") continue starttime.update({'picks': s_pick}) elif _swin == 'all': @@ -743,22 +749,30 @@ def _template_gen(picks, st, length, swin='all', prepick=0.05, starttime.update({'picks': channel_pick}) elif _swin == 'P': p_pick = [pick for pick in station_picks - if pick.phase_hint.upper()[0] == 'P' and + if pick.phase_hint and + pick.phase_hint.upper()[0] == 'P' and pick.waveform_id.channel_code == tr.stats.channel] if len(p_pick) == 0: + Logger.debug( + f"No picks with phase_hint P " + f"found for {tr.stats.station}.{tr.stats.channel}") continue starttime.update({'picks': p_pick}) elif _swin == 'S': if tr.stats.channel[-1] in ['Z', 'U']: continue s_pick = [pick for pick in station_picks - if pick.phase_hint.upper()[0] == 'S'] + if pick.phase_hint and + pick.phase_hint.upper()[0] == 'S'] if not all_horiz: s_pick = [pick for pick in s_pick if pick.waveform_id.channel_code == tr.stats.channel] starttime.update({'picks': s_pick}) if len(starttime['picks']) == 0: + Logger.debug( + f"No picks with phase_hint S " + f"found for {tr.stats.station}.{tr.stats.channel}") continue if not delayed: starttime.update({'picks': [first_pick]}) diff --git a/eqcorrscan/tests/template_gen_test.py b/eqcorrscan/tests/template_gen_test.py index 88b0ebada..86aef6e14 100644 --- a/eqcorrscan/tests/template_gen_test.py +++ b/eqcorrscan/tests/template_gen_test.py @@ -57,7 +57,6 @@ def get_default_catalog(self): return Catalog(events=[event]) - class TestTemplateGeneration(unittest.TestCase): """Test the reading a writing of pick info.""" def test_sac_template_gen(self): @@ -365,6 +364,23 @@ def test_no_matched_picks(self): template = _template_gen(picks, self.st.copy(), 10) self.assertFalse(template) + def test_missing_phase_hints(self): + picks = copy.deepcopy(self.picks) + template = _template_gen(picks, self.st.copy(), 10, swin="P_all") + self.assertIn("WV03", {tr.stats.station for tr in template}) + # Remove phase_hint for WV03 P pick + for pick in picks: + if pick.waveform_id.station_code == "WV03": + pick.phase_hint = None + template_2 = _template_gen(picks, self.st.copy(), 10, swin="P_all") + self.assertNotIn("WV03", {tr.stats.station for tr in template_2}) + for tr in template: + if tr.stats.station == "WV03": + continue + # check that other than WV03 the template is the same + tr2 = template_2.select(id=tr.id)[0] + self.assertEqual(tr, tr2) + def test_misc(self): template = _template_gen(self.picks, self.st.copy(), 10) self.assertEqual(len(template), len(self.picks)) From 0bccecd65aded8c321ef7799efc46387d64251c8 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 10:39:50 +1200 Subject: [PATCH 47/61] Slightly streamline phase-hint checking --- eqcorrscan/core/template_gen.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/eqcorrscan/core/template_gen.py b/eqcorrscan/core/template_gen.py index cfb5d49f1..b431d0ea3 100644 --- a/eqcorrscan/core/template_gen.py +++ b/eqcorrscan/core/template_gen.py @@ -714,10 +714,13 @@ def _template_gen(picks, st, length, swin='all', prepick=0.05, station_picks = [pick for pick in picks_copy if pick.waveform_id.station_code == tr.stats.station] + # Cope with missing phase_hints + if _swin != "all": + station_picks = [p for p in station_picks if p.phase_hint] + if _swin == 'P_all': p_pick = [pick for pick in station_picks - if pick.phase_hint and - pick.phase_hint.upper()[0] == 'P'] + if pick.phase_hint.upper()[0] == 'P'] if len(p_pick) == 0: Logger.debug(f"No picks with phase_hint P " f"found for {tr.stats.station}") @@ -725,8 +728,7 @@ def _template_gen(picks, st, length, swin='all', prepick=0.05, starttime.update({'picks': p_pick}) elif _swin == 'S_all': s_pick = [pick for pick in station_picks - if pick.phase_hint and - pick.phase_hint.upper()[0] == 'S'] + if pick.phase_hint.upper()[0] == 'S'] if len(s_pick) == 0: Logger.debug(f"No picks with phase_hint S " f"found for {tr.stats.station}") @@ -749,8 +751,7 @@ def _template_gen(picks, st, length, swin='all', prepick=0.05, starttime.update({'picks': channel_pick}) elif _swin == 'P': p_pick = [pick for pick in station_picks - if pick.phase_hint and - pick.phase_hint.upper()[0] == 'P' and + if pick.phase_hint.upper()[0] == 'P' and pick.waveform_id.channel_code == tr.stats.channel] if len(p_pick) == 0: Logger.debug( @@ -762,8 +763,7 @@ def _template_gen(picks, st, length, swin='all', prepick=0.05, if tr.stats.channel[-1] in ['Z', 'U']: continue s_pick = [pick for pick in station_picks - if pick.phase_hint and - pick.phase_hint.upper()[0] == 'S'] + if pick.phase_hint.upper()[0] == 'S'] if not all_horiz: s_pick = [pick for pick in s_pick if pick.waveform_id.channel_code == From 97373160882bc1674a2645388a1b935cdcca1c71 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 10:46:38 +1200 Subject: [PATCH 48/61] Ensure duplicates are squashed after template.detect --- eqcorrscan/core/match_filter/template.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eqcorrscan/core/match_filter/template.py b/eqcorrscan/core/match_filter/template.py index 41faa3dcd..8b792de90 100644 --- a/eqcorrscan/core/match_filter/template.py +++ b/eqcorrscan/core/match_filter/template.py @@ -520,7 +520,10 @@ def detect(self, stream, threshold, threshold_type, trig_int, parallel_process=parallel_process, xcorr_func=xcorr_func, concurrency=concurrency, cores=cores, ignore_length=ignore_length, overlap=overlap, full_peaks=full_peaks, **kwargs) - return party[0] + family = party[0] + # Remove duplicates + family.detections = family._uniq().detections + return family def construct(self, method, name, lowcut, highcut, samp_rate, filt_order, length, prepick, swin="all", process_len=86400, From 8592acfd0418564aa5772dbf2f2f8118ac08cb18 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 11:31:55 +1200 Subject: [PATCH 49/61] Detections should not hash --- CHANGES.md | 2 ++ eqcorrscan/core/match_filter/detection.py | 7 ------- eqcorrscan/core/match_filter/family.py | 3 ++- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 288022dda..74ec346bd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Fix: only update peak_cores if it isn't there already. * core.match_filter.tribe - Detect now allows passing of pre-processed data +* core.match_filter.template + - Remove duplicate detections from overlapping windows using `._uniq()` * core.lag_calc._xcorr_interp - CC-interpolation replaced with resampling (more robust), old method deprecated. Use new method with use_new_resamp_method=True as **kwarg. diff --git a/eqcorrscan/core/match_filter/detection.py b/eqcorrscan/core/match_filter/detection.py index 3fa5aa948..5fe689db3 100644 --- a/eqcorrscan/core/match_filter/detection.py +++ b/eqcorrscan/core/match_filter/detection.py @@ -152,13 +152,6 @@ def __gt__(self, other): def __ge__(self, other): return not self.__lt__(other) - def __hash__(self): - """ - Cannot hash Detection objects, they may change. - :return: 0 - """ - return 0 - def __ne__(self, other): return not self.__eq__(other) diff --git a/eqcorrscan/core/match_filter/family.py b/eqcorrscan/core/match_filter/family.py index e9e1978ec..332b02b8d 100644 --- a/eqcorrscan/core/match_filter/family.py +++ b/eqcorrscan/core/match_filter/family.py @@ -304,7 +304,8 @@ def _uniq(self): .. rubric:: Example - >>> from eqcorrscan import Template, Detection + >>> from eqcorrscan import Template, Detection, Family + >>> from obspy import UTCDateTime >>> family = Family( ... template=Template(name='a'), detections=[ ... Detection(template_name='a', detect_time=UTCDateTime(0), From b4477c282addcd9fd79dd8fe1b0a61ea3d0da716 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 11:39:41 +1200 Subject: [PATCH 50/61] Pin Scipy due to hanning renaming issues --- .github/test_conda_env.yml | 2 +- .github/test_conda_env_macOS.yml | 2 +- requirements.txt | 3 ++- setup.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/test_conda_env.yml b/.github/test_conda_env.yml index 83f8ba40d..8ff5c492f 100644 --- a/.github/test_conda_env.yml +++ b/.github/test_conda_env.yml @@ -5,7 +5,7 @@ channels: dependencies: - numpy>=1.12 - matplotlib>=1.3.0 - - scipy>=0.18 + - scipy>=0.18,<1.9.0 # Pinned due to scipy/obspy hanning renaming - mock - obspy>=1.3.0 - h5py diff --git a/.github/test_conda_env_macOS.yml b/.github/test_conda_env_macOS.yml index 199f7f29a..3282a48c6 100644 --- a/.github/test_conda_env_macOS.yml +++ b/.github/test_conda_env_macOS.yml @@ -14,7 +14,7 @@ dependencies: - llvm-openmp>=4.0.1 - numpy>=1.12 - matplotlib>=1.3.0 - - scipy>=0.18 + - scipy>=0.18,<1.9.0 # Pinned due to scipy/obspy hanning renaming - mock - obspy>=1.3.0 - h5py<3.2 # Issue with dep resolution: https://github.com/conda-forge/h5py-feedstock/issues/92 diff --git a/requirements.txt b/requirements.txt index 34adadcb6..13952d965 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ +# Note that this file is not explicitly used by setup.py for EQcorrscan. numpy>=1.12 matplotlib>=1.3.0 -scipy>=0.18 +scipy>=0.18,<1.9.0 # Pinned due to scipy/obspy hanning renaming bottleneck obspy>=1.3.0 # ObsPy <1.3.0 is incompatible with numpy >= 1.22: https://github.com/obspy/obspy/issues/2912 pyfftw # PyFFTW 0.13 on conda has a build issue: https://github.com/conda-forge/pyfftw-feedstock/issues/51 diff --git a/setup.py b/setup.py index afcb8f168..4273d8482 100644 --- a/setup.py +++ b/setup.py @@ -362,7 +362,7 @@ def setup_package(): build_requires = ['numpy>=1.6, <2.0'] if not READ_THE_DOCS: - install_requires = ['matplotlib>=1.3.0', 'scipy>=0.18', + install_requires = ['matplotlib>=1.3.0', 'scipy>=0.18,<1.9.0', 'bottleneck', 'obspy>=1.0.3', 'numpy>=1.12', 'h5py'] else: From 32ccd64955b04cd22e6029a1bd317ff3f426e9b0 Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Tue, 9 Aug 2022 14:59:08 +1200 Subject: [PATCH 51/61] Properly check for libname extensions --- eqcorrscan/utils/libnames.py | 52 +++++++++++++----------------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/eqcorrscan/utils/libnames.py b/eqcorrscan/utils/libnames.py index be88a0d77..adc9576d9 100644 --- a/eqcorrscan/utils/libnames.py +++ b/eqcorrscan/utils/libnames.py @@ -13,35 +13,12 @@ import ctypes import logging from distutils import sysconfig +import importlib.machinery Logger = logging.getLogger(__name__) -def _get_lib_name(lib): - """ - Helper function to get an architecture and Python version specific library - filename. - """ - # append any extension suffix defined by Python for current platform - ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") - # in principle "EXT_SUFFIX" is what we want. - # "SO" seems to be deprecated on newer python - # but: older python seems to have empty "EXT_SUFFIX", so we fall back - if not ext_suffix: - try: - ext_suffix = sysconfig.get_config_var("SO") - except Exception as e: - Logger.warning( - "Empty 'EXT_SUFFIX' encountered while building CDLL " - "filename and fallback to 'SO' variable failed " - "(%s)." % str(e)) - pass - if ext_suffix: - libname = lib + ext_suffix - return libname - - def _load_cdll(name): """ Helper function to load a shared library built during installation @@ -51,10 +28,8 @@ def _load_cdll(name): :param name: Name of the library to load (e.g. 'mseed'). :rtype: :class:`ctypes.CDLL` """ - # our custom defined part of the extension file name - libname = _get_lib_name(name) libdir = os.path.join(os.path.dirname(__file__), 'lib') - libpath = os.path.join(libdir, libname) + # Try and cope with static FFTW libs static_fftw = os.path.join(libdir, 'libfftw3-3.dll') static_fftwf = os.path.join(libdir, 'libfftw3f-3.dll') try: @@ -62,12 +37,23 @@ def _load_cdll(name): fftwf_lib = ctypes.CDLL(str(static_fftwf)) # noqa: F841 except Exception: pass - try: - cdll = ctypes.CDLL(str(libpath), mode=ctypes.RTLD_LOCAL) - except Exception as e: - msg = 'Could not load shared library "%s".\n\n %s' % (libname, str(e)) - raise ImportError(msg) - return cdll + + # Cope with range of possible extensions for different python versions + errs = [] + for ext in importlib.machinery.EXTENSION_SUFFIXES: + libpath = os.path.join(libdir, name + ext) + try: + cdll = ctypes.CDLL(str(libpath), mode=ctypes.RTLD_LOCAL) + except Exception as e: + msg = 'Could not load shared library "%s".\n\n %s' % (name, str(e)) + errs.append(msg) + Logger.debug(msg) + else: + Logger.info(f"Loaded library from {libpath}") + return cdll + raise ImportError( + "Could not load shared library {0} due to " + "errors:\n{1}".format(name, "\n".join(errs))) if __name__ == '__main__': From e6132a60ebebc0fa48d7ad0e7a988ea65c539b91 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 16:27:53 +1200 Subject: [PATCH 52/61] Add recent changes to updates page --- eqcorrscan/doc/updates.rst | 98 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/eqcorrscan/doc/updates.rst b/eqcorrscan/doc/updates.rst index d9681e2fe..f5574dc94 100644 --- a/eqcorrscan/doc/updates.rst +++ b/eqcorrscan/doc/updates.rst @@ -1,6 +1,104 @@ What's new ========== +Version 0.4.4 +------------- +* core.match_filter + - Bug-fix: peak-cores could be defined twice in _group_detect through kwargs. + Fix: only update peak_cores if it isn't there already. +* core.match_filter.tribe + - Detect now allows passing of pre-processed data +* core.match_filter.template + - Remove duplicate detections from overlapping windows using `._uniq()` +* core.lag_calc._xcorr_interp + - CC-interpolation replaced with resampling (more robust), old method + deprecated. Use new method with use_new_resamp_method=True as **kwarg. +* core.lag_calc: + - Fixed bug where minimum CC defined via min_cc_from_mean_cc_factor was not + set correctly for negative correlation sums. +* utils.correlate + - Fast Matched Filter now supported natively for FMF version >= 1.4.0 + - Only full correlation stacks are returned now (e.g. where fewer than than + the full number of channels are in the stack at the end of the stack, zeros + are returned). +* utils.mag_calc.relative_magnitude + - fixed bug where S-picks / traces were used for relative-magnitude calculation + against user's choice. + - implemented full magnitude bias-correction for CC and SNR +* utils.mag_calc.relative_amplitude: + - returns dicts for SNR measurements +* utils.catalog_to_dd.write_correlations + - Fixed bug on execution of parallel execution. + - Added parallel-options for catalog-dt measurements and for stream-preparation + before cross correlation-dt measurements. + - Default parallelization of dt-computation is now across events (loads CPUs + more efficiently), and there is a new option ``max_trace_workers` to use + the old parallelization strategy across traces. + - Now includes `all_horiz`-option that will correlate all matching horizontal + channels no matter to which of these the S-pick is linking. +* utils.clustering + - Allow to handle indirect comparison of event-waveforms when (i.e., events + without matching traces which can be compared indirectly via a third event) + - Allows to set clustering method, metric, and sort_order from + scipy.cluster.hierarchy.linkage. +* tribe, template, template_gen, archive_read, clustering: remove option to read + from seishub (deprecated in obspy). + +Version 0.4.3 +------------- +* core.match_filter + - match_filter: + - Provide option of exporting the cross-correlation sums for additional later + analysis. +* core.match_filter.party.write + - BUG-FIX: When `format='tar'` is selected, added a check for .tgz-file + suffix before checking the filename against an existing file. Previously, + when a filename without '.tgz'-suffix was supplied, then the file was + overwritten against the function's intention. + - Add option `overwrite=True` to allow overwriting of existing files. +* core.match_filter.party.read + - BUG-FIX: Ensure wildcard reading works as expected: #453 +* core.match_filter.party.rethreshold: + - added option to rethreshold based on absolute values to keep relevant + detections with large negative detect_val. +* core.lag_calc: + - Added option to set minimum CC threshold individually for detections based + on: min(detect_val / n_chans * min_cc_from_mean_cc_factor, min_cc). + - Added the ability of saving correlation data of the lag_calc. +* core.template_gen: + - Added support for generating templates from any object with a + get_waveforms method. See #459. +* utils.mag_calc.calc_b_value: + - Added useful information to doc-string regarding method and meaning of + residuals + - Changed the number of magnitudes used to an int (from a string!?) +* utils.mag_calc.relative_magnitude: + - Refactor so that `min_cc` is used regardless of whether + `weight_by_correlation` is set. See issue #455. +* utils.archive_read + - Add support for wildcard-comparisons in the list of requested stations and + channels. + - New option `arctype='SDS'` to read from a SeisComp Data Structure (SDS). + This option is also available in `utils.clustering.extract_detections` and + in `utils.archive_read._check_available_data`. +* utils.catalog_to_dd + - Bug-fixes in #424: + - only P and S phases are used now (previously spurious amplitude picks + were included in correlations); + - Checks for length are done prior to correlations and more helpful error + outputs are provided. + - Progress is not reported within dt.cc computation + - `write_station` now supports writing elevations: #424. +* utils.clustering + - For `cluster`, `distance_matrix` and `cross_chan_correlation`, implemented + full support for `shift_len != 0`. The latter two functions now return, in + addition to the distance-matrix, a shift-matrix (both functions) and a + shift-dictionary (for `distance_matrix`). New option for shifting streams + as a whole or letting traces shift individually + (`allow_individual_trace_shifts=True`). +* utils.plotting + - Function added (twoD_seismplot) for plotting seismicity (#365). + Version 0.4.2 ------------- * Add seed-ids to the _spike_test's message. From 6a0e709bf53b9742f3bfb3d5eb207f5651d9e0d9 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 16:28:22 +1200 Subject: [PATCH 53/61] Bump version number --- eqcorrscan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eqcorrscan/__init__.py b/eqcorrscan/__init__.py index 55a7ae5ac..6225f84ee 100755 --- a/eqcorrscan/__init__.py +++ b/eqcorrscan/__init__.py @@ -25,7 +25,7 @@ __all__ = ['core', 'utils', 'tutorials', 'tests'] -__version__ = '0.4.3' +__version__ = '0.4.4' # Cope with changes to name-space to remove most of the camel-case _import_map = {} From a88c2e79e3ca53e08bf2faf50f9deed8a3bfcc95 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 16:42:33 +1200 Subject: [PATCH 54/61] Reformat updates nested lists --- eqcorrscan/doc/updates.rst | 310 ++++++++++++++++++------------------- 1 file changed, 154 insertions(+), 156 deletions(-) diff --git a/eqcorrscan/doc/updates.rst b/eqcorrscan/doc/updates.rst index f5574dc94..602a804c2 100644 --- a/eqcorrscan/doc/updates.rst +++ b/eqcorrscan/doc/updates.rst @@ -4,152 +4,152 @@ What's new Version 0.4.4 ------------- * core.match_filter - - Bug-fix: peak-cores could be defined twice in _group_detect through kwargs. - Fix: only update peak_cores if it isn't there already. + - Bug-fix: peak-cores could be defined twice in _group_detect through kwargs. + Fix: only update peak_cores if it isn't there already. * core.match_filter.tribe - - Detect now allows passing of pre-processed data + - Detect now allows passing of pre-processed data * core.match_filter.template - - Remove duplicate detections from overlapping windows using `._uniq()` + - Remove duplicate detections from overlapping windows using `._uniq()` * core.lag_calc._xcorr_interp - - CC-interpolation replaced with resampling (more robust), old method - deprecated. Use new method with use_new_resamp_method=True as **kwarg. + - CC-interpolation replaced with resampling (more robust), old method + deprecated. Use new method with use_new_resamp_method=True as **kwarg. * core.lag_calc: - - Fixed bug where minimum CC defined via min_cc_from_mean_cc_factor was not - set correctly for negative correlation sums. + - Fixed bug where minimum CC defined via min_cc_from_mean_cc_factor was not + set correctly for negative correlation sums. * utils.correlate - - Fast Matched Filter now supported natively for FMF version >= 1.4.0 - - Only full correlation stacks are returned now (e.g. where fewer than than - the full number of channels are in the stack at the end of the stack, zeros - are returned). + - Fast Matched Filter now supported natively for FMF version >= 1.4.0 + - Only full correlation stacks are returned now (e.g. where fewer than than + the full number of channels are in the stack at the end of the stack, zeros + are returned). * utils.mag_calc.relative_magnitude - - fixed bug where S-picks / traces were used for relative-magnitude calculation - against user's choice. - - implemented full magnitude bias-correction for CC and SNR + - fixed bug where S-picks / traces were used for relative-magnitude calculation + against user's choice. + - implemented full magnitude bias-correction for CC and SNR * utils.mag_calc.relative_amplitude: - - returns dicts for SNR measurements + - returns dicts for SNR measurements * utils.catalog_to_dd.write_correlations - - Fixed bug on execution of parallel execution. - - Added parallel-options for catalog-dt measurements and for stream-preparation - before cross correlation-dt measurements. - - Default parallelization of dt-computation is now across events (loads CPUs - more efficiently), and there is a new option ``max_trace_workers` to use - the old parallelization strategy across traces. - - Now includes `all_horiz`-option that will correlate all matching horizontal - channels no matter to which of these the S-pick is linking. + - Fixed bug on execution of parallel execution. + - Added parallel-options for catalog-dt measurements and for stream-preparation + before cross correlation-dt measurements. + - Default parallelization of dt-computation is now across events (loads CPUs + more efficiently), and there is a new option ``max_trace_workers` to use + the old parallelization strategy across traces. + - Now includes `all_horiz`-option that will correlate all matching horizontal + channels no matter to which of these the S-pick is linking. * utils.clustering - - Allow to handle indirect comparison of event-waveforms when (i.e., events - without matching traces which can be compared indirectly via a third event) - - Allows to set clustering method, metric, and sort_order from - scipy.cluster.hierarchy.linkage. + - Allow to handle indirect comparison of event-waveforms when (i.e., events + without matching traces which can be compared indirectly via a third event) + - Allows to set clustering method, metric, and sort_order from + scipy.cluster.hierarchy.linkage. * tribe, template, template_gen, archive_read, clustering: remove option to read from seishub (deprecated in obspy). Version 0.4.3 ------------- * core.match_filter - - match_filter: - - Provide option of exporting the cross-correlation sums for additional later - analysis. + - match_filter: + - Provide option of exporting the cross-correlation sums for additional later + analysis. * core.match_filter.party.write - - BUG-FIX: When `format='tar'` is selected, added a check for .tgz-file - suffix before checking the filename against an existing file. Previously, - when a filename without '.tgz'-suffix was supplied, then the file was - overwritten against the function's intention. - - Add option `overwrite=True` to allow overwriting of existing files. + - BUG-FIX: When `format='tar'` is selected, added a check for .tgz-file + suffix before checking the filename against an existing file. Previously, + when a filename without '.tgz'-suffix was supplied, then the file was + overwritten against the function's intention. + - Add option `overwrite=True` to allow overwriting of existing files. * core.match_filter.party.read - - BUG-FIX: Ensure wildcard reading works as expected: #453 + - BUG-FIX: Ensure wildcard reading works as expected: #453 * core.match_filter.party.rethreshold: - - added option to rethreshold based on absolute values to keep relevant - detections with large negative detect_val. + - added option to rethreshold based on absolute values to keep relevant + detections with large negative detect_val. * core.lag_calc: - - Added option to set minimum CC threshold individually for detections based - on: min(detect_val / n_chans * min_cc_from_mean_cc_factor, min_cc). - - Added the ability of saving correlation data of the lag_calc. + - Added option to set minimum CC threshold individually for detections based + on: min(detect_val / n_chans * min_cc_from_mean_cc_factor, min_cc). + - Added the ability of saving correlation data of the lag_calc. * core.template_gen: - - Added support for generating templates from any object with a - get_waveforms method. See #459. + - Added support for generating templates from any object with a + get_waveforms method. See #459. * utils.mag_calc.calc_b_value: - - Added useful information to doc-string regarding method and meaning of - residuals - - Changed the number of magnitudes used to an int (from a string!?) + - Added useful information to doc-string regarding method and meaning of + residuals + - Changed the number of magnitudes used to an int (from a string!?) * utils.mag_calc.relative_magnitude: - - Refactor so that `min_cc` is used regardless of whether - `weight_by_correlation` is set. See issue #455. + - Refactor so that `min_cc` is used regardless of whether + `weight_by_correlation` is set. See issue #455. * utils.archive_read - - Add support for wildcard-comparisons in the list of requested stations and - channels. - - New option `arctype='SDS'` to read from a SeisComp Data Structure (SDS). - This option is also available in `utils.clustering.extract_detections` and - in `utils.archive_read._check_available_data`. + - Add support for wildcard-comparisons in the list of requested stations and + channels. + - New option `arctype='SDS'` to read from a SeisComp Data Structure (SDS). + This option is also available in `utils.clustering.extract_detections` and + in `utils.archive_read._check_available_data`. * utils.catalog_to_dd - - Bug-fixes in #424: - - only P and S phases are used now (previously spurious amplitude picks - were included in correlations); - - Checks for length are done prior to correlations and more helpful error - outputs are provided. - - Progress is not reported within dt.cc computation - - `write_station` now supports writing elevations: #424. + - Bug-fixes in #424: + - only P and S phases are used now (previously spurious amplitude picks + were included in correlations); + - Checks for length are done prior to correlations and more helpful error + outputs are provided. + - Progress is not reported within dt.cc computation + - `write_station` now supports writing elevations: #424. * utils.clustering - - For `cluster`, `distance_matrix` and `cross_chan_correlation`, implemented - full support for `shift_len != 0`. The latter two functions now return, in - addition to the distance-matrix, a shift-matrix (both functions) and a - shift-dictionary (for `distance_matrix`). New option for shifting streams - as a whole or letting traces shift individually - (`allow_individual_trace_shifts=True`). + - For `cluster`, `distance_matrix` and `cross_chan_correlation`, implemented + full support for `shift_len != 0`. The latter two functions now return, in + addition to the distance-matrix, a shift-matrix (both functions) and a + shift-dictionary (for `distance_matrix`). New option for shifting streams + as a whole or letting traces shift individually + (`allow_individual_trace_shifts=True`). * utils.plotting - - Function added (twoD_seismplot) for plotting seismicity (#365). + - Function added (twoD_seismplot) for plotting seismicity (#365). Version 0.4.2 ------------- * Add seed-ids to the _spike_test's message. * utils.correlation - - Cross-correlation normalisation errors no-longer raise an error - - When "out-of-range" correlations occur a warning is given by the C-function - with details of what channel, what template and where in the data vector - the issue occurred for the user to check their data. - - Out-of-range correlations are set to 0.0 - - After extensive testing these errors have always been related to data issues - within regions where correlations should not be computed (spikes, step - artifacts due to incorrectly padding data gaps). - - USERS SHOULD BE CAREFUL TO CHECK THEIR DATA IF THEY SEE THESE WARNINGS + - Cross-correlation normalisation errors no-longer raise an error + - When "out-of-range" correlations occur a warning is given by the C-function + with details of what channel, what template and where in the data vector + the issue occurred for the user to check their data. + - Out-of-range correlations are set to 0.0 + - After extensive testing these errors have always been related to data issues + within regions where correlations should not be computed (spikes, step + artifacts due to incorrectly padding data gaps). + - USERS SHOULD BE CAREFUL TO CHECK THEIR DATA IF THEY SEE THESE WARNINGS * utils.mag_calc.amp_pick_event - - Added option to output IASPEI standard amplitudes, with static amplification - of 1 (rather than 2080 as per Wood Anderson specs). - - Added `filter_id` and `method_id` to amplitudes to make these methods more - traceable. + - Added option to output IASPEI standard amplitudes, with static amplification + of 1 (rather than 2080 as per Wood Anderson specs). + - Added `filter_id` and `method_id` to amplitudes to make these methods more + traceable. * core.match_filter - - Bug-fix - cope with data that are too short with `ignore_bad_data=True`. - This flag is generally not advised, but when used, may attempt to trim all - data to zero length. The expected behaviour is to remove bad data and run - with the remaining data. - - Party: - - decluster now accepts a hypocentral_separation argument. This allows - the inclusion of detections that occur close in time, but not in space. - This is underwritten by a new findpeaks.decluster_dist_time function - based on a new C-function. - - Tribe: - - Add monkey-patching for clients that do not have a `get_waveforms_bulk` - method for use in `.client_detect`. See issue #394. + - Bug-fix - cope with data that are too short with `ignore_bad_data=True`. + This flag is generally not advised, but when used, may attempt to trim all + data to zero length. The expected behaviour is to remove bad data and run + with the remaining data. + - Party: + - decluster now accepts a hypocentral_separation argument. This allows + the inclusion of detections that occur close in time, but not in space. + This is underwritten by a new findpeaks.decluster_dist_time function + based on a new C-function. + - Tribe: + - Add monkey-patching for clients that do not have a `get_waveforms_bulk` + method for use in `.client_detect`. See issue #394. * utils.pre_processing - - Only templates that need to be reshaped are reshaped now - this can be a lot - faster. + - Only templates that need to be reshaped are reshaped now - this can be a lot + faster. Version 0.4.1 ------------- * core.match_filter - - BUG-FIX: Empty families are no longer run through lag-calc when using Party.lag_calc(). Previously this resulted in a "No matching data" error, see #341. + - BUG-FIX: Empty families are no longer run through lag-calc when using Party.lag_calc(). Previously this resulted in a "No matching data" error, see #341. * core.template_gen - - BUG-FIX: Fix bug where events were incorrectly associated with templates in `Tribe().construct()` if the given catalog contained events outside of the time-range of the stream. See issue #381 and PR #382. + - BUG-FIX: Fix bug where events were incorrectly associated with templates in `Tribe().construct()` if the given catalog contained events outside of the time-range of the stream. See issue #381 and PR #382. * utils.catalog_to_dd - - Added ability to turn off parallel processing (this is turned off by default now) for `write_correlations` - parallel processing for moderate to large datasets was copying far too much data and using lots of memory. This is a short-term fix - ideally we will move filtering and resampling to C functions with shared-memory parallelism and GIL releasing. See PR #374. - - Moved parallelism for `_compute_dt_correlations` to the C functions to reduce memory overhead. Using a generator to construct sub-catalogs rather than making a list of lists in memory. See issue #361. + - Added ability to turn off parallel processing (this is turned off by default now) for `write_correlations` - parallel processing for moderate to large datasets was copying far too much data and using lots of memory. This is a short-term fix - ideally we will move filtering and resampling to C functions with shared-memory parallelism and GIL releasing. See PR #374. + - Moved parallelism for `_compute_dt_correlations` to the C functions to reduce memory overhead. Using a generator to construct sub-catalogs rather than making a list of lists in memory. See issue #361. * utils.mag_calc: - - `amp_pick_event` now works on a copy of the data by default - - `amp_pick_event` uses the appropriate digital filter gain to correct the applied filter. See issue #376. - - `amp_pick_event` rewritten for simplicity. - - `amp_pick_event` now has simple synthetic tests for accuracy. - - `_sim_wa` uses the full response information to correct to velocity this includes FIR filters (previously not used), and ensures that the wood-anderson poles (with a single zero) are correctly applied to velocity waveforms. - - `calc_max_curv` is now computed using the non-cumulative distribution. + - `amp_pick_event` now works on a copy of the data by default + - `amp_pick_event` uses the appropriate digital filter gain to correct the applied filter. See issue #376. + - `amp_pick_event` rewritten for simplicity. + - `amp_pick_event` now has simple synthetic tests for accuracy. + - `_sim_wa` uses the full response information to correct to velocity this includes FIR filters (previously not used), and ensures that the wood-anderson poles (with a single zero) are correctly applied to velocity waveforms. + - `calc_max_curv` is now computed using the non-cumulative distribution. * Some problem solved in _match_filter_plot. Now it shows all new detections. * Add plotdir to eqcorrscan.core.lag_calc.lag_calc function to save the images. @@ -160,20 +160,20 @@ Version 0.4.0 alleviate issue related to large-prime length transforms. This requires an additional dependency, but EQcorrscan already depends on FFTW itself (#316). * Refactor of catalog_to_dd functions (#322): - - Speed-ups, using new correlation functions and better resource management - - Removed enforcement of seisan, arguments are now standard obspy objects. + - Speed-ups, using new correlation functions and better resource management + - Removed enforcement of seisan, arguments are now standard obspy objects. * Add plotdir to lag-calc, template construction and matched-filter detection methods and functions (#330, #325). * Wholesale re-write of lag-calc function and methods. External interface is similar, but some arguments have been depreciated as they were unnecessary (#321). - - This was done to make use of the new internal correlation functions which - are faster and more memory efficient. - - Party.lag_calc and Family.lag_calc now work in-place on the events in - the grouping. - - Added relative_mags method to Party and Family; this can be called from - lag-calc to avoid reprocessing data. - - Added lag_calc.xcorr_pick_family as a public facing API to implement - correlation re-picking of a group of events. + - This was done to make use of the new internal correlation functions which + are faster and more memory efficient. + - Party.lag_calc and Family.lag_calc now work in-place on the events in + the grouping. + - Added relative_mags method to Party and Family; this can be called from + lag-calc to avoid reprocessing data. + - Added lag_calc.xcorr_pick_family as a public facing API to implement + correlation re-picking of a group of events. * Renamed utils.clustering.cross_chan_coherence to utils.clustering.cross_chan_correlation to better reflect what it actually does. @@ -194,9 +194,9 @@ Version 0.4.0 an argument for function calls. * `find-peaks` replaced by compiled peak finding routine - more efficient both in memory and time #249 - approx 50x faster - * Note that the results of the C-func and the Python functions are slightly - different. The C function (now the default) is more stable when peaks - are small and close together (e.g. in noisy data). + - Note that the results of the C-func and the Python functions are slightly + different. The C function (now the default) is more stable when peaks + are small and close together (e.g. in noisy data). * multi-find peaks makes use of openMP parallelism for more efficient memory usage #249 * enforce normalization of continuous data before correlation to avoid float32 @@ -214,10 +214,10 @@ Version 0.4.0 massive and was slow to load and process in IDEs. * Refactored `_prep_data_for_correlation` to reduce looping for speed, now approximately six times faster than previously (minor speed-up) - * Now explicitly doesn't allow templates with different length traces - - previously this was ignored and templates with different length - channels to other templates had their channels padded with zeros or - trimmed. + - Now explicitly doesn't allow templates with different length traces - + previously this was ignored and templates with different length + channels to other templates had their channels padded with zeros or + trimmed. * Add `skip_short_channels` option to template generation. This allows users to provide data of unknown length and short channels will not be used, rather than generating an error. This is useful for downloading data from @@ -314,11 +314,10 @@ Version 0.3.2 trends - variance threshold is now raised, and Python checks for low-variance and applies gain to stabilise correlations if needed. * Plotting functions are now tested and have a more consistent interface: - - * All plotting functions accept the keyword arguments `save`, `savefile`, - `show`, `return_figure` and `title`. - * All plotting functions return a figure. - * `SVD_plot` renamed to `svd_plot` + - All plotting functions accept the keyword arguments `save`, `savefile`, + `show`, `return_figure` and `title`. + - All plotting functions return a figure. + - `SVD_plot` renamed to `svd_plot` * Enforce pre-processing even when no filters or resampling is to be done to ensure gaps are properly processed (when called from `Tribe.detect`, `Template.detect` or `Tribe.client_detect`) @@ -326,9 +325,8 @@ Version 0.3.2 one sample too long resulting in minor differences in data processing (due to difference in FFT length) and therefore minor differences in resulting correlations (~0.07 per channel). - - * Includes extra stability check in fftw_normxcorr which affects the - last sample before a gap when that sample is near-zero. + - Includes extra stability check in fftw_normxcorr which affects the + last sample before a gap when that sample is near-zero. * BUG-FIX: fftw correlation dot product was not thread-safe on some systems. The dot-product did not have the inner index protected as a private variable. This did not appear to cause issues for Linux with Python 3.x or Windows, but @@ -393,10 +391,10 @@ Version 0.3.0 to obspy for obspy version 1.1.0. All functions are there and many bugs have been fixed. This also means the removal of nordic-specific functions in EQcorrscan - the following functions have been removed: - * template_gen.from_sfile - * template_gen.from_contbase - * mag_calc.amp_pick_sfile - * mag_calc.pick_db + - template_gen.from_sfile + - template_gen.from_contbase + - mag_calc.amp_pick_sfile + - mag_calc.pick_db All removed functions will error and tell you to use obspy.io.nordic.core. This now means that you can use obspy's `read_events` to read in sfiles. * Added `P_all` and `S_all` options to template generation functions @@ -419,17 +417,17 @@ Version 0.2.6 methods through the parameter xcorr_func of match_filter, Template.detect and Tribe.detect, or using the set_xcorr context manager in the utils.correlate module. Supported options are: - * numpy - * fftw - * time-domain - * or passing a function that implements the xcorr interface. + - numpy + - fftw + - time-domain + - or passing a function that implements the xcorr interface. * Added the ability to change the concurrency strategy of xcorr functions using the paramter concurrency of match_filter, Template.detect and Tribe.detect. Supported options are: - * None - for single-threaded execution in a single process - * multithread - for multi-threaded execution - * multiprocess- for multiprocess execution - * concurrent - allows functions to describe their own preferred currency methods, defaults to multithread + - None - for single-threaded execution in a single process + - multithread - for multi-threaded execution + - multiprocess- for multiprocess execution + - concurrent - allows functions to describe their own preferred currency methods, defaults to multithread * Change debug printing output, it should be a little quieter; * Speed-up time-domain using a threaded C-routine - separate from frequency domain C-routines; @@ -482,16 +480,16 @@ Version 0.2.4 * Rename SVD_moments to lower-case and add depreciation warning; * Increase test coverage in utils.mag_calc; * Add Template, Tribe, Family, Party objects and rename DETECTION to Detection - * Template objects maintain meta-data associated with their creation to stream-line processing of data (e.g. reduce chance of using the wrong filters). - * Template events have a detect method which takes unprocessed data and does the correct processing using the Template meta-data, and computes the matched-filter detections. - * Tribe objects are containers for multiple Templates. - * Tribe objects have a detect method which groups Templates with similar meta-data (processing information) and runs these templates in parallel through the matched-filter routine. Tribe.detect outputs a Party of Family objects. - * The Party object is a container for many Family objects. - * Family objects are containers for detections from the same Template. - * Family and Party objects have a lag_calc method which computes the cross-correlation pick-refinements. - * The upshot of this is that it is possible to, in one line, generate a Tribe of templates, compute their matched-filter detections, and generate cross-correlation pick refinements, which output Event objects, which can be written to a catalog: Tribe.construct(method, **kwargs).detect(st, **kwargs).lag_calc(**kwargs).write() - * Added 25 tests for these methods. - * Add parameters *threshold_type* and *threshold_input* to Detection class. Add support for legacy Detection objects via NaN and unset values. + - Template objects maintain meta-data associated with their creation to stream-line processing of data (e.g. reduce chance of using the wrong filters). + - Template events have a detect method which takes unprocessed data and does the correct processing using the Template meta-data, and computes the matched-filter detections. + - Tribe objects are containers for multiple Templates. + - Tribe objects have a detect method which groups Templates with similar meta-data (processing information) and runs these templates in parallel through the matched-filter routine. Tribe.detect outputs a Party of Family objects. + - The Party object is a container for many Family objects. + - Family objects are containers for detections from the same Template. + - Family and Party objects have a lag_calc method which computes the cross-correlation pick-refinements. + - The upshot of this is that it is possible to, in one line, generate a Tribe of templates, compute their matched-filter detections, and generate cross-correlation pick refinements, which output Event objects, which can be written to a catalog: Tribe.construct(method, **kwargs).detect(st, **kwargs).lag_calc(**kwargs).write() + - Added 25 tests for these methods. + - Add parameters *threshold_type* and *threshold_input* to Detection class. Add support for legacy Detection objects via NaN and unset values. * Removed support for obspy < 1.0.0 * Update / correct doc-strings in template-gen functions when describing processing parameters. @@ -504,10 +502,10 @@ Version 0.2.4 * Fix bug in detection_multiplot which didn't allow streams with fewer traces than template; * Update internals to custom C fftw-based correlation rather than openCV (Major change); - * OpenCV has been removed as a dependancy; - * eqcorrscan.core.match_filter.normxcorr2 now calls a compiled C routine; - * Parallel workflows handled by openMP rather than Python Multiprocessing for matched-filter operations to allow better memory handling. - * It is worth noting that we tried re-writing using SciPy internals which led to a significant speed-up, but with high memory costs, we ended up going with this option, which was the more difficult option, because it allows effective use on SLURM managed systems where python multiprocessing results in un-real memory spikes (issue #88). + - OpenCV has been removed as a dependancy; + - eqcorrscan.core.match_filter.normxcorr2 now calls a compiled C routine; + - Parallel workflows handled by openMP rather than Python Multiprocessing for matched-filter operations to allow better memory handling. + - It is worth noting that we tried re-writing using SciPy internals which led to a significant speed-up, but with high memory costs, we ended up going with this option, which was the more difficult option, because it allows effective use on SLURM managed systems where python multiprocessing results in un-real memory spikes (issue #88). Version 0.2.0-0.2.3 ................... From eee2c39a5ce19199cd59d6f4fda227c261bcc69d Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 16:48:53 +1200 Subject: [PATCH 55/61] Reformat updates nested lists --- eqcorrscan/doc/updates.rst | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/eqcorrscan/doc/updates.rst b/eqcorrscan/doc/updates.rst index 602a804c2..ff7a228de 100644 --- a/eqcorrscan/doc/updates.rst +++ b/eqcorrscan/doc/updates.rst @@ -164,8 +164,7 @@ Version 0.4.0 - Removed enforcement of seisan, arguments are now standard obspy objects. * Add plotdir to lag-calc, template construction and matched-filter detection methods and functions (#330, #325). -* Wholesale re-write of lag-calc function and methods. External interface is - similar, but some arguments have been depreciated as they were unnecessary (#321). +* Wholesale re-write of lag-calc function and methods. External interface is similar, but some arguments have been depreciated as they were unnecessary (#321). - This was done to make use of the new internal correlation functions which are faster and more memory efficient. - Party.lag_calc and Family.lag_calc now work in-place on the events in @@ -192,8 +191,7 @@ Version 0.4.0 outside the time-range. * Changed all prints to calls to logging, as a result, debug is no longer an argument for function calls. -* `find-peaks` replaced by compiled peak finding routine - more efficient - both in memory and time #249 - approx 50x faster +* `find-peaks` replaced by compiled peak finding routine - more efficient both in memory and time #249 - approx 50x faster - Note that the results of the C-func and the Python functions are slightly different. The C function (now the default) is more stable when peaks are small and close together (e.g. in noisy data). @@ -212,8 +210,7 @@ Version 0.4.0 * Refactored match-filter into smaller files. Namespace remains the same. This was done to ease maintenance - the match_filter.py file had become massive and was slow to load and process in IDEs. -* Refactored `_prep_data_for_correlation` to reduce looping for speed, - now approximately six times faster than previously (minor speed-up) +* Refactored `_prep_data_for_correlation` to reduce looping for speed, now approximately six times faster than previously (minor speed-up) - Now explicitly doesn't allow templates with different length traces - previously this was ignored and templates with different length channels to other templates had their channels padded with zeros or From 847982eaecaa12f01e52bdafe98916f2c168ee14 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 16:53:52 +1200 Subject: [PATCH 56/61] Reformat updates nested lists --- eqcorrscan/doc/updates.rst | 470 ++++++++++++++++++------------------- 1 file changed, 235 insertions(+), 235 deletions(-) diff --git a/eqcorrscan/doc/updates.rst b/eqcorrscan/doc/updates.rst index ff7a228de..a50c5db63 100644 --- a/eqcorrscan/doc/updates.rst +++ b/eqcorrscan/doc/updates.rst @@ -3,31 +3,31 @@ What's new Version 0.4.4 ------------- -* core.match_filter +- core.match_filter - Bug-fix: peak-cores could be defined twice in _group_detect through kwargs. Fix: only update peak_cores if it isn't there already. -* core.match_filter.tribe +- core.match_filter.tribe - Detect now allows passing of pre-processed data -* core.match_filter.template +- core.match_filter.template - Remove duplicate detections from overlapping windows using `._uniq()` -* core.lag_calc._xcorr_interp +- core.lag_calc._xcorr_interp - CC-interpolation replaced with resampling (more robust), old method - deprecated. Use new method with use_new_resamp_method=True as **kwarg. -* core.lag_calc: + deprecated. Use new method with use_new_resamp_method=True as --kwarg. +- core.lag_calc: - Fixed bug where minimum CC defined via min_cc_from_mean_cc_factor was not set correctly for negative correlation sums. -* utils.correlate +- utils.correlate - Fast Matched Filter now supported natively for FMF version >= 1.4.0 - Only full correlation stacks are returned now (e.g. where fewer than than the full number of channels are in the stack at the end of the stack, zeros are returned). -* utils.mag_calc.relative_magnitude +- utils.mag_calc.relative_magnitude - fixed bug where S-picks / traces were used for relative-magnitude calculation against user's choice. - implemented full magnitude bias-correction for CC and SNR -* utils.mag_calc.relative_amplitude: +- utils.mag_calc.relative_amplitude: - returns dicts for SNR measurements -* utils.catalog_to_dd.write_correlations +- utils.catalog_to_dd.write_correlations - Fixed bug on execution of parallel execution. - Added parallel-options for catalog-dt measurements and for stream-preparation before cross correlation-dt measurements. @@ -36,52 +36,52 @@ Version 0.4.4 the old parallelization strategy across traces. - Now includes `all_horiz`-option that will correlate all matching horizontal channels no matter to which of these the S-pick is linking. -* utils.clustering +- utils.clustering - Allow to handle indirect comparison of event-waveforms when (i.e., events without matching traces which can be compared indirectly via a third event) - Allows to set clustering method, metric, and sort_order from scipy.cluster.hierarchy.linkage. -* tribe, template, template_gen, archive_read, clustering: remove option to read +- tribe, template, template_gen, archive_read, clustering: remove option to read from seishub (deprecated in obspy). Version 0.4.3 ------------- -* core.match_filter +- core.match_filter - match_filter: - Provide option of exporting the cross-correlation sums for additional later analysis. -* core.match_filter.party.write +- core.match_filter.party.write - BUG-FIX: When `format='tar'` is selected, added a check for .tgz-file suffix before checking the filename against an existing file. Previously, when a filename without '.tgz'-suffix was supplied, then the file was overwritten against the function's intention. - Add option `overwrite=True` to allow overwriting of existing files. -* core.match_filter.party.read +- core.match_filter.party.read - BUG-FIX: Ensure wildcard reading works as expected: #453 -* core.match_filter.party.rethreshold: +- core.match_filter.party.rethreshold: - added option to rethreshold based on absolute values to keep relevant detections with large negative detect_val. -* core.lag_calc: +- core.lag_calc: - Added option to set minimum CC threshold individually for detections based - on: min(detect_val / n_chans * min_cc_from_mean_cc_factor, min_cc). + on: min(detect_val / n_chans - min_cc_from_mean_cc_factor, min_cc). - Added the ability of saving correlation data of the lag_calc. -* core.template_gen: +- core.template_gen: - Added support for generating templates from any object with a get_waveforms method. See #459. -* utils.mag_calc.calc_b_value: +- utils.mag_calc.calc_b_value: - Added useful information to doc-string regarding method and meaning of residuals - Changed the number of magnitudes used to an int (from a string!?) -* utils.mag_calc.relative_magnitude: +- utils.mag_calc.relative_magnitude: - Refactor so that `min_cc` is used regardless of whether `weight_by_correlation` is set. See issue #455. -* utils.archive_read +- utils.archive_read - Add support for wildcard-comparisons in the list of requested stations and channels. - New option `arctype='SDS'` to read from a SeisComp Data Structure (SDS). This option is also available in `utils.clustering.extract_detections` and in `utils.archive_read._check_available_data`. -* utils.catalog_to_dd +- utils.catalog_to_dd - Bug-fixes in #424: - only P and S phases are used now (previously spurious amplitude picks were included in correlations); @@ -89,20 +89,20 @@ Version 0.4.3 outputs are provided. - Progress is not reported within dt.cc computation - `write_station` now supports writing elevations: #424. -* utils.clustering +- utils.clustering - For `cluster`, `distance_matrix` and `cross_chan_correlation`, implemented full support for `shift_len != 0`. The latter two functions now return, in addition to the distance-matrix, a shift-matrix (both functions) and a shift-dictionary (for `distance_matrix`). New option for shifting streams as a whole or letting traces shift individually (`allow_individual_trace_shifts=True`). -* utils.plotting +- utils.plotting - Function added (twoD_seismplot) for plotting seismicity (#365). Version 0.4.2 ------------- -* Add seed-ids to the _spike_test's message. -* utils.correlation +- Add seed-ids to the _spike_test's message. +- utils.correlation - Cross-correlation normalisation errors no-longer raise an error - When "out-of-range" correlations occur a warning is given by the C-function with details of what channel, what template and where in the data vector @@ -112,12 +112,12 @@ Version 0.4.2 within regions where correlations should not be computed (spikes, step artifacts due to incorrectly padding data gaps). - USERS SHOULD BE CAREFUL TO CHECK THEIR DATA IF THEY SEE THESE WARNINGS -* utils.mag_calc.amp_pick_event +- utils.mag_calc.amp_pick_event - Added option to output IASPEI standard amplitudes, with static amplification of 1 (rather than 2080 as per Wood Anderson specs). - Added `filter_id` and `method_id` to amplitudes to make these methods more traceable. -* core.match_filter +- core.match_filter - Bug-fix - cope with data that are too short with `ignore_bad_data=True`. This flag is generally not advised, but when used, may attempt to trim all data to zero length. The expected behaviour is to remove bad data and run @@ -130,41 +130,41 @@ Version 0.4.2 - Tribe: - Add monkey-patching for clients that do not have a `get_waveforms_bulk` method for use in `.client_detect`. See issue #394. -* utils.pre_processing +- utils.pre_processing - Only templates that need to be reshaped are reshaped now - this can be a lot faster. Version 0.4.1 ------------- -* core.match_filter +- core.match_filter - BUG-FIX: Empty families are no longer run through lag-calc when using Party.lag_calc(). Previously this resulted in a "No matching data" error, see #341. -* core.template_gen +- core.template_gen - BUG-FIX: Fix bug where events were incorrectly associated with templates in `Tribe().construct()` if the given catalog contained events outside of the time-range of the stream. See issue #381 and PR #382. -* utils.catalog_to_dd +- utils.catalog_to_dd - Added ability to turn off parallel processing (this is turned off by default now) for `write_correlations` - parallel processing for moderate to large datasets was copying far too much data and using lots of memory. This is a short-term fix - ideally we will move filtering and resampling to C functions with shared-memory parallelism and GIL releasing. See PR #374. - Moved parallelism for `_compute_dt_correlations` to the C functions to reduce memory overhead. Using a generator to construct sub-catalogs rather than making a list of lists in memory. See issue #361. -* utils.mag_calc: +- utils.mag_calc: - `amp_pick_event` now works on a copy of the data by default - `amp_pick_event` uses the appropriate digital filter gain to correct the applied filter. See issue #376. - `amp_pick_event` rewritten for simplicity. - `amp_pick_event` now has simple synthetic tests for accuracy. - `_sim_wa` uses the full response information to correct to velocity this includes FIR filters (previously not used), and ensures that the wood-anderson poles (with a single zero) are correctly applied to velocity waveforms. - `calc_max_curv` is now computed using the non-cumulative distribution. -* Some problem solved in _match_filter_plot. Now it shows all new detections. -* Add plotdir to eqcorrscan.core.lag_calc.lag_calc function to save the images. +- Some problem solved in _match_filter_plot. Now it shows all new detections. +- Add plotdir to eqcorrscan.core.lag_calc.lag_calc function to save the images. Version 0.4.0 ------------- -* Change resampling to use pyFFTW backend for FFT's. This is an attempt to +- Change resampling to use pyFFTW backend for FFT's. This is an attempt to alleviate issue related to large-prime length transforms. This requires an additional dependency, but EQcorrscan already depends on FFTW itself (#316). -* Refactor of catalog_to_dd functions (#322): +- Refactor of catalog_to_dd functions (#322): - Speed-ups, using new correlation functions and better resource management - Removed enforcement of seisan, arguments are now standard obspy objects. -* Add plotdir to lag-calc, template construction and matched-filter detection +- Add plotdir to lag-calc, template construction and matched-filter detection methods and functions (#330, #325). -* Wholesale re-write of lag-calc function and methods. External interface is similar, but some arguments have been depreciated as they were unnecessary (#321). +- Wholesale re-write of lag-calc function and methods. External interface is similar, but some arguments have been depreciated as they were unnecessary (#321). - This was done to make use of the new internal correlation functions which are faster and more memory efficient. - Party.lag_calc and Family.lag_calc now work in-place on the events in @@ -173,83 +173,83 @@ Version 0.4.0 lag-calc to avoid reprocessing data. - Added lag_calc.xcorr_pick_family as a public facing API to implement correlation re-picking of a group of events. -* Renamed utils.clustering.cross_chan_coherence to +- Renamed utils.clustering.cross_chan_coherence to utils.clustering.cross_chan_correlation to better reflect what it actually does. -* Add --no-mkl flag for setup.py to force the FFTW correlation routines not +- Add --no-mkl flag for setup.py to force the FFTW correlation routines not to compile against intels mkl. On NeSI systems mkl is currently causing issues. -* BUG-FIX: `eqcorrscan.utils.mag_calc.dist_calc` calculated the long-way round +- BUG-FIX: `eqcorrscan.utils.mag_calc.dist_calc` calculated the long-way round the Earth when changing hemispheres. We now use the Haversine formula, which should give better results at short distances, and does not use a flat-Earth approximation, so is better suited to larger distances as well. -* Add C-openmp parallel distance-clustering (speed-ups of ~100 times). -* Allow option to not stack correlations in correlation functions. -* Use compiled correlation functions for correlation clustering (speed-up). -* Add time-clustering for catalogs and change how space-time cluster works +- Add C-openmp parallel distance-clustering (speed-ups of ~100 times). +- Allow option to not stack correlations in correlation functions. +- Use compiled correlation functions for correlation clustering (speed-up). +- Add time-clustering for catalogs and change how space-time cluster works so that it uses the time-clustering, rather than just throwing out events outside the time-range. -* Changed all prints to calls to logging, as a result, debug is no longer +- Changed all prints to calls to logging, as a result, debug is no longer an argument for function calls. -* `find-peaks` replaced by compiled peak finding routine - more efficient both in memory and time #249 - approx 50x faster +- `find-peaks` replaced by compiled peak finding routine - more efficient both in memory and time #249 - approx 50x faster - Note that the results of the C-func and the Python functions are slightly different. The C function (now the default) is more stable when peaks are small and close together (e.g. in noisy data). -* multi-find peaks makes use of openMP parallelism for more efficient +- multi-find peaks makes use of openMP parallelism for more efficient memory usage #249 -* enforce normalization of continuous data before correlation to avoid float32 +- enforce normalization of continuous data before correlation to avoid float32 overflow errors that result in correlation errors (see pr #292). -* Add SEC-C style chunked cross-correlations. This is both faster and more +- Add SEC-C style chunked cross-correlations. This is both faster and more memory efficient. This is now used by default with an fft length of - 2 ** 13. This was found to be consistently the fastest length in testing. + 2 -- 13. This was found to be consistently the fastest length in testing. This can be changed by the user by passing the `fft_len` keyword argument. See PR #285. -* Outer-loop parallelism has been disabled for all systems now. This was not +- Outer-loop parallelism has been disabled for all systems now. This was not useful in most situations and is hard to maintain. -* Improved support for compilation on RedHat systems -* Refactored match-filter into smaller files. Namespace remains the same. +- Improved support for compilation on RedHat systems +- Refactored match-filter into smaller files. Namespace remains the same. This was done to ease maintenance - the match_filter.py file had become massive and was slow to load and process in IDEs. -* Refactored `_prep_data_for_correlation` to reduce looping for speed, now approximately six times faster than previously (minor speed-up) +- Refactored `_prep_data_for_correlation` to reduce looping for speed, now approximately six times faster than previously (minor speed-up) - Now explicitly doesn't allow templates with different length traces - previously this was ignored and templates with different length channels to other templates had their channels padded with zeros or trimmed. -* Add `skip_short_channels` option to template generation. This allows users +- Add `skip_short_channels` option to template generation. This allows users to provide data of unknown length and short channels will not be used, rather than generating an error. This is useful for downloading data from datacentres via the `from_client` method. -* Remove pytest_namespace in conftest.py to support pytest 4.x -* Add `ignore_bad_data` kwarg for all processing functions, if set to True +- Remove pytest_namespace in conftest.py to support pytest 4.x +- Add `ignore_bad_data` kwarg for all processing functions, if set to True (defaults to False for continuity) then any errors related to bad data at process-time will be supressed and empty traces returned. This is useful for downloading data from datacentres via the `from_client` method when data quality is not known. -* Added relative amplitude measurements as +- Added relative amplitude measurements as `utils.mag_calc.relative_amplitude` (#306). -* Added relative magnitude calculation using relative amplitudes weighted by +- Added relative magnitude calculation using relative amplitudes weighted by correlations to `utils.mag_calc.relative_magnitude`. -* Added `relative_magnitudes` argument to +- Added `relative_magnitudes` argument to `eqcorrscan.core.match_filter.party.Party.lag_calc` to provide an in-flow way to compute relative magnitudes for detected events. -* Events constructed from detections now include estimated origins alongside +- Events constructed from detections now include estimated origins alongside the picks. These origins are time-shifted versions of the template origin and should be used with caution. They are corrected for prepick (#308). -* Picks in detection.event are now corrected for prepick *if* the template is +- Picks in detection.event are now corrected for prepick -if- the template is given. This is now standard in all Tribe, Party and Family methods. Picks will not be corrected for prepick in match_filter (#308). -* Fix #298 where the header was repeated in detection csv files. Also added +- Fix #298 where the header was repeated in detection csv files. Also added a `write_detections` function to `eqcorrscan.core.match_filter.detection` to streamline writing detections. -* Remove support for Python 2.7. -* Add warning about unused data when using `Tribe.detect` methods with data that +- Remove support for Python 2.7. +- Add warning about unused data when using `Tribe.detect` methods with data that do not fit into chunks. Fixes #291. -* Fix #179 when decimating for cccsum_hist in `_match_filter_plot` -* `utils.pre_processing` now uses the `.interpolate` method rather than +- Fix #179 when decimating for cccsum_hist in `_match_filter_plot` +- `utils.pre_processing` now uses the `.interpolate` method rather than `.resample` to change the sampling rate of data. This is generally more stable and faster than resampling in the frequency domain, but will likely change the quality of correlations. -* Removed depreciated `template_gen` functions and `bright_lights` and +- Removed depreciated `template_gen` functions and `bright_lights` and `seismo_logs`. See #315 --- @@ -259,132 +259,132 @@ Older Versions Version 0.3.3 ............. -* Make test-script more stable - use the installed script for testing. -* Fix bug where `set_xcorr` as context manager did not correctly reset +- Make test-script more stable - use the installed script for testing. +- Fix bug where `set_xcorr` as context manager did not correctly reset stream_xcorr methods. -* Correct test-script (`test_eqcorrscan.py`) to find paths properly. -* BUG-FIX in `Party.decluster` when detections made at exactly the same time +- Correct test-script (`test_eqcorrscan.py`) to find paths properly. +- BUG-FIX in `Party.decluster` when detections made at exactly the same time the first, rather than the highest of these was taken. -* Catch one-sample difference in day properly in pre-processing.dayproc -* Shortproc now clips and pads to the correct length asserted by starttime and +- Catch one-sample difference in day properly in pre-processing.dayproc +- Shortproc now clips and pads to the correct length asserted by starttime and endtime. -* Bug-fix: Match-filter collection objects (Tribe, Party, Family) implemented +- Bug-fix: Match-filter collection objects (Tribe, Party, Family) implemented addition (`__add__`) to alter the main object. Now the main object is left unchanged. -* `Family.catalog` is now an immutable property. +- `Family.catalog` is now an immutable property. Version 0.3.2 ............. -* Implement reading Party objects from multiple files, including wildcard +- Implement reading Party objects from multiple files, including wildcard expansion. This will only read template information if it was not previously read in (which is a little more efficient). -* Allow reading of Party objects without reading the catalog files. -* Check quality of downloaded data in `Tribe.client_detect()` and remove it if it +- Allow reading of Party objects without reading the catalog files. +- Check quality of downloaded data in `Tribe.client_detect()` and remove it if it would otherwise result in errors. -* Add `process_cores` argument to `Tribe.client_detect()` and `Tribe.detect()` +- Add `process_cores` argument to `Tribe.client_detect()` and `Tribe.detect()` to provide a separate number of cores for processing and peak-finding - both functions are less memory efficient that fftw correlation and can result in memory errors if using lots of cores. -* Allow passing of `cores_outer` kwarg through to fftw correlate functions to +- Allow passing of `cores_outer` kwarg through to fftw correlate functions to control inner/outer thread numbers. If given, `cores` will define the number of inner-cores (used for parallel fft calculation) and `cores_outer` sets the number of channels to process in parallel (which results in increased memory usage). -* Allow Tribe and Party IO to use QUAKEML or SC3ML format for catalogs (NORDIC +- Allow Tribe and Party IO to use QUAKEML or SC3ML format for catalogs (NORDIC to come once obspy updates). -* Allow Party IO to not write detection catalogs if so desired, because +- Allow Party IO to not write detection catalogs if so desired, because writing and reading large catalogs can be slow. -* If detection-catalogs are not read in, then the detection events will be +- If detection-catalogs are not read in, then the detection events will be generated on the fly using `Detection._calculate_event`. -* BUG-FIX: When one template in a set of templates had a channel repeated, +- BUG-FIX: When one template in a set of templates had a channel repeated, all detections had an extra, spurious pick in their event object. This should no-longer happen. -* Add `select` method to `Party` and `Tribe` to allow selection of a +- Add `select` method to `Party` and `Tribe` to allow selection of a specific family/template. -* Add ability to "retry" downloading in `Tribe.client_detect`. -* Change behaviour of template_gen for data that are daylong, but do not start +- Add ability to "retry" downloading in `Tribe.client_detect`. +- Change behaviour of template_gen for data that are daylong, but do not start within 1 minute of a day-break - previous versions enforced padding to start and end at day-breaks, which led to zeros in the data and undesirable behaviour. -* BUG-FIX: Normalisation errors not properly passed back from internal fftw +- BUG-FIX: Normalisation errors not properly passed back from internal fftw correlation functions, gaps not always properly handled during long-period trends - variance threshold is now raised, and Python checks for low-variance and applies gain to stabilise correlations if needed. -* Plotting functions are now tested and have a more consistent interface: +- Plotting functions are now tested and have a more consistent interface: - All plotting functions accept the keyword arguments `save`, `savefile`, `show`, `return_figure` and `title`. - All plotting functions return a figure. - `SVD_plot` renamed to `svd_plot` -* Enforce pre-processing even when no filters or resampling is to be done +- Enforce pre-processing even when no filters or resampling is to be done to ensure gaps are properly processed (when called from `Tribe.detect`, `Template.detect` or `Tribe.client_detect`) -* BUG-FIX in `Tribe.client_detect` where data were processed from data +- BUG-FIX in `Tribe.client_detect` where data were processed from data one sample too long resulting in minor differences in data processing (due to difference in FFT length) and therefore minor differences in resulting correlations (~0.07 per channel). - Includes extra stability check in fftw_normxcorr which affects the last sample before a gap when that sample is near-zero. -* BUG-FIX: fftw correlation dot product was not thread-safe on some systems. +- BUG-FIX: fftw correlation dot product was not thread-safe on some systems. The dot-product did not have the inner index protected as a private variable. This did not appear to cause issues for Linux with Python 3.x or Windows, but did cause issues for on Linux for Python 2.7 and Mac OS builds. -* KeyboardInterrupt (e.g. ctrl-c) should now be caught during python parallel +- KeyboardInterrupt (e.g. ctrl-c) should now be caught during python parallel processes. -* Stopped allowing outer-threading on OSX, clang openMP is not thread-safe +- Stopped allowing outer-threading on OSX, clang openMP is not thread-safe for how we have this set-up. Inner threading is faster and more memory efficient anyway. -* Added testing script (`test_eqcorrscan.py`, which will be installed to your +- Added testing script (`test_eqcorrscan.py`, which will be installed to your path on installation of EQcorrscan) that will download all the relevant data and run the tests on the installed package - no need to clone EQcorrscan to run tests! Version 0.3.1 ............. -* Cleaned imports in utils modules -* Removed parallel checking loop in archive_read. -* Add better checks for timing in lag-calc functions (#207) -* Removed gap-threshold of twice the template length in `Tribe.client_detect`, see +- Cleaned imports in utils modules +- Removed parallel checking loop in archive_read. +- Add better checks for timing in lag-calc functions (#207) +- Removed gap-threshold of twice the template length in `Tribe.client_detect`, see issue #224. -* Bug-fix: give multi_find_peaks a cores kwarg to limit thread +- Bug-fix: give multi_find_peaks a cores kwarg to limit thread usage. -* Check for the same value in a row in continuous data when computing +- Check for the same value in a row in continuous data when computing correlations and zero resulting correlations where the whole window is the same value repeated (#224, #230). -* BUG-FIX: template generation `from_client` methods for swin=P_all or S_all +- BUG-FIX: template generation `from_client` methods for swin=P_all or S_all now download all channels and return them (as they should). See #235 and #206 -* Change from raising an error if data from a station are not long enough, to +- Change from raising an error if data from a station are not long enough, to logging a critical warning and not using the station. -* Add ability to give multiple `swin` options as a list. Remains backwards +- Add ability to give multiple `swin` options as a list. Remains backwards compatible with single `swin` arguments. -* Add option to `save_progress` for long running `Tribe` methods. Files +- Add option to `save_progress` for long running `Tribe` methods. Files are written to temporary files local to the caller. -* Fix bug where if gaps overlapped the endtime set in pre_processing an error +- Fix bug where if gaps overlapped the endtime set in pre_processing an error was raised - happened when downloading data with a deliberate pad at either end. Version 0.3.0 ............. -* Compiled peak-finding routine written to speed-up peak-finding. -* Change default match-filter plotting to not decimate unless it has to. -* BUG-FIX: changed minimum variance for fftw correlation backend. -* Do not try to process when no processing needs to be done in +- Compiled peak-finding routine written to speed-up peak-finding. +- Change default match-filter plotting to not decimate unless it has to. +- BUG-FIX: changed minimum variance for fftw correlation backend. +- Do not try to process when no processing needs to be done in core.match_filter._group_process. -* Length checking in core.match_filter._group_process done in samples rather +- Length checking in core.match_filter._group_process done in samples rather than time. -* BUG-FIX: Fix bug where data lengths were not correct in +- BUG-FIX: Fix bug where data lengths were not correct in match_filter.Tribe.detect when sampling time-stamps were inconsistent between channels, which previously resulted in error. -* BUG-FIX: Fix memory-leak in tribe.construct -* Add plotting options for plotting rate to Party.plot -* Add filtering detections by date as Party.filter -* BUG-FIX: Change method for Party.rethreshold: list.remove was not reliable. -* Add option `full_peaks` to detect methods to map to find_peaks. -* pre-processing (and match-filter object methods) are now gap-aware and will +- BUG-FIX: Fix memory-leak in tribe.construct +- Add plotting options for plotting rate to Party.plot +- Add filtering detections by date as Party.filter +- BUG-FIX: Change method for Party.rethreshold: list.remove was not reliable. +- Add option `full_peaks` to detect methods to map to find_peaks. +- pre-processing (and match-filter object methods) are now gap-aware and will accept gappy traces and can return gappy traces. By default gaps are filled to maintain backwards compatibility. Note that the fftw correlation backend requires gaps to be padded with zeros. -* **Removed sfile_utils** This support for Nordic IO has been upgraded and moved +- --Removed sfile_utils-- This support for Nordic IO has been upgraded and moved to obspy for obspy version 1.1.0. All functions are there and many bugs have been fixed. This also means the removal of nordic-specific functions in EQcorrscan - the following functions have been removed: @@ -394,23 +394,23 @@ Version 0.3.0 - mag_calc.pick_db All removed functions will error and tell you to use obspy.io.nordic.core. This now means that you can use obspy's `read_events` to read in sfiles. -* Added `P_all` and `S_all` options to template generation functions +- Added `P_all` and `S_all` options to template generation functions to allow creation of multi-channel templates starting at the P and S times respectively. -* Refactored `template_gen`, all options are available via +- Refactored `template_gen`, all options are available via `template_gen(method=...)`, and depreciation warnings are in place. -* Added some docs for converting older templates and detections into Template +- Added some docs for converting older templates and detections into Template and Party objects. Version 0.2.7 ............. -* Patch multi_corr.c to work with more versions of MSVC; -* Revert to using single-precision floats for correlations (as in previous, +- Patch multi_corr.c to work with more versions of MSVC; +- Revert to using single-precision floats for correlations (as in previous, < 0.2.x versions) for memory efficiency. Version 0.2.6 ............. -* Added the ability to change the correlation functions used in detection +- Added the ability to change the correlation functions used in detection methods through the parameter xcorr_func of match_filter, Template.detect and Tribe.detect, or using the set_xcorr context manager in the utils.correlate module. Supported options are: @@ -418,65 +418,65 @@ Version 0.2.6 - fftw - time-domain - or passing a function that implements the xcorr interface. -* Added the ability to change the concurrency strategy of xcorr functions +- Added the ability to change the concurrency strategy of xcorr functions using the paramter concurrency of match_filter, Template.detect and Tribe.detect. Supported options are: - None - for single-threaded execution in a single process - multithread - for multi-threaded execution - multiprocess- for multiprocess execution - concurrent - allows functions to describe their own preferred currency methods, defaults to multithread -* Change debug printing output, it should be a little quieter; -* Speed-up time-domain using a threaded C-routine - separate from frequency +- Change debug printing output, it should be a little quieter; +- Speed-up time-domain using a threaded C-routine - separate from frequency domain C-routines; -* Expose useful parallel options for all correlation routines; -* Expose cores argument for match-filter objects to allow limits to be placed +- Expose useful parallel options for all correlation routines; +- Expose cores argument for match-filter objects to allow limits to be placed on how much of your machine is used; -* Limit number of workers created during pre-processing to never be more than +- Limit number of workers created during pre-processing to never be more than the number of traces in the stream being processed; -* Implement openMP parallelisation of cross-correlation sum routines - memory +- Implement openMP parallelisation of cross-correlation sum routines - memory consumption reduced by using shared memory, and by computing the cross-correlation sums rather than individual channel cross-correlations. This also leads to a speed-up. This routine is the default concurrent correlation routine; -* Test examples in rst doc files to ensure they are up-to-date; -* Tests that were prone to timeout issues have been migrated to run on circleci +- Test examples in rst doc files to ensure they are up-to-date; +- Tests that were prone to timeout issues have been migrated to run on circleci to allow quick re-starting of fails not due to code errors Version 0.2.5 ............. -* Fix bug with \_group_process that resulted in stalled processes. -* Force NumPy version -* Support indexing of Tribe and Party objects by template-name. -* Add tests for lag-calc issue with preparing data -* Change internals of *eqcorrscan.core.lag_calc._prepare_data* to use a +- Fix bug with \_group_process that resulted in stalled processes. +- Force NumPy version +- Support indexing of Tribe and Party objects by template-name. +- Add tests for lag-calc issue with preparing data +- Change internals of -eqcorrscan.core.lag_calc._prepare_data- to use a dictionary for delays, and to work correctly! Issues arose from not checking for masked data properly and not checking length properly. -* Fix bug in match_filter.match_filter when checking for equal length traces, +- Fix bug in match_filter.match_filter when checking for equal length traces, length count was one sample too short. Version 0.2.4 ............. -* Increase test coverage (edge-cases) in template_gen; -* Fix bug in template_gen.extract_from_stack for duplicate channels in +- Increase test coverage (edge-cases) in template_gen; +- Fix bug in template_gen.extract_from_stack for duplicate channels in template; -* Increase coverage somewhat in bright_lights, remove non-parallel +- Increase coverage somewhat in bright_lights, remove non-parallel option (previously only used for debugging in development); -* Increase test coverage in lag_calc; -* Speed-up tests for brightness; -* Increase test coverage for match_filter including testing io of +- Increase test coverage in lag_calc; +- Speed-up tests for brightness; +- Increase test coverage for match_filter including testing io of detections; -* Increase subspace test coverage for edge cases; -* Speed-up catalog_to_dd_tests; -* Lag-calc will pick S-picks on channels ending E, N, 1 and 2, change +- Increase subspace test coverage for edge cases; +- Speed-up catalog_to_dd_tests; +- Lag-calc will pick S-picks on channels ending E, N, 1 and 2, change from only picking on E and N before; warning added to docs; -* Add full tests for pre-processing; -* Run tests in parallel on ci, speed-up tests dramatically; -* Rename singular-value decomposition functions (with depreciation +- Add full tests for pre-processing; +- Run tests in parallel on ci, speed-up tests dramatically; +- Rename singular-value decomposition functions (with depreciation warnings); -* Rename SVD_moments to lower-case and add depreciation warning; -* Increase test coverage in utils.mag_calc; -* Add Template, Tribe, Family, Party objects and rename DETECTION to Detection +- Rename SVD_moments to lower-case and add depreciation warning; +- Increase test coverage in utils.mag_calc; +- Add Template, Tribe, Family, Party objects and rename DETECTION to Detection - Template objects maintain meta-data associated with their creation to stream-line processing of data (e.g. reduce chance of using the wrong filters). - Template events have a detect method which takes unprocessed data and does the correct processing using the Template meta-data, and computes the matched-filter detections. - Tribe objects are containers for multiple Templates. @@ -484,21 +484,21 @@ Version 0.2.4 - The Party object is a container for many Family objects. - Family objects are containers for detections from the same Template. - Family and Party objects have a lag_calc method which computes the cross-correlation pick-refinements. - - The upshot of this is that it is possible to, in one line, generate a Tribe of templates, compute their matched-filter detections, and generate cross-correlation pick refinements, which output Event objects, which can be written to a catalog: Tribe.construct(method, **kwargs).detect(st, **kwargs).lag_calc(**kwargs).write() + - The upshot of this is that it is possible to, in one line, generate a Tribe of templates, compute their matched-filter detections, and generate cross-correlation pick refinements, which output Event objects, which can be written to a catalog: Tribe.construct(method, --kwargs).detect(st, --kwargs).lag_calc(--kwargs).write() - Added 25 tests for these methods. - - Add parameters *threshold_type* and *threshold_input* to Detection class. Add support for legacy Detection objects via NaN and unset values. -* Removed support for obspy < 1.0.0 -* Update / correct doc-strings in template-gen functions when describing + - Add parameters -threshold_type- and -threshold_input- to Detection class. Add support for legacy Detection objects via NaN and unset values. +- Removed support for obspy < 1.0.0 +- Update / correct doc-strings in template-gen functions when describing processing parameters. -* Add warning message when removing channels from continuous data in +- Add warning message when removing channels from continuous data in match_filter; -* Add min_snr option for template generation routines, if the +- Add min_snr option for template generation routines, if the signal-to-noise ratio is below a user-defined threshold, the channel will not be used. -* Stop enforcing two-channel template channel names. -* Fix bug in detection_multiplot which didn't allow streams with +- Stop enforcing two-channel template channel names. +- Fix bug in detection_multiplot which didn't allow streams with fewer traces than template; -* Update internals to custom C fftw-based correlation rather than openCV (Major change); +- Update internals to custom C fftw-based correlation rather than openCV (Major change); - OpenCV has been removed as a dependancy; - eqcorrscan.core.match_filter.normxcorr2 now calls a compiled C routine; - Parallel workflows handled by openMP rather than Python Multiprocessing for matched-filter operations to allow better memory handling. @@ -506,98 +506,98 @@ Version 0.2.4 Version 0.2.0-0.2.3 ................... -* See 0.2.4: these versions were not fully released while trying to get +- See 0.2.4: these versions were not fully released while trying to get anaconda packages to build properly. Version 0.1.6 ............. -* Fix bug introduced in version 0.1.5 for match_filter where looping +- Fix bug introduced in version 0.1.5 for match_filter where looping through multiple templates did not correctly match image and template data: 0.1.5 fix did not work; -* Bug-fix in catalog_to_dd for events without magnitudes; -* Amend match-filter to not edit the list of template names in place. +- Bug-fix in catalog_to_dd for events without magnitudes; +- Amend match-filter to not edit the list of template names in place. Previously, if a template was not used (due to no matching continuous data) then the name of the template was removed: this now copies the list of template_names internally and does not change the external list. Version 0.1.5 ............. -* Migrate coverage to codecov; -* Fix bug introduced in version 0.1.5 for match_filter where looping +- Migrate coverage to codecov; +- Fix bug introduced in version 0.1.5 for match_filter where looping through multiple templates did not correctly match image and template data. Version 0.1.4 ............. -* Bug-fix in plot_repicked removed where data were not normalized properly; -* Bug-fix in lag_calc where data were missing in the continuous data fixed (this led to incorrect picks, **major bug!**); -* Output cross-channel correlation sum in lag-calc output; -* Add id to DETECTION objects, which is consistent with the events within DETECTION objects and catalog output, and used in lag_calc to allow linking of detections to catalog events; -* Add lots of logging and error messages to lag-calc to ensure user understands limits; -* Add error to day-proc to ensure user is aware of risks of padding; -* Change utils.pre_processing.process to accept different length of data enforcement, not just full day (allow for overlap in processing, which might be useful for reducing day start and end effects); -* Bug-fix in mag_calc.amp_pick_event, broke loop if data were missing; -* Lots of docs adjustment to sort order of doc-strings and hyper-links; -* Allow multiple uses of the same channel in templates (e.g. you can now use a template with two windows from the same channel, such as a P and an S); -* Add evaluation mode filter to utils.catalog_utils.filter_picks; -* Update subspace plot to work when detector is not partitioned; -* Make tests run a little faster; -* Add pep8 testing for all code. +- Bug-fix in plot_repicked removed where data were not normalized properly; +- Bug-fix in lag_calc where data were missing in the continuous data fixed (this led to incorrect picks, --major bug!--); +- Output cross-channel correlation sum in lag-calc output; +- Add id to DETECTION objects, which is consistent with the events within DETECTION objects and catalog output, and used in lag_calc to allow linking of detections to catalog events; +- Add lots of logging and error messages to lag-calc to ensure user understands limits; +- Add error to day-proc to ensure user is aware of risks of padding; +- Change utils.pre_processing.process to accept different length of data enforcement, not just full day (allow for overlap in processing, which might be useful for reducing day start and end effects); +- Bug-fix in mag_calc.amp_pick_event, broke loop if data were missing; +- Lots of docs adjustment to sort order of doc-strings and hyper-links; +- Allow multiple uses of the same channel in templates (e.g. you can now use a template with two windows from the same channel, such as a P and an S); +- Add evaluation mode filter to utils.catalog_utils.filter_picks; +- Update subspace plot to work when detector is not partitioned; +- Make tests run a little faster; +- Add pep8 testing for all code. Version 0.1.3 ............. -* Now testing on OSX (python 2.7 and 3.5) - also added linux python 3.4; -* Add lag-calculation and tests for it; -* Change how lag-calc does the trace splitting to reduce memory usage; -* Added pick-filtering utility to clean up tutorials; -* Change template generation function names for clarity (wrappers for depreciated names); -* Add more useful error messages when picks are not associated with waveforms; -* Add example plots for more plotting functions; -* Add subspace detector including docs and tutorial. -* Add *delayed* option to all template_gen functions, set to True by default which retains old behaviour. +- Now testing on OSX (python 2.7 and 3.5) - also added linux python 3.4; +- Add lag-calculation and tests for it; +- Change how lag-calc does the trace splitting to reduce memory usage; +- Added pick-filtering utility to clean up tutorials; +- Change template generation function names for clarity (wrappers for depreciated names); +- Add more useful error messages when picks are not associated with waveforms; +- Add example plots for more plotting functions; +- Add subspace detector including docs and tutorial. +- Add -delayed- option to all template_gen functions, set to True by default which retains old behaviour. Version 0.1.2 ............. -* Add handling for empty location information in sfiles; -* Added project setup script which creates a useful directory structure and copies a default match-filter script to the directory; -* Add archive reader helper for default script, and parameter classes and definitions for default script; -* Re-write history to make repository smaller, removed trash files that had been added carelessly; -* Now tested on appveyor, so able to be run on Windows; -* Added ability to read hypoDD/tomoDD phase files to obspy events; -* Added simple despiking algorithm - not ideal for correlation as spikes are interpolated around when found: eqcorrscan.utils.despike; -* Option to output catalog object from match_filter - this will become the default once we introduce meta-data to templates - currently the picks for events are the template trace start-times, which will be before the phase-pick by the lag defined in the template creation - also added event into detection class, so you can access the event info from the detections, or create a catalog from a list of detections; -* Add option to extract detections at run-time in match_filter.match_filter; -* Edited multi_event_singlechan to take a catalog with multiple picks, but requires you to specify the station and channel to plot; -* Add normalize option to stacking routines; -* Add tests for stacking - PWS test needs more checks; -* Add many examples to doc-strings, not complete though; -* Change docs to have one page per function. -* Python 3.5 testing underway, all tests pass, but only testing about 65% of codebase. -* Add io functions to match_filter to simplify detection handling including writing detections to catalog and to text file. -* Stricter match_filter testing to enforce exactly the same result with a variety of systems. -* Add hack to template_gen tutorial to fix differences in sorting between python 3.x and python 2. -* Added advanced network triggering routine from Konstantinos, allows different parameters for individual stations - note only uses recursive sta-lta triggering at the moment. Useful for template generations alongside pickers. -* Added magnitude of completeness and b-value calculators to utils.mag_calc +- Add handling for empty location information in sfiles; +- Added project setup script which creates a useful directory structure and copies a default match-filter script to the directory; +- Add archive reader helper for default script, and parameter classes and definitions for default script; +- Re-write history to make repository smaller, removed trash files that had been added carelessly; +- Now tested on appveyor, so able to be run on Windows; +- Added ability to read hypoDD/tomoDD phase files to obspy events; +- Added simple despiking algorithm - not ideal for correlation as spikes are interpolated around when found: eqcorrscan.utils.despike; +- Option to output catalog object from match_filter - this will become the default once we introduce meta-data to templates - currently the picks for events are the template trace start-times, which will be before the phase-pick by the lag defined in the template creation - also added event into detection class, so you can access the event info from the detections, or create a catalog from a list of detections; +- Add option to extract detections at run-time in match_filter.match_filter; +- Edited multi_event_singlechan to take a catalog with multiple picks, but requires you to specify the station and channel to plot; +- Add normalize option to stacking routines; +- Add tests for stacking - PWS test needs more checks; +- Add many examples to doc-strings, not complete though; +- Change docs to have one page per function. +- Python 3.5 testing underway, all tests pass, but only testing about 65% of codebase. +- Add io functions to match_filter to simplify detection handling including writing detections to catalog and to text file. +- Stricter match_filter testing to enforce exactly the same result with a variety of systems. +- Add hack to template_gen tutorial to fix differences in sorting between python 3.x and python 2. +- Added advanced network triggering routine from Konstantinos, allows different parameters for individual stations - note only uses recursive sta-lta triggering at the moment. Useful for template generations alongside pickers. +- Added magnitude of completeness and b-value calculators to utils.mag_calc Version 0.1.1 ............. -* Cope with events not always having time_errors in them in eventtoSfile; -* Convert Quakeml depths from m to km; -* Multiple little fixes to make Sfile conversion play well with GeoNet QuakeML files; -* Add function to convert from obspy.core.inventory.station.Station to string format for Seisan STATION0.HYP file; -* Merged feature branch - hypoDD into develop, this provides mappings for the hypoDD location program, including generation of dt.cc files; -* Added tests for functions in catalog_to_dd; -* Implemented unittest tests; -* Changed name of EQcorrscan_plotting to plotting; -* Added depreciation warnings; -* Changed internal structure of pre-processing to aid long-term upkeep; -* Added warnings in docs for template_gen relating to template generation from set length files; -* Updated template_creation tutorial to use day-long data; -* Renamed Sfile_util to sfile_util, and functions there-in: will warn about name changes; -* Updated template plotting to include pick labels; -* Updated template_creation tutorial to download S-picks as well as P-picks; -* Update sfile_util to cope with many possible unfilled objects; -* Added sac_util to convert from sac headers to useful event information - note, does not convert all things, just origin and pick times; -* Added from_sac function to template_gen. +- Cope with events not always having time_errors in them in eventtoSfile; +- Convert Quakeml depths from m to km; +- Multiple little fixes to make Sfile conversion play well with GeoNet QuakeML files; +- Add function to convert from obspy.core.inventory.station.Station to string format for Seisan STATION0.HYP file; +- Merged feature branch - hypoDD into develop, this provides mappings for the hypoDD location program, including generation of dt.cc files; +- Added tests for functions in catalog_to_dd; +- Implemented unittest tests; +- Changed name of EQcorrscan_plotting to plotting; +- Added depreciation warnings; +- Changed internal structure of pre-processing to aid long-term upkeep; +- Added warnings in docs for template_gen relating to template generation from set length files; +- Updated template_creation tutorial to use day-long data; +- Renamed Sfile_util to sfile_util, and functions there-in: will warn about name changes; +- Updated template plotting to include pick labels; +- Updated template_creation tutorial to download S-picks as well as P-picks; +- Update sfile_util to cope with many possible unfilled objects; +- Added sac_util to convert from sac headers to useful event information - note, does not convert all things, just origin and pick times; +- Added from_sac function to template_gen. From de6db7fdff30475f8083ad80ecd3930cbfef8d78 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 17:03:37 +1200 Subject: [PATCH 57/61] Reformat updates nested lists --- eqcorrscan/doc/updates.rst | 535 +++++++++++++++++++------------------ 1 file changed, 269 insertions(+), 266 deletions(-) diff --git a/eqcorrscan/doc/updates.rst b/eqcorrscan/doc/updates.rst index a50c5db63..cc1e9e3e4 100644 --- a/eqcorrscan/doc/updates.rst +++ b/eqcorrscan/doc/updates.rst @@ -3,85 +3,85 @@ What's new Version 0.4.4 ------------- -- core.match_filter - - Bug-fix: peak-cores could be defined twice in _group_detect through kwargs. - Fix: only update peak_cores if it isn't there already. -- core.match_filter.tribe - - Detect now allows passing of pre-processed data -- core.match_filter.template - - Remove duplicate detections from overlapping windows using `._uniq()` -- core.lag_calc._xcorr_interp - - CC-interpolation replaced with resampling (more robust), old method - deprecated. Use new method with use_new_resamp_method=True as --kwarg. -- core.lag_calc: - - Fixed bug where minimum CC defined via min_cc_from_mean_cc_factor was not - set correctly for negative correlation sums. -- utils.correlate - - Fast Matched Filter now supported natively for FMF version >= 1.4.0 - - Only full correlation stacks are returned now (e.g. where fewer than than - the full number of channels are in the stack at the end of the stack, zeros - are returned). -- utils.mag_calc.relative_magnitude - - fixed bug where S-picks / traces were used for relative-magnitude calculation - against user's choice. - - implemented full magnitude bias-correction for CC and SNR -- utils.mag_calc.relative_amplitude: - - returns dicts for SNR measurements -- utils.catalog_to_dd.write_correlations - - Fixed bug on execution of parallel execution. - - Added parallel-options for catalog-dt measurements and for stream-preparation - before cross correlation-dt measurements. - - Default parallelization of dt-computation is now across events (loads CPUs - more efficiently), and there is a new option ``max_trace_workers` to use - the old parallelization strategy across traces. - - Now includes `all_horiz`-option that will correlate all matching horizontal - channels no matter to which of these the S-pick is linking. -- utils.clustering - - Allow to handle indirect comparison of event-waveforms when (i.e., events - without matching traces which can be compared indirectly via a third event) - - Allows to set clustering method, metric, and sort_order from - scipy.cluster.hierarchy.linkage. -- tribe, template, template_gen, archive_read, clustering: remove option to read +* core.match_filter + * Bug-fix: peak-cores could be defined twice in _group_detect through kwargs. + Fix: only update peak_cores if it isn't there already. +* core.match_filter.tribe + * Detect now allows passing of pre-processed data +* core.match_filter.template + * Remove duplicate detections from overlapping windows using `._uniq()` +* core.lag_calc._xcorr_interp + * CC-interpolation replaced with resampling (more robust), old method + deprecated. Use new method with use_new_resamp_method=True as **kwarg. +* core.lag_calc: + * Fixed bug where minimum CC defined via min_cc_from_mean_cc_factor was not + set correctly for negative correlation sums. +* utils.correlate + * Fast Matched Filter now supported natively for FMF version >= 1.4.0 + * Only full correlation stacks are returned now (e.g. where fewer than than + the full number of channels are in the stack at the end of the stack, zeros + are returned). +* utils.mag_calc.relative_magnitude + * fixed bug where S-picks / traces were used for relative-magnitude calculation + against user's choice. + * implemented full magnitude bias-correction for CC and SNR +* utils.mag_calc.relative_amplitude: + * returns dicts for SNR measurements +* utils.catalog_to_dd.write_correlations + * Fixed bug on execution of parallel execution. + * Added parallel-options for catalog-dt measurements and for stream-preparation + before cross correlation-dt measurements. + * Default parallelization of dt-computation is now across events (loads CPUs + more efficiently), and there is a new option ``max_trace_workers` to use + the old parallelization strategy across traces. + * Now includes `all_horiz`-option that will correlate all matching horizontal + channels no matter to which of these the S-pick is linking. +* utils.clustering + * Allow to handle indirect comparison of event-waveforms when (i.e., events + without matching traces which can be compared indirectly via a third event) + * Allows to set clustering method, metric, and sort_order from + scipy.cluster.hierarchy.linkage. +* tribe, template, template_gen, archive_read, clustering: remove option to read from seishub (deprecated in obspy). Version 0.4.3 ------------- -- core.match_filter +* core.match_filter - match_filter: - Provide option of exporting the cross-correlation sums for additional later analysis. -- core.match_filter.party.write +* core.match_filter.party.write - BUG-FIX: When `format='tar'` is selected, added a check for .tgz-file suffix before checking the filename against an existing file. Previously, when a filename without '.tgz'-suffix was supplied, then the file was overwritten against the function's intention. - Add option `overwrite=True` to allow overwriting of existing files. -- core.match_filter.party.read +* core.match_filter.party.read - BUG-FIX: Ensure wildcard reading works as expected: #453 -- core.match_filter.party.rethreshold: +* core.match_filter.party.rethreshold: - added option to rethreshold based on absolute values to keep relevant detections with large negative detect_val. -- core.lag_calc: +* core.lag_calc: - Added option to set minimum CC threshold individually for detections based - on: min(detect_val / n_chans - min_cc_from_mean_cc_factor, min_cc). + on: min(detect_val / n_chans * min_cc_from_mean_cc_factor, min_cc). - Added the ability of saving correlation data of the lag_calc. -- core.template_gen: +* core.template_gen: - Added support for generating templates from any object with a get_waveforms method. See #459. -- utils.mag_calc.calc_b_value: +* utils.mag_calc.calc_b_value: - Added useful information to doc-string regarding method and meaning of residuals - Changed the number of magnitudes used to an int (from a string!?) -- utils.mag_calc.relative_magnitude: +* utils.mag_calc.relative_magnitude: - Refactor so that `min_cc` is used regardless of whether `weight_by_correlation` is set. See issue #455. -- utils.archive_read +* utils.archive_read - Add support for wildcard-comparisons in the list of requested stations and channels. - New option `arctype='SDS'` to read from a SeisComp Data Structure (SDS). This option is also available in `utils.clustering.extract_detections` and in `utils.archive_read._check_available_data`. -- utils.catalog_to_dd +* utils.catalog_to_dd - Bug-fixes in #424: - only P and S phases are used now (previously spurious amplitude picks were included in correlations); @@ -89,20 +89,20 @@ Version 0.4.3 outputs are provided. - Progress is not reported within dt.cc computation - `write_station` now supports writing elevations: #424. -- utils.clustering +* utils.clustering - For `cluster`, `distance_matrix` and `cross_chan_correlation`, implemented full support for `shift_len != 0`. The latter two functions now return, in addition to the distance-matrix, a shift-matrix (both functions) and a shift-dictionary (for `distance_matrix`). New option for shifting streams as a whole or letting traces shift individually (`allow_individual_trace_shifts=True`). -- utils.plotting +* utils.plotting - Function added (twoD_seismplot) for plotting seismicity (#365). Version 0.4.2 ------------- -- Add seed-ids to the _spike_test's message. -- utils.correlation +* Add seed-ids to the _spike_test's message. +* utils.correlation - Cross-correlation normalisation errors no-longer raise an error - When "out-of-range" correlations occur a warning is given by the C-function with details of what channel, what template and where in the data vector @@ -112,12 +112,12 @@ Version 0.4.2 within regions where correlations should not be computed (spikes, step artifacts due to incorrectly padding data gaps). - USERS SHOULD BE CAREFUL TO CHECK THEIR DATA IF THEY SEE THESE WARNINGS -- utils.mag_calc.amp_pick_event +* utils.mag_calc.amp_pick_event - Added option to output IASPEI standard amplitudes, with static amplification of 1 (rather than 2080 as per Wood Anderson specs). - Added `filter_id` and `method_id` to amplitudes to make these methods more traceable. -- core.match_filter +* core.match_filter - Bug-fix - cope with data that are too short with `ignore_bad_data=True`. This flag is generally not advised, but when used, may attempt to trim all data to zero length. The expected behaviour is to remove bad data and run @@ -130,41 +130,42 @@ Version 0.4.2 - Tribe: - Add monkey-patching for clients that do not have a `get_waveforms_bulk` method for use in `.client_detect`. See issue #394. -- utils.pre_processing +* utils.pre_processing - Only templates that need to be reshaped are reshaped now - this can be a lot faster. Version 0.4.1 ------------- -- core.match_filter +* core.match_filter - BUG-FIX: Empty families are no longer run through lag-calc when using Party.lag_calc(). Previously this resulted in a "No matching data" error, see #341. -- core.template_gen +* core.template_gen - BUG-FIX: Fix bug where events were incorrectly associated with templates in `Tribe().construct()` if the given catalog contained events outside of the time-range of the stream. See issue #381 and PR #382. -- utils.catalog_to_dd +* utils.catalog_to_dd - Added ability to turn off parallel processing (this is turned off by default now) for `write_correlations` - parallel processing for moderate to large datasets was copying far too much data and using lots of memory. This is a short-term fix - ideally we will move filtering and resampling to C functions with shared-memory parallelism and GIL releasing. See PR #374. - Moved parallelism for `_compute_dt_correlations` to the C functions to reduce memory overhead. Using a generator to construct sub-catalogs rather than making a list of lists in memory. See issue #361. -- utils.mag_calc: +* utils.mag_calc: - `amp_pick_event` now works on a copy of the data by default - `amp_pick_event` uses the appropriate digital filter gain to correct the applied filter. See issue #376. - `amp_pick_event` rewritten for simplicity. - `amp_pick_event` now has simple synthetic tests for accuracy. - `_sim_wa` uses the full response information to correct to velocity this includes FIR filters (previously not used), and ensures that the wood-anderson poles (with a single zero) are correctly applied to velocity waveforms. - `calc_max_curv` is now computed using the non-cumulative distribution. -- Some problem solved in _match_filter_plot. Now it shows all new detections. -- Add plotdir to eqcorrscan.core.lag_calc.lag_calc function to save the images. +* Some problem solved in _match_filter_plot. Now it shows all new detections. +* Add plotdir to eqcorrscan.core.lag_calc.lag_calc function to save the images. Version 0.4.0 ------------- -- Change resampling to use pyFFTW backend for FFT's. This is an attempt to +* Change resampling to use pyFFTW backend for FFT's. This is an attempt to alleviate issue related to large-prime length transforms. This requires an additional dependency, but EQcorrscan already depends on FFTW itself (#316). -- Refactor of catalog_to_dd functions (#322): +* Refactor of catalog_to_dd functions (#322): - Speed-ups, using new correlation functions and better resource management - Removed enforcement of seisan, arguments are now standard obspy objects. -- Add plotdir to lag-calc, template construction and matched-filter detection +* Add plotdir to lag-calc, template construction and matched-filter detection methods and functions (#330, #325). -- Wholesale re-write of lag-calc function and methods. External interface is similar, but some arguments have been depreciated as they were unnecessary (#321). +* Wholesale re-write of lag-calc function and methods. External interface is + similar, but some arguments have been depreciated as they were unnecessary (#321). - This was done to make use of the new internal correlation functions which are faster and more memory efficient. - Party.lag_calc and Family.lag_calc now work in-place on the events in @@ -173,83 +174,85 @@ Version 0.4.0 lag-calc to avoid reprocessing data. - Added lag_calc.xcorr_pick_family as a public facing API to implement correlation re-picking of a group of events. -- Renamed utils.clustering.cross_chan_coherence to +* Renamed utils.clustering.cross_chan_coherence to utils.clustering.cross_chan_correlation to better reflect what it actually does. -- Add --no-mkl flag for setup.py to force the FFTW correlation routines not +* Add --no-mkl flag for setup.py to force the FFTW correlation routines not to compile against intels mkl. On NeSI systems mkl is currently causing issues. -- BUG-FIX: `eqcorrscan.utils.mag_calc.dist_calc` calculated the long-way round +* BUG-FIX: `eqcorrscan.utils.mag_calc.dist_calc` calculated the long-way round the Earth when changing hemispheres. We now use the Haversine formula, which should give better results at short distances, and does not use a flat-Earth approximation, so is better suited to larger distances as well. -- Add C-openmp parallel distance-clustering (speed-ups of ~100 times). -- Allow option to not stack correlations in correlation functions. -- Use compiled correlation functions for correlation clustering (speed-up). -- Add time-clustering for catalogs and change how space-time cluster works +* Add C-openmp parallel distance-clustering (speed-ups of ~100 times). +* Allow option to not stack correlations in correlation functions. +* Use compiled correlation functions for correlation clustering (speed-up). +* Add time-clustering for catalogs and change how space-time cluster works so that it uses the time-clustering, rather than just throwing out events outside the time-range. -- Changed all prints to calls to logging, as a result, debug is no longer +* Changed all prints to calls to logging, as a result, debug is no longer an argument for function calls. -- `find-peaks` replaced by compiled peak finding routine - more efficient both in memory and time #249 - approx 50x faster +* `find-peaks` replaced by compiled peak finding routine - more efficient + both in memory and time #249 - approx 50x faster - Note that the results of the C-func and the Python functions are slightly different. The C function (now the default) is more stable when peaks are small and close together (e.g. in noisy data). -- multi-find peaks makes use of openMP parallelism for more efficient +* multi-find peaks makes use of openMP parallelism for more efficient memory usage #249 -- enforce normalization of continuous data before correlation to avoid float32 +* enforce normalization of continuous data before correlation to avoid float32 overflow errors that result in correlation errors (see pr #292). -- Add SEC-C style chunked cross-correlations. This is both faster and more +* Add SEC-C style chunked cross-correlations. This is both faster and more memory efficient. This is now used by default with an fft length of - 2 -- 13. This was found to be consistently the fastest length in testing. + 2 ** 13. This was found to be consistently the fastest length in testing. This can be changed by the user by passing the `fft_len` keyword argument. See PR #285. -- Outer-loop parallelism has been disabled for all systems now. This was not +* Outer-loop parallelism has been disabled for all systems now. This was not useful in most situations and is hard to maintain. -- Improved support for compilation on RedHat systems -- Refactored match-filter into smaller files. Namespace remains the same. +* Improved support for compilation on RedHat systems +* Refactored match-filter into smaller files. Namespace remains the same. This was done to ease maintenance - the match_filter.py file had become massive and was slow to load and process in IDEs. -- Refactored `_prep_data_for_correlation` to reduce looping for speed, now approximately six times faster than previously (minor speed-up) +* Refactored `_prep_data_for_correlation` to reduce looping for speed, + now approximately six times faster than previously (minor speed-up) - Now explicitly doesn't allow templates with different length traces - previously this was ignored and templates with different length channels to other templates had their channels padded with zeros or trimmed. -- Add `skip_short_channels` option to template generation. This allows users +* Add `skip_short_channels` option to template generation. This allows users to provide data of unknown length and short channels will not be used, rather than generating an error. This is useful for downloading data from datacentres via the `from_client` method. -- Remove pytest_namespace in conftest.py to support pytest 4.x -- Add `ignore_bad_data` kwarg for all processing functions, if set to True +* Remove pytest_namespace in conftest.py to support pytest 4.x +* Add `ignore_bad_data` kwarg for all processing functions, if set to True (defaults to False for continuity) then any errors related to bad data at process-time will be supressed and empty traces returned. This is useful for downloading data from datacentres via the `from_client` method when data quality is not known. -- Added relative amplitude measurements as +* Added relative amplitude measurements as `utils.mag_calc.relative_amplitude` (#306). -- Added relative magnitude calculation using relative amplitudes weighted by +* Added relative magnitude calculation using relative amplitudes weighted by correlations to `utils.mag_calc.relative_magnitude`. -- Added `relative_magnitudes` argument to +* Added `relative_magnitudes` argument to `eqcorrscan.core.match_filter.party.Party.lag_calc` to provide an in-flow way to compute relative magnitudes for detected events. -- Events constructed from detections now include estimated origins alongside +* Events constructed from detections now include estimated origins alongside the picks. These origins are time-shifted versions of the template origin and should be used with caution. They are corrected for prepick (#308). -- Picks in detection.event are now corrected for prepick -if- the template is +* Picks in detection.event are now corrected for prepick *if* the template is given. This is now standard in all Tribe, Party and Family methods. Picks will not be corrected for prepick in match_filter (#308). -- Fix #298 where the header was repeated in detection csv files. Also added +* Fix #298 where the header was repeated in detection csv files. Also added a `write_detections` function to `eqcorrscan.core.match_filter.detection` to streamline writing detections. -- Remove support for Python 2.7. -- Add warning about unused data when using `Tribe.detect` methods with data that +* Remove support for Python 2.7. +* Add warning about unused data when using `Tribe.detect` methods with data that do not fit into chunks. Fixes #291. -- Fix #179 when decimating for cccsum_hist in `_match_filter_plot` -- `utils.pre_processing` now uses the `.interpolate` method rather than +* Fix #179 when decimating for cccsum_hist in `_match_filter_plot` +* `utils.pre_processing` now uses the `.interpolate` method rather than `.resample` to change the sampling rate of data. This is generally more stable and faster than resampling in the frequency domain, but will likely change the quality of correlations. -- Removed depreciated `template_gen` functions and `bright_lights` and +* Removed depreciated `template_gen` functions and `bright_lights` and `seismo_logs`. See #315 --- @@ -259,132 +262,132 @@ Older Versions Version 0.3.3 ............. -- Make test-script more stable - use the installed script for testing. -- Fix bug where `set_xcorr` as context manager did not correctly reset +* Make test-script more stable - use the installed script for testing. +* Fix bug where `set_xcorr` as context manager did not correctly reset stream_xcorr methods. -- Correct test-script (`test_eqcorrscan.py`) to find paths properly. -- BUG-FIX in `Party.decluster` when detections made at exactly the same time +* Correct test-script (`test_eqcorrscan.py`) to find paths properly. +* BUG-FIX in `Party.decluster` when detections made at exactly the same time the first, rather than the highest of these was taken. -- Catch one-sample difference in day properly in pre-processing.dayproc -- Shortproc now clips and pads to the correct length asserted by starttime and +* Catch one-sample difference in day properly in pre-processing.dayproc +* Shortproc now clips and pads to the correct length asserted by starttime and endtime. -- Bug-fix: Match-filter collection objects (Tribe, Party, Family) implemented +* Bug-fix: Match-filter collection objects (Tribe, Party, Family) implemented addition (`__add__`) to alter the main object. Now the main object is left unchanged. -- `Family.catalog` is now an immutable property. +* `Family.catalog` is now an immutable property. Version 0.3.2 ............. -- Implement reading Party objects from multiple files, including wildcard +* Implement reading Party objects from multiple files, including wildcard expansion. This will only read template information if it was not previously read in (which is a little more efficient). -- Allow reading of Party objects without reading the catalog files. -- Check quality of downloaded data in `Tribe.client_detect()` and remove it if it +* Allow reading of Party objects without reading the catalog files. +* Check quality of downloaded data in `Tribe.client_detect()` and remove it if it would otherwise result in errors. -- Add `process_cores` argument to `Tribe.client_detect()` and `Tribe.detect()` +* Add `process_cores` argument to `Tribe.client_detect()` and `Tribe.detect()` to provide a separate number of cores for processing and peak-finding - both functions are less memory efficient that fftw correlation and can result in memory errors if using lots of cores. -- Allow passing of `cores_outer` kwarg through to fftw correlate functions to +* Allow passing of `cores_outer` kwarg through to fftw correlate functions to control inner/outer thread numbers. If given, `cores` will define the number of inner-cores (used for parallel fft calculation) and `cores_outer` sets the number of channels to process in parallel (which results in increased memory usage). -- Allow Tribe and Party IO to use QUAKEML or SC3ML format for catalogs (NORDIC +* Allow Tribe and Party IO to use QUAKEML or SC3ML format for catalogs (NORDIC to come once obspy updates). -- Allow Party IO to not write detection catalogs if so desired, because +* Allow Party IO to not write detection catalogs if so desired, because writing and reading large catalogs can be slow. -- If detection-catalogs are not read in, then the detection events will be +* If detection-catalogs are not read in, then the detection events will be generated on the fly using `Detection._calculate_event`. -- BUG-FIX: When one template in a set of templates had a channel repeated, +* BUG-FIX: When one template in a set of templates had a channel repeated, all detections had an extra, spurious pick in their event object. This should no-longer happen. -- Add `select` method to `Party` and `Tribe` to allow selection of a +* Add `select` method to `Party` and `Tribe` to allow selection of a specific family/template. -- Add ability to "retry" downloading in `Tribe.client_detect`. -- Change behaviour of template_gen for data that are daylong, but do not start +* Add ability to "retry" downloading in `Tribe.client_detect`. +* Change behaviour of template_gen for data that are daylong, but do not start within 1 minute of a day-break - previous versions enforced padding to start and end at day-breaks, which led to zeros in the data and undesirable behaviour. -- BUG-FIX: Normalisation errors not properly passed back from internal fftw +* BUG-FIX: Normalisation errors not properly passed back from internal fftw correlation functions, gaps not always properly handled during long-period trends - variance threshold is now raised, and Python checks for low-variance and applies gain to stabilise correlations if needed. -- Plotting functions are now tested and have a more consistent interface: +* Plotting functions are now tested and have a more consistent interface: - All plotting functions accept the keyword arguments `save`, `savefile`, `show`, `return_figure` and `title`. - All plotting functions return a figure. - `SVD_plot` renamed to `svd_plot` -- Enforce pre-processing even when no filters or resampling is to be done +* Enforce pre-processing even when no filters or resampling is to be done to ensure gaps are properly processed (when called from `Tribe.detect`, `Template.detect` or `Tribe.client_detect`) -- BUG-FIX in `Tribe.client_detect` where data were processed from data +* BUG-FIX in `Tribe.client_detect` where data were processed from data one sample too long resulting in minor differences in data processing (due to difference in FFT length) and therefore minor differences in resulting correlations (~0.07 per channel). - Includes extra stability check in fftw_normxcorr which affects the last sample before a gap when that sample is near-zero. -- BUG-FIX: fftw correlation dot product was not thread-safe on some systems. +* BUG-FIX: fftw correlation dot product was not thread-safe on some systems. The dot-product did not have the inner index protected as a private variable. This did not appear to cause issues for Linux with Python 3.x or Windows, but did cause issues for on Linux for Python 2.7 and Mac OS builds. -- KeyboardInterrupt (e.g. ctrl-c) should now be caught during python parallel +* KeyboardInterrupt (e.g. ctrl-c) should now be caught during python parallel processes. -- Stopped allowing outer-threading on OSX, clang openMP is not thread-safe +* Stopped allowing outer-threading on OSX, clang openMP is not thread-safe for how we have this set-up. Inner threading is faster and more memory efficient anyway. -- Added testing script (`test_eqcorrscan.py`, which will be installed to your +* Added testing script (`test_eqcorrscan.py`, which will be installed to your path on installation of EQcorrscan) that will download all the relevant data and run the tests on the installed package - no need to clone EQcorrscan to run tests! Version 0.3.1 ............. -- Cleaned imports in utils modules -- Removed parallel checking loop in archive_read. -- Add better checks for timing in lag-calc functions (#207) -- Removed gap-threshold of twice the template length in `Tribe.client_detect`, see +* Cleaned imports in utils modules +* Removed parallel checking loop in archive_read. +* Add better checks for timing in lag-calc functions (#207) +* Removed gap-threshold of twice the template length in `Tribe.client_detect`, see issue #224. -- Bug-fix: give multi_find_peaks a cores kwarg to limit thread +* Bug-fix: give multi_find_peaks a cores kwarg to limit thread usage. -- Check for the same value in a row in continuous data when computing +* Check for the same value in a row in continuous data when computing correlations and zero resulting correlations where the whole window is the same value repeated (#224, #230). -- BUG-FIX: template generation `from_client` methods for swin=P_all or S_all +* BUG-FIX: template generation `from_client` methods for swin=P_all or S_all now download all channels and return them (as they should). See #235 and #206 -- Change from raising an error if data from a station are not long enough, to +* Change from raising an error if data from a station are not long enough, to logging a critical warning and not using the station. -- Add ability to give multiple `swin` options as a list. Remains backwards +* Add ability to give multiple `swin` options as a list. Remains backwards compatible with single `swin` arguments. -- Add option to `save_progress` for long running `Tribe` methods. Files +* Add option to `save_progress` for long running `Tribe` methods. Files are written to temporary files local to the caller. -- Fix bug where if gaps overlapped the endtime set in pre_processing an error +* Fix bug where if gaps overlapped the endtime set in pre_processing an error was raised - happened when downloading data with a deliberate pad at either end. Version 0.3.0 ............. -- Compiled peak-finding routine written to speed-up peak-finding. -- Change default match-filter plotting to not decimate unless it has to. -- BUG-FIX: changed minimum variance for fftw correlation backend. -- Do not try to process when no processing needs to be done in +* Compiled peak-finding routine written to speed-up peak-finding. +* Change default match-filter plotting to not decimate unless it has to. +* BUG-FIX: changed minimum variance for fftw correlation backend. +* Do not try to process when no processing needs to be done in core.match_filter._group_process. -- Length checking in core.match_filter._group_process done in samples rather +* Length checking in core.match_filter._group_process done in samples rather than time. -- BUG-FIX: Fix bug where data lengths were not correct in +* BUG-FIX: Fix bug where data lengths were not correct in match_filter.Tribe.detect when sampling time-stamps were inconsistent between channels, which previously resulted in error. -- BUG-FIX: Fix memory-leak in tribe.construct -- Add plotting options for plotting rate to Party.plot -- Add filtering detections by date as Party.filter -- BUG-FIX: Change method for Party.rethreshold: list.remove was not reliable. -- Add option `full_peaks` to detect methods to map to find_peaks. -- pre-processing (and match-filter object methods) are now gap-aware and will +* BUG-FIX: Fix memory-leak in tribe.construct +* Add plotting options for plotting rate to Party.plot +* Add filtering detections by date as Party.filter +* BUG-FIX: Change method for Party.rethreshold: list.remove was not reliable. +* Add option `full_peaks` to detect methods to map to find_peaks. +* pre-processing (and match-filter object methods) are now gap-aware and will accept gappy traces and can return gappy traces. By default gaps are filled to maintain backwards compatibility. Note that the fftw correlation backend requires gaps to be padded with zeros. -- --Removed sfile_utils-- This support for Nordic IO has been upgraded and moved +* **Removed sfile_utils** This support for Nordic IO has been upgraded and moved to obspy for obspy version 1.1.0. All functions are there and many bugs have been fixed. This also means the removal of nordic-specific functions in EQcorrscan - the following functions have been removed: @@ -394,23 +397,23 @@ Version 0.3.0 - mag_calc.pick_db All removed functions will error and tell you to use obspy.io.nordic.core. This now means that you can use obspy's `read_events` to read in sfiles. -- Added `P_all` and `S_all` options to template generation functions +* Added `P_all` and `S_all` options to template generation functions to allow creation of multi-channel templates starting at the P and S times respectively. -- Refactored `template_gen`, all options are available via +* Refactored `template_gen`, all options are available via `template_gen(method=...)`, and depreciation warnings are in place. -- Added some docs for converting older templates and detections into Template +* Added some docs for converting older templates and detections into Template and Party objects. Version 0.2.7 ............. -- Patch multi_corr.c to work with more versions of MSVC; -- Revert to using single-precision floats for correlations (as in previous, +* Patch multi_corr.c to work with more versions of MSVC; +* Revert to using single-precision floats for correlations (as in previous, < 0.2.x versions) for memory efficiency. Version 0.2.6 ............. -- Added the ability to change the correlation functions used in detection +* Added the ability to change the correlation functions used in detection methods through the parameter xcorr_func of match_filter, Template.detect and Tribe.detect, or using the set_xcorr context manager in the utils.correlate module. Supported options are: @@ -418,65 +421,65 @@ Version 0.2.6 - fftw - time-domain - or passing a function that implements the xcorr interface. -- Added the ability to change the concurrency strategy of xcorr functions +* Added the ability to change the concurrency strategy of xcorr functions using the paramter concurrency of match_filter, Template.detect and Tribe.detect. Supported options are: - None - for single-threaded execution in a single process - multithread - for multi-threaded execution - multiprocess- for multiprocess execution - concurrent - allows functions to describe their own preferred currency methods, defaults to multithread -- Change debug printing output, it should be a little quieter; -- Speed-up time-domain using a threaded C-routine - separate from frequency +* Change debug printing output, it should be a little quieter; +* Speed-up time-domain using a threaded C-routine - separate from frequency domain C-routines; -- Expose useful parallel options for all correlation routines; -- Expose cores argument for match-filter objects to allow limits to be placed +* Expose useful parallel options for all correlation routines; +* Expose cores argument for match-filter objects to allow limits to be placed on how much of your machine is used; -- Limit number of workers created during pre-processing to never be more than +* Limit number of workers created during pre-processing to never be more than the number of traces in the stream being processed; -- Implement openMP parallelisation of cross-correlation sum routines - memory +* Implement openMP parallelisation of cross-correlation sum routines - memory consumption reduced by using shared memory, and by computing the cross-correlation sums rather than individual channel cross-correlations. This also leads to a speed-up. This routine is the default concurrent correlation routine; -- Test examples in rst doc files to ensure they are up-to-date; -- Tests that were prone to timeout issues have been migrated to run on circleci +* Test examples in rst doc files to ensure they are up-to-date; +* Tests that were prone to timeout issues have been migrated to run on circleci to allow quick re-starting of fails not due to code errors Version 0.2.5 ............. -- Fix bug with \_group_process that resulted in stalled processes. -- Force NumPy version -- Support indexing of Tribe and Party objects by template-name. -- Add tests for lag-calc issue with preparing data -- Change internals of -eqcorrscan.core.lag_calc._prepare_data- to use a +* Fix bug with \_group_process that resulted in stalled processes. +* Force NumPy version +* Support indexing of Tribe and Party objects by template-name. +* Add tests for lag-calc issue with preparing data +* Change internals of *eqcorrscan.core.lag_calc._prepare_data* to use a dictionary for delays, and to work correctly! Issues arose from not checking for masked data properly and not checking length properly. -- Fix bug in match_filter.match_filter when checking for equal length traces, +* Fix bug in match_filter.match_filter when checking for equal length traces, length count was one sample too short. Version 0.2.4 ............. -- Increase test coverage (edge-cases) in template_gen; -- Fix bug in template_gen.extract_from_stack for duplicate channels in +* Increase test coverage (edge-cases) in template_gen; +* Fix bug in template_gen.extract_from_stack for duplicate channels in template; -- Increase coverage somewhat in bright_lights, remove non-parallel +* Increase coverage somewhat in bright_lights, remove non-parallel option (previously only used for debugging in development); -- Increase test coverage in lag_calc; -- Speed-up tests for brightness; -- Increase test coverage for match_filter including testing io of +* Increase test coverage in lag_calc; +* Speed-up tests for brightness; +* Increase test coverage for match_filter including testing io of detections; -- Increase subspace test coverage for edge cases; -- Speed-up catalog_to_dd_tests; -- Lag-calc will pick S-picks on channels ending E, N, 1 and 2, change +* Increase subspace test coverage for edge cases; +* Speed-up catalog_to_dd_tests; +* Lag-calc will pick S-picks on channels ending E, N, 1 and 2, change from only picking on E and N before; warning added to docs; -- Add full tests for pre-processing; -- Run tests in parallel on ci, speed-up tests dramatically; -- Rename singular-value decomposition functions (with depreciation +* Add full tests for pre-processing; +* Run tests in parallel on ci, speed-up tests dramatically; +* Rename singular-value decomposition functions (with depreciation warnings); -- Rename SVD_moments to lower-case and add depreciation warning; -- Increase test coverage in utils.mag_calc; -- Add Template, Tribe, Family, Party objects and rename DETECTION to Detection +* Rename SVD_moments to lower-case and add depreciation warning; +* Increase test coverage in utils.mag_calc; +* Add Template, Tribe, Family, Party objects and rename DETECTION to Detection - Template objects maintain meta-data associated with their creation to stream-line processing of data (e.g. reduce chance of using the wrong filters). - Template events have a detect method which takes unprocessed data and does the correct processing using the Template meta-data, and computes the matched-filter detections. - Tribe objects are containers for multiple Templates. @@ -484,120 +487,120 @@ Version 0.2.4 - The Party object is a container for many Family objects. - Family objects are containers for detections from the same Template. - Family and Party objects have a lag_calc method which computes the cross-correlation pick-refinements. - - The upshot of this is that it is possible to, in one line, generate a Tribe of templates, compute their matched-filter detections, and generate cross-correlation pick refinements, which output Event objects, which can be written to a catalog: Tribe.construct(method, --kwargs).detect(st, --kwargs).lag_calc(--kwargs).write() + - The upshot of this is that it is possible to, in one line, generate a Tribe of templates, compute their matched-filter detections, and generate cross-correlation pick refinements, which output Event objects, which can be written to a catalog: Tribe.construct(method, **kwargs).detect(st, **kwargs).lag_calc(**kwargs).write() - Added 25 tests for these methods. - - Add parameters -threshold_type- and -threshold_input- to Detection class. Add support for legacy Detection objects via NaN and unset values. -- Removed support for obspy < 1.0.0 -- Update / correct doc-strings in template-gen functions when describing + - Add parameters *threshold_type* and *threshold_input* to Detection class. Add support for legacy Detection objects via NaN and unset values. +* Removed support for obspy < 1.0.0 +* Update / correct doc-strings in template-gen functions when describing processing parameters. -- Add warning message when removing channels from continuous data in +* Add warning message when removing channels from continuous data in match_filter; -- Add min_snr option for template generation routines, if the +* Add min_snr option for template generation routines, if the signal-to-noise ratio is below a user-defined threshold, the channel will not be used. -- Stop enforcing two-channel template channel names. -- Fix bug in detection_multiplot which didn't allow streams with +* Stop enforcing two-channel template channel names. +* Fix bug in detection_multiplot which didn't allow streams with fewer traces than template; -- Update internals to custom C fftw-based correlation rather than openCV (Major change); - - OpenCV has been removed as a dependancy; - - eqcorrscan.core.match_filter.normxcorr2 now calls a compiled C routine; - - Parallel workflows handled by openMP rather than Python Multiprocessing for matched-filter operations to allow better memory handling. - - It is worth noting that we tried re-writing using SciPy internals which led to a significant speed-up, but with high memory costs, we ended up going with this option, which was the more difficult option, because it allows effective use on SLURM managed systems where python multiprocessing results in un-real memory spikes (issue #88). +* Update internals to custom C fftw-based correlation rather than openCV (Major change); + * OpenCV has been removed as a dependancy; + * eqcorrscan.core.match_filter.normxcorr2 now calls a compiled C routine; + * Parallel workflows handled by openMP rather than Python Multiprocessing for matched-filter operations to allow better memory handling. + * It is worth noting that we tried re-writing using SciPy internals which led to a significant speed-up, but with high memory costs, we ended up going with this option, which was the more difficult option, because it allows effective use on SLURM managed systems where python multiprocessing results in un-real memory spikes (issue #88). Version 0.2.0-0.2.3 ................... -- See 0.2.4: these versions were not fully released while trying to get +* See 0.2.4: these versions were not fully released while trying to get anaconda packages to build properly. Version 0.1.6 ............. -- Fix bug introduced in version 0.1.5 for match_filter where looping +* Fix bug introduced in version 0.1.5 for match_filter where looping through multiple templates did not correctly match image and template data: 0.1.5 fix did not work; -- Bug-fix in catalog_to_dd for events without magnitudes; -- Amend match-filter to not edit the list of template names in place. +* Bug-fix in catalog_to_dd for events without magnitudes; +* Amend match-filter to not edit the list of template names in place. Previously, if a template was not used (due to no matching continuous data) then the name of the template was removed: this now copies the list of template_names internally and does not change the external list. Version 0.1.5 ............. -- Migrate coverage to codecov; -- Fix bug introduced in version 0.1.5 for match_filter where looping +* Migrate coverage to codecov; +* Fix bug introduced in version 0.1.5 for match_filter where looping through multiple templates did not correctly match image and template data. Version 0.1.4 ............. -- Bug-fix in plot_repicked removed where data were not normalized properly; -- Bug-fix in lag_calc where data were missing in the continuous data fixed (this led to incorrect picks, --major bug!--); -- Output cross-channel correlation sum in lag-calc output; -- Add id to DETECTION objects, which is consistent with the events within DETECTION objects and catalog output, and used in lag_calc to allow linking of detections to catalog events; -- Add lots of logging and error messages to lag-calc to ensure user understands limits; -- Add error to day-proc to ensure user is aware of risks of padding; -- Change utils.pre_processing.process to accept different length of data enforcement, not just full day (allow for overlap in processing, which might be useful for reducing day start and end effects); -- Bug-fix in mag_calc.amp_pick_event, broke loop if data were missing; -- Lots of docs adjustment to sort order of doc-strings and hyper-links; -- Allow multiple uses of the same channel in templates (e.g. you can now use a template with two windows from the same channel, such as a P and an S); -- Add evaluation mode filter to utils.catalog_utils.filter_picks; -- Update subspace plot to work when detector is not partitioned; -- Make tests run a little faster; -- Add pep8 testing for all code. +* Bug-fix in plot_repicked removed where data were not normalized properly; +* Bug-fix in lag_calc where data were missing in the continuous data fixed (this led to incorrect picks, **major bug!**); +* Output cross-channel correlation sum in lag-calc output; +* Add id to DETECTION objects, which is consistent with the events within DETECTION objects and catalog output, and used in lag_calc to allow linking of detections to catalog events; +* Add lots of logging and error messages to lag-calc to ensure user understands limits; +* Add error to day-proc to ensure user is aware of risks of padding; +* Change utils.pre_processing.process to accept different length of data enforcement, not just full day (allow for overlap in processing, which might be useful for reducing day start and end effects); +* Bug-fix in mag_calc.amp_pick_event, broke loop if data were missing; +* Lots of docs adjustment to sort order of doc-strings and hyper-links; +* Allow multiple uses of the same channel in templates (e.g. you can now use a template with two windows from the same channel, such as a P and an S); +* Add evaluation mode filter to utils.catalog_utils.filter_picks; +* Update subspace plot to work when detector is not partitioned; +* Make tests run a little faster; +* Add pep8 testing for all code. Version 0.1.3 ............. -- Now testing on OSX (python 2.7 and 3.5) - also added linux python 3.4; -- Add lag-calculation and tests for it; -- Change how lag-calc does the trace splitting to reduce memory usage; -- Added pick-filtering utility to clean up tutorials; -- Change template generation function names for clarity (wrappers for depreciated names); -- Add more useful error messages when picks are not associated with waveforms; -- Add example plots for more plotting functions; -- Add subspace detector including docs and tutorial. -- Add -delayed- option to all template_gen functions, set to True by default which retains old behaviour. +* Now testing on OSX (python 2.7 and 3.5) - also added linux python 3.4; +* Add lag-calculation and tests for it; +* Change how lag-calc does the trace splitting to reduce memory usage; +* Added pick-filtering utility to clean up tutorials; +* Change template generation function names for clarity (wrappers for depreciated names); +* Add more useful error messages when picks are not associated with waveforms; +* Add example plots for more plotting functions; +* Add subspace detector including docs and tutorial. +* Add *delayed* option to all template_gen functions, set to True by default which retains old behaviour. Version 0.1.2 ............. -- Add handling for empty location information in sfiles; -- Added project setup script which creates a useful directory structure and copies a default match-filter script to the directory; -- Add archive reader helper for default script, and parameter classes and definitions for default script; -- Re-write history to make repository smaller, removed trash files that had been added carelessly; -- Now tested on appveyor, so able to be run on Windows; -- Added ability to read hypoDD/tomoDD phase files to obspy events; -- Added simple despiking algorithm - not ideal for correlation as spikes are interpolated around when found: eqcorrscan.utils.despike; -- Option to output catalog object from match_filter - this will become the default once we introduce meta-data to templates - currently the picks for events are the template trace start-times, which will be before the phase-pick by the lag defined in the template creation - also added event into detection class, so you can access the event info from the detections, or create a catalog from a list of detections; -- Add option to extract detections at run-time in match_filter.match_filter; -- Edited multi_event_singlechan to take a catalog with multiple picks, but requires you to specify the station and channel to plot; -- Add normalize option to stacking routines; -- Add tests for stacking - PWS test needs more checks; -- Add many examples to doc-strings, not complete though; -- Change docs to have one page per function. -- Python 3.5 testing underway, all tests pass, but only testing about 65% of codebase. -- Add io functions to match_filter to simplify detection handling including writing detections to catalog and to text file. -- Stricter match_filter testing to enforce exactly the same result with a variety of systems. -- Add hack to template_gen tutorial to fix differences in sorting between python 3.x and python 2. -- Added advanced network triggering routine from Konstantinos, allows different parameters for individual stations - note only uses recursive sta-lta triggering at the moment. Useful for template generations alongside pickers. -- Added magnitude of completeness and b-value calculators to utils.mag_calc +* Add handling for empty location information in sfiles; +* Added project setup script which creates a useful directory structure and copies a default match-filter script to the directory; +* Add archive reader helper for default script, and parameter classes and definitions for default script; +* Re-write history to make repository smaller, removed trash files that had been added carelessly; +* Now tested on appveyor, so able to be run on Windows; +* Added ability to read hypoDD/tomoDD phase files to obspy events; +* Added simple despiking algorithm - not ideal for correlation as spikes are interpolated around when found: eqcorrscan.utils.despike; +* Option to output catalog object from match_filter - this will become the default once we introduce meta-data to templates - currently the picks for events are the template trace start-times, which will be before the phase-pick by the lag defined in the template creation - also added event into detection class, so you can access the event info from the detections, or create a catalog from a list of detections; +* Add option to extract detections at run-time in match_filter.match_filter; +* Edited multi_event_singlechan to take a catalog with multiple picks, but requires you to specify the station and channel to plot; +* Add normalize option to stacking routines; +* Add tests for stacking - PWS test needs more checks; +* Add many examples to doc-strings, not complete though; +* Change docs to have one page per function. +* Python 3.5 testing underway, all tests pass, but only testing about 65% of codebase. +* Add io functions to match_filter to simplify detection handling including writing detections to catalog and to text file. +* Stricter match_filter testing to enforce exactly the same result with a variety of systems. +* Add hack to template_gen tutorial to fix differences in sorting between python 3.x and python 2. +* Added advanced network triggering routine from Konstantinos, allows different parameters for individual stations - note only uses recursive sta-lta triggering at the moment. Useful for template generations alongside pickers. +* Added magnitude of completeness and b-value calculators to utils.mag_calc Version 0.1.1 ............. -- Cope with events not always having time_errors in them in eventtoSfile; -- Convert Quakeml depths from m to km; -- Multiple little fixes to make Sfile conversion play well with GeoNet QuakeML files; -- Add function to convert from obspy.core.inventory.station.Station to string format for Seisan STATION0.HYP file; -- Merged feature branch - hypoDD into develop, this provides mappings for the hypoDD location program, including generation of dt.cc files; -- Added tests for functions in catalog_to_dd; -- Implemented unittest tests; -- Changed name of EQcorrscan_plotting to plotting; -- Added depreciation warnings; -- Changed internal structure of pre-processing to aid long-term upkeep; -- Added warnings in docs for template_gen relating to template generation from set length files; -- Updated template_creation tutorial to use day-long data; -- Renamed Sfile_util to sfile_util, and functions there-in: will warn about name changes; -- Updated template plotting to include pick labels; -- Updated template_creation tutorial to download S-picks as well as P-picks; -- Update sfile_util to cope with many possible unfilled objects; -- Added sac_util to convert from sac headers to useful event information - note, does not convert all things, just origin and pick times; -- Added from_sac function to template_gen. +* Cope with events not always having time_errors in them in eventtoSfile; +* Convert Quakeml depths from m to km; +* Multiple little fixes to make Sfile conversion play well with GeoNet QuakeML files; +* Add function to convert from obspy.core.inventory.station.Station to string format for Seisan STATION0.HYP file; +* Merged feature branch - hypoDD into develop, this provides mappings for the hypoDD location program, including generation of dt.cc files; +* Added tests for functions in catalog_to_dd; +* Implemented unittest tests; +* Changed name of EQcorrscan_plotting to plotting; +* Added depreciation warnings; +* Changed internal structure of pre-processing to aid long-term upkeep; +* Added warnings in docs for template_gen relating to template generation from set length files; +* Updated template_creation tutorial to use day-long data; +* Renamed Sfile_util to sfile_util, and functions there-in: will warn about name changes; +* Updated template plotting to include pick labels; +* Updated template_creation tutorial to download S-picks as well as P-picks; +* Update sfile_util to cope with many possible unfilled objects; +* Added sac_util to convert from sac headers to useful event information - note, does not convert all things, just origin and pick times; +* Added from_sac function to template_gen. From aa92e6e7ab2d2412ee0c39674017caac63ab08de Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 17:12:54 +1200 Subject: [PATCH 58/61] Reformat updates nested lists --- eqcorrscan/doc/updates.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/eqcorrscan/doc/updates.rst b/eqcorrscan/doc/updates.rst index cc1e9e3e4..9f25f4809 100644 --- a/eqcorrscan/doc/updates.rst +++ b/eqcorrscan/doc/updates.rst @@ -4,30 +4,47 @@ What's new Version 0.4.4 ------------- * core.match_filter + * Bug-fix: peak-cores could be defined twice in _group_detect through kwargs. Fix: only update peak_cores if it isn't there already. + * core.match_filter.tribe + * Detect now allows passing of pre-processed data + * core.match_filter.template + * Remove duplicate detections from overlapping windows using `._uniq()` + * core.lag_calc._xcorr_interp + * CC-interpolation replaced with resampling (more robust), old method deprecated. Use new method with use_new_resamp_method=True as **kwarg. + * core.lag_calc: + * Fixed bug where minimum CC defined via min_cc_from_mean_cc_factor was not set correctly for negative correlation sums. + * utils.correlate + * Fast Matched Filter now supported natively for FMF version >= 1.4.0 * Only full correlation stacks are returned now (e.g. where fewer than than the full number of channels are in the stack at the end of the stack, zeros are returned). + * utils.mag_calc.relative_magnitude + * fixed bug where S-picks / traces were used for relative-magnitude calculation against user's choice. * implemented full magnitude bias-correction for CC and SNR + * utils.mag_calc.relative_amplitude: + * returns dicts for SNR measurements + * utils.catalog_to_dd.write_correlations + * Fixed bug on execution of parallel execution. * Added parallel-options for catalog-dt measurements and for stream-preparation before cross correlation-dt measurements. @@ -36,11 +53,13 @@ Version 0.4.4 the old parallelization strategy across traces. * Now includes `all_horiz`-option that will correlate all matching horizontal channels no matter to which of these the S-pick is linking. + * utils.clustering * Allow to handle indirect comparison of event-waveforms when (i.e., events without matching traces which can be compared indirectly via a third event) * Allows to set clustering method, metric, and sort_order from scipy.cluster.hierarchy.linkage. + * tribe, template, template_gen, archive_read, clustering: remove option to read from seishub (deprecated in obspy). From 63c996e0185ba973ded770b2e5991617c2d9e04c Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 17:19:18 +1200 Subject: [PATCH 59/61] Reformat updates nested lists --- eqcorrscan/doc/updates.rst | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/eqcorrscan/doc/updates.rst b/eqcorrscan/doc/updates.rst index 9f25f4809..f0442fd6e 100644 --- a/eqcorrscan/doc/updates.rst +++ b/eqcorrscan/doc/updates.rst @@ -55,6 +55,7 @@ Version 0.4.4 channels no matter to which of these the S-pick is linking. * utils.clustering + * Allow to handle indirect comparison of event-waveforms when (i.e., events without matching traces which can be compared indirectly via a third event) * Allows to set clustering method, metric, and sort_order from @@ -66,62 +67,89 @@ Version 0.4.4 Version 0.4.3 ------------- * core.match_filter + - match_filter: + - Provide option of exporting the cross-correlation sums for additional later analysis. + * core.match_filter.party.write + - BUG-FIX: When `format='tar'` is selected, added a check for .tgz-file suffix before checking the filename against an existing file. Previously, when a filename without '.tgz'-suffix was supplied, then the file was overwritten against the function's intention. - Add option `overwrite=True` to allow overwriting of existing files. + * core.match_filter.party.read + - BUG-FIX: Ensure wildcard reading works as expected: #453 + * core.match_filter.party.rethreshold: + - added option to rethreshold based on absolute values to keep relevant detections with large negative detect_val. + * core.lag_calc: + - Added option to set minimum CC threshold individually for detections based on: min(detect_val / n_chans * min_cc_from_mean_cc_factor, min_cc). - Added the ability of saving correlation data of the lag_calc. + * core.template_gen: + - Added support for generating templates from any object with a get_waveforms method. See #459. + * utils.mag_calc.calc_b_value: + - Added useful information to doc-string regarding method and meaning of residuals - Changed the number of magnitudes used to an int (from a string!?) + * utils.mag_calc.relative_magnitude: + - Refactor so that `min_cc` is used regardless of whether `weight_by_correlation` is set. See issue #455. + * utils.archive_read + - Add support for wildcard-comparisons in the list of requested stations and channels. - New option `arctype='SDS'` to read from a SeisComp Data Structure (SDS). This option is also available in `utils.clustering.extract_detections` and in `utils.archive_read._check_available_data`. + * utils.catalog_to_dd + - Bug-fixes in #424: + - only P and S phases are used now (previously spurious amplitude picks were included in correlations); - Checks for length are done prior to correlations and more helpful error outputs are provided. - Progress is not reported within dt.cc computation + - `write_station` now supports writing elevations: #424. + * utils.clustering + - For `cluster`, `distance_matrix` and `cross_chan_correlation`, implemented full support for `shift_len != 0`. The latter two functions now return, in addition to the distance-matrix, a shift-matrix (both functions) and a shift-dictionary (for `distance_matrix`). New option for shifting streams as a whole or letting traces shift individually (`allow_individual_trace_shifts=True`). + * utils.plotting + - Function added (twoD_seismplot) for plotting seismicity (#365). Version 0.4.2 ------------- * Add seed-ids to the _spike_test's message. * utils.correlation + - Cross-correlation normalisation errors no-longer raise an error - When "out-of-range" correlations occur a warning is given by the C-function with details of what channel, what template and where in the data vector @@ -131,44 +159,61 @@ Version 0.4.2 within regions where correlations should not be computed (spikes, step artifacts due to incorrectly padding data gaps). - USERS SHOULD BE CAREFUL TO CHECK THEIR DATA IF THEY SEE THESE WARNINGS + * utils.mag_calc.amp_pick_event + - Added option to output IASPEI standard amplitudes, with static amplification of 1 (rather than 2080 as per Wood Anderson specs). - Added `filter_id` and `method_id` to amplitudes to make these methods more traceable. + * core.match_filter + - Bug-fix - cope with data that are too short with `ignore_bad_data=True`. This flag is generally not advised, but when used, may attempt to trim all data to zero length. The expected behaviour is to remove bad data and run with the remaining data. - Party: + - decluster now accepts a hypocentral_separation argument. This allows the inclusion of detections that occur close in time, but not in space. This is underwritten by a new findpeaks.decluster_dist_time function based on a new C-function. + - Tribe: + - Add monkey-patching for clients that do not have a `get_waveforms_bulk` method for use in `.client_detect`. See issue #394. + * utils.pre_processing + - Only templates that need to be reshaped are reshaped now - this can be a lot faster. Version 0.4.1 ------------- * core.match_filter + - BUG-FIX: Empty families are no longer run through lag-calc when using Party.lag_calc(). Previously this resulted in a "No matching data" error, see #341. + * core.template_gen + - BUG-FIX: Fix bug where events were incorrectly associated with templates in `Tribe().construct()` if the given catalog contained events outside of the time-range of the stream. See issue #381 and PR #382. + * utils.catalog_to_dd + - Added ability to turn off parallel processing (this is turned off by default now) for `write_correlations` - parallel processing for moderate to large datasets was copying far too much data and using lots of memory. This is a short-term fix - ideally we will move filtering and resampling to C functions with shared-memory parallelism and GIL releasing. See PR #374. - Moved parallelism for `_compute_dt_correlations` to the C functions to reduce memory overhead. Using a generator to construct sub-catalogs rather than making a list of lists in memory. See issue #361. + * utils.mag_calc: + - `amp_pick_event` now works on a copy of the data by default - `amp_pick_event` uses the appropriate digital filter gain to correct the applied filter. See issue #376. - `amp_pick_event` rewritten for simplicity. - `amp_pick_event` now has simple synthetic tests for accuracy. - `_sim_wa` uses the full response information to correct to velocity this includes FIR filters (previously not used), and ensures that the wood-anderson poles (with a single zero) are correctly applied to velocity waveforms. - `calc_max_curv` is now computed using the non-cumulative distribution. + * Some problem solved in _match_filter_plot. Now it shows all new detections. * Add plotdir to eqcorrscan.core.lag_calc.lag_calc function to save the images. @@ -179,12 +224,15 @@ Version 0.4.0 alleviate issue related to large-prime length transforms. This requires an additional dependency, but EQcorrscan already depends on FFTW itself (#316). * Refactor of catalog_to_dd functions (#322): + - Speed-ups, using new correlation functions and better resource management - Removed enforcement of seisan, arguments are now standard obspy objects. + * Add plotdir to lag-calc, template construction and matched-filter detection methods and functions (#330, #325). * Wholesale re-write of lag-calc function and methods. External interface is similar, but some arguments have been depreciated as they were unnecessary (#321). + - This was done to make use of the new internal correlation functions which are faster and more memory efficient. - Party.lag_calc and Family.lag_calc now work in-place on the events in @@ -193,6 +241,7 @@ Version 0.4.0 lag-calc to avoid reprocessing data. - Added lag_calc.xcorr_pick_family as a public facing API to implement correlation re-picking of a group of events. + * Renamed utils.clustering.cross_chan_coherence to utils.clustering.cross_chan_correlation to better reflect what it actually does. @@ -213,9 +262,11 @@ Version 0.4.0 an argument for function calls. * `find-peaks` replaced by compiled peak finding routine - more efficient both in memory and time #249 - approx 50x faster + - Note that the results of the C-func and the Python functions are slightly different. The C function (now the default) is more stable when peaks are small and close together (e.g. in noisy data). + * multi-find peaks makes use of openMP parallelism for more efficient memory usage #249 * enforce normalization of continuous data before correlation to avoid float32 @@ -233,10 +284,12 @@ Version 0.4.0 massive and was slow to load and process in IDEs. * Refactored `_prep_data_for_correlation` to reduce looping for speed, now approximately six times faster than previously (minor speed-up) + - Now explicitly doesn't allow templates with different length traces - previously this was ignored and templates with different length channels to other templates had their channels padded with zeros or trimmed. + * Add `skip_short_channels` option to template generation. This allows users to provide data of unknown length and short channels will not be used, rather than generating an error. This is useful for downloading data from @@ -333,10 +386,12 @@ Version 0.3.2 trends - variance threshold is now raised, and Python checks for low-variance and applies gain to stabilise correlations if needed. * Plotting functions are now tested and have a more consistent interface: + - All plotting functions accept the keyword arguments `save`, `savefile`, `show`, `return_figure` and `title`. - All plotting functions return a figure. - `SVD_plot` renamed to `svd_plot` + * Enforce pre-processing even when no filters or resampling is to be done to ensure gaps are properly processed (when called from `Tribe.detect`, `Template.detect` or `Tribe.client_detect`) @@ -344,8 +399,10 @@ Version 0.3.2 one sample too long resulting in minor differences in data processing (due to difference in FFT length) and therefore minor differences in resulting correlations (~0.07 per channel). + - Includes extra stability check in fftw_normxcorr which affects the last sample before a gap when that sample is near-zero. + * BUG-FIX: fftw correlation dot product was not thread-safe on some systems. The dot-product did not have the inner index protected as a private variable. This did not appear to cause issues for Linux with Python 3.x or Windows, but @@ -410,10 +467,12 @@ Version 0.3.0 to obspy for obspy version 1.1.0. All functions are there and many bugs have been fixed. This also means the removal of nordic-specific functions in EQcorrscan - the following functions have been removed: + - template_gen.from_sfile - template_gen.from_contbase - mag_calc.amp_pick_sfile - mag_calc.pick_db + All removed functions will error and tell you to use obspy.io.nordic.core. This now means that you can use obspy's `read_events` to read in sfiles. * Added `P_all` and `S_all` options to template generation functions @@ -436,17 +495,21 @@ Version 0.2.6 methods through the parameter xcorr_func of match_filter, Template.detect and Tribe.detect, or using the set_xcorr context manager in the utils.correlate module. Supported options are: + - numpy - fftw - time-domain - or passing a function that implements the xcorr interface. + * Added the ability to change the concurrency strategy of xcorr functions using the paramter concurrency of match_filter, Template.detect and Tribe.detect. Supported options are: + - None - for single-threaded execution in a single process - multithread - for multi-threaded execution - multiprocess- for multiprocess execution - concurrent - allows functions to describe their own preferred currency methods, defaults to multithread + * Change debug printing output, it should be a little quieter; * Speed-up time-domain using a threaded C-routine - separate from frequency domain C-routines; @@ -499,6 +562,7 @@ Version 0.2.4 * Rename SVD_moments to lower-case and add depreciation warning; * Increase test coverage in utils.mag_calc; * Add Template, Tribe, Family, Party objects and rename DETECTION to Detection + - Template objects maintain meta-data associated with their creation to stream-line processing of data (e.g. reduce chance of using the wrong filters). - Template events have a detect method which takes unprocessed data and does the correct processing using the Template meta-data, and computes the matched-filter detections. - Tribe objects are containers for multiple Templates. @@ -509,6 +573,7 @@ Version 0.2.4 - The upshot of this is that it is possible to, in one line, generate a Tribe of templates, compute their matched-filter detections, and generate cross-correlation pick refinements, which output Event objects, which can be written to a catalog: Tribe.construct(method, **kwargs).detect(st, **kwargs).lag_calc(**kwargs).write() - Added 25 tests for these methods. - Add parameters *threshold_type* and *threshold_input* to Detection class. Add support for legacy Detection objects via NaN and unset values. + * Removed support for obspy < 1.0.0 * Update / correct doc-strings in template-gen functions when describing processing parameters. @@ -521,6 +586,7 @@ Version 0.2.4 * Fix bug in detection_multiplot which didn't allow streams with fewer traces than template; * Update internals to custom C fftw-based correlation rather than openCV (Major change); + * OpenCV has been removed as a dependancy; * eqcorrscan.core.match_filter.normxcorr2 now calls a compiled C routine; * Parallel workflows handled by openMP rather than Python Multiprocessing for matched-filter operations to allow better memory handling. From 42cce1211dabbce950be9d66991451b203a24911 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 17:25:11 +1200 Subject: [PATCH 60/61] Reformat updates nested lists --- eqcorrscan/doc/updates.rst | 156 ++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/eqcorrscan/doc/updates.rst b/eqcorrscan/doc/updates.rst index f0442fd6e..9d3625982 100644 --- a/eqcorrscan/doc/updates.rst +++ b/eqcorrscan/doc/updates.rst @@ -68,73 +68,73 @@ Version 0.4.3 ------------- * core.match_filter - - match_filter: + * match_filter: - - Provide option of exporting the cross-correlation sums for additional later + * Provide option of exporting the cross-correlation sums for additional later analysis. * core.match_filter.party.write - - BUG-FIX: When `format='tar'` is selected, added a check for .tgz-file + * BUG-FIX: When `format='tar'` is selected, added a check for .tgz-file suffix before checking the filename against an existing file. Previously, when a filename without '.tgz'-suffix was supplied, then the file was overwritten against the function's intention. - - Add option `overwrite=True` to allow overwriting of existing files. + * Add option `overwrite=True` to allow overwriting of existing files. * core.match_filter.party.read - - BUG-FIX: Ensure wildcard reading works as expected: #453 + * BUG-FIX: Ensure wildcard reading works as expected: #453 * core.match_filter.party.rethreshold: - - added option to rethreshold based on absolute values to keep relevant + * added option to rethreshold based on absolute values to keep relevant detections with large negative detect_val. * core.lag_calc: - - Added option to set minimum CC threshold individually for detections based + * Added option to set minimum CC threshold individually for detections based on: min(detect_val / n_chans * min_cc_from_mean_cc_factor, min_cc). - - Added the ability of saving correlation data of the lag_calc. + * Added the ability of saving correlation data of the lag_calc. * core.template_gen: - - Added support for generating templates from any object with a + * Added support for generating templates from any object with a get_waveforms method. See #459. * utils.mag_calc.calc_b_value: - - Added useful information to doc-string regarding method and meaning of + * Added useful information to doc-string regarding method and meaning of residuals - - Changed the number of magnitudes used to an int (from a string!?) + * Changed the number of magnitudes used to an int (from a string!?) * utils.mag_calc.relative_magnitude: - - Refactor so that `min_cc` is used regardless of whether + * Refactor so that `min_cc` is used regardless of whether `weight_by_correlation` is set. See issue #455. * utils.archive_read - - Add support for wildcard-comparisons in the list of requested stations and + * Add support for wildcard-comparisons in the list of requested stations and channels. - - New option `arctype='SDS'` to read from a SeisComp Data Structure (SDS). + * New option `arctype='SDS'` to read from a SeisComp Data Structure (SDS). This option is also available in `utils.clustering.extract_detections` and in `utils.archive_read._check_available_data`. * utils.catalog_to_dd - - Bug-fixes in #424: + * Bug-fixes in #424: - - only P and S phases are used now (previously spurious amplitude picks + * only P and S phases are used now (previously spurious amplitude picks were included in correlations); - - Checks for length are done prior to correlations and more helpful error + * Checks for length are done prior to correlations and more helpful error outputs are provided. - - Progress is not reported within dt.cc computation + * Progress is not reported within dt.cc computation - - `write_station` now supports writing elevations: #424. + * `write_station` now supports writing elevations: #424. * utils.clustering - - For `cluster`, `distance_matrix` and `cross_chan_correlation`, implemented + * For `cluster`, `distance_matrix` and `cross_chan_correlation`, implemented full support for `shift_len != 0`. The latter two functions now return, in addition to the distance-matrix, a shift-matrix (both functions) and a shift-dictionary (for `distance_matrix`). New option for shifting streams @@ -143,76 +143,76 @@ Version 0.4.3 * utils.plotting - - Function added (twoD_seismplot) for plotting seismicity (#365). + * Function added (twoD_seismplot) for plotting seismicity (#365). Version 0.4.2 ------------- * Add seed-ids to the _spike_test's message. * utils.correlation - - Cross-correlation normalisation errors no-longer raise an error - - When "out-of-range" correlations occur a warning is given by the C-function + * Cross-correlation normalisation errors no-longer raise an error + * When "out-of-range" correlations occur a warning is given by the C-function with details of what channel, what template and where in the data vector the issue occurred for the user to check their data. - - Out-of-range correlations are set to 0.0 - - After extensive testing these errors have always been related to data issues + * Out-of-range correlations are set to 0.0 + * After extensive testing these errors have always been related to data issues within regions where correlations should not be computed (spikes, step artifacts due to incorrectly padding data gaps). - - USERS SHOULD BE CAREFUL TO CHECK THEIR DATA IF THEY SEE THESE WARNINGS + * USERS SHOULD BE CAREFUL TO CHECK THEIR DATA IF THEY SEE THESE WARNINGS * utils.mag_calc.amp_pick_event - - Added option to output IASPEI standard amplitudes, with static amplification + * Added option to output IASPEI standard amplitudes, with static amplification of 1 (rather than 2080 as per Wood Anderson specs). - - Added `filter_id` and `method_id` to amplitudes to make these methods more + * Added `filter_id` and `method_id` to amplitudes to make these methods more traceable. * core.match_filter - - Bug-fix - cope with data that are too short with `ignore_bad_data=True`. + * Bug-fix - cope with data that are too short with `ignore_bad_data=True`. This flag is generally not advised, but when used, may attempt to trim all data to zero length. The expected behaviour is to remove bad data and run with the remaining data. - - Party: + * Party: - - decluster now accepts a hypocentral_separation argument. This allows + * decluster now accepts a hypocentral_separation argument. This allows the inclusion of detections that occur close in time, but not in space. This is underwritten by a new findpeaks.decluster_dist_time function based on a new C-function. - - Tribe: + * Tribe: - - Add monkey-patching for clients that do not have a `get_waveforms_bulk` + * Add monkey-patching for clients that do not have a `get_waveforms_bulk` method for use in `.client_detect`. See issue #394. * utils.pre_processing - - Only templates that need to be reshaped are reshaped now - this can be a lot + * Only templates that need to be reshaped are reshaped now - this can be a lot faster. Version 0.4.1 ------------- * core.match_filter - - BUG-FIX: Empty families are no longer run through lag-calc when using Party.lag_calc(). Previously this resulted in a "No matching data" error, see #341. + * BUG-FIX: Empty families are no longer run through lag-calc when using Party.lag_calc(). Previously this resulted in a "No matching data" error, see #341. * core.template_gen - - BUG-FIX: Fix bug where events were incorrectly associated with templates in `Tribe().construct()` if the given catalog contained events outside of the time-range of the stream. See issue #381 and PR #382. + * BUG-FIX: Fix bug where events were incorrectly associated with templates in `Tribe().construct()` if the given catalog contained events outside of the time-range of the stream. See issue #381 and PR #382. * utils.catalog_to_dd - - Added ability to turn off parallel processing (this is turned off by default now) for `write_correlations` - parallel processing for moderate to large datasets was copying far too much data and using lots of memory. This is a short-term fix - ideally we will move filtering and resampling to C functions with shared-memory parallelism and GIL releasing. See PR #374. - - Moved parallelism for `_compute_dt_correlations` to the C functions to reduce memory overhead. Using a generator to construct sub-catalogs rather than making a list of lists in memory. See issue #361. + * Added ability to turn off parallel processing (this is turned off by default now) for `write_correlations` - parallel processing for moderate to large datasets was copying far too much data and using lots of memory. This is a short-term fix - ideally we will move filtering and resampling to C functions with shared-memory parallelism and GIL releasing. See PR #374. + * Moved parallelism for `_compute_dt_correlations` to the C functions to reduce memory overhead. Using a generator to construct sub-catalogs rather than making a list of lists in memory. See issue #361. * utils.mag_calc: - - `amp_pick_event` now works on a copy of the data by default - - `amp_pick_event` uses the appropriate digital filter gain to correct the applied filter. See issue #376. - - `amp_pick_event` rewritten for simplicity. - - `amp_pick_event` now has simple synthetic tests for accuracy. - - `_sim_wa` uses the full response information to correct to velocity this includes FIR filters (previously not used), and ensures that the wood-anderson poles (with a single zero) are correctly applied to velocity waveforms. - - `calc_max_curv` is now computed using the non-cumulative distribution. + * `amp_pick_event` now works on a copy of the data by default + * `amp_pick_event` uses the appropriate digital filter gain to correct the applied filter. See issue #376. + * `amp_pick_event` rewritten for simplicity. + * `amp_pick_event` now has simple synthetic tests for accuracy. + * `_sim_wa` uses the full response information to correct to velocity this includes FIR filters (previously not used), and ensures that the wood-anderson poles (with a single zero) are correctly applied to velocity waveforms. + * `calc_max_curv` is now computed using the non-cumulative distribution. * Some problem solved in _match_filter_plot. Now it shows all new detections. * Add plotdir to eqcorrscan.core.lag_calc.lag_calc function to save the images. @@ -225,21 +225,21 @@ Version 0.4.0 additional dependency, but EQcorrscan already depends on FFTW itself (#316). * Refactor of catalog_to_dd functions (#322): - - Speed-ups, using new correlation functions and better resource management - - Removed enforcement of seisan, arguments are now standard obspy objects. + * Speed-ups, using new correlation functions and better resource management + * Removed enforcement of seisan, arguments are now standard obspy objects. * Add plotdir to lag-calc, template construction and matched-filter detection methods and functions (#330, #325). * Wholesale re-write of lag-calc function and methods. External interface is similar, but some arguments have been depreciated as they were unnecessary (#321). - - This was done to make use of the new internal correlation functions which + * This was done to make use of the new internal correlation functions which are faster and more memory efficient. - - Party.lag_calc and Family.lag_calc now work in-place on the events in + * Party.lag_calc and Family.lag_calc now work in-place on the events in the grouping. - - Added relative_mags method to Party and Family; this can be called from + * Added relative_mags method to Party and Family; this can be called from lag-calc to avoid reprocessing data. - - Added lag_calc.xcorr_pick_family as a public facing API to implement + * Added lag_calc.xcorr_pick_family as a public facing API to implement correlation re-picking of a group of events. * Renamed utils.clustering.cross_chan_coherence to @@ -263,7 +263,7 @@ Version 0.4.0 * `find-peaks` replaced by compiled peak finding routine - more efficient both in memory and time #249 - approx 50x faster - - Note that the results of the C-func and the Python functions are slightly + * Note that the results of the C-func and the Python functions are slightly different. The C function (now the default) is more stable when peaks are small and close together (e.g. in noisy data). @@ -285,7 +285,7 @@ Version 0.4.0 * Refactored `_prep_data_for_correlation` to reduce looping for speed, now approximately six times faster than previously (minor speed-up) - - Now explicitly doesn't allow templates with different length traces - + * Now explicitly doesn't allow templates with different length traces - previously this was ignored and templates with different length channels to other templates had their channels padded with zeros or trimmed. @@ -387,10 +387,10 @@ Version 0.3.2 and applies gain to stabilise correlations if needed. * Plotting functions are now tested and have a more consistent interface: - - All plotting functions accept the keyword arguments `save`, `savefile`, + * All plotting functions accept the keyword arguments `save`, `savefile`, `show`, `return_figure` and `title`. - - All plotting functions return a figure. - - `SVD_plot` renamed to `svd_plot` + * All plotting functions return a figure. + * `SVD_plot` renamed to `svd_plot` * Enforce pre-processing even when no filters or resampling is to be done to ensure gaps are properly processed (when called from `Tribe.detect`, @@ -400,7 +400,7 @@ Version 0.3.2 (due to difference in FFT length) and therefore minor differences in resulting correlations (~0.07 per channel). - - Includes extra stability check in fftw_normxcorr which affects the + * Includes extra stability check in fftw_normxcorr which affects the last sample before a gap when that sample is near-zero. * BUG-FIX: fftw correlation dot product was not thread-safe on some systems. @@ -468,10 +468,10 @@ Version 0.3.0 been fixed. This also means the removal of nordic-specific functions in EQcorrscan - the following functions have been removed: - - template_gen.from_sfile - - template_gen.from_contbase - - mag_calc.amp_pick_sfile - - mag_calc.pick_db + * template_gen.from_sfile + * template_gen.from_contbase + * mag_calc.amp_pick_sfile + * mag_calc.pick_db All removed functions will error and tell you to use obspy.io.nordic.core. This now means that you can use obspy's `read_events` to read in sfiles. @@ -496,19 +496,19 @@ Version 0.2.6 and Tribe.detect, or using the set_xcorr context manager in the utils.correlate module. Supported options are: - - numpy - - fftw - - time-domain - - or passing a function that implements the xcorr interface. + * numpy + * fftw + * time-domain + * or passing a function that implements the xcorr interface. * Added the ability to change the concurrency strategy of xcorr functions using the paramter concurrency of match_filter, Template.detect and Tribe.detect. Supported options are: - - None - for single-threaded execution in a single process - - multithread - for multi-threaded execution - - multiprocess- for multiprocess execution - - concurrent - allows functions to describe their own preferred currency methods, defaults to multithread + * None - for single-threaded execution in a single process + * multithread - for multi-threaded execution + * multiprocess- for multiprocess execution + * concurrent - allows functions to describe their own preferred currency methods, defaults to multithread * Change debug printing output, it should be a little quieter; * Speed-up time-domain using a threaded C-routine - separate from frequency @@ -563,16 +563,16 @@ Version 0.2.4 * Increase test coverage in utils.mag_calc; * Add Template, Tribe, Family, Party objects and rename DETECTION to Detection - - Template objects maintain meta-data associated with their creation to stream-line processing of data (e.g. reduce chance of using the wrong filters). - - Template events have a detect method which takes unprocessed data and does the correct processing using the Template meta-data, and computes the matched-filter detections. - - Tribe objects are containers for multiple Templates. - - Tribe objects have a detect method which groups Templates with similar meta-data (processing information) and runs these templates in parallel through the matched-filter routine. Tribe.detect outputs a Party of Family objects. - - The Party object is a container for many Family objects. - - Family objects are containers for detections from the same Template. - - Family and Party objects have a lag_calc method which computes the cross-correlation pick-refinements. - - The upshot of this is that it is possible to, in one line, generate a Tribe of templates, compute their matched-filter detections, and generate cross-correlation pick refinements, which output Event objects, which can be written to a catalog: Tribe.construct(method, **kwargs).detect(st, **kwargs).lag_calc(**kwargs).write() - - Added 25 tests for these methods. - - Add parameters *threshold_type* and *threshold_input* to Detection class. Add support for legacy Detection objects via NaN and unset values. + * Template objects maintain meta-data associated with their creation to stream-line processing of data (e.g. reduce chance of using the wrong filters). + * Template events have a detect method which takes unprocessed data and does the correct processing using the Template meta-data, and computes the matched-filter detections. + * Tribe objects are containers for multiple Templates. + * Tribe objects have a detect method which groups Templates with similar meta-data (processing information) and runs these templates in parallel through the matched-filter routine. Tribe.detect outputs a Party of Family objects. + * The Party object is a container for many Family objects. + * Family objects are containers for detections from the same Template. + * Family and Party objects have a lag_calc method which computes the cross-correlation pick-refinements. + * The upshot of this is that it is possible to, in one line, generate a Tribe of templates, compute their matched-filter detections, and generate cross-correlation pick refinements, which output Event objects, which can be written to a catalog: Tribe.construct(method, **kwargs).detect(st, **kwargs).lag_calc(**kwargs).write() + * Added 25 tests for these methods. + * Add parameters *threshold_type* and *threshold_input* to Detection class. Add support for legacy Detection objects via NaN and unset values. * Removed support for obspy < 1.0.0 * Update / correct doc-strings in template-gen functions when describing From 1fd2a12646ba5ecbe89226e60a3699cc6b8ca3a0 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Aug 2022 17:38:01 +1200 Subject: [PATCH 61/61] Reformat updates nested lists --- eqcorrscan/doc/updates.rst | 245 +++++++++++++++++++------------------ 1 file changed, 123 insertions(+), 122 deletions(-) diff --git a/eqcorrscan/doc/updates.rst b/eqcorrscan/doc/updates.rst index 9d3625982..afb3ef523 100644 --- a/eqcorrscan/doc/updates.rst +++ b/eqcorrscan/doc/updates.rst @@ -68,151 +68,152 @@ Version 0.4.3 ------------- * core.match_filter - * match_filter: + * match_filter: + + * Provide option of exporting the cross-correlation sums for additional later + analysis. - * Provide option of exporting the cross-correlation sums for additional later - analysis. * core.match_filter.party.write - * BUG-FIX: When `format='tar'` is selected, added a check for .tgz-file - suffix before checking the filename against an existing file. Previously, - when a filename without '.tgz'-suffix was supplied, then the file was - overwritten against the function's intention. - * Add option `overwrite=True` to allow overwriting of existing files. + * BUG-FIX: When `format='tar'` is selected, added a check for .tgz-file + suffix before checking the filename against an existing file. Previously, + when a filename without '.tgz'-suffix was supplied, then the file was + overwritten against the function's intention. + * Add option `overwrite=True` to allow overwriting of existing files. * core.match_filter.party.read - * BUG-FIX: Ensure wildcard reading works as expected: #453 + * BUG-FIX: Ensure wildcard reading works as expected: #453 * core.match_filter.party.rethreshold: - * added option to rethreshold based on absolute values to keep relevant - detections with large negative detect_val. + * added option to rethreshold based on absolute values to keep relevant + detections with large negative detect_val. * core.lag_calc: - * Added option to set minimum CC threshold individually for detections based - on: min(detect_val / n_chans * min_cc_from_mean_cc_factor, min_cc). - * Added the ability of saving correlation data of the lag_calc. + * Added option to set minimum CC threshold individually for detections based + on: min(detect_val / n_chans * min_cc_from_mean_cc_factor, min_cc). + * Added the ability of saving correlation data of the lag_calc. * core.template_gen: - * Added support for generating templates from any object with a - get_waveforms method. See #459. + * Added support for generating templates from any object with a + get_waveforms method. See #459. * utils.mag_calc.calc_b_value: - * Added useful information to doc-string regarding method and meaning of - residuals - * Changed the number of magnitudes used to an int (from a string!?) + * Added useful information to doc-string regarding method and meaning of + residuals + * Changed the number of magnitudes used to an int (from a string!?) * utils.mag_calc.relative_magnitude: - * Refactor so that `min_cc` is used regardless of whether - `weight_by_correlation` is set. See issue #455. + * Refactor so that `min_cc` is used regardless of whether + `weight_by_correlation` is set. See issue #455. * utils.archive_read - * Add support for wildcard-comparisons in the list of requested stations and - channels. - * New option `arctype='SDS'` to read from a SeisComp Data Structure (SDS). - This option is also available in `utils.clustering.extract_detections` and - in `utils.archive_read._check_available_data`. + * Add support for wildcard-comparisons in the list of requested stations and + channels. + * New option `arctype='SDS'` to read from a SeisComp Data Structure (SDS). + This option is also available in `utils.clustering.extract_detections` and + in `utils.archive_read._check_available_data`. * utils.catalog_to_dd - * Bug-fixes in #424: + * Bug-fixes in #424: - * only P and S phases are used now (previously spurious amplitude picks - were included in correlations); - * Checks for length are done prior to correlations and more helpful error - outputs are provided. - * Progress is not reported within dt.cc computation + * only P and S phases are used now (previously spurious amplitude picks + were included in correlations); + * Checks for length are done prior to correlations and more helpful error + outputs are provided. + * Progress is not reported within dt.cc computation - * `write_station` now supports writing elevations: #424. + * `write_station` now supports writing elevations: #424. * utils.clustering - * For `cluster`, `distance_matrix` and `cross_chan_correlation`, implemented - full support for `shift_len != 0`. The latter two functions now return, in - addition to the distance-matrix, a shift-matrix (both functions) and a - shift-dictionary (for `distance_matrix`). New option for shifting streams - as a whole or letting traces shift individually - (`allow_individual_trace_shifts=True`). + * For `cluster`, `distance_matrix` and `cross_chan_correlation`, implemented + full support for `shift_len != 0`. The latter two functions now return, in + addition to the distance-matrix, a shift-matrix (both functions) and a + shift-dictionary (for `distance_matrix`). New option for shifting streams + as a whole or letting traces shift individually + (`allow_individual_trace_shifts=True`). * utils.plotting - * Function added (twoD_seismplot) for plotting seismicity (#365). + * Function added (twoD_seismplot) for plotting seismicity (#365). Version 0.4.2 ------------- * Add seed-ids to the _spike_test's message. * utils.correlation - * Cross-correlation normalisation errors no-longer raise an error - * When "out-of-range" correlations occur a warning is given by the C-function - with details of what channel, what template and where in the data vector - the issue occurred for the user to check their data. - * Out-of-range correlations are set to 0.0 - * After extensive testing these errors have always been related to data issues - within regions where correlations should not be computed (spikes, step - artifacts due to incorrectly padding data gaps). - * USERS SHOULD BE CAREFUL TO CHECK THEIR DATA IF THEY SEE THESE WARNINGS + * Cross-correlation normalisation errors no-longer raise an error + * When "out-of-range" correlations occur a warning is given by the C-function + with details of what channel, what template and where in the data vector + the issue occurred for the user to check their data. + * Out-of-range correlations are set to 0.0 + * After extensive testing these errors have always been related to data issues + within regions where correlations should not be computed (spikes, step + artifacts due to incorrectly padding data gaps). + * USERS SHOULD BE CAREFUL TO CHECK THEIR DATA IF THEY SEE THESE WARNINGS * utils.mag_calc.amp_pick_event - * Added option to output IASPEI standard amplitudes, with static amplification - of 1 (rather than 2080 as per Wood Anderson specs). - * Added `filter_id` and `method_id` to amplitudes to make these methods more - traceable. + * Added option to output IASPEI standard amplitudes, with static amplification + of 1 (rather than 2080 as per Wood Anderson specs). + * Added `filter_id` and `method_id` to amplitudes to make these methods more + traceable. * core.match_filter - * Bug-fix - cope with data that are too short with `ignore_bad_data=True`. - This flag is generally not advised, but when used, may attempt to trim all - data to zero length. The expected behaviour is to remove bad data and run - with the remaining data. - * Party: + * Bug-fix - cope with data that are too short with `ignore_bad_data=True`. + This flag is generally not advised, but when used, may attempt to trim all + data to zero length. The expected behaviour is to remove bad data and run + with the remaining data. + * Party: - * decluster now accepts a hypocentral_separation argument. This allows - the inclusion of detections that occur close in time, but not in space. - This is underwritten by a new findpeaks.decluster_dist_time function - based on a new C-function. + * decluster now accepts a hypocentral_separation argument. This allows + the inclusion of detections that occur close in time, but not in space. + This is underwritten by a new findpeaks.decluster_dist_time function + based on a new C-function. - * Tribe: + * Tribe: - * Add monkey-patching for clients that do not have a `get_waveforms_bulk` - method for use in `.client_detect`. See issue #394. + * Add monkey-patching for clients that do not have a `get_waveforms_bulk` + method for use in `.client_detect`. See issue #394. * utils.pre_processing - * Only templates that need to be reshaped are reshaped now - this can be a lot - faster. + * Only templates that need to be reshaped are reshaped now - this can be a lot + faster. Version 0.4.1 ------------- * core.match_filter - * BUG-FIX: Empty families are no longer run through lag-calc when using Party.lag_calc(). Previously this resulted in a "No matching data" error, see #341. + * BUG-FIX: Empty families are no longer run through lag-calc when using Party.lag_calc(). Previously this resulted in a "No matching data" error, see #341. * core.template_gen - * BUG-FIX: Fix bug where events were incorrectly associated with templates in `Tribe().construct()` if the given catalog contained events outside of the time-range of the stream. See issue #381 and PR #382. + * BUG-FIX: Fix bug where events were incorrectly associated with templates in `Tribe().construct()` if the given catalog contained events outside of the time-range of the stream. See issue #381 and PR #382. * utils.catalog_to_dd - * Added ability to turn off parallel processing (this is turned off by default now) for `write_correlations` - parallel processing for moderate to large datasets was copying far too much data and using lots of memory. This is a short-term fix - ideally we will move filtering and resampling to C functions with shared-memory parallelism and GIL releasing. See PR #374. - * Moved parallelism for `_compute_dt_correlations` to the C functions to reduce memory overhead. Using a generator to construct sub-catalogs rather than making a list of lists in memory. See issue #361. + * Added ability to turn off parallel processing (this is turned off by default now) for `write_correlations` - parallel processing for moderate to large datasets was copying far too much data and using lots of memory. This is a short-term fix - ideally we will move filtering and resampling to C functions with shared-memory parallelism and GIL releasing. See PR #374. + * Moved parallelism for `_compute_dt_correlations` to the C functions to reduce memory overhead. Using a generator to construct sub-catalogs rather than making a list of lists in memory. See issue #361. * utils.mag_calc: - * `amp_pick_event` now works on a copy of the data by default - * `amp_pick_event` uses the appropriate digital filter gain to correct the applied filter. See issue #376. - * `amp_pick_event` rewritten for simplicity. - * `amp_pick_event` now has simple synthetic tests for accuracy. - * `_sim_wa` uses the full response information to correct to velocity this includes FIR filters (previously not used), and ensures that the wood-anderson poles (with a single zero) are correctly applied to velocity waveforms. - * `calc_max_curv` is now computed using the non-cumulative distribution. + * `amp_pick_event` now works on a copy of the data by default + * `amp_pick_event` uses the appropriate digital filter gain to correct the applied filter. See issue #376. + * `amp_pick_event` rewritten for simplicity. + * `amp_pick_event` now has simple synthetic tests for accuracy. + * `_sim_wa` uses the full response information to correct to velocity this includes FIR filters (previously not used), and ensures that the wood-anderson poles (with a single zero) are correctly applied to velocity waveforms. + * `calc_max_curv` is now computed using the non-cumulative distribution. * Some problem solved in _match_filter_plot. Now it shows all new detections. * Add plotdir to eqcorrscan.core.lag_calc.lag_calc function to save the images. @@ -225,22 +226,22 @@ Version 0.4.0 additional dependency, but EQcorrscan already depends on FFTW itself (#316). * Refactor of catalog_to_dd functions (#322): - * Speed-ups, using new correlation functions and better resource management - * Removed enforcement of seisan, arguments are now standard obspy objects. + * Speed-ups, using new correlation functions and better resource management + * Removed enforcement of seisan, arguments are now standard obspy objects. * Add plotdir to lag-calc, template construction and matched-filter detection methods and functions (#330, #325). * Wholesale re-write of lag-calc function and methods. External interface is similar, but some arguments have been depreciated as they were unnecessary (#321). - * This was done to make use of the new internal correlation functions which - are faster and more memory efficient. - * Party.lag_calc and Family.lag_calc now work in-place on the events in - the grouping. - * Added relative_mags method to Party and Family; this can be called from - lag-calc to avoid reprocessing data. - * Added lag_calc.xcorr_pick_family as a public facing API to implement - correlation re-picking of a group of events. + * This was done to make use of the new internal correlation functions which + are faster and more memory efficient. + * Party.lag_calc and Family.lag_calc now work in-place on the events in + the grouping. + * Added relative_mags method to Party and Family; this can be called from + lag-calc to avoid reprocessing data. + * Added lag_calc.xcorr_pick_family as a public facing API to implement + correlation re-picking of a group of events. * Renamed utils.clustering.cross_chan_coherence to utils.clustering.cross_chan_correlation to better reflect what it actually @@ -263,9 +264,9 @@ Version 0.4.0 * `find-peaks` replaced by compiled peak finding routine - more efficient both in memory and time #249 - approx 50x faster - * Note that the results of the C-func and the Python functions are slightly - different. The C function (now the default) is more stable when peaks - are small and close together (e.g. in noisy data). + * Note that the results of the C-func and the Python functions are slightly + different. The C function (now the default) is more stable when peaks + are small and close together (e.g. in noisy data). * multi-find peaks makes use of openMP parallelism for more efficient memory usage #249 @@ -285,10 +286,10 @@ Version 0.4.0 * Refactored `_prep_data_for_correlation` to reduce looping for speed, now approximately six times faster than previously (minor speed-up) - * Now explicitly doesn't allow templates with different length traces - - previously this was ignored and templates with different length - channels to other templates had their channels padded with zeros or - trimmed. + * Now explicitly doesn't allow templates with different length traces - + previously this was ignored and templates with different length + channels to other templates had their channels padded with zeros or + trimmed. * Add `skip_short_channels` option to template generation. This allows users to provide data of unknown length and short channels will not be used, rather @@ -387,10 +388,10 @@ Version 0.3.2 and applies gain to stabilise correlations if needed. * Plotting functions are now tested and have a more consistent interface: - * All plotting functions accept the keyword arguments `save`, `savefile`, - `show`, `return_figure` and `title`. - * All plotting functions return a figure. - * `SVD_plot` renamed to `svd_plot` + * All plotting functions accept the keyword arguments `save`, `savefile`, + `show`, `return_figure` and `title`. + * All plotting functions return a figure. + * `SVD_plot` renamed to `svd_plot` * Enforce pre-processing even when no filters or resampling is to be done to ensure gaps are properly processed (when called from `Tribe.detect`, @@ -400,8 +401,8 @@ Version 0.3.2 (due to difference in FFT length) and therefore minor differences in resulting correlations (~0.07 per channel). - * Includes extra stability check in fftw_normxcorr which affects the - last sample before a gap when that sample is near-zero. + * Includes extra stability check in fftw_normxcorr which affects the + last sample before a gap when that sample is near-zero. * BUG-FIX: fftw correlation dot product was not thread-safe on some systems. The dot-product did not have the inner index protected as a private variable. @@ -468,10 +469,10 @@ Version 0.3.0 been fixed. This also means the removal of nordic-specific functions in EQcorrscan - the following functions have been removed: - * template_gen.from_sfile - * template_gen.from_contbase - * mag_calc.amp_pick_sfile - * mag_calc.pick_db + * template_gen.from_sfile + * template_gen.from_contbase + * mag_calc.amp_pick_sfile + * mag_calc.pick_db All removed functions will error and tell you to use obspy.io.nordic.core. This now means that you can use obspy's `read_events` to read in sfiles. @@ -496,19 +497,19 @@ Version 0.2.6 and Tribe.detect, or using the set_xcorr context manager in the utils.correlate module. Supported options are: - * numpy - * fftw - * time-domain - * or passing a function that implements the xcorr interface. + * numpy + * fftw + * time-domain + * or passing a function that implements the xcorr interface. * Added the ability to change the concurrency strategy of xcorr functions using the paramter concurrency of match_filter, Template.detect and Tribe.detect. Supported options are: - * None - for single-threaded execution in a single process - * multithread - for multi-threaded execution - * multiprocess- for multiprocess execution - * concurrent - allows functions to describe their own preferred currency methods, defaults to multithread + * None - for single-threaded execution in a single process + * multithread - for multi-threaded execution + * multiprocess- for multiprocess execution + * concurrent - allows functions to describe their own preferred currency methods, defaults to multithread * Change debug printing output, it should be a little quieter; * Speed-up time-domain using a threaded C-routine - separate from frequency @@ -563,16 +564,16 @@ Version 0.2.4 * Increase test coverage in utils.mag_calc; * Add Template, Tribe, Family, Party objects and rename DETECTION to Detection - * Template objects maintain meta-data associated with their creation to stream-line processing of data (e.g. reduce chance of using the wrong filters). - * Template events have a detect method which takes unprocessed data and does the correct processing using the Template meta-data, and computes the matched-filter detections. - * Tribe objects are containers for multiple Templates. - * Tribe objects have a detect method which groups Templates with similar meta-data (processing information) and runs these templates in parallel through the matched-filter routine. Tribe.detect outputs a Party of Family objects. - * The Party object is a container for many Family objects. - * Family objects are containers for detections from the same Template. - * Family and Party objects have a lag_calc method which computes the cross-correlation pick-refinements. - * The upshot of this is that it is possible to, in one line, generate a Tribe of templates, compute their matched-filter detections, and generate cross-correlation pick refinements, which output Event objects, which can be written to a catalog: Tribe.construct(method, **kwargs).detect(st, **kwargs).lag_calc(**kwargs).write() - * Added 25 tests for these methods. - * Add parameters *threshold_type* and *threshold_input* to Detection class. Add support for legacy Detection objects via NaN and unset values. + * Template objects maintain meta-data associated with their creation to stream-line processing of data (e.g. reduce chance of using the wrong filters). + * Template events have a detect method which takes unprocessed data and does the correct processing using the Template meta-data, and computes the matched-filter detections. + * Tribe objects are containers for multiple Templates. + * Tribe objects have a detect method which groups Templates with similar meta-data (processing information) and runs these templates in parallel through the matched-filter routine. Tribe.detect outputs a Party of Family objects. + * The Party object is a container for many Family objects. + * Family objects are containers for detections from the same Template. + * Family and Party objects have a lag_calc method which computes the cross-correlation pick-refinements. + * The upshot of this is that it is possible to, in one line, generate a Tribe of templates, compute their matched-filter detections, and generate cross-correlation pick refinements, which output Event objects, which can be written to a catalog: Tribe.construct(method, **kwargs).detect(st, **kwargs).lag_calc(**kwargs).write() + * Added 25 tests for these methods. + * Add parameters *threshold_type* and *threshold_input* to Detection class. Add support for legacy Detection objects via NaN and unset values. * Removed support for obspy < 1.0.0 * Update / correct doc-strings in template-gen functions when describing