Introduction of new threshold for rms histogram and fixing errors

tkp/config/job_template/job_params.cfg

@@ -6,6 +6,7 @@ rms_est_fraction = 8         ; Determines size of image subsection used for RMS
 rms_est_history = 100        ; how many images used for calculating rms histogram
 rms_est_max = 100            ; global maximum acceptable rms
 rms_est_min = 0.0            ; global minimum acceptable rms
+rms_rej_sigma = 3	     ; threshold for rejecting images using rms histogram
 bandwidth_max = 0.0          ; if non zero override bandwidth of image, determines which images fall in same band
 
 [quality_lofar]

tkp/main.py

@@ -201,11 +201,12 @@ def quality_check(db_images, accessors, job_config, runner):
     rms_max = job_config.persistence.rms_est_max
     rms_min = job_config.persistence.rms_est_min
     est_sigma = job_config.persistence.rms_est_sigma
+    rej_sigma = job_config.persistence.rms_rej_sigma
     good_images = []
     for db_image, rejected, accessor in zip(db_images, rejecteds, accessors):
         if not rejected:
             rejected = reject_historical_rms(db_image.id, db.session,
-                                             history, est_sigma, rms_max, rms_min)
+                                             history, est_sigma, rms_max, rms_min, rej_sigma)
 
         if rejected:
             reason, comment = rejected

tkp/quality/rms.py

@@ -79,7 +79,7 @@ def rms_with_clipped_subregion(data, rms_est_sigma=3, rms_est_fraction=4):
     return rms(clip(subregion(data, rms_est_fraction), rms_est_sigma))
 
 
-def reject_historical_rms(image_id, session, history=100, est_sigma=4, rms_max=100., rms_min=0.0):
+def reject_historical_rms(image_id, session, history=100, est_sigma=4, rms_max=100., rms_min=0.0, rej_sigma=3.0):
     """
     Check if the RMS value of an image lies within a range defined
     by a gaussian fit on the histogram calculated from the last x RMS
@@ -103,14 +103,14 @@ def reject_historical_rms(image_id, session, history=100, est_sigma=4, rms_max=1
     if len(rmss) < history:
         return False
     mu, sigma = norm.fit(rmss)
-    t_low = mu - sigma * est_sigma
-    t_high = mu + sigma * est_sigma
+    t_low = mu - sigma * rej_sigma
+    t_high = mu + sigma * rej_sigma
 
     if not rms_min < image.rms_qc < rms_max:
         return reject_reasons['rms'],\
-               "RMS value not within {} and {}".format(0.0, rms_max)
+               "RMS value not within {} and {}".format(rms_min, rms_max)
 
-    if not t_low < image.rms_qc < t_high or not 0.0 < image.rms_qc < rms_max:
+    if not t_low < image.rms_qc < t_high or not rms_min < image.rms_qc < rms_max:
         return reject_reasons['rms'],\
                "RMS value not within {} and {}".format(t_low, t_high)