PYCBC-446: Gracefully fail on signal receipt

Motivation
----------
If the Python process is interrupted by a signal,
lcb_wait3 may return before all remaining jobs
have completed/timed out. This can lead to an exception
being thrown after the wait, as normally
this would only return after this point.

We should gracefully handle this scenario.

Changes
-------

Add a field 'check_type' to the Bucket class.

Instead of using pycbc_assert to raise an abort() if there are
remaining operations to complete after lcb_wait has
returned, according to the check_type value, do the following:

PYCBC_CHECK_NONE: don't raise an exception.
PYCBC_CHECK_STRICT: (the default): raise an SDKInternalError
PYCBC_CHECK_FAIL: (for testing): always fail and raise an exception

Results
-------
It is difficult to automatically reproduce this, but as
we are now raising a Python exception rather than an abort(),
as verified by the tests, the end user should be able
to handle this by catching the exception if necessary. Or
they can turn off checking by setting the flag to
PYCBC_CHECK_NONE.

Change-Id: I5497267ba58496c57fe03b040fbc4eeb67fb16bd
Reviewed-on: http://review.couchbase.org/112725
Tested-by: Ellis Breen <[email protected]>
Reviewed-by: Brett Lawson <[email protected]>
Reviewed-by: Sergey Avseyev <[email protected]>
Tested-by: Build Bot <[email protected]>

couchbase_v2/tests/cases/misc_t.py

@@ -28,7 +28,7 @@
 
 from couchbase_tests.base import ConnectionTestCaseBase
 from couchbase_core.user_constants import FMT_AUTO, FMT_JSON, FMT_PICKLE
-from couchbase_v2.exceptions import ClientTemporaryFailError
+from couchbase_v2.exceptions import ClientTemporaryFailError, InternalSDKError
 from couchbase_v2.exceptions import CouchbaseError
 import re
 import couchbase_core._libcouchbase as _LCB

src/bucket.c

@@ -1026,7 +1026,8 @@ Bucket__init__(pycbc_Bucket *self,
     X("lockmode", &self->lockmode, "i")                    \
     X("_flags", &self->flags, "I")                         \
     X("_conntype", &conntype, "i")                         \
-    X("_iops", &iops_O, "O")
+    X("_iops", &iops_O, "O")                               \
+    X("_check_inconsistent", &self->check_type, "I")
 
 #define XCTOR_ARGS(X)\
     XCTOR_ARGS_NOTRACING(X)\

src/constants.c

@@ -200,6 +200,11 @@ do_all_constants(PyObject *module, pycbc_constant_handler handler)
     ADD_MACRO(LCB_CNTL_N1QL_TIMEOUT);
 	ADD_MACRO(LCB_CNTL_COMPRESSION_OPTS);
     ADD_MACRO(LCB_CNTL_LOG_REDACTION);
+
+    ADD_MACRO(PYCBC_CHECK_NONE);
+    ADD_MACRO(PYCBC_CHECK_STRICT);
+    ADD_MACRO(PYCBC_CHECK_FAIL);
+
     ADD_STRING(LCB_LOG_MD_OTAG);
     ADD_STRING(LCB_LOG_MD_CTAG);
     ADD_STRING(LCB_LOG_SD_OTAG);

src/oputil.c

@@ -204,18 +204,26 @@ pycbc_common_vars_wait, struct pycbc_common_vars *cv, pycbc_Bucket *self)
     }
     pycbc_oputil_wait_common(self, context);
 
-    if (!pycbc_assert(self->nremaining == 0)) {
+    if (self->nremaining || self->check_type == PYCBC_CHECK_FAIL) {
         fprintf(stderr,
                 "Remaining count %d!= 0. Adjusting",
                 (int)self->nremaining);
         self->nremaining = 0;
+        if (self->check_type != PYCBC_CHECK_NONE) {
+            cv->ret = NULL;
+            PYCBC_EXC_WRAP(PYCBC_EXC_INTERNAL,
+                           0,
+                           "self->nremaining!=0, resetting to 0");
+            goto FAIL;
+        }
     }
 
     if (pycbc_multiresult_maybe_raise(cv->mres)) {
         return -1;
     }
 
     cv->ret = pycbc_multiresult_get_result(cv->mres);
+FAIL:
     Py_DECREF(cv->mres);
     cv->mres = NULL;
 

src/pycbc.h

@@ -526,6 +526,12 @@ lcb_STATUS pycbc_logging_monad_verb(const char *FILE,
                                 __VA_ARGS__)
 
 
+typedef enum {
+    PYCBC_CHECK_STRICT,
+    PYCBC_CHECK_NONE,
+    PYCBC_CHECK_FAIL
+} pycbc_check_type;
+
 typedef struct {
     PyObject_HEAD
 
@@ -596,6 +602,8 @@ typedef struct {
     pycbc_dur_params dur_global;
     unsigned long dur_timeout;
 
+    pycbc_check_type check_type;
+
 } pycbc_Bucket;
 
 /**