[libc][printf] Handle case where snprintf underflows for len = 0

The code would write a null pointer always, even if len = 0, or if the
buffer pointer is null.

Test for this condition and add a unit test.

fixes #407

app/tests/printf_tests.c

@@ -96,16 +96,31 @@ int printf_tests(int argc, const console_cmd_args *argv) {
     /* make sure snprintf terminates at the right spot */
     char buf[32];
 
-    memset(buf, 0, sizeof(buf));
+    memset(buf, 0x99, sizeof(buf));
     err = sprintf(buf, "0123456789abcdef012345678");
     printf("sprintf returns %d\n", err);
     hexdump8(buf, sizeof(buf));
 
-    memset(buf, 0, sizeof(buf));
+    memset(buf, 0x99, sizeof(buf));
     err = snprintf(buf, 15, "0123456789abcdef012345678");
     printf("snprintf returns %d\n", err);
     hexdump8(buf, sizeof(buf));
 
+    memset(buf, 0x99, sizeof(buf));
+    err = snprintf(buf, 1, "0123456789abcdef012345678");
+    printf("snprintf returns %d\n", err);
+    hexdump8(buf, sizeof(buf));
+
+    /* zero length is special case, should not write anything */
+    memset(buf, 0x99, sizeof(buf));
+    err = snprintf(buf, 0, "0123456789abcdef012345678");
+    printf("snprintf returns %d\n", err);
+    hexdump8(buf, sizeof(buf));
+
+    /* shold be able to pass null to the output buffer if zero length */
+    err = snprintf(NULL, 0, "0123456789abcdef012345678");
+    printf("snprintf returns %d\n", err);
+
     return NO_ERROR;
 }
 

lib/libc/printf.c

@@ -66,6 +66,8 @@ static int _vsnprintf_output(const char *str, size_t len, void *state) {
         count++;
     }
 
+    // Return the count of the number of bytes that would be written even if the buffer
+    // wasn't large enough.
     return count;
 }
 
@@ -78,10 +80,13 @@ int vsnprintf(char *str, size_t len, const char *fmt, va_list ap) {
     args.pos = 0;
 
     wlen = _printf_engine(&_vsnprintf_output, (void *)&args, fmt, ap);
-    if (args.pos >= len)
+    if (len == 0) {
+        // do nothing, we can't null terminate the output
+    } else if (args.pos >= len) {
         str[len-1] = '\0';
-    else
+    } else {
         str[wlen] = '\0';
+    }
     return wlen;
 }