upstream unencrypted headers

YapDatabase/Utilities/YapDatabaseCryptoUtils.h

@@ -13,8 +13,12 @@ extern const NSUInteger kSQLCipherSaltLength;
 extern const NSUInteger kSQLCipherDerivedKeyLength;
 extern const NSUInteger kSQLCipherKeySpecLength;
 
-typedef void (^YapDatabaseSaltBlock)(NSData *saltData);
-typedef void (^YapDatabaseKeySpecBlock)(NSData *keySpecData);
+// User specified block used to notify the caller of a database's salt when
+// converting SQLCipher headers to plaintext. Failing to properly record the
+// salt will leave the database unreadable.
+// @returns BOOL indicating if the salt was successfully recorded. Conversion
+//          will not proceed if recording the salt fails.
+typedef BOOL (^YapRecordDatabaseSaltBlock)(NSData *saltData);
 
 // This class contains utility methods for use with SQLCipher encrypted
 // databases, specifically to address an issue around database files that
@@ -58,13 +62,14 @@ typedef void (^YapDatabaseKeySpecBlock)(NSData *keySpecData);
 // The header does not contain any user data.  See:
 // https://www.sqlite.org/fileformat.html#the_database_header
 //
-// However, Sqlite normally uses the first 16 bytes of the Sqlite header to store
+// However, SQLCipher normally uses the first 16 bytes of the Sqlite header to store
 // a salt value.  Therefore when using unencrypted headers, it is also necessary
 // to explicitly specify a salt value.
 //
 // It is possible to convert SQLCipher databases with encrypted headers to use
 // unencrypted headers.  However, during this conversion, the salt must be extracted
-// and preserved by reading the first 16 bytes of the unconverted file.
+// by reading the first 16 bytes of the unconverted file and preserving it elsewhere,
+// e.g. the keychain.
 //
 //
 // Implementation
@@ -99,9 +104,9 @@ typedef void (^YapDatabaseKeySpecBlock)(NSData *keySpecData);
 // * This method should always be pretty fast, and should be safe to
 //   call from within [UIApplicationDelegate application: didFinishLaunchingWithOptions:].
 // * If convertDatabaseIfNecessary converts the database, it will use its
-//   saltBlock and keySpecBlock parameters to inform you of the salt
+//   recordSaltBlock to inform you of the salt
 //   and keyspec for this database.  These values will be needed when
-//   opening the database, so they should presumably stored in the
+//   opening the database, so they should presumably be stored in the
 //   keychain (like the database password).
 //
 //
@@ -130,23 +135,21 @@ typedef void (^YapDatabaseKeySpecBlock)(NSData *keySpecData);
 // * This method will have no effect if the YapDatabase has already been converted.
 // * This method should always be pretty fast, and should be safe to
 //   call from within [UIApplicationDelegate application: didFinishLaunchingWithOptions:].
-// * If convertDatabaseIfNecessary converts the database, it will use its
-//   saltBlock and keySpecBlock parameters to inform you of the salt
-//   and keyspec for this database.  These values will be needed when
-//   opening the database, so they should presumably stored in the
-//   keychain (like the database password).
+// * IMPORTANT: If you fail to record the salt during conversion you will not be able to decrypt
+//   the database in the future, effectively losing all data.  If convertDatabaseIfNecessary
+//   converts the database, it will use its recordSaltBlock parameter to inform you of the salt
+//   for this database. Within that block you must store the salt somewhere durable.
 + (nullable NSError *)convertDatabaseIfNecessary:(NSString *)databaseFilePath
                                 databasePassword:(NSData *)databasePassword
-                                       saltBlock:(YapDatabaseSaltBlock)saltBlock
-                                    keySpecBlock:(YapDatabaseKeySpecBlock)keySpecBlock;
+                                 recordSaltBlock:(YapRecordDatabaseSaltBlock)recordSaltBlock;
 
 // This method can be used to derive a SQLCipher "key spec" from a
 // database password and salt.  Key spec derivation is somewhat costly.
 // The key spec is needed every time the database file is opened
-// (including every time YapDatabse makes a new database connection),
+// (including every time YapDatabase makes a new database connection),
 // So it benefits performance to pass a pre-derived key spec to
 // YapDatabase.
-+ (nullable NSData *)databaseKeySpecForPassword:(NSData *)passwordData saltData:(NSData *)saltData;
++ (nullable NSData *)deriveDatabaseKeySpecForPassword:(NSData *)passwordData saltData:(NSData *)saltData;
 
 #pragma mark - Utils
 

YapDatabase/Utilities/YapDatabaseCryptoUtils.m

@@ -140,7 +140,7 @@ + (BOOL)doesDatabaseNeedToBeConverted:(NSString *)databaseFilePath
 
     if (![[NSFileManager defaultManager] fileExistsAtPath:databaseFilePath]) {
         YDBLogVerbose(@"%@ database file not found.", self.logTag);
-        return nil;
+        return NO;
     }
 
     NSData *headerData = [self readFirstNBytesOfDatabaseFile:databaseFilePath byteCount:kSqliteHeaderLength];
@@ -160,28 +160,27 @@ + (BOOL)doesDatabaseNeedToBeConverted:(NSString *)databaseFilePath
 
 + (nullable NSError *)convertDatabaseIfNecessary:(NSString *)databaseFilePath
                                 databasePassword:(NSData *)databasePassword
-                                       saltBlock:(YapDatabaseSaltBlock)saltBlock
-                                    keySpecBlock:(YapDatabaseKeySpecBlock)keySpecBlock
+                                 recordSaltBlock:(YapRecordDatabaseSaltBlock)recordSaltBlock
 {
     if (![self doesDatabaseNeedToBeConverted:databaseFilePath]) {
+        YDBLogInfo(@"%@ convertDatabaseIfNecessary: database does not need to be converted.", self.logTag);
         return nil;
     }
 
     return [self convertDatabase:databaseFilePath
                 databasePassword:databasePassword
-                       saltBlock:saltBlock
-                    keySpecBlock:keySpecBlock];
+                 recordSaltBlock:recordSaltBlock];
 }
 
 + (nullable NSError *)convertDatabase:(NSString *)databaseFilePath
                      databasePassword:(NSData *)databasePassword
-                            saltBlock:(YapDatabaseSaltBlock)saltBlock
-                         keySpecBlock:(YapDatabaseKeySpecBlock)keySpecBlock
+                      recordSaltBlock:(YapRecordDatabaseSaltBlock)recordSaltBlock
 {
     YapAssert(databaseFilePath.length > 0);
     YapAssert(databasePassword.length > 0);
-    YapAssert(saltBlock);
-    YapAssert(keySpecBlock);
+    YapAssert(recordSaltBlock);
+
+    YDBLogInfo(@"%@ convertDatabase.", self.logTag);
 
     NSData *saltData;
     {
@@ -194,23 +193,15 @@ + (nullable NSError *)convertDatabase:(NSString *)databaseFilePath
         // Make sure we successfully persist the salt (persumably in the keychain) before
         // proceeding with the database conversion or we could leave the app in an
         // unrecoverable state.
-        saltBlock(saltData);
-    }
-
-    {
-        NSData *_Nullable keySpecData = [self databaseKeySpecForPassword:databasePassword saltData:saltData];
-        if (!keySpecData || keySpecData.length != kSQLCipherKeySpecLength) {
-            YDBLogError(@"Error deriving key spec");
-            return YDBErrorWithDescription(@"Invalid key spec");
+        YDBLogInfo(@"%@ convertDatabase: salt extracted.", self.logTag);
+        BOOL success = recordSaltBlock(saltData);
+        if (!success) {
+            YDBLogError(@"Failed to record salt, aborting conversion");
+            return YDBErrorWithDescription(@"Failed to record salt");
         }
-
-        YapAssert(keySpecData.length == kSQLCipherKeySpecLength);
-
-        // Make sure we successfully persist the key spec (persumably in the keychain) before
-        // proceeding with the database conversion or we could leave the app in an
-        // unrecoverable state.
-        keySpecBlock(keySpecData);
     }
+
+    YDBLogInfo(@"%@ convertDatabase: key spec derived.", self.logTag);
 
     // -----------------------------------------------------------
     //
@@ -235,6 +226,8 @@ + (nullable NSError *)convertDatabase:(NSString *)databaseFilePath
             return YDBErrorWithDescription(@"Failed to open database");
         }
     }
+
+    YDBLogInfo(@"%@ convertDatabase: database open.", self.logTag);
 
     // -----------------------------------------------------------
     //
@@ -248,6 +241,8 @@ + (nullable NSError *)convertDatabase:(NSString *)databaseFilePath
             return YDBErrorWithDescription(@"Failed to set SQLCipher key");
         }
     }
+
+    YDBLogInfo(@"%@ convertDatabase: database keyed.", self.logTag);
 
     // -----------------------------------------------------------
     //
@@ -302,6 +297,8 @@ + (nullable NSError *)convertDatabase:(NSString *)databaseFilePath
         // END DB setup copied from YapDatabase
         // BEGIN SQLCipher migration
     }
+
+    YDBLogInfo(@"%@ convertDatabase: database configured.", self.logTag);
 
 #ifdef DEBUG
     // We can obtain the database salt in two ways: by reading the first 16 bytes of the encrypted
@@ -313,6 +310,8 @@ + (nullable NSError *)convertDatabase:(NSString *)databaseFilePath
 
         YapAssert([[self hexadecimalStringForData:saltData] isEqualToString:saltString]);
     }
+
+    YDBLogInfo(@"%@ convertDatabase: salt confirmed.", self.logTag);
 #endif
 
     // -----------------------------------------------------------
@@ -327,6 +326,8 @@ + (nullable NSError *)convertDatabase:(NSString *)databaseFilePath
         if (error) {
             return error;
         }
+
+        YDBLogInfo(@"%@ convertDatabase: encrypted header configured.", self.logTag);
 
         // Modify the first page, so that SQLCipher will overwrite, respecting our new cipher_plaintext_header_size
         NSString *tableName = [NSString stringWithFormat:@"signal-migration-%@", [NSUUID new].UUIDString];
@@ -340,6 +341,8 @@ + (nullable NSError *)convertDatabase:(NSString *)databaseFilePath
         if (error) {
             return error;
         }
+
+        YDBLogInfo(@"%@ convertDatabase: database dirtied.", self.logTag);
 
         // Force a checkpoint so that the plaintext is written to the actual DB file, not just living in the WAL.
         int log, ckpt;
@@ -348,8 +351,12 @@ + (nullable NSError *)convertDatabase:(NSString *)databaseFilePath
             YDBLogError(@"%@ Error forcing checkpoint. status: %d, log: %d, ckpt: %d, error: %s", self.logTag, status, log, ckpt, sqlite3_errmsg(db));
             return YDBErrorWithDescription(@"Error forcing checkpoint.");
         }
+
+        YDBLogInfo(@"%@ convertDatabase: checkpoint completed.", self.logTag);
 
         sqlite3_close(db);
+
+        YDBLogInfo(@"%@ convertDatabase: database closed.", self.logTag);
     }
 
     return nil;
@@ -408,9 +415,13 @@ + (nullable NSData *)deriveDatabaseKeyForPassword:(NSData *)passwordData saltDat
     YapAssert(passwordData.length > 0);
     YapAssert(saltData.length == kSQLCipherSaltLength);
 
-    unsigned char *derivedKeyBytes = malloc((size_t)kSQLCipherDerivedKeyLength);
-    YapAssert(derivedKeyBytes);
-    // See: PBKDF2_ITER.
+    NSMutableData *_Nullable derivedKeyData = [NSMutableData dataWithLength:kSQLCipherDerivedKeyLength];
+    if (!derivedKeyData) {
+        YapFail(@"failed to allocate derivedKeyData");
+        return nil;
+    }
+
+    // See: PBKDF2_ITER from SQLCipher.
     const unsigned int workfactor = 64000;
 
     int result = CCKeyDerivationPBKDF(kCCPBKDF2,
@@ -420,23 +431,18 @@ + (nullable NSData *)deriveDatabaseKeyForPassword:(NSData *)passwordData saltDat
         (size_t)saltData.length,
         kCCPRFHmacAlgSHA1,
         workfactor,
-        derivedKeyBytes,
-        kSQLCipherDerivedKeyLength);
+        derivedKeyData.mutableBytes,
+        (size_t)derivedKeyData.length);
+
     if (result != kCCSuccess) {
         YDBLogError(@"Error deriving key: %d", result);
         return nil;
     }
 
-    NSData *_Nullable derivedKeyData = [NSData dataWithBytes:derivedKeyBytes length:kSQLCipherDerivedKeyLength];
-    if (!derivedKeyData || derivedKeyData.length != kSQLCipherDerivedKeyLength) {
-        YDBLogError(@"Invalid derived key: %d", result);
-        return nil;
-    }
-
-    return derivedKeyData;
+    return [derivedKeyData copy];
 }
 
-+ (nullable NSData *)databaseKeySpecForPassword:(NSData *)passwordData saltData:(NSData *)saltData
++ (nullable NSData *)deriveDatabaseKeySpecForPassword:(NSData *)passwordData saltData:(NSData *)saltData
 {
     YapAssert(passwordData.length > 0);
     YapAssert(saltData.length == kSQLCipherSaltLength);
@@ -452,7 +458,7 @@ + (nullable NSData *)databaseKeySpecForPassword:(NSData *)passwordData saltData:
 
     YapAssert(keySpecData.length == kSQLCipherKeySpecLength);
 
-    return keySpecData;
+    return [keySpecData copy];
 }
 
 #pragma mark - Utils

YapDatabase/YapDatabase.m

@@ -7,6 +7,7 @@
 #import "YapDatabaseConnectionState.h"
 #import "YapDatabaseLogging.h"
 #import "YapDatabaseString.h"
+#import "YapDatabaseCryptoUtils.h"
 
 #import "sqlite3.h"
 
@@ -830,18 +831,49 @@ - (BOOL)configureDatabase:(BOOL)isNewDatabaseFile
 **/
 - (BOOL)configureEncryptionForDatabase:(sqlite3 *)sqlite
 {
+    if (options.cipherUnencryptedHeaderLength > 0) {
+        if (options.cipherKeySpecBlock)
+        {
+            // Do nothing.
+        } else if (!(options.cipherKeyBlock && options.cipherSaltBlock)) {
+            NSAssert(NO, @"If you're using YapDatabaseOptions.cipherUnencryptedHeaderLength, you need to set either cipherKeySpecBlock or both cipherKeyBlock and cipherSaltBlock.");
+            return NO;
+        }
+    }        
+
     if (options.cipherKeyBlock ||
         options.cipherKeySpecBlock)
 	{
         NSData *_Nullable keyData = nil;
         if (options.cipherKeySpecBlock)
         {
-            keyData = options.cipherKeySpecBlock();
-            if (!keyData)
+            if (options.cipherKeyBlock) {
+                NSAssert(NO, @"If you're using YapDatabaseOptions.cipherKeySpecBlock, you don't need to set a cipherKeySpecBlock.");
+                return NO;
+            }
+            if (options.cipherSaltBlock) {
+                NSAssert(NO, @"If you're using YapDatabaseOptions.cipherKeySpecBlock, you don't need to set a cipherSaltBlock.");
+                return NO;
+            }
+
+            NSData *_Nullable keySpecData = options.cipherKeySpecBlock();
+            if (!keySpecData)
             {
                 NSAssert(NO, @"YapDatabaseOptions.cipherKeySpecBlock cannot return nil!");
                 return NO;
             }
+            if (keySpecData.length != kSQLCipherKeySpecLength) {
+                NSAssert(NO, @"YapDatabaseOptions.cipherKeySpecBlock returned a key spec of unexpected length: %zd.", keySpecData.length);
+                return NO;
+            }
+
+            // Use a raw key spec, where the 96 hexadecimal digits are provided
+            // (i.e. 64 hex for the 256 bit key, followed by 32 hex for the 128 bit salt)
+            // using explicit BLOB syntax, e.g.:
+            //
+            // x'98483C6EB40B6C31A448C22A66DED3B5E5E8D5119CAC8327B655C8B5C483648101010101010101010101010101010101'
+            NSString *keySpecString = [NSString stringWithFormat:@"x'%@'", [self hexadecimalStringForData:keySpecData]];
+            keyData = [keySpecString dataUsingEncoding:NSUTF8StringEncoding];
         } else {
             keyData = options.cipherKeyBlock();
             if (!keyData)
@@ -884,27 +916,11 @@ - (BOOL)configureEncryptionForDatabase:(sqlite3 *)sqlite
             }
         }
 
-        if (options.cipherKeySpecBlock) {
-            // Use a raw key spec, where the 96 hexadecimal digits are provided
-            // (i.e. 64 hex for the 256 bit key, followed by 32 hex for the 128 bit salt)
-            // using explicit BLOB syntax, e.g.:
-            //
-            // x'98483C6EB40B6C31A448C22A66DED3B5E5E8D5119CAC8327B655C8B5C483648101010101010101010101010101010101'
-            NSString *keySpecString = [NSString stringWithFormat:@"x'%@'", [self hexadecimalStringForData:keyData]];
-            NSData *keySpecStringData = [keySpecString dataUsingEncoding:NSUTF8StringEncoding];
-            int status = sqlite3_key(sqlite, [keySpecStringData bytes], (int)[keySpecStringData length]);
-            if (status != SQLITE_OK)
-            {
-                YDBLogError(@"Error setting SQLCipher key: %d %s", status, sqlite3_errmsg(sqlite));
-                return NO;
-            }
-        } else {
-            int status = sqlite3_key(sqlite, [keyData bytes], (int)[keyData length]);
-            if (status != SQLITE_OK)
-            {
-                YDBLogError(@"Error setting SQLCipher key: %d %s", status, sqlite3_errmsg(sqlite));
-                return NO;
-            }
+        int status = sqlite3_key(sqlite, [keyData bytes], (int)[keyData length]);
+        if (status != SQLITE_OK)
+        {
+            YDBLogError(@"Error setting SQLCipher key: %d %s", status, sqlite3_errmsg(sqlite));
+            return NO;
         }
 
         if (options.cipherUnencryptedHeaderLength > 0 &&
@@ -923,6 +939,10 @@ - (BOOL)configureEncryptionForDatabase:(sqlite3 *)sqlite
                     NSAssert(NO, @"YapDatabaseOptions.cipherSaltBlock cannot return nil!");
                     return NO;
                 }
+                if (saltData.length != kSQLCipherSaltLength) {
+                    NSAssert(NO, @"YapDatabaseOptions.cipherSaltBlock returned a salt of unexpected length: %zd.", saltData.length);
+                    return NO;
+                }
 
                 {
                     char *errorMsg;

YapDatabase/YapDatabaseOptions.h

@@ -244,7 +244,19 @@ typedef NSData *_Nonnull (^YapDatabaseCipherKeyBlock)(void);
 /**
  * Set a block here that returns the key spec (not the key) for the SQLCipher database.
  *
- * This key spec incorporates the "derived key" and the "salt".
+ * The key spec incorporates the "derived key" and the "salt".
+ *
+ * The key spec should be kSQLCipherKeySpecLength bytes in length.
+ *
+ * If you a key spec, you do NOT need to specify the salt (using cipherSaltBlock)
+ * and "key/password" (using cipherKeyBlock).
+ *
+ * For new databases, the key spec can be any N bytes where N is kSQLCipherKeySpecLength.
+ * You should consider generating them with SecRandomCopyBytes().
+ *
+ * For existing databases that were created using a "key/password" (i.e. cipherKeyBlock),
+ * you can derive a key spec using that key/password and the database's salt.  See
+ * comments in YapDatabaseCryptoUtils.h.
  *
  * This block allows you to fetch the key spec from the keychain (or elsewhere)
  * only when you need it, instead of persisting it in memory.