Fix Cobalt certificate verification issues

This change gets Cobalt to talk to Google servers without needing
`--ignore_certificate_errors` flag. Changes include:
1. Disabled AIA cert net fetcher since we do should not need it.
2. Added implementation for trust store which lazily loads certificate
   from disk and provides it to the net module during cert path building.

Note that we should upstream our trust store implementation if possible.

b/330355565

Change-Id: I721e747c0bd02a7ae9a86f224b2a4d9fc451627f

net/BUILD.gn

@@ -1075,6 +1075,8 @@ component("net") {
   if (is_starboard) {
     sources += [
       "base/file_stream_context_starboard.cc",
+      "cert/internal/trust_store_in_memory_starboard.cc",
+      "cert/internal/trust_store_in_memory_starboard.h",
     ]
     if (sb_is_modular || is_android || is_win) {
       # TODO: Either always build or don't build with this.

net/cert/internal/system_trust_store.cc

@@ -51,6 +51,10 @@
 #include "net/cert/internal/trust_store_win.h"
 #elif BUILDFLAG(IS_ANDROID)
 #include "net/cert/internal/trust_store_android.h"
+#elif defined(STARBOARD)
+#include "base/lazy_instance.h"
+#include "net/cert/internal/trust_store_in_memory_starboard.h"
+#include "net/cert/test_root_certs.h"
 #endif
 #if BUILDFLAG(CHROME_ROOT_STORE_SUPPORTED)
 #include "net/cert/internal/trust_store_chrome.h"
@@ -396,6 +400,52 @@ void InitializeTrustStoreAndroid() {}
 
 #endif  // CHROME_ROOT_STORE_SUPPORTED
 
+#elif defined(STARBOARD)
+
+namespace {
+
+class StarboardSystemCerts {
+ public:
+  StarboardSystemCerts() {}
+
+  TrustStoreInMemoryStarboard* system_trust_store() {
+    return &system_trust_store_;
+  }
+
+ private:
+  TrustStoreInMemoryStarboard system_trust_store_;
+};
+
+base::LazyInstance<StarboardSystemCerts>::Leaky g_root_certs_starboard =
+    LAZY_INSTANCE_INITIALIZER;
+
+}  // namespace
+
+// Starboard does not use OS certificate stores, instead Cobalt ships with a
+// set of trusted CA certificates to be used for validations.
+class SystemTrustStoreStarboard : public SystemTrustStore {
+ public:
+  SystemTrustStoreStarboard() = default;
+
+  TrustStore* GetTrustStore() override {
+    if (TestRootCerts::HasInstance()) {
+      return TestRootCerts::GetInstance()->test_trust_store();
+    }
+    return g_root_certs_starboard.Get().system_trust_store();
+  }
+
+  bool UsesSystemTrustStore() const override { return true; }
+
+  bool IsKnownRoot(const ParsedCertificate* trust_anchor) const override {
+    return g_root_certs_starboard.Get().system_trust_store()->Contains(
+        trust_anchor);
+  }
+};
+
+std::unique_ptr<SystemTrustStore> CreateSslSystemTrustStore() {
+  return std::make_unique<SystemTrustStoreStarboard>();
+}
+
 #else
 
 std::unique_ptr<SystemTrustStore> CreateSslSystemTrustStore() {

net/cert/internal/trust_store_in_memory_starboard.cc

@@ -0,0 +1,178 @@
+// Copyright 2019 The Cobalt Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "net/cert/internal/trust_store_in_memory_starboard.h"
+
+#include "base/files/file_path.h"
+#include "base/path_service.h"
+#include "base/logging.h"
+#include "base/time/time.h"
+#include "net/cert/pki/cert_errors.h"
+#include "net/cert/pem.h"
+#include "net/cert/x509_certificate.h"
+#include "net/cert/x509_util.h"
+#include "starboard/configuration_constants.h"
+#include "starboard/directory.h"
+#include "starboard/file.h"
+#include "starboard/string.h"
+#include "third_party/boringssl/src/include/openssl/digest.h"
+#include "third_party/boringssl/src/include/openssl/sha.h"
+#include "third_party/boringssl/src/include/openssl/x509.h"
+
+namespace net {
+
+namespace {
+// PEM encoded DER cert is usually around or less than 2000 bytes long.
+const short kCertBufferSize = 8 * 1024;
+// Each certificate file name is 8 bit hash + ".0" suffix.
+const short kCertFileNameLength = 10;
+const char kCertificateHeader[] = "CERTIFICATE";
+const char kSSLDirName[] = "ssl";
+const char kCertsDirName[] = "certs";
+
+// Essentially an X509_NAME_hash without using X509_NAME. We did not use the
+// boringSSL function directly because net does not store certs with X509
+// struct anymore and converting binary certs to X509_NAME is slightly
+// expensive.
+unsigned long CertNameHash(const void* data, size_t length) {
+  unsigned long ret = 0;
+  unsigned char md[SHA_DIGEST_LENGTH];
+  if (!EVP_Digest(data, length, md, NULL, EVP_sha1(), NULL))
+    return 0;
+
+  ret = (((unsigned long)md[0]) | ((unsigned long)md[1] << 8L) |
+         ((unsigned long)md[2] << 16L) | ((unsigned long)md[3] << 24L)) &
+        0xffffffffL;
+  return ret;
+}
+
+base::FilePath GetCertificateDirPath() {
+  base::FilePath cert_path;
+  base::PathService::Get(base::DIR_EXE, &cert_path);
+  cert_path = cert_path.Append(kSSLDirName).Append(kCertsDirName);
+  return std::move(cert_path);
+}
+
+std::unordered_set<std::string> GetCertNamesOnDisk() {
+  auto sb_certs_directory =
+      SbDirectoryOpen(GetCertificateDirPath().value().c_str(), nullptr);
+  if (!SbDirectoryIsValid(sb_certs_directory)) {
+// Unit tests, for example, do not use production certificates.
+#if defined(STARBOARD_BUILD_TYPE_QA) || defined(STARBOARD_BUILD_TYPE_GOLD)
+    SB_CHECK(false);
+#else
+    DLOG(WARNING) << "ssl/certs directory is not valid, no root certificates"
+                     " will be loaded";
+#endif
+    return std::unordered_set<std::string>();
+  }
+  std::unordered_set<std::string> trusted_certs_on_disk;
+  std::vector<char> dir_entry(kSbFileMaxName);
+
+  while (SbDirectoryGetNext(sb_certs_directory, dir_entry.data(),
+                            dir_entry.size())) {
+    if (strlen(dir_entry.data()) != kCertFileNameLength) {
+      continue;
+    }
+    trusted_certs_on_disk.emplace(dir_entry.data());
+  }
+  SbDirectoryClose(sb_certs_directory);
+  return std::move(trusted_certs_on_disk);
+}
+}  // namespace
+
+std::shared_ptr<const ParsedCertificate> TrustStoreInMemoryStarboard::TryLoadCert(
+    const base::StringPiece& cert_name) const {
+  auto hash = CertNameHash(cert_name.data(), cert_name.length());
+  char cert_file_name[256];
+  snprintf(cert_file_name, 256, "%08lx.%d", hash, 0);
+
+  if (trusted_cert_names_on_disk_.find(cert_file_name) ==
+      trusted_cert_names_on_disk_.end()) {
+    // The requested certificate is not found.
+    return nullptr;
+  }
+
+  SbFileError out_error;
+  char cert_buffer[kCertBufferSize];
+  base::FilePath cert_path = GetCertificateDirPath().Append(cert_file_name);
+  SbFile sb_cert_file =
+      SbFileOpen(cert_path.value().c_str(), kSbFileOpenOnly | kSbFileRead,
+                 nullptr, &out_error);
+  // The file was in certs directory when we iterated the directory at startup,
+  // opening it should not fail.
+  if (!SbFileIsValid(sb_cert_file)) {
+    NOTREACHED() << "ssl/certs/" << cert_path << " failed to open.";
+    return nullptr;
+  }
+  int cert_size = SbFileReadAll(sb_cert_file, cert_buffer, kCertBufferSize);
+  SbFileClose(sb_cert_file);
+  PEMTokenizer pem_tokenizer(base::StringPiece(cert_buffer, cert_size),
+                             {kCertificateHeader});
+  pem_tokenizer.GetNext();
+  std::string decoded(pem_tokenizer.data());
+  DCHECK(!pem_tokenizer.GetNext());
+  auto crypto_buffer = x509_util::CreateCryptoBuffer(decoded);
+  CertErrors errors;
+  auto parsed = ParsedCertificate::Create(
+      std::move(crypto_buffer),
+      x509_util::DefaultParseCertificateOptions(), &errors);
+  CHECK(parsed) << errors.ToDebugString();
+  return parsed;
+}
+
+TrustStoreInMemoryStarboard::TrustStoreInMemoryStarboard()
+    : trusted_cert_names_on_disk_(std::move(GetCertNamesOnDisk())) {}
+
+TrustStoreInMemoryStarboard::~TrustStoreInMemoryStarboard() = default;
+
+void TrustStoreInMemoryStarboard::SyncGetIssuersOf(
+    const ParsedCertificate* cert,
+    ParsedCertificateList* issuers) {
+  DCHECK(issuers);
+  DCHECK(issuers->empty());
+  starboard::ScopedLock scoped_lock(load_mutex_);
+  // Look up the request certificate first in the trust store in memory.
+  underlying_trust_store_.SyncGetIssuersOf(cert, issuers);
+  if (issuers->empty()) {
+    // If the requested certificate is not found, compute certificate hash name
+    // and see if the certificate is stored on disk.
+    auto parsed_cert = TryLoadCert(cert->normalized_issuer().AsStringView());
+    if (parsed_cert.get()) {
+      issuers->push_back(parsed_cert);
+      underlying_trust_store_.AddTrustAnchor(parsed_cert);
+    }
+  }
+}
+
+CertificateTrust TrustStoreInMemoryStarboard::GetTrust(const ParsedCertificate* cert,
+                          base::SupportsUserData* debug_data) {
+  starboard::ScopedLock scoped_lock(load_mutex_);
+  // Loop up the request certificate first in the trust store in memory.
+  CertificateTrust trust = underlying_trust_store_.GetTrust(cert, debug_data);
+  if (trust.HasUnspecifiedTrust()) {
+    // If the requested certificate is not found, compute certificate hash name
+    // and see if the certificate is stored on disk.
+    auto parsed_cert = TryLoadCert(cert->normalized_subject().AsStringView());
+    if (parsed_cert.get()) {
+      trust = CertificateTrust::ForTrustAnchor();
+      const_cast<TrustStoreInMemoryStarboard*>(this)
+          ->underlying_trust_store_.AddTrustAnchor(parsed_cert);
+    }
+  }
+  return trust;
+}
+
+}  // namespace net
+

net/cert/internal/trust_store_in_memory_starboard.h

@@ -0,0 +1,69 @@
+// Copyright 2019 The Cobalt Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NET_CERT_INTERNAL_TRUST_STORE_IN_MEMORY_STARBOARD_H_
+#define NET_CERT_INTERNAL_TRUST_STORE_IN_MEMORY_STARBOARD_H_
+
+#include <unordered_set>
+
+#include "base/strings/string_piece.h"
+#include "net/cert/pki/trust_store_in_memory.h"
+#include "starboard/common/mutex.h"
+
+namespace net {
+
+// Wrapper around TrustStoreInMemory to lazily load trusted root certificates.
+class NET_EXPORT TrustStoreInMemoryStarboard : public TrustStore {
+ public:
+  TrustStoreInMemoryStarboard();
+  ~TrustStoreInMemoryStarboard() override;
+
+  // TrustStore implementation:
+  void SyncGetIssuersOf(const ParsedCertificate* cert,
+                        ParsedCertificateList* issuers) override;
+  CertificateTrust GetTrust(const ParsedCertificate* cert,
+                                    base::SupportsUserData* debug_data) override;
+
+  // Returns true if the trust store contains the given ParsedCertificate
+  // (matches by DER).
+  bool Contains(const ParsedCertificate* cert) const {
+    starboard::ScopedLock scoped_lock(load_mutex_);
+    return underlying_trust_store_.Contains(cert);
+  }
+
+ private:
+  TrustStoreInMemoryStarboard(const TrustStoreInMemoryStarboard&) = delete;
+  TrustStoreInMemoryStarboard& operator=(const TrustStoreInMemoryStarboard&) =
+      delete;
+
+  TrustStoreInMemory underlying_trust_store_;
+
+  // Given a certificate's canonical name, try to load this cert from trusted
+  // certs on disk if it is found.
+  std::shared_ptr<const ParsedCertificate> TryLoadCert(
+      const base::StringPiece& cert_name) const;
+
+  // The memory trust store can be accessed by multiple threads, in Chromium,
+  // the synchronization issue is solved by initializing trust store at startup
+  // and passing constant reference to consumers. Cobalt loads certs lazily and
+  // therefore guards the underlying_trust_store_ with mutex.
+  starboard::Mutex load_mutex_;
+
+  const std::unordered_set<std::string> trusted_cert_names_on_disk_;
+};
+
+}  // namespace net
+
+#endif  // NET_CERT_INTERNAL_TRUST_STORE_IN_MEMORY_STARBOARD_H_
+

net/cert/multi_threaded_cert_verifier.cc

@@ -59,6 +59,9 @@ int GetFlagsForConfig(const CertVerifier::Config& config) {
     flags |= CertVerifyProc::VERIFY_ENABLE_SHA1_LOCAL_ANCHORS;
   if (config.disable_symantec_enforcement)
     flags |= CertVerifyProc::VERIFY_DISABLE_SYMANTEC_ENFORCEMENT;
+#if defined(STARBOARD)
+  flags |= CertVerifyProc::VERIFY_DISABLE_NETWORK_FETCHES;
+#endif
 
   return flags;
 }