Merge pull request #4859 from lukewarmtemp/luyang-filelists

disable downloading filelists by default

src/app/rpmostree-compose-builtin-tree.cxx

@@ -397,7 +397,7 @@ install_packages (RpmOstreeTreeComposeContext *self, gboolean *out_unmodified,
         }
     }
 
-  if (!rpmostree_context_prepare (self->corectx, cancellable, error))
+  if (!rpmostree_context_prepare (self->corectx, TRUE, cancellable, error))
     return FALSE;
 
   rpmostree_print_transaction (dnfctx);
@@ -1685,7 +1685,7 @@ rpmostree_compose_builtin_extensions (int argc, char **argv, RpmOstreeCommandInv
 
 #undef TMP_EXTENSIONS_ROOTFS
 
-  if (!rpmostree_context_prepare (ctx, cancellable, error))
+  if (!rpmostree_context_prepare (ctx, FALSE, cancellable, error))
     return FALSE;
 
   if (!glnx_shutil_mkdir_p_at (AT_FDCWD, opt_extensions_output_dir, 0755, cancellable, error))

src/daemon/rpm-ostreed.service

@@ -28,3 +28,5 @@ TimeoutStartSec=5m
 # with the rpm-ostree user.
 ExecStart=+rpm-ostree start-daemon
 ExecReload=rpm-ostree reload
+# disable/enable downloading filelists
+Environment="DOWNLOAD_FILELISTS=false"

src/daemon/rpmostree-sysroot-upgrader.cxx

@@ -953,7 +953,7 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self, GCancellable *cancellable,
 
   if (rpmostree_origin_has_any_packages (self->computed_origin))
     {
-      if (!rpmostree_context_prepare (self->ctx, cancellable, error))
+      if (!rpmostree_context_prepare (self->ctx, FALSE, cancellable, error))
         return FALSE;
       self->layering_type = RPMOSTREE_SYSROOT_UPGRADER_LAYERING_RPMMD_REPOS;
 
@@ -967,7 +967,7 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self, GCancellable *cancellable,
        * See comment in rpmostree_origin_may_require_local_assembly(). */
       if (rpmostree_origin_has_modules_enable (self->computed_origin))
         {
-          if (!rpmostree_context_prepare (self->ctx, cancellable, error))
+          if (!rpmostree_context_prepare (self->ctx, FALSE, cancellable, error))
             return FALSE;
         }
       rpmostree_context_set_is_empty (self->ctx);

src/daemon/rpmostreed-os.cxx

@@ -911,8 +911,8 @@ os_handle_modify_yum_repo (RPMOSTreeOS *interface, GDBusMethodInvocation *invoca
 }
 
 static DnfContext *
-os_create_dnf_context_simple (RPMOSTreeOS *interface, gboolean with_sack, GCancellable *cancellable,
-                              GError **error)
+os_create_dnf_context_simple (RPMOSTreeOS *interface, gboolean with_sack, gboolean enable_filelists,
+                              GCancellable *cancellable, GError **error)
 {
   glnx_unref_object OstreeSysroot *ot_sysroot = NULL;
   const gchar *os_name = rpmostree_os_get_name (interface);
@@ -947,14 +947,29 @@ os_create_dnf_context_simple (RPMOSTreeOS *interface, gboolean with_sack, GCance
   /* point libdnf to our repos dir */
   rpmostree_context_configure_from_deployment (ctx, ot_sysroot, cfg_merge_deployment);
 
-  if (with_sack
-      && !rpmostree_context_download_metadata (
-          ctx,
-          static_cast<DnfContextSetupSackFlags> (DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB
-                                                 | DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO),
-          cancellable, error))
-    return NULL;
+  auto flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB
+                                          | DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_FILELISTS
+                                          | DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO);
+
+  char *download_filelists = (char *)"false";
+  if (g_getenv ("DOWNLOAD_FILELISTS"))
+    {
+      download_filelists = (char *)(g_getenv ("DOWNLOAD_FILELISTS"));
+      for (int i = 0; i < strlen (download_filelists); i++)
+        {
+          download_filelists[i] = tolower (download_filelists[i]);
+        }
+    }
 
+  /* check if filelist optimization is disabled */
+  if (strcmp (download_filelists, "true") == 0 || enable_filelists)
+    {
+      flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB
+                                         | DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO);
+    }
+
+  if (with_sack && !rpmostree_context_download_metadata (ctx, flags, cancellable, error))
+    return NULL;
   DnfContext *dnfctx = rpmostree_context_get_dnf (ctx);
   return static_cast<DnfContext *> (g_object_ref (dnfctx));
 }
@@ -968,7 +983,7 @@ os_handle_list_repos (RPMOSTreeOS *interface, GDBusMethodInvocation *invocation)
   sd_journal_print (LOG_INFO, "Handling ListRepos for caller %s",
                     g_dbus_method_invocation_get_sender (invocation));
 
-  dnfctx = os_create_dnf_context_simple (interface, FALSE, NULL, &local_error);
+  dnfctx = os_create_dnf_context_simple (interface, FALSE, FALSE, NULL, &local_error);
   if (dnfctx == NULL)
     return os_throw_dbus_invocation_error (invocation, &local_error);
 
@@ -1073,7 +1088,7 @@ os_handle_what_provides (RPMOSTreeOS *interface, GDBusMethodInvocation *invocati
   sd_journal_print (LOG_INFO, "Handling WhatProvides for caller %s",
                     g_dbus_method_invocation_get_sender (invocation));
 
-  dnfctx = os_create_dnf_context_simple (interface, TRUE, cancellable, &local_error);
+  dnfctx = os_create_dnf_context_simple (interface, TRUE, FALSE, cancellable, &local_error);
   if (dnfctx == NULL)
     return os_throw_dbus_invocation_error (invocation, &local_error);
 
@@ -1112,7 +1127,7 @@ os_handle_get_packages (RPMOSTreeOS *interface, GDBusMethodInvocation *invocatio
   sd_journal_print (LOG_INFO, "Handling GetPackages for caller %s",
                     g_dbus_method_invocation_get_sender (invocation));
 
-  dnfctx = os_create_dnf_context_simple (interface, TRUE, cancellable, &local_error);
+  dnfctx = os_create_dnf_context_simple (interface, TRUE, FALSE, cancellable, &local_error);
   if (dnfctx == NULL)
     return os_throw_dbus_invocation_error (invocation, &local_error);
 
@@ -1317,7 +1332,7 @@ os_handle_search (RPMOSTreeOS *interface, GDBusMethodInvocation *invocation,
     }
 
   g_autoptr (DnfContext) dnfctx
-      = os_create_dnf_context_simple (interface, TRUE, cancellable, &local_error);
+      = os_create_dnf_context_simple (interface, TRUE, FALSE, cancellable, &local_error);
   if (dnfctx == NULL)
     return os_throw_dbus_invocation_error (invocation, &local_error);
 

src/daemon/rpmostreed-transaction-types.cxx

@@ -2279,8 +2279,30 @@ refresh_md_transaction_execute (RpmostreedTransaction *transaction, GCancellable
   rpmostree_context_configure_from_deployment (ctx, sysroot, cfg_merge_deployment);
 
   /* don't even bother loading the rpmdb */
-  if (!rpmostree_context_download_metadata (ctx, DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB,
-                                            cancellable, error))
+
+  /* skip filelists if filelists have not been previously downloaded */
+  auto flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB
+                                          | DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_FILELISTS);
+
+  /* update filelists if filelists were previously downloaded or filelist optimization is disabled
+   */
+
+  char *download_filelists = (char *)"false";
+  if (g_getenv ("DOWNLOAD_FILELISTS"))
+    {
+      download_filelists = (char *)(g_getenv ("DOWNLOAD_FILELISTS"));
+      for (int i = 0; i < strlen (download_filelists); i++)
+        {
+          download_filelists[i] = tolower (download_filelists[i]);
+        }
+    }
+
+  if (rpmostree_context_get_filelists_exist (ctx) || strcmp (download_filelists, "true") == 0)
+    {
+      flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB);
+    }
+
+  if (!rpmostree_context_download_metadata (ctx, flags, cancellable, error))
     return FALSE;
 
   return TRUE;

src/libpriv/rpmostree-container.cxx

@@ -59,7 +59,7 @@ rpmostree_container_rebuild (rpmostreecxx::Treefile &treefile, GCancellable *can
   if (!rpmostree_context_setup (ctx, "/", "/", cancellable, error))
     return FALSE;
 
-  if (!rpmostree_context_prepare (ctx, cancellable, error))
+  if (!rpmostree_context_prepare (ctx, FALSE, cancellable, error))
     return FALSE;
 
   if (!rpmostree_context_download (ctx, cancellable, error))
@@ -93,4 +93,4 @@ container_rebuild (rust::Str treefile)
   if (!rpmostree_container_rebuild (*tf, NULL, &local_error))
     util::throw_gerror (local_error);
 }
-}
+}

src/libpriv/rpmostree-core-private.h

@@ -73,6 +73,8 @@ struct _RpmOstreeContext
 
   GHashTable *fileoverride_pkgs; /* set of nevras */
 
+  gboolean filelists_exist;
+
   std::optional<rust::Box<rpmostreecxx::LockfileConfig> > lockfile;
   gboolean lockfile_strict;
 

src/libpriv/rpmostree-core.cxx

@@ -119,6 +119,7 @@ rpmostree_context_init (RpmOstreeContext *self)
   self->dnf_cache_policy = RPMOSTREE_CONTEXT_DNF_CACHE_DEFAULT;
   self->enable_rofiles = TRUE;
   self->unprivileged = getuid () != 0;
+  self->filelists_exist = FALSE;
 }
 
 static void
@@ -428,6 +429,18 @@ rpmostree_context_get_dnf (RpmOstreeContext *self)
   return self->dnfctx;
 }
 
+gboolean
+rpmostree_context_get_filelists_exist (RpmOstreeContext *self)
+{
+  return self->filelists_exist;
+}
+
+void
+rpmostree_context_set_filelists_exist (RpmOstreeContext *self, gboolean filelists_exist)
+{
+  self->filelists_exist = filelists_exist;
+}
+
 /* Add rpmmd repo information, since it's very useful for determining
  * state.  See also:
  *
@@ -1055,7 +1068,6 @@ rpmostree_context_download_metadata (RpmOstreeContext *self, DnfContextSetupSack
                                 dnf_repo_get_id (repo), !updated ? " (cached)" : "", repo_ts_str,
                                 dnf_repo_get_n_solvables (repo));
     }
-
   return TRUE;
 }
 
@@ -1724,7 +1736,8 @@ find_locked_packages (RpmOstreeContext *self, GPtrArray **out_pkgs, GError **err
 
 /* Check for/download new rpm-md, then depsolve */
 gboolean
-rpmostree_context_prepare (RpmOstreeContext *self, GCancellable *cancellable, GError **error)
+rpmostree_context_prepare (RpmOstreeContext *self, gboolean enable_filelists,
+                           GCancellable *cancellable, GError **error)
 {
   g_assert (!self->empty);
 
@@ -1759,10 +1772,43 @@ rpmostree_context_prepare (RpmOstreeContext *self, GCancellable *cancellable, GE
   /* setup sack if not yet set up */
   if (dnf_context_get_sack (dnfctx) == NULL)
     {
+      auto flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO
+                                              | DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_FILELISTS);
+
+      char *download_filelists = (char *)"false";
+      if (g_getenv ("DOWNLOAD_FILELISTS"))
+        {
+          download_filelists = (char *)(g_getenv ("DOWNLOAD_FILELISTS"));
+          for (int i = 0; i < strlen (download_filelists); i++)
+            {
+              download_filelists[i] = tolower (download_filelists[i]);
+            }
+        }
+
+      /* check if filelist optimization is disabled */
+      if (strcmp (download_filelists, "true") == 0 || enable_filelists)
+        {
+          flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO);
+        }
+      else
+        {
+          auto pkg = "";
+          for (auto &pkg_str : packages)
+            {
+              pkg = std::string (pkg_str).c_str ();
+              char *query = strchr ((char *)pkg, '/');
+              if (query)
+                {
+                  flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO);
+                  rpmostree_context_set_filelists_exist (self, TRUE);
+                  break;
+                }
+            }
+        }
+
       /* default to loading updateinfo in this path; this allows the sack to be used later
        * on for advisories -- it's always downloaded anyway */
-      if (!rpmostree_context_download_metadata (self, DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO,
-                                                cancellable, error))
+      if (!rpmostree_context_download_metadata (self, flags, cancellable, error))
         return FALSE;
       journal_rpmmd_info (self);
     }
@@ -2295,8 +2341,26 @@ rpmostree_find_and_download_packages (const char *const *packages, const char *s
   g_autofree char *reposdir = g_build_filename (repo_root ?: source_root, "etc/yum.repos.d", NULL);
   dnf_context_set_repo_dir (ctx->dnfctx, reposdir);
 
-  if (!rpmostree_context_download_metadata (ctx, DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB,
-                                            cancellable, error))
+  auto flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB
+                                          | DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_FILELISTS);
+
+  char *download_filelists = (char *)"false";
+  if (g_getenv ("DOWNLOAD_FILELISTS"))
+    {
+      download_filelists = (char *)(g_getenv ("DOWNLOAD_FILELISTS"));
+      for (int i = 0; i < strlen (download_filelists); i++)
+        {
+          download_filelists[i] = tolower (download_filelists[i]);
+        }
+    }
+
+  /* check if filelist optimization is disabled */
+  if (strcmp (download_filelists, "true") == 0)
+    {
+      flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB);
+    }
+
+  if (!rpmostree_context_download_metadata (ctx, flags, cancellable, error))
     return glnx_prefix_error (error, "Downloading metadata");
 
   g_autoptr (GPtrArray) pkgs = g_ptr_array_new_with_free_func (g_object_unref);

src/libpriv/rpmostree-core.h

@@ -134,8 +134,8 @@ gboolean rpmostree_context_download_metadata (RpmOstreeContext *context,
                                               GCancellable *cancellable, GError **error);
 
 /* This API allocates an install context, use with one of the later ones */
-gboolean rpmostree_context_prepare (RpmOstreeContext *self, GCancellable *cancellable,
-                                    GError **error);
+gboolean rpmostree_context_prepare (RpmOstreeContext *self, gboolean enable_filelists,
+                                    GCancellable *cancellable, GError **error);
 
 GPtrArray *rpmostree_context_get_packages (RpmOstreeContext *self);
 
@@ -166,6 +166,10 @@ gboolean rpmostree_context_import (RpmOstreeContext *self, GCancellable *cancell
 gboolean rpmostree_context_force_relabel (RpmOstreeContext *self, GCancellable *cancellable,
                                           GError **error);
 
+gboolean rpmostree_context_get_filelists_exist (RpmOstreeContext *);
+
+void rpmostree_context_set_filelists_exist (RpmOstreeContext *, gboolean filelists_exist);
+
 typedef enum
 {
   RPMOSTREE_ASSEMBLE_TYPE_SERVER_BASE,

tests/common/libtest.sh

@@ -537,5 +537,5 @@ rpmostree_busctl_call_os() {
   stateroot=$(rpm-ostree status --booted --json | jq -r '.deployments[0].osname')
   ospath=/org/projectatomic/rpmostree1/${stateroot//-/_}
   busctl call org.projectatomic.rpmostree1 $ospath \
-    org.projectatomic.rpmostree1.OS "$@"
+    org.projectatomic.rpmostree1.OS "$@" --timeout=240s
 }