Disable downloading filelists by default

Disable filelists from being downloaded by default for Fedora 40. Adds
the `DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_FILELISTS` where applicable,
which prevents the filelists from being downloaded.

For the changes in `rpmostree_context_prepare()`, disabling filelists is
conditional on whether a filepath was specified for `rpm-ostree install`
(ie. rpm-ostree install /usr/bin/htop).

For the changes in refresh_md_transaction_execute(), `rpm-ostree
refresh-md` will not download filelists if they were not previously
downloaded. If they were previously downloaded, the filelists will be
updated.

Since there are many functions affected by this change, the option to
return to the previous filelist behaviour is also implemented. This is
done by adding:

[Service]
Environment="DOWNLOAD_FILELISTS=true"

in a newly created conf file under /etc/systemd/system/rpm-ostreed.service.d
For example: /etc/systemd/system/rpm-ostreed.service.d/filelists.conf

src/app/rpmostree-compose-builtin-tree.cxx

@@ -397,7 +397,7 @@ install_packages (RpmOstreeTreeComposeContext *self, gboolean *out_unmodified,
         }
     }
 
-  if (!rpmostree_context_prepare (self->corectx, cancellable, error))
+  if (!rpmostree_context_prepare (self->corectx, TRUE, cancellable, error))
     return FALSE;
 
   rpmostree_print_transaction (dnfctx);
@@ -1685,7 +1685,7 @@ rpmostree_compose_builtin_extensions (int argc, char **argv, RpmOstreeCommandInv
 
 #undef TMP_EXTENSIONS_ROOTFS
 
-  if (!rpmostree_context_prepare (ctx, cancellable, error))
+  if (!rpmostree_context_prepare (ctx, FALSE, cancellable, error))
     return FALSE;
 
   if (!glnx_shutil_mkdir_p_at (AT_FDCWD, opt_extensions_output_dir, 0755, cancellable, error))

src/daemon/rpm-ostreed.service

@@ -28,3 +28,5 @@ TimeoutStartSec=5m
 # with the rpm-ostree user.
 ExecStart=+rpm-ostree start-daemon
 ExecReload=rpm-ostree reload
+# disable/enable downloading filelists
+Environment="DOWNLOAD_FILELISTS=false"

src/daemon/rpmostree-sysroot-upgrader.cxx

@@ -953,7 +953,7 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self, GCancellable *cancellable,
 
   if (rpmostree_origin_has_any_packages (self->computed_origin))
     {
-      if (!rpmostree_context_prepare (self->ctx, cancellable, error))
+      if (!rpmostree_context_prepare (self->ctx, FALSE, cancellable, error))
         return FALSE;
       self->layering_type = RPMOSTREE_SYSROOT_UPGRADER_LAYERING_RPMMD_REPOS;
 
@@ -967,7 +967,7 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self, GCancellable *cancellable,
        * See comment in rpmostree_origin_may_require_local_assembly(). */
       if (rpmostree_origin_has_modules_enable (self->computed_origin))
         {
-          if (!rpmostree_context_prepare (self->ctx, cancellable, error))
+          if (!rpmostree_context_prepare (self->ctx, FALSE, cancellable, error))
             return FALSE;
         }
       rpmostree_context_set_is_empty (self->ctx);

src/daemon/rpmostreed-os.cxx

@@ -911,8 +911,8 @@ os_handle_modify_yum_repo (RPMOSTreeOS *interface, GDBusMethodInvocation *invoca
 }
 
 static DnfContext *
-os_create_dnf_context_simple (RPMOSTreeOS *interface, gboolean with_sack, GCancellable *cancellable,
-                              GError **error)
+os_create_dnf_context_simple (RPMOSTreeOS *interface, gboolean with_sack, gboolean enable_filelists,
+                              GCancellable *cancellable, GError **error)
 {
   glnx_unref_object OstreeSysroot *ot_sysroot = NULL;
   const gchar *os_name = rpmostree_os_get_name (interface);
@@ -947,14 +947,29 @@ os_create_dnf_context_simple (RPMOSTreeOS *interface, gboolean with_sack, GCance
   /* point libdnf to our repos dir */
   rpmostree_context_configure_from_deployment (ctx, ot_sysroot, cfg_merge_deployment);
 
-  if (with_sack
-      && !rpmostree_context_download_metadata (
-          ctx,
-          static_cast<DnfContextSetupSackFlags> (DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB
-                                                 | DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO),
-          cancellable, error))
-    return NULL;
+  auto flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB
+                                          | DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_FILELISTS
+                                          | DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO);
+
+  char *download_filelists = (char *)"false";
+  if (g_getenv ("DOWNLOAD_FILELISTS"))
+    {
+      download_filelists = (char *)(g_getenv ("DOWNLOAD_FILELISTS"));
+      for (int i = 0; i < strlen (download_filelists); i++)
+        {
+          download_filelists[i] = tolower (download_filelists[i]);
+        }
+    }
 
+  /* check if filelist optimization is disabled */
+  if (strcmp (download_filelists, "true") == 0 || enable_filelists)
+    {
+      flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB
+                                         | DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO);
+    }
+
+  if (with_sack && !rpmostree_context_download_metadata (ctx, flags, cancellable, error))
+    return NULL;
   DnfContext *dnfctx = rpmostree_context_get_dnf (ctx);
   return static_cast<DnfContext *> (g_object_ref (dnfctx));
 }
@@ -968,7 +983,7 @@ os_handle_list_repos (RPMOSTreeOS *interface, GDBusMethodInvocation *invocation)
   sd_journal_print (LOG_INFO, "Handling ListRepos for caller %s",
                     g_dbus_method_invocation_get_sender (invocation));
 
-  dnfctx = os_create_dnf_context_simple (interface, FALSE, NULL, &local_error);
+  dnfctx = os_create_dnf_context_simple (interface, FALSE, FALSE, NULL, &local_error);
   if (dnfctx == NULL)
     return os_throw_dbus_invocation_error (invocation, &local_error);
 
@@ -1073,7 +1088,7 @@ os_handle_what_provides (RPMOSTreeOS *interface, GDBusMethodInvocation *invocati
   sd_journal_print (LOG_INFO, "Handling WhatProvides for caller %s",
                     g_dbus_method_invocation_get_sender (invocation));
 
-  dnfctx = os_create_dnf_context_simple (interface, TRUE, cancellable, &local_error);
+  dnfctx = os_create_dnf_context_simple (interface, TRUE, FALSE, cancellable, &local_error);
   if (dnfctx == NULL)
     return os_throw_dbus_invocation_error (invocation, &local_error);
 
@@ -1112,7 +1127,7 @@ os_handle_get_packages (RPMOSTreeOS *interface, GDBusMethodInvocation *invocatio
   sd_journal_print (LOG_INFO, "Handling GetPackages for caller %s",
                     g_dbus_method_invocation_get_sender (invocation));
 
-  dnfctx = os_create_dnf_context_simple (interface, TRUE, cancellable, &local_error);
+  dnfctx = os_create_dnf_context_simple (interface, TRUE, FALSE, cancellable, &local_error);
   if (dnfctx == NULL)
     return os_throw_dbus_invocation_error (invocation, &local_error);
 
@@ -1317,7 +1332,7 @@ os_handle_search (RPMOSTreeOS *interface, GDBusMethodInvocation *invocation,
     }
 
   g_autoptr (DnfContext) dnfctx
-      = os_create_dnf_context_simple (interface, TRUE, cancellable, &local_error);
+      = os_create_dnf_context_simple (interface, TRUE, FALSE, cancellable, &local_error);
   if (dnfctx == NULL)
     return os_throw_dbus_invocation_error (invocation, &local_error);
 

src/daemon/rpmostreed-transaction-types.cxx

@@ -2279,8 +2279,30 @@ refresh_md_transaction_execute (RpmostreedTransaction *transaction, GCancellable
   rpmostree_context_configure_from_deployment (ctx, sysroot, cfg_merge_deployment);
 
   /* don't even bother loading the rpmdb */
-  if (!rpmostree_context_download_metadata (ctx, DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB,
-                                            cancellable, error))
+
+  /* skip filelists if filelists have not been previously downloaded */
+  auto flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB
+                                          | DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_FILELISTS);
+
+  /* update filelists if filelists were previously downloaded or filelist optimization is disabled
+   */
+
+  char *download_filelists = (char *)"false";
+  if (g_getenv ("DOWNLOAD_FILELISTS"))
+    {
+      download_filelists = (char *)(g_getenv ("DOWNLOAD_FILELISTS"));
+      for (int i = 0; i < strlen (download_filelists); i++)
+        {
+          download_filelists[i] = tolower (download_filelists[i]);
+        }
+    }
+
+  if (rpmostree_context_get_filelists_exist (ctx) || strcmp (download_filelists, "true") == 0)
+    {
+      flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB);
+    }
+
+  if (!rpmostree_context_download_metadata (ctx, flags, cancellable, error))
     return FALSE;
 
   return TRUE;

src/libpriv/rpmostree-container.cxx

@@ -59,7 +59,7 @@ rpmostree_container_rebuild (rpmostreecxx::Treefile &treefile, GCancellable *can
   if (!rpmostree_context_setup (ctx, "/", "/", cancellable, error))
     return FALSE;
 
-  if (!rpmostree_context_prepare (ctx, cancellable, error))
+  if (!rpmostree_context_prepare (ctx, FALSE, cancellable, error))
     return FALSE;
 
   if (!rpmostree_context_download (ctx, cancellable, error))
@@ -93,4 +93,4 @@ container_rebuild (rust::Str treefile)
   if (!rpmostree_container_rebuild (*tf, NULL, &local_error))
     util::throw_gerror (local_error);
 }
-}
+}

src/libpriv/rpmostree-core-private.h

@@ -73,6 +73,8 @@ struct _RpmOstreeContext
 
   GHashTable *fileoverride_pkgs; /* set of nevras */
 
+  gboolean filelists_exist;
+
   std::optional<rust::Box<rpmostreecxx::LockfileConfig> > lockfile;
   gboolean lockfile_strict;
 

src/libpriv/rpmostree-core.cxx

@@ -119,6 +119,7 @@ rpmostree_context_init (RpmOstreeContext *self)
   self->dnf_cache_policy = RPMOSTREE_CONTEXT_DNF_CACHE_DEFAULT;
   self->enable_rofiles = TRUE;
   self->unprivileged = getuid () != 0;
+  self->filelists_exist = FALSE;
 }
 
 static void
@@ -428,6 +429,18 @@ rpmostree_context_get_dnf (RpmOstreeContext *self)
   return self->dnfctx;
 }
 
+gboolean
+rpmostree_context_get_filelists_exist (RpmOstreeContext *self)
+{
+  return self->filelists_exist;
+}
+
+void
+rpmostree_context_set_filelists_exist (RpmOstreeContext *self, gboolean filelists_exist)
+{
+  self->filelists_exist = filelists_exist;
+}
+
 /* Add rpmmd repo information, since it's very useful for determining
  * state.  See also:
  *
@@ -1055,7 +1068,6 @@ rpmostree_context_download_metadata (RpmOstreeContext *self, DnfContextSetupSack
                                 dnf_repo_get_id (repo), !updated ? " (cached)" : "", repo_ts_str,
                                 dnf_repo_get_n_solvables (repo));
     }
-
   return TRUE;
 }
 
@@ -1724,7 +1736,8 @@ find_locked_packages (RpmOstreeContext *self, GPtrArray **out_pkgs, GError **err
 
 /* Check for/download new rpm-md, then depsolve */
 gboolean
-rpmostree_context_prepare (RpmOstreeContext *self, GCancellable *cancellable, GError **error)
+rpmostree_context_prepare (RpmOstreeContext *self, gboolean enable_filelists,
+                           GCancellable *cancellable, GError **error)
 {
   g_assert (!self->empty);
 
@@ -1759,10 +1772,43 @@ rpmostree_context_prepare (RpmOstreeContext *self, GCancellable *cancellable, GE
   /* setup sack if not yet set up */
   if (dnf_context_get_sack (dnfctx) == NULL)
     {
+      auto flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO
+                                              | DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_FILELISTS);
+
+      char *download_filelists = (char *)"false";
+      if (g_getenv ("DOWNLOAD_FILELISTS"))
+        {
+          download_filelists = (char *)(g_getenv ("DOWNLOAD_FILELISTS"));
+          for (int i = 0; i < strlen (download_filelists); i++)
+            {
+              download_filelists[i] = tolower (download_filelists[i]);
+            }
+        }
+
+      /* check if filelist optimization is disabled */
+      if (strcmp (download_filelists, "true") == 0 || enable_filelists)
+        {
+          flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO);
+        }
+      else
+        {
+          auto pkg = "";
+          for (auto &pkg_str : packages)
+            {
+              pkg = std::string (pkg_str).c_str ();
+              char *query = strchr ((char *)pkg, '/');
+              if (query)
+                {
+                  flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO);
+                  rpmostree_context_set_filelists_exist (self, TRUE);
+                  break;
+                }
+            }
+        }
+
       /* default to loading updateinfo in this path; this allows the sack to be used later
        * on for advisories -- it's always downloaded anyway */
-      if (!rpmostree_context_download_metadata (self, DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO,
-                                                cancellable, error))
+      if (!rpmostree_context_download_metadata (self, flags, cancellable, error))
         return FALSE;
       journal_rpmmd_info (self);
     }
@@ -2295,8 +2341,26 @@ rpmostree_find_and_download_packages (const char *const *packages, const char *s
   g_autofree char *reposdir = g_build_filename (repo_root ?: source_root, "etc/yum.repos.d", NULL);
   dnf_context_set_repo_dir (ctx->dnfctx, reposdir);
 
-  if (!rpmostree_context_download_metadata (ctx, DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB,
-                                            cancellable, error))
+  auto flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB
+                                          | DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_FILELISTS);
+
+  char *download_filelists = (char *)"false";
+  if (g_getenv ("DOWNLOAD_FILELISTS"))
+    {
+      download_filelists = (char *)(g_getenv ("DOWNLOAD_FILELISTS"));
+      for (int i = 0; i < strlen (download_filelists); i++)
+        {
+          download_filelists[i] = tolower (download_filelists[i]);
+        }
+    }
+
+  /* check if filelist optimization is disabled */
+  if (strcmp (download_filelists, "true") == 0)
+    {
+      flags = (DnfContextSetupSackFlags)(DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB);
+    }
+
+  if (!rpmostree_context_download_metadata (ctx, flags, cancellable, error))
     return glnx_prefix_error (error, "Downloading metadata");
 
   g_autoptr (GPtrArray) pkgs = g_ptr_array_new_with_free_func (g_object_unref);

src/libpriv/rpmostree-core.h

@@ -134,8 +134,8 @@ gboolean rpmostree_context_download_metadata (RpmOstreeContext *context,
                                               GCancellable *cancellable, GError **error);
 
 /* This API allocates an install context, use with one of the later ones */
-gboolean rpmostree_context_prepare (RpmOstreeContext *self, GCancellable *cancellable,
-                                    GError **error);
+gboolean rpmostree_context_prepare (RpmOstreeContext *self, gboolean enable_filelists,
+                                    GCancellable *cancellable, GError **error);
 
 GPtrArray *rpmostree_context_get_packages (RpmOstreeContext *self);
 
@@ -166,6 +166,10 @@ gboolean rpmostree_context_import (RpmOstreeContext *self, GCancellable *cancell
 gboolean rpmostree_context_force_relabel (RpmOstreeContext *self, GCancellable *cancellable,
                                           GError **error);
 
+gboolean rpmostree_context_get_filelists_exist (RpmOstreeContext *);
+
+void rpmostree_context_set_filelists_exist (RpmOstreeContext *, gboolean filelists_exist);
+
 typedef enum
 {
   RPMOSTREE_ASSEMBLE_TYPE_SERVER_BASE,

tests/common/libtest.sh

@@ -537,5 +537,5 @@ rpmostree_busctl_call_os() {
   stateroot=$(rpm-ostree status --booted --json | jq -r '.deployments[0].osname')
   ospath=/org/projectatomic/rpmostree1/${stateroot//-/_}
   busctl call org.projectatomic.rpmostree1 $ospath \
-    org.projectatomic.rpmostree1.OS "$@"
+    org.projectatomic.rpmostree1.OS "$@" --timeout=240s
 }