Merge branch 'acidanthera:master' into YamingNetwork

Changelog.md

@@ -5,6 +5,7 @@ OpenCore Changelog
 - Fixed EHCI handoff logic in OpenDuet, causing older machines to hang at start
 - Added Arrow Lake CPU detection
 - Fixed Raptor Lake CPU detection
+- Supported booting with TuneD in Fedora 41 in OpenLinuxBoot
 
 #### v1.0.2
 - Fixed error in macrecovery when running headless, thx @mkorje

Docs/Configuration.md5

@@ -1 +1 @@
-29ba6cfd103b4a2b7c4e935fcf700cd3
+c025c15e04dc77d69d19ce3e12047a79

Docs/Configuration.tex

@@ -6920,7 +6920,8 @@ \subsubsection{Configuration}
   \begin{itemize}
   \tightlist
     \item \texttt{LINUX\_BOOT\_ADD\_RW},
-    \item \texttt{LINUX\_BOOT\_LOG\_VERBOSE} and
+    \item \texttt{LINUX\_BOOT\_LOG\_VERBOSE},
+    \item \texttt{LINUX\_BOOT\_LOG\_GRUB\_VARS} and
     \item \texttt{LINUX\_BOOT\_ADD\_DEBUG\_INFO}.
   \end{itemize}
   \medskip
@@ -7001,6 +7002,17 @@ \subsubsection{Configuration}
     partition's unique partition uuid, to each generated entry name. Can help with debugging
     the origin of entries generated by the driver when there are multiple Linux installs on
     one system.
+    \item \texttt{0x00010000} (bit \texttt{16}) --- \texttt{LINUX\_BOOT\_LOG\_GRUB\_VARS},
+    When a \texttt{BootLoaderSpecByDefault} setup is detected, log available GRUB variables
+    found in \texttt{grub2/grubenv} and \texttt{grub2/grub.cfg}.
+    \item \texttt{0x00020000} (bit \texttt{17}) --- \texttt{LINUX\_BOOT\_FIX\_TUNED},
+    In some circumstances, such as after upgrades which add TuneD to existing systems, the TuneD
+    system tuning plugin may add its GRUB variables to \texttt{loader/entries/*.conf} files but not
+    initialise them in \texttt{grub2/grub.cfg}. In order to avoid incorrect boots, OpenLinuxBoot
+    treats used, non-initialised GRUB variables as an error. When this flag is set, empty values
+    are added for the TuneD variables \texttt{tuned\_params} and \texttt{tuned\_initrd} if they
+    are not present. This is required for OpenLinuxBoot on TuneD systems with this problem, and
+    harmless otherwise.
   \end{itemize} \medskip
 
   Flag values can be specified in hexadecimal beginning with \texttt{0x} or in decimal,
@@ -7048,7 +7060,7 @@ \subsubsection{Additional information}
 
 OpenLinuxBoot can detect the \texttt{loader/entries/*.conf} files created according to the
 \href{https://systemd.io/BOOT_LOADER_SPECIFICATION/}{Boot Loader Specification} or the closely related
-\href{https://fedoraproject.org/wiki/Changes/BootLoaderSpecByDefault}{systemd BootLoaderSpecByDefault}. The
+\href{https://fedoraproject.org/wiki/Changes/BootLoaderSpecByDefault}{Fedora BootLoaderSpecByDefault}. The
 former is specific to systemd-boot and is used by Arch Linux, the latter applies to most Fedora-related distros
 including Fedora itself, RHEL and variants.
 
@@ -7064,13 +7076,13 @@ \subsubsection{Additional information}
 \texttt{autoopts:\{partuuid\}=...} (\texttt{+=} variants of these options will not work, as these only add additional
 arguments).
 
-BootLoaderSpecByDefault (but not pure Boot Loader Specification) can expand GRUB variables
+Fedora \texttt{BootLoaderSpecByDefault} (but not pure Boot Loader Specification) can expand GRUB variables
 in the \texttt{*.conf} files -- and this is used in practice in certain distros such as CentOS.
 In order to handle this correctly, when this situation is detected OpenLinuxBoot extracts all variables from
 \texttt{\{boot\}/grub2/grubenv} and also any unconditionally set variables from
 \texttt{\{boot\}/grub2/grub.cfg}, and then expands these where required in \texttt{*.conf} file entries.
 
-The only currently supported method of starting Linux kernels relies on their being compiled with EFISTUB.
+The only currently supported method of starting Linux kernels from OpenLinuxBoot relies on their being compiled with EFISTUB.
 This applies to almost all modern distros, particularly those which use systemd. Note that most modern
 distros use systemd as their system manager, even though most do not use systemd-boot as
 their bootloader.

Docs/Differences/Differences.tex

@@ -1,7 +1,7 @@
 \documentclass[]{article}
 %DIF LATEXDIFF DIFFERENCE FILE
-%DIF DEL PreviousConfiguration.tex   Thu Oct 10 12:37:13 2024
-%DIF ADD ../Configuration.tex        Thu Oct 10 12:37:13 2024
+%DIF DEL PreviousConfiguration.tex   Sat Nov  9 05:47:31 2024
+%DIF ADD ../Configuration.tex        Mon Nov 18 22:46:55 2024
 
 \usepackage{lmodern}
 \usepackage{amssymb,amsmath}
@@ -118,7 +118,7 @@
 %DIF HYPERREF PREAMBLE %DIF PREAMBLE
 \providecommand{\DIFadd}[1]{\texorpdfstring{\DIFaddtex{#1}}{#1}} %DIF PREAMBLE
 \providecommand{\DIFdel}[1]{\texorpdfstring{\DIFdeltex{#1}}{}} %DIF PREAMBLE
-%DIF LISTINGS PREAMBLE %DIF PREAMBLE
+%DIF COLORLISTINGS PREAMBLE %DIF PREAMBLE
 \RequirePackage{listings} %DIF PREAMBLE
 \RequirePackage{color} %DIF PREAMBLE
 \lstdefinelanguage{DIFcode}{ %DIF PREAMBLE
@@ -6980,7 +6980,8 @@ \subsubsection{Configuration}
   \begin{itemize}
   \tightlist
     \item \texttt{LINUX\_BOOT\_ADD\_RW},
-    \item \texttt{LINUX\_BOOT\_LOG\_VERBOSE} and
+    \item \texttt{LINUX\_BOOT\_LOG\_VERBOSE}\DIFaddbegin \DIFadd{,
+    }\item \texttt{\DIFadd{LINUX\_BOOT\_LOG\_GRUB\_VARS}} \DIFaddend and
     \item \texttt{LINUX\_BOOT\_ADD\_DEBUG\_INFO}.
   \end{itemize}
   \medskip
@@ -7061,7 +7062,18 @@ \subsubsection{Configuration}
     partition's unique partition uuid, to each generated entry name. Can help with debugging
     the origin of entries generated by the driver when there are multiple Linux installs on
     one system.
-  \end{itemize} \medskip
+    \DIFaddbegin \item \texttt{\DIFadd{0x00010000}} \DIFadd{(bit }\texttt{\DIFadd{16}}\DIFadd{) --- }\texttt{\DIFadd{LINUX\_BOOT\_LOG\_GRUB\_VARS}}\DIFadd{,
+    When a }\texttt{\DIFadd{BootLoaderSpecByDefault}} \DIFadd{setup is detected, log available GRUB variables
+    found in }\texttt{\DIFadd{grub2/grubenv}} \DIFadd{and }\texttt{\DIFadd{grub2/grub.cfg}}\DIFadd{.
+    }\item \texttt{\DIFadd{0x00020000}} \DIFadd{(bit }\texttt{\DIFadd{17}}\DIFadd{) --- }\texttt{\DIFadd{LINUX\_BOOT\_FIX\_TUNED}}\DIFadd{,
+    In some circumstances, such as after upgrades which add TuneD to existing systems, the TuneD
+    system tuning plugin may add its GRUB variables to }\texttt{\DIFadd{loader/entries/*.conf}} \DIFadd{files but not
+    initialise them in }\texttt{\DIFadd{grub2/grub.cfg}}\DIFadd{. In order to avoid incorrect boots, OpenLinuxBoot
+    treats used, non-initialised GRUB variables as an error. When this flag is set, empty values
+    are added for the TuneD variables }\texttt{\DIFadd{tuned\_params}} \DIFadd{and }\texttt{\DIFadd{tuned\_initrd}} \DIFadd{if they
+    are not present. This is required for OpenLinuxBoot on TuneD systems with this problem, and
+    harmless otherwise.
+  }\DIFaddend \end{itemize} \medskip
 
   Flag values can be specified in hexadecimal beginning with \texttt{0x} or in decimal,
   e.g. \texttt{flags=0x80} or \texttt{flags=128}. It is also possible to specify flags to
@@ -7108,7 +7120,7 @@ \subsubsection{Additional information}
 
 OpenLinuxBoot can detect the \texttt{loader/entries/*.conf} files created according to the
 \href{https://systemd.io/BOOT_LOADER_SPECIFICATION/}{Boot Loader Specification} or the closely related
-\href{https://fedoraproject.org/wiki/Changes/BootLoaderSpecByDefault}{systemd BootLoaderSpecByDefault}. The
+\href{https://fedoraproject.org/wiki/Changes/BootLoaderSpecByDefault}{\DIFdelbegin \DIFdel{systemd }\DIFdelend \DIFaddbegin \DIFadd{Fedora }\DIFaddend BootLoaderSpecByDefault}. The
 former is specific to systemd-boot and is used by Arch Linux, the latter applies to most Fedora-related distros
 including Fedora itself, RHEL and variants.
 
@@ -7124,13 +7136,13 @@ \subsubsection{Additional information}
 \texttt{autoopts:\{partuuid\}=...} (\texttt{+=} variants of these options will not work, as these only add additional
 arguments).
 
-BootLoaderSpecByDefault (but not pure Boot Loader Specification) can expand GRUB variables
+\DIFdelbegin \DIFdel{BootLoaderSpecByDefault }\DIFdelend \DIFaddbegin \DIFadd{Fedora }\texttt{\DIFadd{BootLoaderSpecByDefault}} \DIFaddend (but not pure Boot Loader Specification) can expand GRUB variables
 in the \texttt{*.conf} files -- and this is used in practice in certain distros such as CentOS.
 In order to handle this correctly, when this situation is detected OpenLinuxBoot extracts all variables from
 \texttt{\{boot\}/grub2/grubenv} and also any unconditionally set variables from
 \texttt{\{boot\}/grub2/grub.cfg}, and then expands these where required in \texttt{*.conf} file entries.
 
-The only currently supported method of starting Linux kernels relies on their being compiled with EFISTUB.
+The only currently supported method of starting Linux kernels \DIFaddbegin \DIFadd{from OpenLinuxBoot }\DIFaddend relies on their being compiled with EFISTUB.
 This applies to almost all modern distros, particularly those which use systemd. Note that most modern
 distros use systemd as their system manager, even though most do not use systemd-boot as
 their bootloader.

Include/Acidanthera/Library/OcBootManagementLib.h

@@ -1926,7 +1926,7 @@ OcParseLoadOptions (
 
 /**
   Parse Unix-style var file or string. Parses a couple of useful ASCII
-  GRUB config files (multi-line, name=var, with optinal comments) and
+  GRUB config files (multi-line, name=var, with optional comments) and
   defines a standard format for Unicode UEFI LoadOptions.
 
   Assumes CHAR_NULL terminated Unicode string of space separated options,
@@ -1940,9 +1940,10 @@ OcParseLoadOptions (
 
   @param[in]   StrVars            Raw var string.
   @param[out]  ParsedVars         Parsed variables if successful, NULL otherwise.
-                                  Caller may free after use with OcFlexArrayFree
-                                  if required.
+                                  Caller may free after use with OcFlexArrayFree.
   @param[in]   StringFormat       Are option names and values Unicode or ASCII?
+  @param[in]   TokensOnly         If TRUE parse as a sequence of token values only,
+                                  rather than as a sequence of name[=[value]] pairs.
 
   @retval EFI_SUCCESS             Success.
   @retval EFI_NOT_FOUND           Missing or empty load options.
@@ -1953,7 +1954,8 @@ EFI_STATUS
 OcParseVars (
   IN           VOID              *StrVars,
   OUT       OC_FLEX_ARRAY        **ParsedVars,
-  IN     CONST OC_STRING_FORMAT  StringFormat
+  IN     CONST OC_STRING_FORMAT  StringFormat,
+  IN     CONST BOOLEAN           TokensOnly
   );
 
 /**

Include/Acidanthera/Library/OcStringLib.h

@@ -742,4 +742,16 @@ OcIsSpace (
   CHAR16  Ch
   );
 
+/**
+  Determine if a particular character is whitespace or CHAR_NULL.
+
+  @param[in] Ch               The character to check.
+
+  @return  Returns TRUE if Ch is a whitespace character or CHAR_NULL.
+**/
+BOOLEAN
+OcIsSpaceOrNull (
+  CHAR16  Ch
+  );
+
 #endif // OC_STRING_LIB_H

Library/OcBootManagementLib/BootArguments.c

@@ -1,5 +1,5 @@
 /** @file
-  Copyright (C) 2019-2021, vit9696, mikebeaton. All rights reserved.<BR>
+  Copyright (C) 2019-2024, vit9696, mikebeaton. All rights reserved.<BR>
   SPDX-License-Identifier: BSD-3-Clause
 **/
 
@@ -24,19 +24,19 @@ typedef enum PARSE_VARS_STATE_ {
   PARSE_VARS_WHITE_SPACE,
   PARSE_VARS_COMMENT,
   PARSE_VARS_NAME,
-  PARSE_VARS_VALUE_FIRST,
   PARSE_VARS_VALUE,
   PARSE_VARS_QUOTED_VALUE,
   PARSE_VARS_SHELL_EXPANSION
 } PARSE_VARS_STATE;
 
 //
-// Shift from token start to current position forwards by offset characters.
+// Shift memory from token start to current position forwards by offset bytes
+// and update token to point to shifted start (thereby discarding offset bytes
+// from the token ending at current position).
 //
 #define SHIFT_TOKEN(pos, token, offset)  do {\
   CopyMem ((UINT8 *)(token) + (offset), (token), (UINT8 *)(pos) - (UINT8 *)(token)); \
   (token) = (UINT8 *)(token) + (offset); \
-  (pos)   = (UINT8 *)(pos) + (offset); \
 } while (0)
 
 VOID
@@ -434,7 +434,7 @@ OcParseLoadOptions (
     return EFI_NOT_FOUND;
   }
 
-  Status = OcParseVars (LoadedImage->LoadOptions, ParsedVars, OcStringFormatUnicode);
+  Status = OcParseVars (LoadedImage->LoadOptions, ParsedVars, OcStringFormatUnicode, FALSE);
 
   if (Status == EFI_INVALID_PARAMETER) {
     DEBUG ((DEBUG_ERROR, "OCB: Failed to parse LoadOptions (%p:%u)\n", LoadedImage->LoadOptions, LoadedImage->LoadOptionsSize));
@@ -449,16 +449,21 @@ EFI_STATUS
 OcParseVars (
   IN           VOID              *StrVars,
   OUT       OC_FLEX_ARRAY        **ParsedVars,
-  IN     CONST OC_STRING_FORMAT  StringFormat
+  IN     CONST OC_STRING_FORMAT  StringFormat,
+  IN     CONST BOOLEAN           TokensOnly
   )
 {
   VOID              *Pos;
+  VOID              *NewPos;
   PARSE_VARS_STATE  State;
   PARSE_VARS_STATE  PushState;
   BOOLEAN           Retake;
   CHAR16            Ch;
+  CHAR16            NewCh;
+  CHAR16            QuoteChar;
   VOID              *Name;
   VOID              *Value;
+  VOID              *OriginalValue;
   OC_PARSED_VAR     *Option;
 
   if ((StrVars == NULL) || ((StringFormat == OcStringFormatUnicode) ? (((CHAR16 *)StrVars)[0] == CHAR_NULL) : (((CHAR8 *)StrVars)[0] == '\0'))) {
@@ -475,16 +480,32 @@ OcParseVars (
   State     = PARSE_VARS_WHITE_SPACE;
   PushState = PARSE_VARS_WHITE_SPACE;
   Retake    = FALSE;
+  QuoteChar = CHAR_NULL;
 
   do {
     Ch = (StringFormat == OcStringFormatUnicode) ? *((CHAR16 *)Pos) : *((CHAR8 *)Pos);
     switch (State) {
       case PARSE_VARS_WHITE_SPACE:
         if (Ch == '#') {
           State = PARSE_VARS_COMMENT;
-        } else if (!(OcIsSpace (Ch) || (Ch == CHAR_NULL))) {
-          Name  = Pos;
-          State = PARSE_VARS_NAME;
+        } else if (!OcIsSpaceOrNull (Ch)) {
+          if (TokensOnly) {
+            Option = OcFlexArrayAddItem (*ParsedVars);
+            if (Option == NULL) {
+              OcFlexArrayFree (ParsedVars);
+              return EFI_OUT_OF_RESOURCES;
+            }
+
+            DEBUG ((OC_TRACE_PARSE_VARS, "OCB: Value-only token\n"));
+
+            State         = PARSE_VARS_VALUE;
+            Value         = Pos;
+            OriginalValue = Value;
+            Retake        = TRUE;
+          } else {
+            State = PARSE_VARS_NAME;
+            Name  = Pos;
+          }
         }
 
         break;
@@ -497,15 +518,17 @@ OcParseVars (
         break;
 
       case PARSE_VARS_NAME:
-        if ((Ch == L'=') || OcIsSpace (Ch) || (Ch == CHAR_NULL)) {
+        if ((Ch == L'=') || OcIsSpaceOrNull (Ch)) {
           if (StringFormat == OcStringFormatUnicode) {
             *((CHAR16 *)Pos) = CHAR_NULL;
           } else {
             *((CHAR8 *)Pos) = '\0';
           }
 
           if (Ch == L'=') {
-            State = PARSE_VARS_VALUE_FIRST;
+            State         = PARSE_VARS_VALUE;
+            Value         = (UINT8 *)Pos + ((StringFormat == OcStringFormatUnicode) ? sizeof (CHAR16) : sizeof (CHAR8));
+            OriginalValue = Value;
           } else {
             State = PARSE_VARS_WHITE_SPACE;
           }
@@ -533,18 +556,6 @@ OcParseVars (
 
         break;
 
-      case PARSE_VARS_VALUE_FIRST:
-        if (Ch == L'"') {
-          State = PARSE_VARS_QUOTED_VALUE;
-          Value = (UINT8 *)Pos + ((StringFormat == OcStringFormatUnicode) ? sizeof (CHAR16) : sizeof (CHAR8));
-        } else {
-          State  = PARSE_VARS_VALUE;
-          Value  = Pos;
-          Retake = TRUE;
-        }
-
-        break;
-
       case PARSE_VARS_SHELL_EXPANSION:
         if (Ch == '`') {
           ASSERT (PushState != PARSE_VARS_WHITE_SPACE);
@@ -553,23 +564,39 @@ OcParseVars (
 
         break;
 
+      //
+      // In token value (but not name) we handle sh and grub quoting and string concatenation, e.g. 'abc\'"'\""def becomes abc\'"def.
+      //
       case PARSE_VARS_VALUE:
       case PARSE_VARS_QUOTED_VALUE:
-        if (Ch == L'`') {
+        if ((State != PARSE_VARS_QUOTED_VALUE) && ((Ch == L'\'') || (Ch == L'"'))) {
+          QuoteChar = Ch;
+          SHIFT_TOKEN (Pos, Value, (StringFormat == OcStringFormatUnicode) ? sizeof (CHAR16) : sizeof (CHAR8));
+          State = PARSE_VARS_QUOTED_VALUE;
+        } else if ((State == PARSE_VARS_QUOTED_VALUE) && (Ch == QuoteChar)) {
+          SHIFT_TOKEN (Pos, Value, (StringFormat == OcStringFormatUnicode) ? sizeof (CHAR16) : sizeof (CHAR8));
+          QuoteChar = CHAR_NULL;
+          State     = PARSE_VARS_VALUE;
+        } else if (((State != PARSE_VARS_QUOTED_VALUE) || (QuoteChar == L'"')) && (Ch == L'\\')) {
+          NewPos = (UINT8 *)Pos + ((StringFormat == OcStringFormatUnicode) ? sizeof (CHAR16) : sizeof (CHAR8));
+          NewCh  = (StringFormat == OcStringFormatUnicode) ? *((CHAR16 *)Pos) : *((CHAR8 *)Pos);
+          //
+          // https://www.gnu.org/software/bash/manual/html_node/Double-Quotes.html
+          //
+          if ((State != PARSE_VARS_QUOTED_VALUE) || (NewCh == '"') || (NewCh == '\\') || (NewCh == '$') || (NewCh == '`')) {
+            SHIFT_TOKEN (Pos, Value, (StringFormat == OcStringFormatUnicode) ? sizeof (CHAR16) : sizeof (CHAR8));
+            Pos = NewPos;
+            Ch  = NewCh;
+          }
+        } else if (Ch == L'`') {
           PushState = State;
           State     = PARSE_VARS_SHELL_EXPANSION;
-        } else if (Ch == L'\\') {
-          SHIFT_TOKEN (Pos, Value, (StringFormat == OcStringFormatUnicode) ? sizeof (CHAR16) : sizeof (CHAR8));
-          Ch = (StringFormat == OcStringFormatUnicode) ? *((CHAR16 *)Pos) : *((CHAR8 *)Pos);
-        } else if (
-                   ((State == PARSE_VARS_VALUE) && (OcIsSpace (Ch) || (Ch == CHAR_NULL))) ||
-                   ((State == PARSE_VARS_QUOTED_VALUE) && (Ch == '"')))
-        {
+        } else if ((State == PARSE_VARS_VALUE) && OcIsSpaceOrNull (Ch)) {
           //
-          // Explicitly quoted empty string needs to be stored detectably
-          // differently from missing value.
+          // Explicitly quoted empty string (e.g. `var=""`) is stored detectably differently from missing value (i.e. `var=`, or just `var`).
           //
-          if ((State != PARSE_VARS_QUOTED_VALUE) && (Pos == Value)) {
+          if (Pos == OriginalValue) {
+            ASSERT (!TokensOnly);
             DEBUG ((OC_TRACE_PARSE_VARS, "OCB: No value %u\n", 2));
           } else {
             if (PushState != PARSE_VARS_WHITE_SPACE) {
@@ -588,9 +615,10 @@ OcParseVars (
             }
           }
 
-          Value  = NULL;
-          Option = NULL;
-          State  = PARSE_VARS_WHITE_SPACE;
+          Value         = NULL;
+          OriginalValue = NULL;
+          Option        = NULL;
+          State         = PARSE_VARS_WHITE_SPACE;
         }
 
         break;