From 1a45dfff488777522315c2112bc146bbf1c0cd6f Mon Sep 17 00:00:00 2001
From: Alexander Hulpke <hulpke@math.colostate.edu>
Date: Tue, 28 May 2024 06:01:09 +0200
Subject: [PATCH] FIX: MinimalFaithfulPermutationDegree

If normal subgroups have abelian factors, intersection can only be avoided,
if both normals have abelian factor, not if only one has.
This fixes #5729
---
 lib/grplatt.gi                       | 17 ++++++++++-------
 tst/testbugfix/2024-05-28-Mindeg.tst |  3 +++
 2 files changed, 13 insertions(+), 7 deletions(-)
 create mode 100644 tst/testbugfix/2024-05-28-Mindeg.tst

diff --git a/lib/grplatt.gi b/lib/grplatt.gi
index 0b3c3b9349..4f7a0d5569 100644
--- a/lib/grplatt.gi
+++ b/lib/grplatt.gi
@@ -3467,13 +3467,13 @@ end);
 BindGlobal("DoMinimalFaithfulPermutationDegree",
 function(G,dorep)
 local c,n,deg,ind,core,i,j,sum,ma,h,ig,bm,m,sel,ds,ise,cnt,
-  start,cind,nind,sl,idfun,spos,select,bla;
+  start,cind,nind,sl,idfun,spos,select,bla,iswith;
 
   if Size(G)=1 then
     # option allows to calculate actual representation -- maybe access under
     # different name
     if dorep=false then
-      return 1;
+      return 0;
     else
       return GroupHomomorphismByImages(G,Group(()),[One(G)],[()]);
     fi;
@@ -3555,10 +3555,7 @@ local c,n,deg,ind,core,i,j,sum,ma,h,ig,bm,m,sel,ds,ise,cnt,
     ind:=IndexNC(G,c[i]);
     spos:=PositionSorted(sl,Size(c[i]))-1;
 
-
-    if i in sel then
-      # do nothing --already processed
-    elif IsNormal(G,c[i]) then # subgroup normal, must be in other case
+    if IsNormal(G,c[i]) then # subgroup normal, must be in other case
       #core:=Position(n,c[i]);
       core:=nind[i];
 
@@ -3569,7 +3566,13 @@ local c,n,deg,ind,core,i,j,sum,ma,h,ig,bm,m,sel,ds,ise,cnt,
 #Print("|select|=",Length(select),"\n");
       fi;
 
-      for j in [2..core-1] do # Intersect with all prior normals
+      iswith:=[2..core-1]; # what to intersect with
+      # avoid intersecting abelians -- there might be many
+      if i in sel then
+        iswith:=Difference(iswith,nind{sel});
+      fi;
+
+      for j in iswith do # Intersect with all prior normals
         sum:=deg[core][1]+deg[j][1];
         if sum<deg[Length(n)][1] then # otherwise too big for new optimal
           if select=fail then
diff --git a/tst/testbugfix/2024-05-28-Mindeg.tst b/tst/testbugfix/2024-05-28-Mindeg.tst
new file mode 100644
index 0000000000..11555db4f0
--- /dev/null
+++ b/tst/testbugfix/2024-05-28-Mindeg.tst
@@ -0,0 +1,3 @@
+# Fix #5729
+gap> MinimalFaithfulPermutationDegree(SmallGroup(42,3));
+10