Fix an error message when computing subgroup lattices of certain inso…

…lvable groups (#5723)

Special permgroup method for multiple conjugation tests only applies if the
groups are subgroups of the acting one.

Also added an option to cut short abelian test.
This test can become expensive in pcgs of length several hundred, as they
might arise in the hybrid quotient algorithm.

lib/oprtglat.gi

@@ -450,7 +450,7 @@ end);
 InstallMethod(SubgroupsOrbitsAndNormalizers,"perm group on list",true,
   [IsPermGroup,IsList,IsBool],0,
 function(G,dom,all)
-local n,l, o, b, t, r;
+local n,l, o, b, t, r,sub;
 
   if Length(dom)=0 then
     return dom;
@@ -463,8 +463,8 @@ local n,l, o, b, t, r;
   # new code -- without `all` option
 
   n:=Length(dom);
-  if n>20 and ForAll(dom,x->IsSubset(G,x))
-    and NrMovedPoints(G)>1000 then
+  sub:=ForAll(dom,x->IsSubset(G,x));
+  if n>20 and sub and NrMovedPoints(G)>1000 then
     #and NrMovedPoints(G)*1000>Size(G) then
 
     b:=SmallerDegreePermutationRepresentation(G:cheap);
@@ -477,6 +477,8 @@ local n,l, o, b, t, r;
     fi;
   fi;
 
+  if not sub then TryNextMethod();fi;
+
   l:=ClusterConjugacyPermgroups(G,ShallowCopy(dom));
   l:=RefineClusterConjugacyPermgroups(l);
   o:=[];

lib/pcgsind.gd

@@ -136,6 +136,9 @@ DeclareOperation( "InducedPcgsByGeneratorsNC", [ IsPcgs, IsCollection ] );
 ##  <ManSection>
 ##  <Oper Name="InducedPcgsByGeneratorsWithImages" Arg='pcgs, gens, imgs'/>
 ##
+##  The option `abeliandomain` can be used to avoid a test for the group to
+##  be abelian (which can be costly if the group has hundreds of generators).
+##
 ##  <Description>
 ##  </Description>
 ##  </ManSection>

lib/pcgsind.gi

@@ -458,7 +458,11 @@ function( pcgs, gens, imgs )
     if gens = AsList( pcgs ) then return [pcgs, imgs]; fi;
 
     #catch special case: abelian
-    nonab:=not IsAbelian(Group(pcgs,OneOfPcgs(pcgs)));
+    f:=ValueOption("abeliandomain");
+    if not (f in [true,false]) then
+      f:=IsAbelian(Group(pcgs,OneOfPcgs(pcgs)));
+    fi;
+    nonab:=not f;
 
     # get relative orders and composition length
     ro  := RelativeOrders(pcgs);

tst/testbugfix/2024-05-24-Cluster.tst

@@ -0,0 +1,3 @@
+# Fix #5717
+gap> g := TransitiveGroup(12,288);;
+gap> ConjugacyClassesSubgroups(g);;