Initial push

src/jdk.compiler/share/classes/com/sun/tools/javac/comp/LambdaToMethod.java

@@ -2116,11 +2116,12 @@ public Symbol baseSymbol() {
             }
 
             void addSymbol(Symbol sym, LambdaSymbolKind skind) {
-                if (skind == CAPTURED_THIS && sym != null && sym.kind == TYP && !typesUnderConstruction.isEmpty()) {
+                if (skind == CAPTURED_THIS && sym != null && sym.kind == TYP) {
                     ClassSymbol currentClass = currentClass();
-                    if (currentClass != null && typesUnderConstruction.contains(currentClass)) {
-                        // reference must be to enclosing outer instance, mutate capture kind.
-                        Assert.check(sym != currentClass); // should have been caught right in Attr
+                    if (currentClass != null && currentClass != sym && !outerThisReachable(currentClass.type, sym)) {
+                        // This is a reference to an enclosing instance that is not reachable via this$n.
+                        // This can happen when a lambda expression occurs in a pre-construction context,
+                        // either directly, or indirectly (e.g. via one or more enclosing local/anonymous classes).
                         skind = CAPTURED_OUTER_THIS;
                     }
                 }
@@ -2130,6 +2131,17 @@ void addSymbol(Symbol sym, LambdaSymbolKind skind) {
                 }
             }
 
+            private boolean outerThisReachable(Type current, Symbol target) {
+                if (current == Type.noType || typesUnderConstruction.contains(current.tsym)) {
+                    return false;
+                } else if (current.tsym.isSubClass(target, types)) {
+                    return true;
+                } else {
+                    return current.tsym.hasOuterInstance() &&
+                            outerThisReachable(current.getEnclosingType(), target);
+                }
+            }
+
             Map<Symbol, Symbol> getSymbolMap(LambdaSymbolKind skind) {
                 Map<Symbol, Symbol> m = translatedSymbols.get(skind);
                 Assert.checkNonNull(m);

test/langtools/tools/javac/lambda/enclosingSuperCapture/EnclosingSuperCapture01.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.util.function.Supplier;
+
+/*
+ * @test
+ * @bug 8333766
+ * @summary Stack overflow with anonymous class in super() parameter
+ */
+public class EnclosingSuperCapture01 {
+
+    interface Getter {
+        int get();
+    }
+
+    class Inner {
+        int i;
+
+        Inner() {
+            this(() -> new Getter() {
+                public int get() {
+                    return m();
+                }
+            });
+        }
+        Inner(Supplier<Getter> r) {
+            i = r.get().get();
+        }
+    }
+
+    int m() {
+        return 42;
+    }
+
+    public static void main(String[] args) {
+        Inner inner = new EnclosingSuperCapture01().new Inner();
+        if (inner.i != 42) {
+            throw new AssertionError("Unexpected value: " + inner.i);
+        }
+    }
+} 

test/langtools/tools/javac/lambda/enclosingSuperCapture/EnclosingSuperCapture02.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.util.function.Supplier;
+
+/*
+ * @test
+ * @bug 8333766
+ * @summary Stack overflow with anonymous class in super() parameter
+ */
+public class EnclosingSuperCapture02 {
+
+    interface Getter {
+        int get();
+    }
+
+    static class Sup {
+        int i;
+
+        Sup(Supplier<Getter> r) {
+            i = r.get().get();
+        }
+    }
+
+    class Inner extends Sup {
+        Inner() {
+            super(() -> new Getter() {
+                public int get() {
+                    return m();
+                }
+            });
+        }
+    }
+
+    int m() {
+        return 42;
+    }
+
+    public static void main(String[] args) {
+        Inner inner = new EnclosingSuperCapture02().new Inner();
+        if (inner.i != 42) {
+            throw new AssertionError("Unexpected value: " + inner.i);
+        }
+    }
+} 

test/langtools/tools/javac/lambda/enclosingSuperCapture/EnclosingSuperCapture03.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.util.function.Supplier;
+
+/*
+ * @test
+ * @bug 8333766
+ * @summary Stack overflow with anonymous class in super() parameter
+ */
+public class EnclosingSuperCapture03 {
+
+    interface Getter {
+        int get();
+    }
+
+    interface GetterGetter {
+        Supplier<Getter> get();
+    }
+
+    class Inner {
+        int i;
+
+        Inner() {
+            this(() -> new GetterGetter() {
+                public Supplier<Getter> get() {
+                    return () -> new Getter() {
+                        public int get() { return m(); }
+                    };
+                }
+            });
+        }
+
+        Inner(Supplier<GetterGetter> r) {
+            i = r.get().get().get().get();
+        }
+    }
+
+    int m() {
+        return 42;
+    }
+
+    public static void main(String[] args) {
+        Inner inner = new EnclosingSuperCapture03().new Inner();
+        if (inner.i != 42) {
+            throw new AssertionError("Unexpected value: " + inner.i);
+        }
+    }
+}

test/langtools/tools/javac/lambda/enclosingSuperCapture/EnclosingSuperCapture04.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.util.function.Supplier;
+
+/*
+ * @test
+ * @bug 8333766
+ * @summary Stack overflow with anonymous class in super() parameter
+ */
+public class EnclosingSuperCapture04 {
+
+    interface Getter {
+        int get();
+    }
+
+    interface GetterGetter {
+        Supplier<Getter> get();
+    }
+
+    static class Sup {
+        int i;
+
+        Sup(Supplier<GetterGetter> r) {
+            i = r.get().get().get().get();
+        }
+    }
+
+    class Inner extends Sup {
+        Inner() {
+            super(() -> new GetterGetter() {
+                public Supplier<Getter> get() {
+                    return () -> new Getter() {
+                        public int get() { return m(); }
+                    };
+                }
+            });
+        }
+    }
+
+    int m() {
+        return 42;
+    }
+
+    public static void main(String[] args) {
+        Inner inner = new EnclosingSuperCapture04().new Inner();
+        if (inner.i != 42) {
+            throw new AssertionError("Unexpected value: " + inner.i);
+        }
+    }
+}