4-byte headers

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

@@ -2520,8 +2520,8 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
 
   if (UseCompressedClassPointers) {
     if (UseCompactObjectHeaders) {
-      __ ldr(result, Address(obj, oopDesc::mark_offset_in_bytes()));
-      __ lsr(result, result, markWord::klass_shift);
+      __ ldrw(result, Address(obj, oopDesc::mark_offset_in_bytes()));
+      __ lsrw(result, result, markWord::klass_shift);
     } else {
       __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
     }

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp

@@ -2574,10 +2574,10 @@ void C2_MacroAssembler::load_nklass_compact(Register dst, Register obj, Register
   // and offset to load the mark-word.
   int offset = oopDesc::mark_offset_in_bytes() + disp - Type::klass_offset();
   if (index == noreg) {
-    ldr(dst, Address(obj, offset));
+    ldrw(dst, Address(obj, offset));
   } else {
     lea(dst, Address(obj, index, Address::lsl(scale)));
-    ldr(dst, Address(dst, offset));
+    ldrw(dst, Address(dst, offset));
   }
-  lsr(dst, dst, markWord::klass_shift);
+  lsrw(dst, dst, markWord::klass_shift);
 }

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -4850,8 +4850,8 @@ void MacroAssembler::load_method_holder(Register holder, Register method) {
 // dst - output nklass.
 void MacroAssembler::load_nklass_compact(Register dst, Register src) {
   assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
-  ldr(dst, Address(src, oopDesc::mark_offset_in_bytes()));
-  lsr(dst, dst, markWord::klass_shift);
+  ldrw(dst, Address(src, oopDesc::mark_offset_in_bytes()));
+  lsrw(dst, dst, markWord::klass_shift);
 }
 
 void MacroAssembler::load_klass(Register dst, Register src) {
@@ -5596,22 +5596,12 @@ address MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
   Label DONE, SAME;
   Register tmp1 = rscratch1;
   Register tmp2 = rscratch2;
+  Register cnt2 = tmp2;  // cnt2 only used in array length compare
   int elem_per_word = wordSize/elem_size;
   int log_elem_size = exact_log2(elem_size);
-  int klass_offset  = arrayOopDesc::klass_offset_in_bytes();
   int length_offset = arrayOopDesc::length_offset_in_bytes();
   int base_offset
     = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
-  // When the length offset is not aligned to 8 bytes,
-  // then we align it down. This is valid because the new
-  // offset will always be the klass which is the same
-  // for type arrays.
-  int start_offset = align_down(length_offset, BytesPerWord);
-  int extra_length = base_offset - start_offset;
-  assert(start_offset == length_offset || start_offset == klass_offset,
-         "start offset must be 8-byte-aligned or be the klass offset");
-  assert(base_offset != start_offset, "must include the length field");
-  extra_length = extra_length / elem_size; // We count in elements, not bytes.
   int stubBytesThreshold = 3 * 64 + (UseSIMDForArrayEquals ? 0 : 16);
 
   assert(elem_size == 1 || elem_size == 2, "must be char or byte");
@@ -5645,10 +5635,11 @@ address MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
     //      return false;
     bind(A_IS_NOT_NULL);
     ldrw(cnt1, Address(a1, length_offset));
-    // Increase loop counter by diff between base- and actual start-offset.
-    addw(cnt1, cnt1, extra_length);
-    lea(a1, Address(a1, start_offset));
-    lea(a2, Address(a2, start_offset));
+    ldrw(cnt2, Address(a2, length_offset));
+    eorw(tmp5, cnt1, cnt2);
+    cbnzw(tmp5, DONE);
+    lea(a1, Address(a1, base_offset));
+    lea(a2, Address(a2, base_offset));
     // Check for short strings, i.e. smaller than wordSize.
     subs(cnt1, cnt1, elem_per_word);
     br(Assembler::LT, SHORT);
@@ -5711,18 +5702,18 @@ address MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
     cbz(a1, DONE);
     ldrw(cnt1, Address(a1, length_offset));
     cbz(a2, DONE);
-    // Increase loop counter by diff between base- and actual start-offset.
-    addw(cnt1, cnt1, extra_length);
-
+    ldrw(cnt2, Address(a2, length_offset));
     // on most CPUs a2 is still "locked"(surprisingly) in ldrw and it's
     // faster to perform another branch before comparing a1 and a2
     cmp(cnt1, (u1)elem_per_word);
     br(LE, SHORT); // short or same
-    ldr(tmp3, Address(pre(a1, start_offset)));
+    ldr(tmp3, Address(pre(a1, base_offset)));
     subs(zr, cnt1, stubBytesThreshold);
     br(GE, STUB);
-    ldr(tmp4, Address(pre(a2, start_offset)));
+    ldr(tmp4, Address(pre(a2, base_offset)));
     sub(tmp5, zr, cnt1, LSL, 3 + log_elem_size);
+    cmp(cnt2, cnt1);
+    br(NE, DONE);
 
     // Main 16 byte comparison loop with 2 exits
     bind(NEXT_DWORD); {
@@ -5754,7 +5745,9 @@ address MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
     b(LAST_CHECK);
 
     bind(STUB);
-    ldr(tmp4, Address(pre(a2, start_offset)));
+    ldr(tmp4, Address(pre(a2, base_offset)));
+    cmp(cnt2, cnt1);
+    br(NE, DONE);
     if (elem_size == 2) { // convert to byte counter
       lsl(cnt1, cnt1, 1);
     }
@@ -5775,9 +5768,12 @@ address MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
     mov(result, a2);
     b(DONE);
     bind(SHORT);
+    cmp(cnt2, cnt1);
+    br(NE, DONE);
+    cbz(cnt1, SAME);
     sub(tmp5, zr, cnt1, LSL, 3 + log_elem_size);
-    ldr(tmp3, Address(a1, start_offset));
-    ldr(tmp4, Address(a2, start_offset));
+    ldr(tmp3, Address(a1, base_offset));
+    ldr(tmp4, Address(a2, base_offset));
     bind(LAST_CHECK);
     eor(tmp4, tmp3, tmp4);
     lslv(tmp5, tmp4, tmp5);

src/hotspot/cpu/aarch64/templateTable_aarch64.cpp

@@ -3629,8 +3629,7 @@ void TemplateTable::_new() {
     // The object is initialized before the header.  If the object size is
     // zero, go directly to the header initialization.
     if (UseCompactObjectHeaders) {
-      assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
-      __ sub(r3, r3, oopDesc::base_offset_in_bytes());
+      __ sub(r3, r3, oopDesc::header_size() * BytesPerWord);
     } else {
       __ sub(r3, r3, sizeof(oopDesc));
     }
@@ -3639,8 +3638,7 @@ void TemplateTable::_new() {
     // Initialize object fields
     {
       if (UseCompactObjectHeaders) {
-        assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
-        __ add(r2, r0, oopDesc::base_offset_in_bytes());
+        __ add(r2, r0, oopDesc::header_size() * BytesPerWord);
       } else {
         __ add(r2, r0, sizeof(oopDesc));
       }

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

@@ -3509,8 +3509,8 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
     assert_different_registers(tmp, obj);
     assert_different_registers(tmp, result);
 
-    __ movq(result, Address(obj, oopDesc::mark_offset_in_bytes()));
-    __ shrq(result, markWord::klass_shift);
+    __ movl(result, Address(obj, oopDesc::mark_offset_in_bytes()));
+    __ shrl(result, markWord::klass_shift);
     __ decode_klass_not_null(result, tmp);
   } else if (UseCompressedClassPointers) {
     __ movl(result, Address(obj, oopDesc::klass_offset_in_bytes()));

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

@@ -6408,7 +6408,7 @@ void C2_MacroAssembler::load_nklass_compact_c2(Register dst, Register obj, Regis
   // then passes that register as obj and 0 in disp. The following code extracts the base
   // and offset to load the mark-word.
   int offset = oopDesc::mark_offset_in_bytes() + disp - Type::klass_offset();
-  movq(dst, Address(obj, index, scale, offset));
-  shrq(dst, markWord::klass_shift);
+  movl(dst, Address(obj, index, scale, offset));
+  shrl(dst, markWord::klass_shift);
 }
 #endif

src/hotspot/cpu/x86/macroAssembler_x86.cpp

@@ -5679,8 +5679,8 @@ void MacroAssembler::load_method_holder(Register holder, Register method) {
 #ifdef _LP64
 void MacroAssembler::load_nklass_compact(Register dst, Register src) {
   assert(UseCompactObjectHeaders, "expect compact object headers");
-  movq(dst, Address(src, oopDesc::mark_offset_in_bytes()));
-  shrq(dst, markWord::klass_shift);
+  movl(dst, Address(src, oopDesc::mark_offset_in_bytes()));
+  shrl(dst, markWord::klass_shift);
 }
 #endif
 

src/hotspot/cpu/x86/templateTable_x86.cpp

@@ -4085,8 +4085,7 @@ void TemplateTable::_new() {
     // The object is initialized before the header.  If the object size is
     // zero, go directly to the header initialization.
     if (UseCompactObjectHeaders) {
-      assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
-      __ decrement(rdx, oopDesc::base_offset_in_bytes());
+      __ decrement(rdx, align_up(oopDesc::base_offset_in_bytes(), BytesPerLong));
     } else {
       __ decrement(rdx, sizeof(oopDesc));
     }
@@ -4112,8 +4111,7 @@ void TemplateTable::_new() {
     { Label loop;
     __ bind(loop);
     if (UseCompactObjectHeaders) {
-      assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
-      int header_size = oopDesc::base_offset_in_bytes();
+      int header_size = align_up(oopDesc::base_offset_in_bytes(), BytesPerLong);
       __ movptr(Address(rax, rdx, Address::times_8, header_size - 1*oopSize), rcx);
       NOT_LP64(__ movptr(Address(rax, rdx, Address::times_8, header_size - 2*oopSize), rcx));
     } else {
@@ -4129,6 +4127,7 @@ void TemplateTable::_new() {
     if (UseCompactObjectHeaders) {
       __ pop(rcx);   // get saved klass back in the register.
       __ movptr(rbx, Address(rcx, Klass::prototype_header_offset()));
+      // Also clears the 'gap'.
       __ movptr(Address(rax, oopDesc::mark_offset_in_bytes()), rbx);
     } else {
       __ movptr(Address(rax, oopDesc::mark_offset_in_bytes()),

src/hotspot/share/gc/serial/defNewGeneration.cpp

@@ -558,6 +558,7 @@ static HeapWord* block_start_const(const ContiguousSpace* cs, const void* p) {
     HeapWord* cur = last;
     while (cur <= p) {
       last = cur;
+      assert(!cast_to_oop(cur)->is_forwarded(), "can not deal with forwarded object here");
       cur += cast_to_oop(cur)->size();
     }
     assert(oopDesc::is_oop(cast_to_oop(last)), PTR_FORMAT " should be an object start", p2i(last));
@@ -799,7 +800,7 @@ oop DefNewGeneration::copy_to_survivor_space(oop old) {
   bool new_obj_is_tenured = false;
   // Otherwise try allocating obj tenured
   if (obj == nullptr) {
-    obj = _old_gen->promote(old, s);
+    obj = _old_gen->promote(old, old_size, s);
     if (obj == nullptr) {
       handle_promotion_failure(old);
       return old;
@@ -816,7 +817,7 @@ oop DefNewGeneration::copy_to_survivor_space(oop old) {
     // Copy obj
     Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(old), cast_from_oop<HeapWord*>(obj), old_size);
     markWord new_mark = obj->mark();
-    assert(!UseCompactObjectHeaders || (!(new_mark.hash_is_hashed() && new_mark.hash_is_copied())), "must not be simultaneously hashed and copied state");
+    assert(!UseCompactObjectHeaders || !(new_mark.hash_is_hashed() && new_mark.hash_is_copied()), "must not be simultaneously hashed and copied state");
 
     ContinuationGCSupport::transform_stack_chunk(obj);
 

src/hotspot/share/gc/serial/tenuredGeneration.cpp

@@ -269,6 +269,7 @@ HeapWord* TenuredGeneration::block_start(const void* addr) const {
   HeapWord* cur_block = _bts->block_start_reaching_into_card(addr);
 
   while (true) {
+    assert(!cast_to_oop(cur_block)->is_forwarded(), "can not deal with forwarded object here");
     HeapWord* next_block = cur_block + cast_to_oop(cur_block)->size();
     if (next_block > addr) {
       assert(cur_block <= addr, "postcondition");
@@ -387,8 +388,8 @@ bool TenuredGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes)
   return res;
 }
 
-oop TenuredGeneration::promote(oop obj, size_t obj_size) {
-  assert(obj_size == obj->size() || UseCompactObjectHeaders, "bad obj_size passed in");
+oop TenuredGeneration::promote(oop obj, size_t old_size, size_t new_size) {
+  assert(old_size == obj->size(), "bad obj_size passed in");
 
 #ifndef PRODUCT
   if (SerialHeap::heap()->promotion_should_fail()) {
@@ -397,17 +398,17 @@ oop TenuredGeneration::promote(oop obj, size_t obj_size) {
 #endif  // #ifndef PRODUCT
 
   // Allocate new object.
-  HeapWord* result = allocate(obj_size, false);
+  HeapWord* result = allocate(new_size, false);
   if (result == nullptr) {
     // Promotion of obj into gen failed.  Try to expand and allocate.
-    result = expand_and_allocate(obj_size, false);
+    result = expand_and_allocate(new_size, false);
     if (result == nullptr) {
       return nullptr;
     }
   }
 
   // Copy to new location.
-  Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(obj), result, obj_size);
+  Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(obj), result, old_size);
   oop new_obj = cast_to_oop<HeapWord*>(result);
   return new_obj;
 }

src/hotspot/share/gc/serial/tenuredGeneration.hpp

@@ -168,7 +168,7 @@ class TenuredGeneration: public Generation {
   //
   // The "obj_size" argument is just obj->size(), passed along so the caller can
   // avoid repeating the virtual call to retrieve it.
-  oop promote(oop obj, size_t obj_size);
+  oop promote(oop obj, size_t old_size, size_t new_size);
 
   virtual void verify();
   virtual void print_on(outputStream* st) const;

src/hotspot/share/gc/shared/c2/barrierSetC2.cpp

@@ -711,15 +711,18 @@ int BarrierSetC2::arraycopy_payload_base_offset(bool is_array) {
   // 16 - 64-bit VM, normal klass
   if (base_off % BytesPerLong != 0) {
     assert(UseCompressedClassPointers, "");
-    assert(!UseCompactObjectHeaders, "");
     if (is_array) {
       // Exclude length to copy by 8 bytes words.
       base_off += sizeof(int);
     } else {
-      // Include klass to copy by 8 bytes words.
-      base_off = instanceOopDesc::klass_offset_in_bytes();
+      if (UseCompactObjectHeaders) {
+        base_off = 0; /* FIXME */
+      } else {
+        // Include klass to copy by 8 bytes words.
+        base_off = instanceOopDesc::klass_offset_in_bytes();
+      }
     }
-    assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
+    assert(base_off % BytesPerLong == 0 || UseCompactObjectHeaders, "expect 8 bytes alignment");
   }
   return base_off;
 }

src/hotspot/share/gc/shared/memAllocator.cpp

@@ -361,9 +361,7 @@ void MemAllocator::mem_clear(HeapWord* mem) const {
   assert(mem != nullptr, "cannot initialize null object");
   const size_t hs = oopDesc::header_size();
   assert(_word_size >= hs, "unexpected object size");
-  if (!UseCompactObjectHeaders) {
-    oopDesc::set_klass_gap(mem, 0);
-  }
+  oopDesc::set_klass_gap(mem, 0);
   Copy::fill_to_aligned_words(mem + hs, _word_size - hs);
 }
 

src/hotspot/share/gc/shared/space.cpp

@@ -100,9 +100,10 @@ void ContiguousSpace::object_iterate(ObjectClosure* blk) {
   oop last = nullptr;
   while (addr < top()) {
     oop obj = cast_to_oop(addr);
+    size_t size = obj->size();
     blk->do_object(obj);
     assert(!UseCompactObjectHeaders || obj->mark().narrow_klass() != 0, "null narrow klass, mark: " INTPTR_FORMAT ", last mark: " INTPTR_FORMAT, obj->mark().value(), last->mark().value());
-    addr += obj->size();
+    addr += size;
     last = obj;
   }
 }

src/hotspot/share/oops/arrayOop.hpp

@@ -90,6 +90,7 @@ class arrayOopDesc : public oopDesc {
     if (arrayoopdesc_hs == 0) arrayoopdesc_hs = hs;
     assert(arrayoopdesc_hs == hs, "header size can't change");
 #endif // ASSERT
+    if (UseCompactObjectHeaders) assert(hs == 8, "array header must be 8 bytes");
     return (int)hs;
   }
 
@@ -98,6 +99,7 @@ class arrayOopDesc : public oopDesc {
   // it occupies the second half of the _klass field in oopDesc.
   static int length_offset_in_bytes() {
     if (UseCompactObjectHeaders) {
+      assert(oopDesc::base_offset_in_bytes() == 4, "array length must be as 4 bytes");
       return oopDesc::base_offset_in_bytes();
     } else if (UseCompressedClassPointers) {
       return klass_gap_offset_in_bytes();

src/hotspot/share/oops/markWord.hpp

@@ -123,6 +123,7 @@ class markWord {
 
   // Conversion
   uintptr_t value() const { return _value; }
+  uint32_t value32() const { return (uint32_t)_value; }
 
   // Constants
   static const int age_bits                       = 4;

src/hotspot/share/oops/markWord.inline.hpp

@@ -31,7 +31,7 @@
 narrowKlass markWord::narrow_klass() const {
 #ifdef _LP64
   assert(UseCompactObjectHeaders, "only used with compact object headers");
-  return narrowKlass(value() >> klass_shift);
+  return narrowKlass(value32() >> klass_shift);
 #else
   ShouldNotReachHere();
   return 0;

src/hotspot/share/oops/oop.cpp

@@ -139,7 +139,7 @@ markWord oopDesc::initialize_hash_if_necessary(oop obj, Klass* k, markWord m) {
     assert(!m.hash_is_copied(), "must not be installed");
     uint32_t hash = static_cast<uint32_t>(ObjectSynchronizer::get_next_hash(nullptr, obj));
     int offset = k->hash_offset_in_bytes(cast_to_oop(this));
-    assert(offset >= 8, "hash offset must not be in header");
+    assert(offset >= 4, "hash offset must not be in header");
     //log_info(gc)("Initializing hash for " PTR_FORMAT ", old: " PTR_FORMAT ", hash: %d, offset: %d", p2i(this), p2i(obj), hash, offset);
     int_field_put(offset, (jint)hash);
     m = m.hash_set_copied();
@@ -159,7 +159,7 @@ bool oopDesc::initialize_hash_if_necessary(oop obj) {
     uint32_t hash = static_cast<uint32_t>(ObjectSynchronizer::get_next_hash(nullptr, obj));
     Klass* k = m.klass();
     int offset = k->hash_offset_in_bytes(cast_to_oop(this));
-    assert(offset >= 8, "hash offset must not be in header");
+    assert(offset >= 4, "hash offset must not be in header");
     log_trace(gc)("Initializing hash for " PTR_FORMAT ", old: " PTR_FORMAT ", hash: %d, offset: %d", p2i(this), p2i(obj), hash, offset);
     int_field_put(offset, (jint)hash);
     m = m.hash_set_copied();
@@ -196,8 +196,7 @@ bool oopDesc::is_typeArray_noinline()   const { return is_typeArray();   }
 
 bool oopDesc::has_klass_gap() {
   // Only has a klass gap when compressed class pointers are used.
-  // Except when using compact headers.
-  return UseCompressedClassPointers && !UseCompactObjectHeaders;
+  return UseCompressedClassPointers || UseCompactObjectHeaders;
 }
 
 #if INCLUDE_CDS_JAVA_HEAP