8320761: [Lilliput] Implement compact identity hashcode

src/hotspot/cpu/x86/sharedRuntime_x86.cpp

@@ -58,6 +58,11 @@ void SharedRuntime::inline_check_hashcode_from_object_header(MacroAssembler* mas
     __ bind(Continue);
   }
 
+  if (UseCompactObjectHeaders) {
+    // Don't generate anything else and always take the slow-path for now.
+    return;
+  }
+
   __ movptr(result, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
 
 

src/hotspot/share/cds/archiveBuilder.cpp

@@ -1298,7 +1298,9 @@ class ArchiveBuilder::CDSMapLogger : AllStatic {
         // Example:
         // 0x00000007ffd27938: @@ Object (0xfffa4f27) java.util.HashMap
         print_oop_info_cr(&st, source_oop, /*print_requested_addr=*/false);
-        byte_size = source_oop->size() * BytesPerWord;
+        size_t old_size = source_oop->size();
+        size_t new_size = source_oop->copy_size_cds(old_size, source_oop->mark());
+        byte_size = new_size * BytesPerWord;
       } else if ((byte_size = ArchiveHeapWriter::get_filler_size_at(start)) > 0) {
         // We have a filler oop, which also does not exist in BufferOffsetToSourceObjectTable.
         // Example:

src/hotspot/share/cds/archiveHeapWriter.cpp

@@ -417,7 +417,9 @@ void update_buffered_object_field(address buffered_obj, int field_offset, T valu
 
 size_t ArchiveHeapWriter::copy_one_source_obj_to_buffer(oop src_obj) {
   assert(!is_too_large_to_archive(src_obj), "already checked");
-  size_t byte_size = src_obj->size() * HeapWordSize;
+  size_t old_size = src_obj->size();
+  size_t new_size = src_obj->copy_size_cds(old_size, src_obj->mark());
+  size_t byte_size = new_size * HeapWordSize;
   assert(byte_size > 0, "no zero-size objects");
 
   // For region-based collectors such as G1, the archive heap may be mapped into
@@ -436,9 +438,11 @@ size_t ArchiveHeapWriter::copy_one_source_obj_to_buffer(oop src_obj) {
 
   address from = cast_from_oop<address>(src_obj);
   address to = offset_to_buffered_address<address>(_buffer_used);
+  log_info(gc)("Copying obj: " PTR_FORMAT ", to: " PTR_FORMAT ", old_size: " SIZE_FORMAT ", new_size: " SIZE_FORMAT, p2i(src_obj), p2i(to), old_size, new_size);
+
   assert(is_object_aligned(_buffer_used), "sanity");
   assert(is_object_aligned(byte_size), "sanity");
-  memcpy(to, from, byte_size);
+  memcpy(to, from, old_size * HeapWordSize);
 
   // These native pointers will be restored explicitly at run time.
   if (java_lang_Module::is_instance(src_obj)) {
@@ -574,6 +578,7 @@ void ArchiveHeapWriter::update_header_for_requested_obj(oop requested_obj, oop s
   oop fake_oop = cast_to_oop(buffered_addr);
   if (UseCompactObjectHeaders) {
     fake_oop->set_mark(markWord::prototype().set_narrow_klass(nk));
+    assert(fake_oop->mark().narrow_klass() != 0, "must not be null");
   } else {
     fake_oop->set_narrow_klass(nk);
   }
@@ -586,15 +591,22 @@ void ArchiveHeapWriter::update_header_for_requested_obj(oop requested_obj, oop s
   if (!src_obj->fast_no_hash_check()) {
     intptr_t src_hash = src_obj->identity_hash();
     if (UseCompactObjectHeaders) {
-      fake_oop->set_mark(markWord::prototype().set_narrow_klass(nk).copy_set_hash(src_hash));
+      markWord m = markWord::prototype().set_narrow_klass(nk);
+      m = m.copy_hashctrl_from(src_obj->mark());
+      fake_oop->set_mark(m);
+      if (m.is_hashed_not_expanded()) {
+        fake_oop->initialize_hash_if_necessary(src_obj, src_klass, m);
+      }
     } else {
       fake_oop->set_mark(markWord::prototype().copy_set_hash(src_hash));
     }
     assert(fake_oop->mark().is_unlocked(), "sanity");
 
-    DEBUG_ONLY(intptr_t archived_hash = fake_oop->identity_hash());
-    assert(src_hash == archived_hash, "Different hash codes: original " INTPTR_FORMAT ", archived " INTPTR_FORMAT, src_hash, archived_hash);
+    //log_trace(gc)("fake_oop: " PTR_FORMAT, p2i(fake_oop));
+    //DEBUG_ONLY(intptr_t archived_hash = fake_oop->identity_hash());
+    //assert(src_hash == archived_hash, "Different hash codes: original " INTPTR_FORMAT ", archived " INTPTR_FORMAT, src_hash, archived_hash);
   }
+  assert(!UseCompactObjectHeaders || (!fake_oop->mark().is_not_hashed_expanded() && !fake_oop->mark().is_hashed_not_expanded()), "must not be not-hashed-moved and not be hashed-not-moved");
   // Strip age bits.
   fake_oop->set_mark(fake_oop->mark().set_age(0));
 }

src/hotspot/share/cds/heapShared.cpp

@@ -378,7 +378,7 @@ void HeapShared::init_scratch_objects(TRAPS) {
   for (int i = T_BOOLEAN; i < T_VOID+1; i++) {
     BasicType bt = (BasicType)i;
     if (!is_reference_type(bt)) {
-      oop m = java_lang_Class::create_basic_type_mirror(type2name(bt), bt, CHECK);
+      oop m = java_lang_Class::create_basic_type_mirror(type2name(bt), bt, true, CHECK);
       _scratch_basic_type_mirrors[i] = OopHandle(Universe::vm_global(), m);
     }
   }
@@ -530,18 +530,38 @@ void HeapShared::copy_aot_initialized_mirror(Klass* orig_k, oop orig_mirror, oop
 static void copy_java_mirror_hashcode(oop orig_mirror, oop scratch_m) {
   // We need to retain the identity_hash, because it may have been used by some hashtables
   // in the shared heap.
+  assert(!UseCompactObjectHeaders || scratch_m->mark().is_not_hashed_expanded(), "scratch mirror must have not-hashed-expanded state");
   if (!orig_mirror->fast_no_hash_check()) {
+    intptr_t orig_mark = orig_mirror->mark().value();
     intptr_t src_hash = orig_mirror->identity_hash();
     if (UseCompactObjectHeaders) {
-      narrowKlass nk = CompressedKlassPointers::encode(orig_mirror->klass());
-      scratch_m->set_mark(markWord::prototype().set_narrow_klass(nk).copy_set_hash(src_hash));
+      // We leave the cases not_hashed/not_hashed_expanded as they are.
+      assert(orig_mirror->mark().is_hashed_not_expanded() || orig_mirror->mark().is_hashed_expanded(), "must be hashed");
+      Klass* orig_klass = orig_mirror->klass();
+      narrowKlass nk = CompressedKlassPointers::encode(orig_klass);
+      markWord mark = markWord::prototype().set_narrow_klass(nk);
+      mark = mark.copy_hashctrl_from(orig_mirror->mark());
+      if (mark.is_hashed_not_expanded()) {
+        scratch_m->initialize_hash_if_necessary(orig_mirror, orig_klass, mark);
+      } else {
+        assert(mark.is_hashed_expanded(), "must be hashed & moved");
+        int offset = orig_klass->hash_offset_in_bytes(orig_mirror);
+        assert(offset >= 8, "hash offset must not be in header");
+        scratch_m->int_field_put(offset, (jint) src_hash);
+        scratch_m->set_mark(mark);
+      }
+      assert(scratch_m->mark().is_hashed_expanded(), "must be hashed & moved");
     } else {
       scratch_m->set_mark(markWord::prototype().copy_set_hash(src_hash));
     }
     assert(scratch_m->mark().is_unlocked(), "sanity");
 
     DEBUG_ONLY(intptr_t archived_hash = scratch_m->identity_hash());
-    assert(src_hash == archived_hash, "Different hash codes: original " INTPTR_FORMAT ", archived " INTPTR_FORMAT, src_hash, archived_hash);
+    assert(src_hash == archived_hash, "Different hash codes, orig_mark: " INTPTR_FORMAT ", scratch mark: " INTPTR_FORMAT ", orig hash: " INTPTR_FORMAT ", new hash: " INTPTR_FORMAT, orig_mark, scratch_m->mark().value(), src_hash, archived_hash);
+  }
+  assert(!UseCompactObjectHeaders || scratch_m->mark().is_not_hashed_expanded() || scratch_m->mark().is_hashed_expanded(), "must be not hashed and expanded");
+  if (UseCompactObjectHeaders) {
+    log_trace(gc)("Updated hashctrl of scratch mirror: " PTR_FORMAT ", mark: " INTPTR_FORMAT, p2i(scratch_m), scratch_m->mark().value());
   }
 }
 

src/hotspot/share/ci/ciInstanceKlass.hpp

@@ -296,6 +296,10 @@ class ciInstanceKlass : public ciKlass {
   }
   GrowableArray<ciInstanceKlass*>* transitive_interfaces() const;
 
+  int hash_offset_in_bytes() const {
+    return get_instanceKlass()->hash_offset_in_bytes(nullptr);
+  }
+
   // Replay support
 
   // Dump the current state of this klass for compilation replay.

src/hotspot/share/ci/ciKlass.hpp

@@ -75,6 +75,9 @@ class ciKlass : public ciType {
 public:
   ciKlass(Klass* k);
 
+  bool is_mirror_instance_klass() { return get_Klass()->is_mirror_instance_klass(); }
+  bool is_reference_instance_klass() { return get_Klass()->is_reference_instance_klass(); }
+
   // What is the name of this klass?
   ciSymbol* name() const { return _name; }
 

src/hotspot/share/classfile/altHashing.cpp

@@ -56,6 +56,9 @@
 // objects.  We don't want to call the synchronizer hash code to install
 // this value because it may safepoint.
 static intptr_t object_hash(Klass* k) {
+  if (UseCompactObjectHeaders) {
+    return os::random();
+  }
   intptr_t hc = k->java_mirror()->mark().hash();
   return hc != markWord::no_hash ? hc : os::random();
 }

src/hotspot/share/classfile/classFileParser.cpp

@@ -4886,6 +4886,14 @@ jint ClassFileParser::layout_size() const {
   return _field_info->_instance_size;
 }
 
+int ClassFileParser::hash_offset() const {
+  return _field_info->_hash_offset;
+}
+
+int ClassFileParser::static_hash_offset() const {
+  return _field_info->_static_hash_offset;
+}
+
 static void check_methods_for_intrinsics(const InstanceKlass* ik,
                                          const Array<Method*>* methods) {
   assert(ik != nullptr, "invariant");

src/hotspot/share/classfile/classFileParser.hpp

@@ -72,6 +72,8 @@ class FieldLayoutInfo : public ResourceObj {
  public:
   OopMapBlocksBuilder* oop_map_blocks;
   int _instance_size;
+  int _hash_offset;
+  int _static_hash_offset;
   int _nonstatic_field_size;
   int _static_field_size;
   bool  _has_nonstatic_fields;
@@ -500,6 +502,8 @@ class ClassFileParser {
   int static_field_size() const;
   int total_oop_map_count() const;
   jint layout_size() const;
+  int hash_offset() const;
+  int static_hash_offset() const;
 
   int vtable_size() const { return _vtable_size; }
   int itable_size() const { return _itable_size; }

src/hotspot/share/classfile/fieldLayoutBuilder.cpp

@@ -209,6 +209,23 @@ void FieldLayout::add(GrowableArray<LayoutRawBlock*>* list, LayoutRawBlock* star
   }
 }
 
+// Finds a slot for the identity hash-code.
+// Same basic algorithm as above add() method, but simplified
+// and does not actually insert the field.
+int FieldLayout::find_hash_offset() {
+  LayoutRawBlock* start = this->_start;
+  LayoutRawBlock* last = last_block();
+  LayoutRawBlock* cursor = start;
+  while (cursor != last) {
+    assert(cursor != nullptr, "Sanity check");
+    if (cursor->kind() == LayoutRawBlock::EMPTY && cursor->fit(4, 1)) {
+      break;
+    }
+    cursor = cursor->next_block();
+  }
+  return cursor->offset();
+}
+
 // Used for classes with hard coded field offsets, insert a field at the specified offset */
 void FieldLayout::add_field_at_offset(LayoutRawBlock* block, int offset, LayoutRawBlock* start) {
   assert(block != nullptr, "Sanity check");
@@ -674,6 +691,10 @@ void FieldLayoutBuilder::epilogue() {
 
   _info->oop_map_blocks = nonstatic_oop_maps;
   _info->_instance_size = align_object_size(instance_end / wordSize);
+  if (UseCompactObjectHeaders) {
+    _info->_hash_offset   = _layout->find_hash_offset();
+    _info->_static_hash_offset = _static_layout->find_hash_offset();
+  }
   _info->_static_field_size = static_fields_size;
   _info->_nonstatic_field_size = (nonstatic_field_end - instanceOopDesc::base_offset_in_bytes()) / heapOopSize;
   _info->_has_nonstatic_fields = _has_nonstatic_fields;

src/hotspot/share/classfile/fieldLayoutBuilder.hpp

@@ -185,6 +185,7 @@ class FieldLayout : public ResourceObj {
 
   LayoutRawBlock* first_field_block();
   void add(GrowableArray<LayoutRawBlock*>* list, LayoutRawBlock* start = nullptr);
+  int find_hash_offset();
   void add_field_at_offset(LayoutRawBlock* blocks, int offset, LayoutRawBlock* start = nullptr);
   void add_contiguously(GrowableArray<LayoutRawBlock*>* list, LayoutRawBlock* start = nullptr);
   LayoutRawBlock* insert_field_block(LayoutRawBlock* slot, LayoutRawBlock* block);

src/hotspot/share/classfile/javaClasses.cpp

@@ -951,6 +951,8 @@ void java_lang_Class::fixup_mirror(Klass* k, TRAPS) {
       int java_fields;
       int injected_fields;
       InstanceKlass* ik = InstanceKlass::cast(k);
+      ik->fix_static_hash_offset();
+
       GrowableArray<FieldInfo>* fields =
         FieldInfoStream::create_FieldInfoArray(ik->fieldinfo_stream(),
                                                &java_fields, &injected_fields);
@@ -1055,7 +1057,7 @@ void java_lang_Class::allocate_fixup_lists() {
 void java_lang_Class::allocate_mirror(Klass* k, bool is_scratch, Handle protection_domain, Handle classData,
                                       Handle& mirror, Handle& comp_mirror, TRAPS) {
   // Allocate mirror (java.lang.Class instance)
-  oop mirror_oop = InstanceMirrorKlass::cast(vmClasses::Class_klass())->allocate_instance(k, CHECK);
+  oop mirror_oop = InstanceMirrorKlass::cast(vmClasses::Class_klass())->allocate_instance(k, is_scratch, CHECK);
   mirror = Handle(THREAD, mirror_oop);
 
   // Setup indirection from mirror->klass
@@ -1349,10 +1351,10 @@ void java_lang_Class::set_source_file(oop java_class, oop source_file) {
   java_class->obj_field_put(_source_file_offset, source_file);
 }
 
-oop java_lang_Class::create_basic_type_mirror(const char* basic_type_name, BasicType type, TRAPS) {
+oop java_lang_Class::create_basic_type_mirror(const char* basic_type_name, BasicType type, bool is_scratch, TRAPS) {
   // This should be improved by adding a field at the Java level or by
   // introducing a new VM klass (see comment in ClassFileParser)
-  oop java_class = InstanceMirrorKlass::cast(vmClasses::Class_klass())->allocate_instance(nullptr, CHECK_NULL);
+  oop java_class = InstanceMirrorKlass::cast(vmClasses::Class_klass())->allocate_instance(nullptr, is_scratch, CHECK_NULL);
   if (type != T_VOID) {
     Klass* aklass = Universe::typeArrayKlass(type);
     assert(aklass != nullptr, "correct bootstrap");

src/hotspot/share/classfile/javaClasses.hpp

@@ -280,7 +280,7 @@ class java_lang_Class : AllStatic {
   static void create_mirror(Klass* k, Handle class_loader, Handle module,
                             Handle protection_domain, Handle classData, TRAPS);
   static void fixup_mirror(Klass* k, TRAPS);
-  static oop  create_basic_type_mirror(const char* basic_type_name, BasicType type, TRAPS);
+  static oop  create_basic_type_mirror(const char* basic_type_name, BasicType type, bool is_scratch, TRAPS);
 
   // Archiving
   static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN;

src/hotspot/share/gc/g1/g1ConcurrentMark.cpp

@@ -1051,7 +1051,8 @@ void G1ConcurrentMark::scan_root_region(const MemRegion* region, uint worker_id)
     Prefetch::read(curr, interval);
     oop obj = cast_to_oop(curr);
     size_t size = obj->oop_iterate_size(&cl);
-    assert(size == obj->size(), "sanity");
+    if (UseCompactObjectHeaders) log_trace(gc)("Scan object : " PTR_FORMAT ", with size: " SIZE_FORMAT, p2i(obj), size);
+    assert(size == obj->size(), "sanity: size: " SIZE_FORMAT ", obj-size: " SIZE_FORMAT, size, obj->size());
     curr += size;
   }
 }

src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp

@@ -60,10 +60,12 @@ void G1FullGCCompactTask::copy_object_to_new_location(oop obj) {
   // Copy object and reinit its mark.
   HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
   HeapWord* destination = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(obj));
+  assert(obj_addr != destination, "only copy actually-moving objects");
   Copy::aligned_conjoint_words(obj_addr, destination, size);
 
   // There is no need to transform stack chunks - marking already did that.
   cast_to_oop(destination)->init_mark();
+  cast_to_oop(destination)->initialize_hash_if_necessary(obj);
   assert(cast_to_oop(destination)->klass() != nullptr, "should have a class");
 }
 

src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp

@@ -97,9 +97,14 @@ void G1FullGCCompactionPoint::switch_region() {
 void G1FullGCCompactionPoint::forward(oop object, size_t size) {
   assert(_current_region != nullptr, "Must have been initialized");
 
+  size_t old_size = size;
+  size_t new_size = object->copy_size(old_size, object->mark());
+  size = cast_from_oop<HeapWord*>(object) != _compaction_top ? new_size : old_size;
+
   // Ensure the object fit in the current region.
   while (!object_will_fit(size)) {
     switch_region();
+    size = cast_from_oop<HeapWord*>(object) != _compaction_top ? new_size : old_size;
   }
 
   // Store a forwarding pointer if the object should be moved.
@@ -154,8 +159,10 @@ void G1FullGCCompactionPoint::forward_humongous(G1HeapRegion* hr) {
   assert(hr->is_starts_humongous(), "Sanity!");
 
   oop obj = cast_to_oop(hr->bottom());
-  size_t obj_size = obj->size();
-  uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(obj_size);
+  size_t old_size = obj->size();
+  size_t new_size = obj->copy_size(old_size, obj->mark());
+
+  uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(new_size);
 
   if (!has_regions()) {
     return;
@@ -173,6 +180,7 @@ void G1FullGCCompactionPoint::forward_humongous(G1HeapRegion* hr) {
   preserved_stack()->push_if_necessary(obj, obj->mark());
 
   G1HeapRegion* dest_hr = _compaction_regions->at(range_begin);
+  assert(hr->bottom() != dest_hr->bottom(), "assuming actual humongous move");
   FullGCForwarding::forward_to(obj, cast_to_oop(dest_hr->bottom()));
   assert(FullGCForwarding::is_forwarded(obj), "Object must be forwarded!");
 

src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp

@@ -69,7 +69,7 @@ template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
   if (FullGCForwarding::is_forwarded(obj)) {
     oop forwardee = FullGCForwarding::forwardee(obj);
     // Forwarded, just update.
-    assert(G1CollectedHeap::heap()->is_in_reserved(forwardee), "should be in object space");
+    assert(G1CollectedHeap::heap()->is_in_reserved(forwardee), "should be in object space, obj: " PTR_FORMAT ", forwardee: " PTR_FORMAT ", mark: " INTPTR_FORMAT ", pre: " INTPTR_FORMAT ", post: " INTPTR_FORMAT, p2i(obj), p2i(forwardee), obj->mark().value(), *(cast_from_oop<intptr_t*>(obj)) - 1, *(cast_from_oop<intptr_t*>(obj) + 1));
     RawAccess<IS_NOT_NULL>::oop_store(p, forwardee);
   }
 

src/hotspot/share/gc/g1/g1HeapRegion.cpp

@@ -738,6 +738,7 @@ void G1HeapRegion::object_iterate(ObjectClosure* blk) {
   HeapWord* p = bottom();
   while (p < top()) {
     if (block_is_obj(p, parsable_bottom())) {
+      log_trace(gc)("Iterate object: " PTR_FORMAT, p2i(p));
       blk->do_object(cast_to_oop(p));
     }
     p += block_size(p);

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -478,7 +478,8 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
       ? old_mark.klass()
       : old->klass();
 
-  const size_t word_sz = old->size_given_klass(klass);
+  const size_t old_size = old->size_given_mark_and_klass(old_mark, klass);
+  const size_t word_sz = old->copy_size(old_size, old_mark);
 
   // JNI only allows pinning of typeArrays, so we only need to keep those in place.
   if (region_attr.is_pinned() && klass->is_typeArray_klass()) {
@@ -514,9 +515,13 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
     return handle_evacuation_failure_par(old, old_mark, word_sz, false /* cause_pinned */);
   }
 
+  if (old_size != word_sz) {
+    log_trace(gc)("expanding obj: " PTR_FORMAT ", old_size: " SIZE_FORMAT ", new object: " PTR_FORMAT ", word_sz: " SIZE_FORMAT, p2i(old), old_size, p2i(obj_ptr), word_sz);
+  }
+
   // We're going to allocate linearly, so might as well prefetch ahead.
   Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes);
-  Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(old), obj_ptr, word_sz);
+  Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(old), obj_ptr, old_size);
 
   const oop obj = cast_to_oop(obj_ptr);
   // Because the forwarding is done with memory_order_relaxed there is no
@@ -533,6 +538,9 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
       _surviving_young_words[young_index] += word_sz;
     }
 
+    // Initialize i-hash if necessary
+    obj->initialize_hash_if_necessary(old);
+
     if (dest_attr.is_young()) {
       if (age < markWord::max_age) {
         age++;

src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp

@@ -177,7 +177,8 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
       ? test_mark.klass()
       : o->klass();
 
-  size_t new_obj_size = o->size_given_klass(klass);
+  size_t old_obj_size = o->size_given_mark_and_klass(test_mark, klass);
+  size_t new_obj_size = o->copy_size(old_obj_size, test_mark);
 
   // Find the objects age, MT safe.
   uint age = (test_mark.has_displaced_mark_helper() /* o->has_displaced_mark() */) ?