Issue 6554 - During import of entries without nsUniqueId, a supplier …

…generates duplicate nsUniqueId (LMDB only)

Bug description:
	During an import the entry is prepared (schema, operational
	attributes, password encryption,...) before starting the
	update of the database and indexes.
	A step of the preparation is to assign a value to 'nsuniqueid'
	operational attribute. 'nsuniqueid' must be unique.
	In LMDB the preparation is done by multiple threads (workers).
	In such case the 'nsuniqueid' are generated in parallel and
	as it is time based several values can be duplicated.

Fix description:
	To prevent that the routine dbmdb_import_generate_uniqueid
	should make sure to synchronize the workers.

fixes: #6554

Reviewed by:

dirsrvtests/tests/suites/import/import_test.py

@@ -14,11 +14,13 @@
 import pytest
 import time
 import glob
+import re
 import logging
 import subprocess
 from datetime import datetime
 from lib389.topologies import topology_st as topo
-from lib389._constants import DEFAULT_SUFFIX, TaskWarning
+from lib389.topologies import topology_m2 as topo_m2
+from lib389._constants import DEFAULT_BENAME, DEFAULT_SUFFIX, TaskWarning
 from lib389.dbgen import dbgen_users
 from lib389.tasks import ImportTask
 from lib389.index import Indexes
@@ -690,6 +692,81 @@ def test_online_import_under_load(topo):
     assert import_task.get_exit_code() == 0
 
 
+def test_duplicate_nsuniqueid(topo_m2, request):
+    """Test that after an offline import all
+    nsuniqueid are different
+
+    :id: a2541677-a288-4633-bacf-4050cc56016d
+    :setup: MMR with 2 suppliers
+    :steps:
+        1. stop the instance to do offline operations
+        2. Generate a 5K users LDIF file
+        3. Check that no uniqueid are present in the generated file
+        4. import the generated LDIF
+        5. export the database
+        6. Check that that exported LDIF contains more than 5K nsuniqueid
+        7. Check that there is no duplicate nsuniqued in exported LDIF
+    :expectedresults:
+        1. Should succeeds
+        2. Should succeeds
+        3. Should succeeds
+        4. Should succeeds
+        5. Should succeeds
+        6. Should succeeds
+        7. Should succeeds
+    """
+    m1 = topo_m2.ms["supplier1"]
+
+    # Stop the instance
+    m1.stop()
+
+    # Generate a test ldif (5k entries)
+    log.info("Generating LDIF...")
+    ldif_dir = m1.get_ldif_dir()
+    import_ldif = ldif_dir + '/5k_users_import.ldif'
+    dbgen_users(m1, 5000, import_ldif, DEFAULT_SUFFIX)
+
+    # Check that the generated LDIF does not contain nsuniqueid
+    all_nsuniqueid = []
+    with open(import_ldif, 'r') as file:
+        for line in file:
+            if line.lower().startswith("nsuniqueid: "):
+                all_nsuniqueid.append(line.split(': ')[1])
+    log.info("import file contains " + str(len(all_nsuniqueid)) + " nsuniqueid")
+    assert len(all_nsuniqueid) == 0
+
+    # Import the "nsuniquied free" LDIF file
+    if not m1.ldif2db('userRoot', None, None, None, import_ldif):
+        assert False
+
+    # Export the DB that now should contain nsuniqueid
+    export_ldif = ldif_dir + '/5k_user_export.ldif'
+    log.info("export to file " + export_ldif)
+    m1.db2ldif(bename=DEFAULT_BENAME, suffixes=[DEFAULT_SUFFIX],
+               excludeSuffixes=None, repl_data=False,
+               outputfile=export_ldif, encrypt=False)
+
+    # Check that the export LDIF contain nsuniqueid
+    all_nsuniqueid = []
+    with open(export_ldif, 'r') as file:
+        for line in file:
+            if line.lower().startswith("nsuniqueid: "):
+                all_nsuniqueid.append(line.split(': ')[1])
+    log.info("export file " + export_ldif + " contains " + str(len(all_nsuniqueid)) + " nsuniqueid")
+    assert len(all_nsuniqueid) >= 5000
+
+    # Check that the nsuniqueid are unique
+    assert len(set(all_nsuniqueid)) == len(all_nsuniqueid)
+
+    def fin():
+        if os.path.exists(import_ldif):
+            os.remove(import_ldif)
+        if os.path.exists(export_ldif):
+            os.remove(export_ldif)
+        m1.start
+
+    request.addfinalizer(fin)
+
 if __name__ == '__main__':
     # Run isolated
     # -s for DEBUG mode

ldap/servers/slapd/back-ldbm/db-mdb/mdb_import_threads.c

@@ -610,10 +610,20 @@ dbmdb_import_generate_uniqueid(ImportJob *job, Slapi_Entry *e)
 {
     const char *uniqueid = slapi_entry_get_uniqueid(e);
     int rc = UID_SUCCESS;
+    static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
 
     if (!uniqueid && (job->uuid_gen_type != SLAPI_UNIQUEID_GENERATE_NONE)) {
         char *newuniqueid;
 
+        /* With 'mdb' we have several workers generating nsuniqueid
+         * we need to serialize them to prevent generating duplicate value
+         * From performance pov it only impacts import
+         * The default value is SLAPI_UNIQUEID_GENERATE_TIME_BASED so
+         * the only syscall is clock_gettime and then string formating
+         * that should limit contention
+         */
+        pthread_mutex_lock(&mutex);
+
         /* generate id based on dn */
         if (job->uuid_gen_type == SLAPI_UNIQUEID_GENERATE_NAME_BASED) {
             char *dn = slapi_entry_get_dn(e);
@@ -624,6 +634,7 @@ dbmdb_import_generate_uniqueid(ImportJob *job, Slapi_Entry *e)
             /* time based */
             rc = slapi_uniqueIDGenerateString(&newuniqueid);
         }
+        pthread_mutex_unlock(&mutex);
 
         if (rc == UID_SUCCESS) {
             slapi_entry_set_uniqueid(e, newuniqueid);