Skip to content

Commit

Permalink
Merge pull request ClickHouse#69563 from vitlibar/restore-access-depe…
Browse files Browse the repository at this point in the history
…ndencies-2

Improve restoring of access entities' dependencies #2
  • Loading branch information
alexey-milovidov authored Sep 28, 2024
2 parents 0e03c94 + 8a7a411 commit cbfbdbf
Show file tree
Hide file tree
Showing 60 changed files with 1,199 additions and 468 deletions.
691 changes: 471 additions & 220 deletions src/Access/AccessBackup.cpp

Large diffs are not rendered by default.

114 changes: 98 additions & 16 deletions src/Access/AccessBackup.h
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#pragma once

#include <Common/Logger.h>
#include <Core/UUID.h>
#include <unordered_map>
#include <unordered_set>


namespace DB
Expand All @@ -12,6 +12,7 @@ enum class AccessEntityType : uint8_t;
struct IAccessEntity;
using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;
class AccessRightsElements;
class IAccessStorage;
class IBackup;
using BackupPtr = std::shared_ptr<const IBackup>;
class IBackupEntry;
Expand All @@ -20,37 +21,118 @@ struct RestoreSettings;
enum class RestoreAccessCreationMode : uint8_t;


/// Makes a backup of access entities of a specified type.
std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
const std::vector<std::pair<UUID, AccessEntityPtr>> & access_entities,
const String & data_path_in_backup,
size_t counter,
const AccessControl & access_control);
/// Makes a backup entry for of a set of access entities.
std::pair<String, BackupEntryPtr> makeBackupEntryForAccessEntities(
const std::vector<UUID> & entities_ids,
const std::unordered_map<UUID, AccessEntityPtr> & all_entities,
bool write_dependents,
const String & data_path_in_backup);

struct AccessEntitiesToRestore
{
/// Access entities loaded from backup with new randomly generated UUIDs.
std::vector<std::pair<UUID /* new_id */, AccessEntityPtr /* new_entity */>> new_entities;

/// Dependents are access entities which exist already and they should be updated after restoring.
/// For example, if there were a role granted to a user: `CREATE USER user1; CREATE ROLE role1; GRANT role1 TO user1`,
/// and we're restoring only role `role1` because user `user1` already exists,
/// then user `user1` should be modified after restoring role `role1` to add this grant `GRANT role1 TO user1`.
struct Dependent
{
/// UUID of an existing access entities.
UUID existing_id;

/// Source access entity from backup to copy dependencies from.
AccessEntityPtr source;
};
using Dependents = std::vector<Dependent>;
Dependents dependents;
};

/// Restores access entities from a backup.
void restoreAccessEntitiesFromBackup(
IAccessStorage & access_storage,
const AccessEntitiesToRestore & entities_to_restore,
const RestoreSettings & restore_settings);


/// Loads access entities from a backup and prepares them for insertion into an access storage.
class AccessRestorerFromBackup
{
public:
AccessRestorerFromBackup(const BackupPtr & backup_, const RestoreSettings & restore_settings_);
~AccessRestorerFromBackup();

/// Adds a data path to loads access entities from.
void addDataPath(const String & data_path);
void addDataPath(const String & data_path_in_backup);

/// Loads access entities from the backup.
void loadFromBackup();

/// Checks that the current user can do restoring.
/// Function loadFromBackup() must be called before that.
AccessRightsElements getRequiredAccess() const;

/// Inserts all access entities loaded from all the paths added by addDataPath().
std::vector<std::pair<UUID, AccessEntityPtr>> getAccessEntities(const AccessControl & access_control) const;
/// Generates random IDs for access entities we're restoring to insert them into an access storage;
/// and finds IDs of existing access entities which are used as dependencies.
void generateRandomIDsAndResolveDependencies(const AccessControl & access_control);

/// Returns access entities loaded from backup and prepared for insertion into an access storage.
/// Both functions loadFromBackup() and generateRandomIDsAndResolveDependencies() must be called before that.
AccessEntitiesToRestore getEntitiesToRestore(const String & data_path_in_backup) const;

private:
BackupPtr backup;
RestoreAccessCreationMode creation_mode;
bool allow_unresolved_dependencies = false;
std::vector<std::pair<UUID, AccessEntityPtr>> entities;
std::unordered_map<UUID, std::pair<String, AccessEntityType>> dependencies;
std::unordered_set<String> data_paths;
const BackupPtr backup;
const RestoreAccessCreationMode creation_mode;
const bool skip_unresolved_dependencies;
const bool update_dependents;
const LoggerPtr log;

/// Whether loadFromBackup() finished.
bool loaded = false;

/// Whether generateRandomIDsAndResolveDependencies() finished.
bool ids_assigned = false;

Strings data_paths_in_backup;
String data_path_with_entities_to_restore;

/// Information about an access entity loaded from the backup.
struct EntityInfo
{
UUID id;
String name;
AccessEntityType type;

AccessEntityPtr entity = nullptr; /// Can be nullptr if `restore=false`.

/// Index in `data_paths_in_backup`.
size_t data_path_index = 0;

/// Whether we're going to restore this entity.
/// For example,
/// in case of `RESTORE TABLE system.roles` this flag is true for all the roles loaded from the backup, and
/// in case of `RESTORE ALL` this flag is always true.
bool restore = false;

/// Whether this entity info was added as a dependency of another entity which we're going to restore.
/// For example, if we're going to restore the following user: `CREATE USER user1 DEFAULT ROLE role1, role2 SETTINGS PROFILE profile1, profile2`
/// then `restore=true` for `user1` and `is_dependency=true` for `role1`, `role2`, `profile1`, `profile2`.
/// Flags `restore` and `is_dependency` both can be set at the same time.
bool is_dependency = false;

/// Whether this entity info is a dependent of another entity which we're going to restore.
/// For example, if we're going to restore role `role1` and there is also the following user stored in the backup:
/// `CREATE USER user1 DEFAULT ROLE role1`, then `is_dependent=true` for `user1`.
/// This flags is set by generateRandomIDsAndResolveDependencies().
bool is_dependent = false;

/// New UUID for this entity - either randomly generated or copied from an existing entity.
/// This UUID is assigned by generateRandomIDsAndResolveDependencies().
std::optional<UUID> new_id = std::nullopt;
};

std::unordered_map<UUID, EntityInfo> entity_infos;
};

}
4 changes: 2 additions & 2 deletions src/Access/AccessControl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -629,9 +629,9 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po
}
}

void AccessControl::restoreFromBackup(RestorerFromBackup & restorer)
void AccessControl::restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup)
{
MultipleAccessStorage::restoreFromBackup(restorer);
MultipleAccessStorage::restoreFromBackup(restorer, data_path_in_backup);
changes_notifier->sendNotifications();
}

Expand Down
2 changes: 1 addition & 1 deletion src/Access/AccessControl.h
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ class AccessControl : public MultipleAccessStorage
AuthResult authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const String & forwarded_address) const;

/// Makes a backup of access entities.
void restoreFromBackup(RestorerFromBackup & restorer) override;
void restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup) override;

void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config);

Expand Down
2 changes: 1 addition & 1 deletion src/Access/DiskAccessStorage.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -676,7 +676,7 @@ bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_
if (!entry.entity)
entry.entity = readAccessEntityFromDisk(id);
auto old_entity = entry.entity;
auto new_entity = update_func(old_entity);
auto new_entity = update_func(old_entity, id);

if (!new_entity->isTypeOf(old_entity->getType()))
throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType());
Expand Down
62 changes: 62 additions & 0 deletions src/Access/GrantedRoles.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,16 @@ std::vector<UUID> GrantedRoles::findDependencies() const
return res;
}

bool GrantedRoles::hasDependencies(const std::unordered_set<UUID> & ids) const
{
for (const auto & role_id : roles)
{
if (ids.contains(role_id))
return true;
}
return false;
}

void GrantedRoles::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
{
std::vector<UUID> new_ids;
Expand Down Expand Up @@ -221,4 +231,56 @@ void GrantedRoles::replaceDependencies(const std::unordered_map<UUID, UUID> & ol
}
}

void GrantedRoles::copyDependenciesFrom(const GrantedRoles & src, const std::unordered_set<UUID> & ids)
{
bool found = false;

for (const auto & role_id : src.roles)
{
if (ids.contains(role_id))
{
roles.emplace(role_id);
found = true;
}
}

if (found)
{
for (const auto & role_id : src.roles_with_admin_option)
{
if (ids.contains(role_id))
roles_with_admin_option.emplace(role_id);
}
}
}

void GrantedRoles::removeDependencies(const std::unordered_set<UUID> & ids)
{
bool found = false;

for (auto it = roles.begin(); it != roles.end();)
{
if (ids.contains(*it))
{
it = roles.erase(it);
found = true;
}
else
{
++it;
}
}

if (found)
{
for (auto it = roles_with_admin_option.begin(); it != roles_with_admin_option.end();)
{
if (ids.contains(*it))
it = roles_with_admin_option.erase(it);
else
++it;
}
}
}

}
3 changes: 3 additions & 0 deletions src/Access/GrantedRoles.h
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,10 @@ class GrantedRoles
friend bool operator !=(const GrantedRoles & left, const GrantedRoles & right) { return !(left == right); }

std::vector<UUID> findDependencies() const;
bool hasDependencies(const std::unordered_set<UUID> & ids) const;
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids);
void copyDependenciesFrom(const GrantedRoles & src, const std::unordered_set<UUID> & ids);
void removeDependencies(const std::unordered_set<UUID> & ids);

private:
boost::container::flat_set<UUID> roles;
Expand Down
24 changes: 0 additions & 24 deletions src/Access/IAccessEntity.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,28 +9,4 @@ bool IAccessEntity::equal(const IAccessEntity & other) const
return (name == other.name) && (getType() == other.getType());
}

void IAccessEntity::replaceDependencies(std::shared_ptr<const IAccessEntity> & entity, const std::unordered_map<UUID, UUID> & old_to_new_ids)
{
if (old_to_new_ids.empty())
return;

bool need_replace_dependencies = false;
auto dependencies = entity->findDependencies();
for (const auto & dependency : dependencies)
{
if (old_to_new_ids.contains(dependency))
{
need_replace_dependencies = true;
break;
}
}

if (!need_replace_dependencies)
return;

auto new_entity = entity->clone();
new_entity->replaceDependencies(old_to_new_ids);
entity = new_entity;
}

}
9 changes: 5 additions & 4 deletions src/Access/IAccessEntity.h
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,13 @@ struct IAccessEntity

/// Finds all dependencies.
virtual std::vector<UUID> findDependencies() const { return {}; }
virtual bool hasDependencies(const std::unordered_set<UUID> & /* ids */) const { return false; }

/// Replaces dependencies according to a specified map.
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) { doReplaceDependencies(old_to_new_ids); }
static void replaceDependencies(std::shared_ptr<const IAccessEntity> & entity, const std::unordered_map<UUID, UUID> & old_to_new_ids);
virtual void replaceDependencies(const std::unordered_map<UUID, UUID> & /* old_to_new_ids */) {}
virtual void copyDependenciesFrom(const IAccessEntity & /* src */, const std::unordered_set<UUID> & /* ids */) {}
virtual void removeDependencies(const std::unordered_set<UUID> & /* ids */) {}
virtual void clearAllExceptDependencies() {}

/// Whether this access entity should be written to a backup.
virtual bool isBackupAllowed() const { return false; }
Expand All @@ -67,8 +70,6 @@ struct IAccessEntity
{
return std::make_shared<EntityClassT>(typeid_cast<const EntityClassT &>(*this));
}

virtual void doReplaceDependencies(const std::unordered_map<UUID, UUID> & /* old_to_new_ids */) {}
};

using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;
Expand Down
Loading

0 comments on commit cbfbdbf

Please sign in to comment.