Skip to content

Commit

Permalink
Merge pull request #602 from iExecBlockchainComputing/hotfix/worker-s…
Browse files Browse the repository at this point in the history 
…ervice-race-condition

Release version 8.1.2
  • Loading branch information
@mcornaton
mcornaton authored Jun 29, 2023
2 parents 2b9bfe4 + 2647a7f commit d10faf0
Show file tree
Hide file tree
Showing 4 changed files with 235 additions and 75 deletions.
7 changes: 7 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@

All notable changes to this project will be documented in this file.

## [[8.1.2]](https://github.com/iExecBlockchainComputing/iexec-core/releases/tag/v8.1.2) 2023-06-29

## Bug fixes
- Prevent race conditions in `WorkerService`. (#602)
### Dependency Upgrades
- Upgrade to `iexec-commons-poco` 3.0.5. (#602)

## [[8.1.1]](https://github.com/iExecBlockchainComputing/iexec-core/releases/tag/v8.1.1) 2023-06-23

### Dependency Upgrades
Expand Down
4 changes: 2 additions & 2 deletions gradle.properties
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
version=8.1.1
version=8.1.2
iexecCommonVersion=8.2.1
iexecCommonsPocoVersion=3.0.4
iexecCommonsPocoVersion=3.0.5
iexecBlockchainAdapterVersion=8.1.1
iexecResultVersion=8.1.1
iexecSmsVersion=8.1.1
Expand Down
202 changes: 129 additions & 73 deletions src/main/java/com/iexec/core/worker/WorkerService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

package com.iexec.core.worker;

import com.iexec.common.utils.ContextualLockRunner;
import com.iexec.core.configuration.WorkerConfiguration;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
Expand All @@ -27,39 +28,33 @@

import static com.iexec.common.utils.DateTimeUtils.addMinutesToDate;

/**
* Manage {@link Worker} objects.
* <p>
* /!\ Private read-and-write methods are not thread-safe.
* They can sometime lead to race conditions.
* Please use the public, thread-safe, versions of these methods instead.
*/
@Slf4j
@Service
public class WorkerService {

private final WorkerRepository workerRepository;
private final WorkerConfiguration workerConfiguration;
private final ContextualLockRunner<String> contextualLockRunner;

public WorkerService(WorkerRepository workerRepository,
WorkerConfiguration workerConfiguration) {
this.workerRepository = workerRepository;
this.workerConfiguration = workerConfiguration;
this.contextualLockRunner = new ContextualLockRunner<>();
}

// region Read methods
public Optional<Worker> getWorker(String walletAddress) {
return workerRepository.findByWalletAddress(walletAddress);
}

public Worker addWorker(Worker worker) {
Optional<Worker> oWorker = workerRepository.findByWalletAddress(worker.getWalletAddress());

if (oWorker.isPresent()) {
Worker existingWorker = oWorker.get();
log.info("The worker is already registered [workerId:{}]", existingWorker.getId());
worker.setId(existingWorker.getId());
worker.setParticipatingChainTaskIds(existingWorker.getParticipatingChainTaskIds());
worker.setComputingChainTaskIds(existingWorker.getComputingChainTaskIds());
} else {
log.info("Registering new worker");
}

return workerRepository.save(worker);
}

public boolean isAllowedToJoin(String workerAddress){
List<String> whitelist = workerConfiguration.getWhitelist();
// if the whitelist is empty, there is no restriction on the workers
Expand All @@ -69,18 +64,6 @@ public boolean isAllowedToJoin(String workerAddress){
return whitelist.contains(workerAddress);
}

public Optional<Worker> updateLastAlive(String walletAddress) {
Optional<Worker> optional = workerRepository.findByWalletAddress(walletAddress);
if (optional.isPresent()) {
Worker worker = optional.get();
worker.setLastAliveDate(new Date());
workerRepository.save(worker);
return Optional.of(worker);
}

return Optional.empty();
}

public boolean isWorkerAllowedToAskReplicate(String walletAddress) {
Optional<Date> oDate = getLastReplicateDemand(walletAddress);
if (oDate.isEmpty()) {
Expand All @@ -105,29 +88,6 @@ public Optional<Date> getLastReplicateDemand(String walletAddress) {
return Optional.ofNullable(worker.getLastReplicateDemandDate());
}

public Optional<Worker> updateLastReplicateDemandDate(String walletAddress) {
Optional<Worker> optional = workerRepository.findByWalletAddress(walletAddress);
if (optional.isPresent()) {
Worker worker = optional.get();
worker.setLastReplicateDemandDate(new Date());
workerRepository.save(worker);
return Optional.of(worker);
}

return Optional.empty();
}

public Optional<Worker> addChainTaskIdToWorker(String chainTaskId, String walletAddress) {
Optional<Worker> optional = workerRepository.findByWalletAddress(walletAddress);
if (optional.isPresent()) {
Worker worker = optional.get();
worker.addChainTaskId(chainTaskId);
log.info("Added chainTaskId to worker [chainTaskId:{}, workerName:{}]", chainTaskId, walletAddress);
return Optional.of(workerRepository.save(worker));
}
return Optional.empty();
}

public List<String> getChainTaskIds(String walletAddress) {
Optional<Worker> optional = workerRepository.findByWalletAddress(walletAddress);
if (optional.isPresent()) {
Expand All @@ -146,28 +106,6 @@ public List<String> getComputingTaskIds(String walletAddress) {
return Collections.emptyList();
}

public Optional<Worker> removeChainTaskIdFromWorker(String chainTaskId, String walletAddress) {
Optional<Worker> optional = workerRepository.findByWalletAddress(walletAddress);
if (optional.isPresent()) {
Worker worker = optional.get();
worker.removeChainTaskId(chainTaskId);
log.info("Removed chainTaskId from worker [chainTaskId:{}, walletAddress:{}]", chainTaskId, walletAddress);
return Optional.of(workerRepository.save(worker));
}
return Optional.empty();
}

public Optional<Worker> removeComputedChainTaskIdFromWorker(String chainTaskId, String walletAddress) {
Optional<Worker> optional = workerRepository.findByWalletAddress(walletAddress);
if (optional.isPresent()) {
Worker worker = optional.get();
worker.removeComputedChainTaskId(chainTaskId);
log.info("Removed computed chainTaskId from worker [chainTaskId:{}, walletAddress:{}]", chainTaskId, walletAddress);
return Optional.of(workerRepository.save(worker));
}
return Optional.empty();
}


// worker is considered lost if it didn't ping for 1 minute
public List<Worker> getLostWorkers() {
Expand Down Expand Up @@ -249,4 +187,122 @@ public int getAliveAvailableGpu () {
}
return availableGpus;
}
// endregion

// region Read-and-write methods
public Worker addWorker(Worker worker) {
return contextualLockRunner.applyWithLock(
worker.getWalletAddress(),
address -> addWorkerWithoutThreadSafety(worker)
);
}

private Worker addWorkerWithoutThreadSafety(Worker worker) {
Optional<Worker> oWorker = workerRepository.findByWalletAddress(worker.getWalletAddress());

if (oWorker.isPresent()) {
Worker existingWorker = oWorker.get();
log.info("The worker is already registered [workerId:{}]", existingWorker.getId());
worker.setId(existingWorker.getId());
worker.setParticipatingChainTaskIds(existingWorker.getParticipatingChainTaskIds());
worker.setComputingChainTaskIds(existingWorker.getComputingChainTaskIds());
} else {
log.info("Registering new worker");
}

return workerRepository.save(worker);
}

public Optional<Worker> updateLastAlive(String walletAddress) {
return contextualLockRunner.applyWithLock(
walletAddress,
this::updateLastAliveWithoutThreadSafety
);
}

private Optional<Worker> updateLastAliveWithoutThreadSafety(String walletAddress) {
Optional<Worker> optional = workerRepository.findByWalletAddress(walletAddress);
if (optional.isPresent()) {
Worker worker = optional.get();
worker.setLastAliveDate(new Date());
workerRepository.save(worker);
return Optional.of(worker);
}

return Optional.empty();
}

public Optional<Worker> updateLastReplicateDemandDate(String walletAddress) {
return contextualLockRunner.applyWithLock(
walletAddress,
this::updateLastReplicateDemandDateWithoutThreadSafety
);
}

private Optional<Worker> updateLastReplicateDemandDateWithoutThreadSafety(String walletAddress) {
Optional<Worker> optional = workerRepository.findByWalletAddress(walletAddress);
if (optional.isPresent()) {
Worker worker = optional.get();
worker.setLastReplicateDemandDate(new Date());
workerRepository.save(worker);
return Optional.of(worker);
}

return Optional.empty();
}

public Optional<Worker> addChainTaskIdToWorker(String chainTaskId, String walletAddress) {
return contextualLockRunner.applyWithLock(
walletAddress,
address -> addChainTaskIdToWorkerWithoutThreadSafety(chainTaskId, address)
);
}

private Optional<Worker> addChainTaskIdToWorkerWithoutThreadSafety(String chainTaskId, String walletAddress) {
Optional<Worker> optional = workerRepository.findByWalletAddress(walletAddress);
if (optional.isPresent()) {
Worker worker = optional.get();
worker.addChainTaskId(chainTaskId);
log.info("Added chainTaskId to worker [chainTaskId:{}, workerName:{}]", chainTaskId, walletAddress);
return Optional.of(workerRepository.save(worker));
}
return Optional.empty();
}

public Optional<Worker> removeChainTaskIdFromWorker(String chainTaskId, String walletAddress) {
return contextualLockRunner.applyWithLock(
walletAddress,
address -> removeChainTaskIdFromWorkerWithoutThreadSafety(chainTaskId, address)
);
}

private Optional<Worker> removeChainTaskIdFromWorkerWithoutThreadSafety(String chainTaskId, String walletAddress) {
Optional<Worker> optional = workerRepository.findByWalletAddress(walletAddress);
if (optional.isPresent()) {
Worker worker = optional.get();
worker.removeChainTaskId(chainTaskId);
log.info("Removed chainTaskId from worker [chainTaskId:{}, walletAddress:{}]", chainTaskId, walletAddress);
return Optional.of(workerRepository.save(worker));
}
return Optional.empty();
}

public Optional<Worker> removeComputedChainTaskIdFromWorker(String chainTaskId, String walletAddress) {
return contextualLockRunner.applyWithLock(
walletAddress,
address -> removeComputedChainTaskIdFromWorkerWithoutThreadSafety(chainTaskId, address)
);
}

private Optional<Worker> removeComputedChainTaskIdFromWorkerWithoutThreadSafety(String chainTaskId, String walletAddress) {
Optional<Worker> optional = workerRepository.findByWalletAddress(walletAddress);
if (optional.isPresent()) {
Worker worker = optional.get();
worker.removeComputedChainTaskId(chainTaskId);
log.info("Removed computed chainTaskId from worker [chainTaskId:{}, walletAddress:{}]", chainTaskId, walletAddress);
return Optional.of(workerRepository.save(worker));
}
return Optional.empty();
}
// endregion
}
97 changes: 97 additions & 0 deletions src/test/java/com/iexec/core/worker/WorkerServiceRealRepositoryTests.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/*
* Copyright 2023-2023 IEXEC BLOCKCHAIN TECH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.iexec.core.worker;

import com.iexec.core.configuration.WorkerConfiguration;
import lombok.extern.slf4j.Slf4j;
import org.assertj.core.api.Assertions;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.test.context.DynamicPropertyRegistry;
import org.springframework.test.context.DynamicPropertySource;
import org.testcontainers.containers.MongoDBContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;

import java.time.Duration;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static com.iexec.commons.poco.utils.TestUtils.WALLET_WORKER_1;

@Slf4j
@DataMongoTest
@Testcontainers
class WorkerServiceRealRepositoryTests {
@Container
private static final MongoDBContainer mongoDBContainer = new MongoDBContainer(DockerImageName.parse("mongo:4.2"));

@DynamicPropertySource
static void registerProperties(DynamicPropertyRegistry registry) {
registry.add("spring.data.mongodb.host", mongoDBContainer::getContainerIpAddress);
registry.add("spring.data.mongodb.port", () -> mongoDBContainer.getMappedPort(27017));
}

@SpyBean
private WorkerRepository workerRepository;
@Mock
private WorkerConfiguration workerConfiguration;
private WorkerService workerService;

@BeforeEach
void init() {
MockitoAnnotations.openMocks(this);
workerService = new WorkerService(workerRepository, workerConfiguration);
}

/**
* Try and add N tasks to a single worker at the same time.
* If everything goes right, the Worker should finally have been assigned N tasks.
*/
@Test
void addMultipleTaskIds() {
workerService.addWorker(
Worker.builder()
.walletAddress(WALLET_WORKER_1)
.build()
);

final int nThreads = 10;
final ExecutorService executor = Executors.newFixedThreadPool(nThreads);

final List<Future<Optional<Worker>>> futures = IntStream.range(0, nThreads)
.mapToObj(i -> executor.submit(() -> workerService.addChainTaskIdToWorker(new Date().getTime() + "", WALLET_WORKER_1)))
.collect(Collectors.toList());

Awaitility.await()
.atMost(Duration.ofMinutes(1))
.until(() -> futures.stream().map(Future::isDone).reduce(Boolean::logicalAnd).orElse(false));

Assertions.assertThat(workerService.getWorker(WALLET_WORKER_1).get().getComputingChainTaskIds())
.hasSize(nThreads);
}
}

0 comments on commit d10faf0

Please sign in to comment.