diff --git a/include/wrench/job/StandardJob.h b/include/wrench/job/StandardJob.h index c0e64d8979..b31eb6a8d5 100755 --- a/include/wrench/job/StandardJob.h +++ b/include/wrench/job/StandardJob.h @@ -65,7 +65,7 @@ namespace wrench { unsigned long getMinimumRequiredNumCores() const; - double getMinimumRequiredMemory() const; + sg_size_t getMinimumRequiredMemory() const; unsigned long getNumCompletedTasks() const; diff --git a/include/wrench/services/compute/batch/BatchComputeService.h b/include/wrench/services/compute/batch/BatchComputeService.h index 74585fb700..b97eb3e2d9 100755 --- a/include/wrench/services/compute/batch/BatchComputeService.h +++ b/include/wrench/services/compute/batch/BatchComputeService.h @@ -64,6 +64,7 @@ namespace wrench { {BatchComputeServiceProperty::BATCH_QUEUE_ORDERING_ALGORITHM, "fcfs"}, #else {BatchComputeServiceProperty::BATCH_SCHEDULING_ALGORITHM, "fcfs"}, + {BatchComputeServiceProperty::BACKFILLING_DEPTH, "infinity"}, #endif {BatchComputeServiceProperty::BATCH_RJMS_PADDING_DELAY, "5"}, {BatchComputeServiceProperty::SIMULATED_WORKLOAD_TRACE_FILE, ""}, diff --git a/include/wrench/services/compute/batch/BatchComputeServiceProperty.h b/include/wrench/services/compute/batch/BatchComputeServiceProperty.h index 82ef1d3c42..e9eb0af6a3 100755 --- a/include/wrench/services/compute/batch/BatchComputeServiceProperty.h +++ b/include/wrench/services/compute/batch/BatchComputeServiceProperty.h @@ -55,6 +55,18 @@ namespace wrench { **/ DECLARE_PROPERTY_NAME(BATCH_SCHEDULING_ALGORITHM); + /** + * @brief The backfilling depth used by backfilling algorithms, i.e., how far they go down + * the batch queue to consider a waiting job for backfilling (e.g., "0" (no backfilling done), "10", "infinity"). + * Default: "infinity". + * + * - If ENABLE_BATSCHED is set to off / not set: + * - if the BATCH_SCHEDULING_ALGORITHM is "fcfs": ignored + * + * - If ENABLE_BATSCHED is set to on: ignored + **/ + DECLARE_PROPERTY_NAME(BACKFILLING_DEPTH); + /** * @brief The batch queue ordering algorithm. Can be: * - If ENABLE_BATSCHED is set to off / not set: ignored diff --git a/include/wrench/services/compute/batch/BatchJob.h b/include/wrench/services/compute/batch/BatchJob.h index fbf4c822ec..6df6134bcb 100755 --- a/include/wrench/services/compute/batch/BatchJob.h +++ b/include/wrench/services/compute/batch/BatchJob.h @@ -25,7 +25,7 @@ namespace wrench { void setRequestedTime(unsigned long time); unsigned long getRequestedCoresPerNode() const; std::string getUsername(); - double getMemoryRequirement(); + sg_size_t getMemoryRequirement(); double getBeginTimestamp() const; void setBeginTimestamp(double time_stamp); double getEndingTimestamp() const; diff --git a/include/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf/ConservativeBackfillingBatchScheduler.h b/include/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf/ConservativeBackfillingBatchScheduler.h index abefff1ad7..6ad0bfbb06 100755 --- a/include/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf/ConservativeBackfillingBatchScheduler.h +++ b/include/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf/ConservativeBackfillingBatchScheduler.h @@ -26,7 +26,7 @@ namespace wrench { class ConservativeBackfillingBatchScheduler : public HomegrownBatchScheduler { public: - explicit ConservativeBackfillingBatchScheduler(BatchComputeService *cs); + explicit ConservativeBackfillingBatchScheduler(BatchComputeService *cs, unsigned long backfilling_depth); void processQueuedJobs() override; @@ -43,6 +43,7 @@ namespace wrench { getStartTimeEstimates(std::set> set_of_jobs) override; private: + unsigned long _backfilling_depth; std::unique_ptr schedule; }; diff --git a/include/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf_core_level/ConservativeBackfillingBatchSchedulerCoreLevel.h b/include/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf_core_level/ConservativeBackfillingBatchSchedulerCoreLevel.h index 94faaec191..42ad251444 100755 --- a/include/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf_core_level/ConservativeBackfillingBatchSchedulerCoreLevel.h +++ b/include/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf_core_level/ConservativeBackfillingBatchSchedulerCoreLevel.h @@ -26,7 +26,7 @@ namespace wrench { class ConservativeBackfillingBatchSchedulerCoreLevel : public HomegrownBatchScheduler { public: - explicit ConservativeBackfillingBatchSchedulerCoreLevel(BatchComputeService *cs); + explicit ConservativeBackfillingBatchSchedulerCoreLevel(BatchComputeService *cs, unsigned long backfilling_depth); void processQueuedJobs() override; @@ -44,6 +44,7 @@ namespace wrench { private: std::unique_ptr schedule; + unsigned long _backfilling_depth; }; diff --git a/include/wrench/services/compute/batch/batch_schedulers/homegrown/easy_bf/EasyBackfillingBatchScheduler.h b/include/wrench/services/compute/batch/batch_schedulers/homegrown/easy_bf/EasyBackfillingBatchScheduler.h index 448c12fc33..b3efbfab47 100755 --- a/include/wrench/services/compute/batch/batch_schedulers/homegrown/easy_bf/EasyBackfillingBatchScheduler.h +++ b/include/wrench/services/compute/batch/batch_schedulers/homegrown/easy_bf/EasyBackfillingBatchScheduler.h @@ -26,7 +26,7 @@ namespace wrench { class EasyBackfillingBatchScheduler : public HomegrownBatchScheduler { public: - explicit EasyBackfillingBatchScheduler(BatchComputeService *cs, int depth); + explicit EasyBackfillingBatchScheduler(BatchComputeService *cs, int depth, unsigned long backfilling_depth); void processQueuedJobs() override; @@ -42,6 +42,7 @@ namespace wrench { private: std::unique_ptr schedule; + unsigned long _backfilling_depth; int _depth; }; diff --git a/src/wrench/job/StandardJob.cpp b/src/wrench/job/StandardJob.cpp index 5e437403b7..cad48a8ebd 100755 --- a/src/wrench/job/StandardJob.cpp +++ b/src/wrench/job/StandardJob.cpp @@ -88,10 +88,10 @@ namespace wrench { * one task in the job cannot run if less ram than this minimum is available) * @return the number of cores */ - double StandardJob::getMinimumRequiredMemory() const { - double max_ram = 0; + sg_size_t StandardJob::getMinimumRequiredMemory() const { + sg_size_t max_ram = 0; for (auto const &t: tasks) { - max_ram = std::max(max_ram, (double) (t->getMemoryRequirement())); + max_ram = std::max(max_ram, t->getMemoryRequirement()); } return max_ram; } diff --git a/src/wrench/services/compute/batch/BatchComputeService.cpp b/src/wrench/services/compute/batch/BatchComputeService.cpp index 52a4809f66..da37885998 100644 --- a/src/wrench/services/compute/batch/BatchComputeService.cpp +++ b/src/wrench/services/compute/batch/BatchComputeService.cpp @@ -61,8 +61,8 @@ namespace wrench { std::string scratch_space_mount_point, WRENCH_PROPERTY_COLLECTION_TYPE property_list, WRENCH_MESSAGE_PAYLOAD_COLLECTION_TYPE messagepayload_list) : BatchComputeService(hostname, std::move(compute_hosts), ComputeService::ALL_CORES, - ComputeService::ALL_RAM, scratch_space_mount_point, std::move(property_list), - std::move(messagepayload_list), "") {} + ComputeService::ALL_RAM, scratch_space_mount_point, std::move(property_list), + std::move(messagepayload_list), "") {} /** * @brief Constructor @@ -201,13 +201,27 @@ namespace wrench { if (batch_scheduling_alg == "fcfs") { this->scheduler = std::unique_ptr(new FCFSBatchScheduler(this)); } else if (batch_scheduling_alg == "conservative_bf") { - this->scheduler = std::unique_ptr(new ConservativeBackfillingBatchScheduler(this)); + this->scheduler = std::unique_ptr( + new ConservativeBackfillingBatchScheduler( + this, + this->getPropertyValueAsUnsignedLong(BatchComputeServiceProperty::BACKFILLING_DEPTH))); } else if (batch_scheduling_alg == "easy_bf_depth0") { - this->scheduler = std::unique_ptr(new EasyBackfillingBatchScheduler(this, 0)); + this->scheduler = std::unique_ptr( + new EasyBackfillingBatchScheduler( + this, + 0, + this->getPropertyValueAsUnsignedLong(BatchComputeServiceProperty::BACKFILLING_DEPTH))); } else if (batch_scheduling_alg == "easy_bf_depth1") { - this->scheduler = std::unique_ptr(new EasyBackfillingBatchScheduler(this, 1)); + this->scheduler = std::unique_ptr( + new EasyBackfillingBatchScheduler( + this, + 1, + this->getPropertyValueAsUnsignedLong(BatchComputeServiceProperty::BACKFILLING_DEPTH))); } else if (batch_scheduling_alg == "conservative_bf_core_level") { - this->scheduler = std::unique_ptr(new ConservativeBackfillingBatchSchedulerCoreLevel(this)); + this->scheduler = std::unique_ptr( + new ConservativeBackfillingBatchSchedulerCoreLevel( + this, + this->getPropertyValueAsUnsignedLong(BatchComputeServiceProperty::BACKFILLING_DEPTH))); } #endif @@ -772,7 +786,7 @@ namespace wrench { return true; #ifdef ENABLE_BATSCHED - } else if (auto msg = std::dynamic_pointer_cast(message)) { + } else if (auto msg = std::dynamic_pointer_cast(message)) { processExecuteJobFromBatSched(msg->batsched_decision_reply); return true; #endif @@ -804,7 +818,7 @@ namespace wrench { unsigned long requested_hosts = job->getRequestedNumNodes(); unsigned long requested_num_cores_per_host = job->getRequestedCoresPerNode(); - double required_ram_per_host = job->getMemoryRequirement(); + sg_size_t required_ram_per_host = job->getMemoryRequirement(); if ((requested_hosts > this->available_nodes_to_cores.size()) or (requested_num_cores_per_host > @@ -1066,7 +1080,7 @@ namespace wrench { } else if (key == "ram_availabilities") { // RAM availability per host (0 if something is running, full otherwise) for (const auto &h: this->available_nodes_to_cores) { - if ((double) h.second < S4U_Simulation::getHostMemoryCapacity(h.first)) { + if (h.second < S4U_Simulation::getHostMemoryCapacity(h.first)) { dict.insert(std::make_pair(h.first->get_name(), 0.0)); } else { dict.insert(std::make_pair(h.first->get_name(), S4U_Simulation::getHostMemoryCapacity(h.first))); diff --git a/src/wrench/services/compute/batch/BatchComputeServiceProperty.cpp b/src/wrench/services/compute/batch/BatchComputeServiceProperty.cpp index 5db29d99e2..cc1b993770 100755 --- a/src/wrench/services/compute/batch/BatchComputeServiceProperty.cpp +++ b/src/wrench/services/compute/batch/BatchComputeServiceProperty.cpp @@ -16,6 +16,7 @@ namespace wrench { SET_PROPERTY_NAME(BatchComputeServiceProperty, TASK_SELECTION_ALGORITHM); SET_PROPERTY_NAME(BatchComputeServiceProperty, BATCH_SCHEDULING_ALGORITHM); + SET_PROPERTY_NAME(BatchComputeServiceProperty, BACKFILLING_DEPTH); SET_PROPERTY_NAME(BatchComputeServiceProperty, BATCH_QUEUE_ORDERING_ALGORITHM); SET_PROPERTY_NAME(BatchComputeServiceProperty, BATCH_RJMS_PADDING_DELAY); diff --git a/src/wrench/services/compute/batch/BatchJob.cpp b/src/wrench/services/compute/batch/BatchJob.cpp index d20a13f183..45da14eaea 100755 --- a/src/wrench/services/compute/batch/BatchJob.cpp +++ b/src/wrench/services/compute/batch/BatchJob.cpp @@ -93,7 +93,7 @@ namespace wrench { * @brief Get the memory_manager_service requirement * @return a size in bytes */ - double BatchJob::getMemoryRequirement() { + sg_size_t BatchJob::getMemoryRequirement() { return this->compound_job->getMinimumRequiredMemory(); } diff --git a/src/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf/ConservativeBackfillingBatchScheduler.cpp b/src/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf/ConservativeBackfillingBatchScheduler.cpp index d3839a4275..18ba0bf4a1 100644 --- a/src/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf/ConservativeBackfillingBatchScheduler.cpp +++ b/src/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf/ConservativeBackfillingBatchScheduler.cpp @@ -23,9 +23,13 @@ namespace wrench { /** * @brief Constructor * @param cs: The BatchComputeService for which this scheduler is working + * @param backfilling_depth: the backfilling depth */ - ConservativeBackfillingBatchScheduler::ConservativeBackfillingBatchScheduler(BatchComputeService *cs) : HomegrownBatchScheduler(cs) { + ConservativeBackfillingBatchScheduler::ConservativeBackfillingBatchScheduler( + BatchComputeService *cs, + unsigned long backfilling_depth) : HomegrownBatchScheduler(cs) { this->schedule = std::make_unique(cs->total_num_of_nodes); + this->_backfilling_depth = backfilling_depth; } /** @@ -125,11 +129,13 @@ namespace wrench { // Reset the time origin auto now = (u_int32_t) Simulation::getCurrentSimulatedDate(); - this->schedule->setTimeOrigin(now); + this->schedule->setTimeOrigin(now);; // Go through the BatchComputeService queue - for (auto const &batch_job: this->cs->batch_queue) { +// for (auto const &batch_job: this->cs->batch_queue) { + for (unsigned long i=0; i < std::min(this->_backfilling_depth, this->cs->batch_queue.size()); i++) { // WRENCH_INFO("DEALING WITH JOB %lu", batch_job->getJobID()); + auto batch_job = this->cs->batch_queue.at(i); // Remove the job from the schedule // WRENCH_INFO("REMOVING IT FROM SCHEDULE"); diff --git a/src/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf_core_level/ConservativeBackfillingBatchSchedulerCoreLevel.cpp b/src/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf_core_level/ConservativeBackfillingBatchSchedulerCoreLevel.cpp index 9695e2da51..73005d3372 100644 --- a/src/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf_core_level/ConservativeBackfillingBatchSchedulerCoreLevel.cpp +++ b/src/wrench/services/compute/batch/batch_schedulers/homegrown/conservative_bf_core_level/ConservativeBackfillingBatchSchedulerCoreLevel.cpp @@ -23,9 +23,12 @@ namespace wrench { /** * @brief Constructor * @param cs: The BatchComputeService for which this scheduler is working + * @param backfilling_depth: the backfilling depth */ - ConservativeBackfillingBatchSchedulerCoreLevel::ConservativeBackfillingBatchSchedulerCoreLevel(BatchComputeService *cs) : HomegrownBatchScheduler(cs) { + ConservativeBackfillingBatchSchedulerCoreLevel::ConservativeBackfillingBatchSchedulerCoreLevel(BatchComputeService *cs, + unsigned long backfilling_depth) : HomegrownBatchScheduler(cs) { this->schedule = std::make_unique(cs->total_num_of_nodes, cs->num_cores_per_node); + this->_backfilling_depth = backfilling_depth; } /** @@ -137,7 +140,9 @@ namespace wrench { this->schedule->setTimeOrigin(now); // Go through the BatchComputeService queue - for (auto const &batch_job: this->cs->batch_queue) { +// for (auto const &batch_job: this->cs->batch_queue) { + for (unsigned long i=0; i < std::min(this->_backfilling_depth, this->cs->batch_queue.size()); i++) { + auto batch_job = this->cs->batch_queue.at(i); // WRENCH_INFO("DEALING WITH JOB %lu", batch_job->getJobID()); // Remove the job from the schedule diff --git a/src/wrench/services/compute/batch/batch_schedulers/homegrown/easy_bf/EasyBackfillingBatchScheduler.cpp b/src/wrench/services/compute/batch/batch_schedulers/homegrown/easy_bf/EasyBackfillingBatchScheduler.cpp index a56485f682..e9e28ece53 100644 --- a/src/wrench/services/compute/batch/batch_schedulers/homegrown/easy_bf/EasyBackfillingBatchScheduler.cpp +++ b/src/wrench/services/compute/batch/batch_schedulers/homegrown/easy_bf/EasyBackfillingBatchScheduler.cpp @@ -23,10 +23,15 @@ namespace wrench { /** * @brief Constructor * @param cs: The BatchComputeService for which this scheduler is working + * @param backfilling_depth: the backfilling depth */ - EasyBackfillingBatchScheduler::EasyBackfillingBatchScheduler(BatchComputeService *cs, int depth) : HomegrownBatchScheduler(cs) { + EasyBackfillingBatchScheduler::EasyBackfillingBatchScheduler( + BatchComputeService *cs, + int depth, + unsigned long backfilling_depth) : HomegrownBatchScheduler(cs) { this->schedule = std::make_unique(cs->total_num_of_nodes); this->_depth = depth; + this->_backfilling_depth = backfilling_depth; } /** @@ -106,7 +111,13 @@ namespace wrench { // BACKFILLING: Go through all the other jobs, and start each one that can start right now // (without hurting the first job in the queue if the depth is 1) unsigned long num_nodes_available_now = this->schedule->getNumAvailableNodesInFirstSlot(); - for (unsigned int i = first_job_not_started + 1; i < this->cs->batch_queue.size(); i++) { + unsigned long loop_upper_bound; + if (this->_backfilling_depth > this->cs->batch_queue.size() - first_job_not_started + 1) { + loop_upper_bound = this->cs->batch_queue.size(); + } else { + loop_upper_bound = std::min(first_job_not_started + 1 + this->_backfilling_depth, this->cs->batch_queue.size()); + } + for (unsigned int i = first_job_not_started + 1; i < loop_upper_bound; i++) { auto candidate_job = this->cs->batch_queue.at(i); // If the job's already started, forget it diff --git a/src/wrench/simgrid_S4U_util/S4U_Simulation.cpp b/src/wrench/simgrid_S4U_util/S4U_Simulation.cpp index 8a1ee38849..cfa0506420 100755 --- a/src/wrench/simgrid_S4U_util/S4U_Simulation.cpp +++ b/src/wrench/simgrid_S4U_util/S4U_Simulation.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -23,10 +22,8 @@ #include -#include #include #include "smpi/smpi.h" -#include WRENCH_LOG_CATEGORY(wrench_core_s4u_simulation, "Log category for S4U_Simulation"); diff --git a/test/services/compute_services/batch_standard_and_pilot_jobs/BatchServiceEASYBFTest.cpp b/test/services/compute_services/batch_standard_and_pilot_jobs/BatchServiceEASYBFTest.cpp index 328bde047d..73f2be0d8b 100644 --- a/test/services/compute_services/batch_standard_and_pilot_jobs/BatchServiceEASYBFTest.cpp +++ b/test/services/compute_services/batch_standard_and_pilot_jobs/BatchServiceEASYBFTest.cpp @@ -201,7 +201,8 @@ void BatchServiceEASY_BFTest::do_EASY_BF_test(int num_compute_nodes, ASSERT_NO_THROW(compute_service = simulation->add( new wrench::BatchComputeService(hostname, compute_hosts, "", {{wrench::BatchComputeServiceProperty::BATCH_SCHEDULING_ALGORITHM, scheduling_algorithm}, - {wrench::BatchComputeServiceProperty::BATCH_RJMS_PADDING_DELAY, "0"}}))); + {wrench::BatchComputeServiceProperty::BATCH_RJMS_PADDING_DELAY, "0"}, + {wrench::BatchComputeServiceProperty::BACKFILLING_DEPTH, "infinity"}}))); simulation->add(new wrench::FileRegistryService(hostname));