diff --git a/CMakeLists.txt b/CMakeLists.txt index 11aa7d5706..424f66df50 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -266,8 +266,6 @@ set(SOURCE_FILES include/wrench/services/compute/batch/batch_schedulers/BatchScheduler.h include/wrench/services/compute/batch/batch_schedulers/homegrown/HomegrownBatchScheduler.h include/wrench/services/storage/storage_helpers/FileTransferThread.h - include/wrench/services/storage/storage_helpers/LogicalFileSystemNoCaching.h - include/wrench/services/storage/storage_helpers/LogicalFileSystemLRUCaching.h src/wrench/logging/TerminalOutput.cpp src/wrench/managers/data_movement_manager/DataMovementManager.cpp src/wrench/managers/data_movement_manager/DataMovementManagerMessage.cpp @@ -461,9 +459,9 @@ set(TEST_FILES test/services/compute_services/bare_metal_compound_jobs/BareMetalComputeServiceMultiActionTests.cpp test/services/compute_services/bare_metal_compound_jobs/BareMetalComputeServiceMultiJobTests.cpp test/services/compute_services/bare_metal_standard_jobs/BareMetalComputeServiceOneTaskTest.cpp - test/services/storage_services/LogicalFileSystem/LogicalFileSystemTest.cpp test/services/storage_services/SimpleStorageService/SimpleStorageServiceCachingTest.cpp test/services/storage_services/SimpleStorageService/SimpleStorageServiceFunctionalTest.cpp + test/services/storage_services/SimpleStorageService/SimpleStorageServiceWrongServiceTest.cpp test/services/storage_services/SimpleStorageService/SimpleStorageServicePerformanceTest.cpp test/services/storage_services/SimpleStorageService/SimpleStorageServiceLimitedConnectionsTest.cpp test/services/storage_services/SimpleStorageService/StorageServiceDeleteRegisterTest.cpp diff --git a/include/wrench/services/storage/storage_helpers/LogicalFileSystemLRUCaching.h b/include/wrench/services/storage/storage_helpers/LogicalFileSystemLRUCaching.h deleted file mode 100755 index 2e4b7faea2..0000000000 --- a/include/wrench/services/storage/storage_helpers/LogicalFileSystemLRUCaching.h +++ /dev/null @@ -1,112 +0,0 @@ -///** -// * Copyright (c) 2017. The WRENCH Team. -// * -// * This program is free software: you can redistribute it and/or modify -// * it under the terms of the GNU General Public License as published by -// * the Free Software Foundation, either version 3 of the License, or -// * (at your option) any later version. -// */ -// -//#ifndef WRENCH_LOGICALFILESYSTEMLRUCACHING_H -//#define WRENCH_LOGICALFILESYSTEMLRUCACHING_H -// -//#include -//#include -//#include -//#include -//#include -//#include -// -//#include -// -// -//#include -// -//namespace wrench { -// -// /***********************/ -// /** \cond INTERNAL */ -// /***********************/ -// -// -// class StorageService; -// -// /** -// * @brief A class that implements a weak file system abstraction -// */ -// class LogicalFileSystemLRUCaching : public LogicalFileSystem { -// -// public: -// /** -// * @brief A helper struct to describe a file instance on disk -// */ -// struct FileOnDiskLRUCaching : public FileOnDisk { -// public: -// /** -// * @brief Constructor -// * @param last_write_date Last write date -// * @param lru_sequence_number LRU sequence number -// * @param num_current_transactions Number of current transactions using this file on disk -// */ -// FileOnDiskLRUCaching(double last_write_date, -// unsigned int lru_sequence_number, -// unsigned short num_current_transactions) : FileOnDisk(last_write_date), -// lru_sequence_number(lru_sequence_number), -// num_current_transactions(num_current_transactions) {} -// -// /** -// * @brief The LRU sequence number (lower means older) -// */ -// unsigned int lru_sequence_number; -// /** -// * @brief The number of transactions that involve this file, meaning that it's not evictable is > 0 -// */ -// unsigned short num_current_transactions; -// }; -// -// public: -// /** -// * @brief Next LRU sequence number -// */ -// unsigned int next_lru_sequence_number = 0; -// -// void storeFileInDirectory(const std::shared_ptr &file, const std::string &absolute_path) override; -// void removeFileFromDirectory(const std::shared_ptr &file, const std::string &absolute_path) override; -// void removeAllFilesInDirectory(const std::string &absolute_path) override; -// void updateReadDate(const std::shared_ptr &file, const std::string &absolute_path) override; -// void incrementNumRunningTransactionsForFileInDirectory(const std::shared_ptr &file, const std::string &absolute_path) override; -// void decrementNumRunningTransactionsForFileInDirectory(const std::shared_ptr &file, const std::string &absolute_path) override; -// -// protected: -// bool evictFiles(double needed_free_space) override; -// -// private: -// friend class StorageService; -// friend class LogicalFileSystem; -// -// explicit LogicalFileSystemLRUCaching(const std::string &hostname, -// StorageService *storage_service, -// const std::string &mount_point); -// -// -// std::map>> lru_list; -// -// void print_lru_list() { -// std::cerr << "LRU LIST:\n"; -// for (auto const &lru: this->lru_list) { -// std::cerr << "[" << lru.first << "] " << std::get<0>(lru.second) << ":" << std::get<1>(lru.second)->getID() << "\n"; -// } -// } -// -// private: -// }; -// -// -// /***********************/ -// /** \endcond */ -// /***********************/ -// -//}// namespace wrench -// -// -//#endif//WRENCH_LOGICALFILESYSTEMLRUCACHING_H diff --git a/include/wrench/services/storage/storage_helpers/LogicalFileSystemNoCaching.h b/include/wrench/services/storage/storage_helpers/LogicalFileSystemNoCaching.h deleted file mode 100755 index 3ab6a77f01..0000000000 --- a/include/wrench/services/storage/storage_helpers/LogicalFileSystemNoCaching.h +++ /dev/null @@ -1,75 +0,0 @@ -///** -// * Copyright (c) 2017. The WRENCH Team. -// * -// * This program is free software: you can redistribute it and/or modify -// * it under the terms of the GNU General Public License as published by -// * the Free Software Foundation, either version 3 of the License, or -// * (at your option) any later version. -// */ -// -//#ifndef WRENCH_LOGICALFILESYSTEMNOCACHING_H -//#define WRENCH_LOGICALFILESYSTEMNOCACHING_H -// -//#include -//#include -//#include -//#include -//#include -//#include -// -//#include -// -// -//#include -//#include -// -//namespace wrench { -// -// /***********************/ -// /** \cond INTERNAL */ -// /***********************/ -// -// -// class StorageService; -// -// /** -// * @brief A class that implements a weak file system abstraction -// */ -// class LogicalFileSystemNoCaching : public LogicalFileSystem { -// -// class FileOnDiskNoCaching : public FileOnDisk { -// public: -// explicit FileOnDiskNoCaching(double last_write_date) : FileOnDisk(last_write_date) {} -// }; -// -// public: -// void storeFileInDirectory(const std::shared_ptr &file, const std::string &absolute_path) override; -// void removeFileFromDirectory(const std::shared_ptr &file, const std::string &absolute_path) override; -// void removeAllFilesInDirectory(const std::string &absolute_path) override; -// void updateReadDate(const std::shared_ptr &file, const std::string &absolute_path) override; -// void incrementNumRunningTransactionsForFileInDirectory(const std::shared_ptr &file, const std::string &absolute_path) override; -// void decrementNumRunningTransactionsForFileInDirectory(const std::shared_ptr &file, const std::string &absolute_path) override; -// -// protected: -// friend class StorageService; -// bool evictFiles(double needed_free_space) override; -// -// private: -// friend class LogicalFileSystem; -// explicit LogicalFileSystemNoCaching(const std::string &hostname, -// StorageService *storage_service, -// const std::string &mount_point); -// -// -// private: -// }; -// -// -// /***********************/ -// /** \endcond */ -// /***********************/ -// -//}// namespace wrench -// -// -//#endif//WRENCH_LOGICALFILESYSTEMNOCACHING_H diff --git a/src/wrench/services/storage/StorageService.cpp b/src/wrench/services/storage/StorageService.cpp index 4af9bb21b3..50f5bef113 100755 --- a/src/wrench/services/storage/StorageService.cpp +++ b/src/wrench/services/storage/StorageService.cpp @@ -110,6 +110,10 @@ namespace wrench { throw std::invalid_argument("StorageService::writeFile(): Invalid arguments"); } + if (location->getStorageService() != this->getSharedPtr()) { + throw std::invalid_argument("StorageService::writeFile(): Can only read from a location at that same storage service"); + } + this->assertServiceIsUp(); this->commport->putMessage( @@ -249,6 +253,10 @@ namespace wrench { throw std::invalid_argument("StorageService::readFile(): Invalid nullptr/0 arguments"); } + if (location->getStorageService() != this->getSharedPtr()) { + throw std::invalid_argument("StorageService::readFile(): Can only read from a location at that same storage service"); + } + assertServiceIsUp(this->getSharedPtr()); this->commport->putMessage( diff --git a/test/services/storage_services/LogicalFileSystem/LogicalFileSystemTest.cpp b/test/services/storage_services/LogicalFileSystem/LogicalFileSystemTest.cpp deleted file mode 100755 index 80fe3b1a25..0000000000 --- a/test/services/storage_services/LogicalFileSystem/LogicalFileSystemTest.cpp +++ /dev/null @@ -1,276 +0,0 @@ -//#include -//#include -// -//#include "../../../include/TestWithFork.h" -//#include "../../../include/UniqueTmpPathPrefix.h" -// -//WRENCH_LOG_CATEGORY(logical_file_system_test, "Log category for LogicalFileSystemTest"); -// -// -//class LogicalFileSystemTest : public ::testing::Test { -// -//public: -// void do_BasicTests(); -// void do_DevNullTests(); -// void do_LRUTests(); -// -//protected: -// LogicalFileSystemTest() { -// -// // Create a 2-host platform file -// // [WMSHost]-----[StorageHost] -// std::string xml = "" -// "" -// " " -// " " -// " " -// " " -// " " -// " " -// " " -// " " -// " " -// " " -// " " -// " " -// " " -// ""; -// FILE *platform_file = fopen(platform_file_path.c_str(), "w"); -// fprintf(platform_file, "%s", xml.c_str()); -// fclose(platform_file); -// } -// -// std::string platform_file_path = UNIQUE_TMP_PATH_PREFIX + "platform.xml"; -//}; -// -// -//TEST_F(LogicalFileSystemTest, BasicTests) { -// DO_TEST_WITH_FORK(do_BasicTests); -//} -// -//void LogicalFileSystemTest::do_BasicTests() { -// // Create and initialize the simulation -// auto simulation = wrench::Simulation::createSimulation(); -// -// int argc = 1; -// char **argv = (char **) calloc(argc, sizeof(char *)); -// argv[0] = strdup("unit_test"); -// // argv[1] = strdup("--wrench-full-log"); -// -// ASSERT_NO_THROW(simulation->init(&argc, argv)); -// auto workflow = wrench::Workflow::createWorkflow(); -// -// // set up the platform -// ASSERT_NO_THROW(simulation->instantiatePlatform(platform_file_path)); -// -// // Create two Storage Services -// std::shared_ptr storage_service1, storage_service2; -// ASSERT_NO_THROW(storage_service1 = simulation->add( -// wrench::SimpleStorageService::createSimpleStorageService("Host", {"/"}))); -// -// ASSERT_THROW(wrench::LogicalFileSystem::createLogicalFileSystem("Host", nullptr, "/tmp", "NONE"), std::invalid_argument); -// ASSERT_THROW(wrench::LogicalFileSystem::createLogicalFileSystem("Host", storage_service1.get(), "/bogus"), std::invalid_argument); -// -// // Create a Logical File System -// try { -// wrench::LogicalFileSystem::createLogicalFileSystem("Host", storage_service1.get(), "/tmp", "BOGUS"); -// throw std::runtime_error("Should not be able to create a logical file system with a bogus caching policy"); -// } catch (std::invalid_argument &ignore) {} -// -// auto fs1 = wrench::LogicalFileSystem::createLogicalFileSystem("Host", storage_service1.get(), "/tmp", "NONE"); -// -// // Attempt to create a redundant Logical File System -// try { -// wrench::LogicalFileSystem::createLogicalFileSystem("Host", storage_service1.get(), "/tmp"); -// throw std::runtime_error("Initializing a redundant file system should have thrown"); -// } catch (std::invalid_argument &ignore) { -// } -// -// fs1->createDirectory(("/foo")); -// fs1->removeAllFilesInDirectory("/foo"); -// fs1->listFilesInDirectory("/foo"); -// fs1->removeEmptyDirectory("/foo"); -// -// ASSERT_DOUBLE_EQ(100, fs1->getTotalCapacity()); -// auto file_80 = wrench::Simulation::addFile("file_80", 80); -// ASSERT_TRUE(fs1->reserveSpace(file_80, "/files/")); -// fs1->unreserveSpace(file_80, "/files/"); -// ASSERT_DOUBLE_EQ(100, fs1->getFreeSpace()); -// ASSERT_TRUE(fs1->reserveSpace(file_80, "/files/")); -// fs1->storeFileInDirectory(file_80, "/files/"); -// ASSERT_DOUBLE_EQ(20, fs1->getFreeSpace()); -// fs1->incrementNumRunningTransactionsForFileInDirectory(file_80, "/files");// coverage -// fs1->decrementNumRunningTransactionsForFileInDirectory(file_80, "/files");// coverage -// -// auto file_50 = wrench::Simulation::addFile("file_50", 50); -// ASSERT_FALSE(fs1->reserveSpace(file_50, "/files/")); -// ASSERT_DOUBLE_EQ(20, fs1->getFreeSpace()); -// fs1->removeFileFromDirectory(file_80, "/files/"); -// ASSERT_DOUBLE_EQ(100, fs1->getFreeSpace()); -// -// fs1->storeFileInDirectory(file_50, "/faa"); -// fs1->removeAllFilesInDirectory("/faa/");// coverage -// -// -// workflow->clear(); -// wrench::Simulation::removeAllFiles(); -// -// for (int i = 0; i < argc; i++) -// free(argv[i]); -// free(argv); -//} -// -// -//TEST_F(LogicalFileSystemTest, DevNullTests) { -// DO_TEST_WITH_FORK(do_DevNullTests); -//} -// -//void LogicalFileSystemTest::do_DevNullTests() { -// // Create and initialize the simulation -// auto simulation = wrench::Simulation::createSimulation(); -// -// int argc = 1; -// char **argv = (char **) calloc(argc, sizeof(char *)); -// argv[0] = strdup("unit_test"); -// // argv[1] = strdup("--wrench-full-log"); -// -// ASSERT_NO_THROW(simulation->init(&argc, argv)); -// auto workflow = wrench::Workflow::createWorkflow(); -// -// // set up the platform -// ASSERT_NO_THROW(simulation->instantiatePlatform(platform_file_path)); -// -// // Create a Storage Services -// std::shared_ptr storage_service; -// ASSERT_NO_THROW(storage_service = simulation->add( -// wrench::SimpleStorageService::createSimpleStorageService("Host", {"/"}))); -// -// // Create a Logical File System -// auto fs1 = wrench::LogicalFileSystem::createLogicalFileSystem("Host", storage_service.get(), "/dev/null"); -// -// auto file = wrench::Simulation::addFile("file", 1); -// -// fs1->createDirectory(("/foo")); -// fs1->createFile(file, "/foo"); -// ASSERT_FALSE(fs1->doesDirectoryExist(("/foo"))); -// ASSERT_TRUE(fs1->isDirectoryEmpty(("/foo"))); -// ASSERT_FALSE(fs1->isFileInDirectory(file, "/foo")); -// fs1->removeEmptyDirectory("/foo"); -// fs1->storeFileInDirectory(file, "/foo"); -// fs1->removeFileFromDirectory(file, "/foo"); -// fs1->removeAllFilesInDirectory("/foo"); -// ASSERT_TRUE(fs1->listFilesInDirectory("/foo").empty()); -// fs1->reserveSpace(file, "/foo"); -// fs1->unreserveSpace(file, "/foo"); -// fs1->getFileLastWriteDate(file, "/foo"); -// -// // Create a Logical File System -// auto fs2 = wrench::LogicalFileSystem::createLogicalFileSystem("Host", storage_service.get(), "/dev/null", "LRU"); -// -// fs2->createDirectory(("/foo")); -// ASSERT_FALSE(fs2->doesDirectoryExist(("/foo"))); -// ASSERT_TRUE(fs2->isDirectoryEmpty(("/foo"))); -// ASSERT_FALSE(fs2->isFileInDirectory(file, "/foo")); -// fs2->removeEmptyDirectory("/foo"); -// fs2->storeFileInDirectory(file, "/foo"); -// fs2->removeFileFromDirectory(file, "/foo"); -// fs2->removeAllFilesInDirectory("/foo"); -// ASSERT_TRUE(fs2->listFilesInDirectory("/foo").empty()); -// fs2->reserveSpace(file, "/foo"); -// fs2->unreserveSpace(file, "/foo"); -// fs2->getFileLastWriteDate(file, "/foo"); -// -// workflow->clear(); -// wrench::Simulation::removeAllFiles(); -// -// for (int i = 0; i < argc; i++) -// free(argv[i]); -// free(argv); -//} -// -// -//TEST_F(LogicalFileSystemTest, LRUTests) { -// DO_TEST_WITH_FORK(do_LRUTests); -//} -// -//void LogicalFileSystemTest::do_LRUTests() { -// // Create and initialize the simulation -// auto simulation = wrench::Simulation::createSimulation(); -// -// int argc = 1; -// char **argv = (char **) calloc(argc, sizeof(char *)); -// argv[0] = strdup("unit_test"); -// // argv[1] = strdup("--wrench-full-log"); -// -// ASSERT_NO_THROW(simulation->init(&argc, argv)); -// -// // set up the platform -// ASSERT_NO_THROW(simulation->instantiatePlatform(platform_file_path)); -// -// // Create a Storage Services -// std::shared_ptr storage_service; -// ASSERT_NO_THROW(storage_service = simulation->add( -// wrench::SimpleStorageService::createSimpleStorageService("Host", {"/"}))); -// -// // Create a Logical File System with LRU eviction -// auto fs1 = wrench::LogicalFileSystem::createLogicalFileSystem("Host", storage_service.get(), "/tmp", "LRU"); -// -// auto file_60 = wrench::Simulation::addFile("file_60", 60); -// auto file_50 = wrench::Simulation::addFile("file_50", 50); -// auto file_30 = wrench::Simulation::addFile("file_30", 30); -// auto file_20 = wrench::Simulation::addFile("file_20", 20); -// auto file_10 = wrench::Simulation::addFile("file_10", 10); -// -// -// fs1->createDirectory(("/foo")); -// ASSERT_TRUE(fs1->reserveSpace(file_60, "/foo")); -// ASSERT_FALSE(fs1->reserveSpace(file_50, "/foo")); -// fs1->storeFileInDirectory(file_60, "/foo"); -// ASSERT_DOUBLE_EQ(40, fs1->getFreeSpace()); -// fs1->storeFileInDirectory(file_10, "/foo"); -// ASSERT_DOUBLE_EQ(30, fs1->getFreeSpace()); -// -// ASSERT_TRUE(fs1->reserveSpace(file_50, "/foo")); -// // Check that file_60 has been evicted -// ASSERT_FALSE(fs1->isFileInDirectory(file_60, "/foo")); -// // Check that file_10 is still there evicted -// ASSERT_TRUE(fs1->isFileInDirectory(file_10, "/foo")); -// fs1->storeFileInDirectory(file_50, "/foo"); -// ASSERT_DOUBLE_EQ(40, fs1->getFreeSpace()); -// -// // At this point the content is: -// // If I store another file that requires 50 bytes, but make file_10 unevictable, file_50 should be evicted -// auto other_file_50 = wrench::Simulation::addFile("other_file_50", 50); -// fs1->incrementNumRunningTransactionsForFileInDirectory(file_10, "/foo"); -// ASSERT_TRUE(fs1->reserveSpace(other_file_50, "/foo")); -// ASSERT_TRUE(fs1->isFileInDirectory(file_10, "/foo")); -// ASSERT_FALSE(fs1->isFileInDirectory(file_50, "/foo")); -// fs1->storeFileInDirectory(other_file_50, "/foo"); -// fs1->updateReadDate(other_file_50, "/foo"); -// fs1->updateReadDate(other_file_50, "/faa");// coverage -// -// // At this point the content is; -// // LRU: file_10 (UNEVICTABLE), other_file_50 (EVICTABLE) -// fs1->incrementNumRunningTransactionsForFileInDirectory(other_file_50, "/foo"); -// // At this point the content is; -// // LRU: file_10 (UNEVICTABLE), other_file_50 (UNEVICTABLE) -// // I should not be able to store/reserve space for file_50 -// ASSERT_FALSE(fs1->reserveSpace(file_50, "/foo")); -// ASSERT_DOUBLE_EQ(fs1->getFreeSpace(), 40); -// // Make other_file_50 EVICTABLE again -// fs1->decrementNumRunningTransactionsForFileInDirectory(other_file_50, "/foo"); -// ASSERT_TRUE(fs1->reserveSpace(file_50, "/foo")); -// ASSERT_FALSE(fs1->isFileInDirectory(file_50, "/foo")); -// ASSERT_DOUBLE_EQ(fs1->getFreeSpace(), 40); -// -// -// fs1->removeFileFromDirectory(file_10, "/foo");// coverage -// fs1->storeFileInDirectory(file_10, "/foo"); // coverage -// fs1->removeAllFilesInDirectory("/foo"); // coverage -// -// fs1->storeFileInDirectory(file_10, "/faa");// coverage -// -// for (int i = 0; i < argc; i++) -// free(argv[i]); -// free(argv); -//} diff --git a/test/services/storage_services/SimpleStorageService/SimpleStorageServiceFunctionalTest.cpp b/test/services/storage_services/SimpleStorageService/SimpleStorageServiceFunctionalTest.cpp index c838d6211b..4730506a4e 100755 --- a/test/services/storage_services/SimpleStorageService/SimpleStorageServiceFunctionalTest.cpp +++ b/test/services/storage_services/SimpleStorageService/SimpleStorageServiceFunctionalTest.cpp @@ -37,6 +37,8 @@ class SimpleStorageServiceFunctionalTest : public ::testing::Test { void do_BasicFunctionality_test(double buffer_size); + void do_WrongService_test(double buffer_size); + void do_SynchronousFileCopy_test(double buffer_size); void do_AsynchronousFileCopy_test(double buffer_size); @@ -724,7 +726,6 @@ void SimpleStorageServiceFunctionalTest::do_BasicFunctionality_test(double buffe free(argv); } - /**********************************************************************/ /** SYNCHRONOUS FILE COPY TEST **/ /**********************************************************************/