From 1d94bdc28d419fa411915e0203c1c7cea340f880 Mon Sep 17 00:00:00 2001 From: mikkeldamsgaard Date: Sun, 13 Nov 2022 10:49:02 +0100 Subject: [PATCH] Implementation of the full host package on windows hosts (#1185) * Ready for PR * buildbot issue * buildbot issue * fix toitp tests for windws * fix profiler tests * toitp test fixes for windows * More windows test case fixes * Added LINE_TERMINATOR constant * Updated bases on review comments --- lib/core/utils.toit | 2 + src/CMakeLists.txt | 4 +- src/{resources => }/error_win.cc | 57 +- src/{resources => }/error_win.h | 2 +- src/event_sources/event_win.cc | 134 ++--- src/event_sources/event_win.h | 11 +- src/primitive.h | 3 + src/primitive_file.h | 5 +- src/primitive_file_win.cc | 275 +++++++--- src/process.cc | 19 +- src/process.h | 11 + src/resources/ble_darwin.mm | 23 +- src/resources/{pipe.cc => pipe_posix.cc} | 0 src/resources/pipe_win.cc | 497 ++++++++++++++++++ src/resources/subprocess.h | 34 +- .../{subprocess.cc => subprocess_posix.cc} | 0 src/resources/subprocess_win.cc | 103 ++++ src/resources/tcp_win.cc | 2 +- src/resources/uart_win.cc | 6 +- src/resources/udp_win.cc | 5 +- src/tags.h | 2 + src/utils.h | 21 + src/vm_win.cc | 4 + tests/image/fail.cmake | 5 - tests/profiler/fail.cmake | 7 - tests/profiler/utils.toit | 2 +- tests/toitp/bytecodes_toitp_test.toit | 2 +- tests/toitp/dispatch_toitp_test.toit | 2 +- tests/toitp/fail.cmake | 12 - tests/toitp/literal_index_toitp_test.toit | 2 +- tests/toitp/literal_toitp_test.toit | 2 +- tests/toitp/senders_toitp_test.toit | 2 +- tests/toitp/utils.toit | 2 +- 33 files changed, 1041 insertions(+), 217 deletions(-) rename src/{resources => }/error_win.cc (56%) rename src/{resources => }/error_win.h (97%) rename src/resources/{pipe.cc => pipe_posix.cc} (100%) create mode 100644 src/resources/pipe_win.cc rename src/resources/{subprocess.cc => subprocess_posix.cc} (100%) create mode 100644 src/resources/subprocess_win.cc diff --git a/lib/core/utils.toit b/lib/core/utils.toit index c3f347d6a..3dfbba6de 100644 --- a/lib/core/utils.toit +++ b/lib/core/utils.toit @@ -256,6 +256,8 @@ PLATFORM_WINDOWS ::= "Windows" PLATFORM_MACOS ::= "macOS" PLATFORM_LINUX ::= "Linux" +LINE_TERMINATOR ::= platform == PLATFORM_WINDOWS ? "\r\n" : "\n" + /// Index for $process_stats. STATS_INDEX_GC_COUNT ::= 0 /// Index for $process_stats. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4b02dd6b4..25c2d7e9c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -170,9 +170,9 @@ else() endif() if ("${CMAKE_SYSTEM_NAME}" MATCHES "MSYS") - set(TOIT_WINDOWS_LIBS ws2_32 rpcrt4) + set(TOIT_WINDOWS_LIBS ws2_32 rpcrt4 shlwapi) elseif ("${CMAKE_SYSTEM_NAME}" MATCHES "Windows") - set(TOIT_WINDOWS_LIBS rpcrt4) + set(TOIT_WINDOWS_LIBS rpcrt4 shlwapi) else() set(TOIT_WINDOWS_LIBS ) endif() diff --git a/src/resources/error_win.cc b/src/error_win.cc similarity index 56% rename from src/resources/error_win.cc rename to src/error_win.cc index 5c72f556a..4bd061c7e 100644 --- a/src/resources/error_win.cc +++ b/src/error_win.cc @@ -13,18 +13,59 @@ // The license can be found in the file `LICENSE` in the top level // directory of this repository. -#include "../top.h" +#include "top.h" #ifdef TOIT_WINDOWS #include "windows.h" -#include "../objects.h" -#include "../objects_inline.h" - +#include "objects.h" +#include "objects_inline.h" +#include +#include namespace toit { -HeapObject* windows_error(Process* process, DWORD error_number) { - if (WSAGetLastError() == ERROR_NOT_ENOUGH_MEMORY) MALLOC_FAILED; +static HeapObject* custom_error(Process* process, const char* txt) { + String* error = process->allocate_string(txt); + if (error == null) ALLOCATION_FAILED; + return Primitive::mark_as_error(error); +} +HeapObject* windows_error(Process* process, DWORD error_number) { + DWORD err = GetLastError(); + if (err == ERROR_FILE_NOT_FOUND || + err == ERROR_INVALID_DRIVE || + err == ERROR_DEV_NOT_EXIST) { + FILE_NOT_FOUND; + } + if (err == ERROR_TOO_MANY_OPEN_FILES || + err == ERROR_SHARING_BUFFER_EXCEEDED || + err == ERROR_TOO_MANY_NAMES || + err == ERROR_NO_PROC_SLOTS || + err == ERROR_TOO_MANY_SEMAPHORES) { + QUOTA_EXCEEDED; + } + if (err == ERROR_ACCESS_DENIED || + err == ERROR_WRITE_PROTECT || + err == ERROR_NETWORK_ACCESS_DENIED) { + PERMISSION_DENIED; + } + if (err == ERROR_INVALID_HANDLE) { + ALREADY_CLOSED; + } + if (err == ERROR_NOT_ENOUGH_MEMORY || + err == ERROR_OUTOFMEMORY) { + MALLOC_FAILED; + } + if (err == ERROR_BAD_COMMAND || + err == ERROR_INVALID_PARAMETER) { + INVALID_ARGUMENT; + } + if (err == ERROR_FILE_EXISTS || + err == ERROR_ALREADY_ASSIGNED) { + ALREADY_EXISTS; + } + if (err == ERROR_NO_DATA) { + return custom_error(process, "Broken pipe"); + } LPVOID lpMsgBuf; FormatMessage( FORMAT_MESSAGE_ALLOCATE_BUFFER | @@ -43,9 +84,7 @@ HeapObject* windows_error(Process* process, DWORD error_number) { } else { char buf[80]; snprintf(buf, 80, "Low-level win32 error: %lu", error_number); - String* error = process->allocate_string(buf); - if (error == null) ALLOCATION_FAILED; - return Primitive::mark_as_error(error); + return custom_error(process, buf); } } diff --git a/src/resources/error_win.h b/src/error_win.h similarity index 97% rename from src/resources/error_win.h rename to src/error_win.h index 98015ac58..56a454a0c 100644 --- a/src/resources/error_win.h +++ b/src/error_win.h @@ -15,7 +15,7 @@ #pragma once -#include "../top.h" +#include "top.h" #ifdef TOIT_WINDOWS #include "windows.h" diff --git a/src/event_sources/event_win.cc b/src/event_sources/event_win.cc index d042c5c7b..a5808d2e3 100644 --- a/src/event_sources/event_win.cc +++ b/src/event_sources/event_win.cc @@ -22,79 +22,83 @@ namespace toit { -WindowsEventSource* WindowsEventSource::_instance = null; +WindowsEventSource* WindowsEventSource::instance_ = null; class WindowsEventThread; class WindowsResourceEvent { public: WindowsResourceEvent(WindowsResource* resource, HANDLE event, WindowsEventThread* thread) - : _resource(resource) - , _event(event) - , _thread(thread) {} - WindowsResource* resource() const { return _resource; } - HANDLE event() const { return _event; } - WindowsEventThread* thread() const { return _thread; } + : resource_(resource) + , event_(event) + , thread_(thread) {} + WindowsResource* resource() const { return resource_; } + HANDLE event() const { return event_; } + WindowsEventThread* thread() const { return thread_; } + bool is_event_enabled() { return resource_->is_event_enabled(event_); } private: - WindowsResource* _resource; - HANDLE _event; - WindowsEventThread* _thread; + WindowsResource* resource_; + HANDLE event_; + WindowsEventThread* thread_; }; class WindowsEventThread: public Thread { public: explicit WindowsEventThread(WindowsEventSource* event_source) - : Thread("WindowsEventThread") - , _handles() - , _resources() - , _count(1) - , _event_source(event_source) - , _recalculated(OS::allocate_condition_variable(event_source->mutex())) { - _control_event = CreateEvent(NULL, true, false, NULL); - _handles[0] = _control_event; + : Thread("WindowsEventThread") + , handles_() + , resources_() + , count_(1) + , event_source_(event_source) + , recalculated_(OS::allocate_condition_variable(event_source->mutex())) { + control_event_ = CreateEvent(NULL, true, false, NULL); + handles_[0] = control_event_; } ~WindowsEventThread() override { - CloseHandle(_control_event); + CloseHandle(control_event_); } void stop() { - Locker locker(_event_source->mutex()); - _stopped = true; - SetEvent(_control_event); + Locker locker(event_source_->mutex()); + stopped_ = true; + SetEvent(control_event_); } size_t size() { - return _resource_events.size(); + return resource_events_.size(); } void add_resource_event(Locker& event_source_locker, WindowsResourceEvent* resource_event) { - ASSERT(_resource_events.size() < MAXIMUM_WAIT_OBJECTS - 2); - _resource_events.insert(resource_event); - SetEvent(_control_event); // Recalculate the wait objects. - OS::wait(_recalculated); + ASSERT(resource_events_.size() < MAXIMUM_WAIT_OBJECTS - 2); + resource_events_.insert(resource_event); + SetEvent(control_event_); // Recalculate the wait objects. + OS::wait(recalculated_); } void remove_resource_event(Locker& event_source_locker, WindowsResourceEvent* resource_event) { - size_t number_erased = _resource_events.erase(resource_event); + size_t number_erased = resource_events_.erase(resource_event); if (number_erased > 0) { - SetEvent(_control_event); // Recalculate the wait objects. - OS::wait(_recalculated); + SetEvent(control_event_); // Recalculate the wait objects. + OS::wait(recalculated_); } } protected: void entry() override { while (true) { - DWORD result = WaitForMultipleObjects(_count, _handles, false, INFINITE); + DWORD result = WaitForMultipleObjects(count_, handles_, false, INFINITE); { - Locker locker(_event_source->mutex()); + Locker locker(event_source_->mutex()); if (result == WAIT_OBJECT_0 + 0) { - if (_stopped) break; + if (stopped_) break; recalculate_handles(); } else if (result != WAIT_FAILED) { size_t index = result - WAIT_OBJECT_0; - ResetEvent(_handles[index]); - _event_source->on_event(locker, _resources[index], _handles[index]); + ResetEvent(handles_[index]); + if (resources_[index]->is_event_enabled(handles_[index])) + event_source_->on_event(locker, resources_[index], handles_[index]); + else + recalculate_handles(); } else { FATAL("wait failed. error=%lu", GetLastError()); } @@ -104,78 +108,76 @@ class WindowsEventThread: public Thread { private: void recalculate_handles() { - _count = _resource_events.size() + 1; int index = 1; - for (auto resource_event : _resource_events) { - _handles[index] = resource_event->event(); - _resources[index] = resource_event->resource(); - index++; + for (auto resource_event : resource_events_) { + if (resource_event->is_event_enabled()) { + handles_[index] = resource_event->event(); + resources_[index] = resource_event->resource(); + index++; + } } - ResetEvent(_control_event); - OS::signal_all(_recalculated); + count_ = index; + ResetEvent(control_event_); + OS::signal_all(recalculated_); } - bool _stopped = false; - HANDLE _control_event; - HANDLE _handles[MAXIMUM_WAIT_OBJECTS]; - WindowsResource* _resources[MAXIMUM_WAIT_OBJECTS]; - DWORD _count; - std::unordered_set _resource_events; - WindowsEventSource* _event_source; - ConditionVariable* _recalculated; + bool stopped_ = false; + HANDLE control_event_; + HANDLE handles_[MAXIMUM_WAIT_OBJECTS]; + WindowsResource* resources_[MAXIMUM_WAIT_OBJECTS]; + DWORD count_; + std::unordered_set resource_events_; + WindowsEventSource* event_source_; + ConditionVariable* recalculated_; }; -WindowsEventSource::WindowsEventSource() : LazyEventSource("WindowsEvents", 1), _threads(), _resource_events() { - ASSERT(_instance == null); - _instance = this; +WindowsEventSource::WindowsEventSource() : LazyEventSource("WindowsEvents", 1), threads_(), resource_events_() { + ASSERT(instance_ == null); + instance_ = this; } WindowsEventSource::~WindowsEventSource() { - for (auto item : _resource_events) { + for (auto item : resource_events_) { delete item.second; } } void WindowsEventSource::on_register_resource(Locker &locker, Resource* r) { - AllowThrowingNew host_only; - auto windows_resource = reinterpret_cast(r); for (auto event : windows_resource->events()) { WindowsResourceEvent* resource_event; // Find a thread with capacity. bool placed_it = false; - for(auto thread : _threads) { + for(auto thread : threads_) { if (thread->size() < MAXIMUM_WAIT_OBJECTS - 2) { resource_event = _new WindowsResourceEvent(windows_resource, event, thread); - thread->add_resource_event(locker, resource_event); placed_it = true; break; } } + if (!placed_it) { // No worker thread with capacity was found. Spawn a new thread. auto thread = _new WindowsEventThread(this); - _threads.push_back(thread); + threads_.push_back(thread); thread->spawn(); resource_event = _new WindowsResourceEvent(windows_resource, event, thread); - thread->add_resource_event(locker, resource_event); } - _resource_events.insert(std::make_pair(windows_resource, resource_event)); + resource_events_.insert(std::make_pair(windows_resource, resource_event)); + resource_event->thread()->add_resource_event(locker, resource_event); } } void WindowsEventSource::on_unregister_resource(Locker &locker, Resource* r) { - AllowThrowingNew host_only; - auto windows_resource = reinterpret_cast(r); - auto range = _resource_events.equal_range(windows_resource); + auto range = resource_events_.equal_range(windows_resource); for (auto it = range.first; it != range.second; ++it) { it->second->thread()->remove_resource_event(locker, it->second); delete it->second; } - _resource_events.erase(windows_resource); + resource_events_.erase(windows_resource); windows_resource->do_close(); // sending an event to let the resource update its state, typically to a CLOSE state. @@ -194,7 +196,7 @@ bool WindowsEventSource::start() { } void WindowsEventSource::stop() { - for (auto thread : _threads) { + for (auto thread : threads_) { thread->stop(); thread->join(); delete thread; diff --git a/src/event_sources/event_win.h b/src/event_sources/event_win.h index d86245a9d..7f9f1d972 100644 --- a/src/event_sources/event_win.h +++ b/src/event_sources/event_win.h @@ -13,6 +13,8 @@ // The license can be found in the file `LICENSE` in the top level // directory of this repository. +#pragma once + #include "../top.h" #if defined(TOIT_WINDOWS) @@ -30,6 +32,7 @@ class WindowsResource : public Resource { virtual std::vector events() = 0; virtual uint32_t on_event(HANDLE event, uint32_t state) = 0; virtual void do_close() = 0; + virtual bool is_event_enabled(HANDLE event) { return true; } }; class WindowsEventThread; @@ -37,7 +40,7 @@ class WindowsResourceEvent; class WindowsEventSource : public LazyEventSource { public: - static WindowsEventSource* instance() { return _instance; } + static WindowsEventSource* instance() { return instance_; } WindowsEventSource(); ~WindowsEventSource() override; @@ -53,10 +56,10 @@ class WindowsEventSource : public LazyEventSource { void on_register_resource(Locker& locker, Resource* r) override; void on_unregister_resource(Locker& locker, Resource* r) override; - static WindowsEventSource* _instance; + static WindowsEventSource* instance_; - std::vector _threads; - std::unordered_multimap _resource_events; + std::vector threads_; + std::unordered_multimap resource_events_; }; } diff --git a/src/primitive.h b/src/primitive.h index ecd545987..34c94117d 100644 --- a/src/primitive.h +++ b/src/primitive.h @@ -949,6 +949,9 @@ namespace toit { #define _A_T_UDPSocketResource(N, name) MAKE_UNPACKING_MACRO(UDPSocketResource, N, name) #define _A_T_TCPSocketResource(N, name) MAKE_UNPACKING_MACRO(TCPSocketResource, N, name) #define _A_T_TCPServerSocketResource(N, name) MAKE_UNPACKING_MACRO(TCPServerSocketResource, N, name) +#define _A_T_SubprocessResource(N, name) MAKE_UNPACKING_MACRO(SubprocessResource, N, name) +#define _A_T_ReadPipeResource(N, name) MAKE_UNPACKING_MACRO(ReadPipeResource, N, name) +#define _A_T_WritePipeResource(N, name) MAKE_UNPACKING_MACRO(WritePipeResource, N, name) #define _A_T_I2SResource(N, name) MAKE_UNPACKING_MACRO(I2SResource, N, name) #define _A_T_AdcResource(N, name) MAKE_UNPACKING_MACRO(AdcResource, N, name) #define _A_T_DacResource(N, name) MAKE_UNPACKING_MACRO(DacResource, N, name) diff --git a/src/primitive_file.h b/src/primitive_file.h index 774a1ff80..b4f84205e 100644 --- a/src/primitive_file.h +++ b/src/primitive_file.h @@ -19,6 +19,9 @@ namespace toit { +#if defined(TOIT_WINDOWS) +const char* current_dir(Process* process); +#else int current_dir(Process* process); - +#endif } diff --git a/src/primitive_file_win.cc b/src/primitive_file_win.cc index 677dc40e8..43b9857af 100644 --- a/src/primitive_file_win.cc +++ b/src/primitive_file_win.cc @@ -15,9 +15,7 @@ #include "top.h" -#ifdef TOIT_WINDOWS - -#define _FILE_OFFSET_BITS 64 +#if defined(TOIT_WINDOWS) #include "objects.h" #include "primitive_file.h" @@ -25,7 +23,7 @@ #include "process.h" #include -#include +#include #include #include // For rpcdce.h. #include // For UuidCreate. @@ -33,10 +31,13 @@ #include #include #include -#include +#include +#include #include "objects_inline.h" +#include "error_win.h" + namespace toit { MODULE_IMPLEMENTATION(file, MODULE_FILE) @@ -71,44 +72,6 @@ static Object* return_open_error(Process* process, int err) { OTHER_ERROR; } -// For Windows API calls. -static Object* return_windows_error(Process* process) { - DWORD err = GetLastError(); - if (err == ERROR_FILE_NOT_FOUND || - err == ERROR_INVALID_DRIVE || - err == ERROR_DEV_NOT_EXIST) { - FILE_NOT_FOUND; - } - if (err == ERROR_TOO_MANY_OPEN_FILES || - err == ERROR_SHARING_BUFFER_EXCEEDED || - err == ERROR_TOO_MANY_NAMES || - err == ERROR_NO_PROC_SLOTS || - err == ERROR_TOO_MANY_SEMAPHORES) { - QUOTA_EXCEEDED; - } - if (err == ERROR_ACCESS_DENIED || - err == ERROR_WRITE_PROTECT || - err == ERROR_NETWORK_ACCESS_DENIED) { - PERMISSION_DENIED; - } - if (err == ERROR_INVALID_HANDLE) { - ALREADY_CLOSED; - } - if (err == ERROR_NOT_ENOUGH_MEMORY || - err == ERROR_OUTOFMEMORY) { - MALLOC_FAILED; - } - if (err == ERROR_BAD_COMMAND || - err == ERROR_INVALID_PARAMETER) { - INVALID_ARGUMENT; - } - if (err == ERROR_FILE_EXISTS || - err == ERROR_ALREADY_ASSIGNED) { - ALREADY_EXISTS; - } - OTHER_ERROR; -} - // Coordinate with utils.toit. static const int FILE_RDONLY = 1; static const int FILE_WRONLY = 2; @@ -129,8 +92,45 @@ static const int FILE_ST_ATIME = 8; static const int FILE_ST_MTIME = 9; static const int FILE_ST_CTIME = 10; +const char* current_dir(Process* process) { + const char* current_directory = process->current_directory(); + if (current_directory) return current_directory; + DWORD length = GetCurrentDirectory(0, NULL); + if (length == 0) { + FATAL("Failed to get current dir"); + } + current_directory = reinterpret_cast(malloc(length)); + if (!current_directory) return null; + if (GetCurrentDirectory(length, const_cast(current_directory)) == 0) { + FATAL("Failed to get current dir"); + } + process->set_current_directory(current_directory); + return current_directory; +} + +HeapObject* get_relative_path(Process* process, const char* pathname, char* output) { + size_t pathname_length = strlen(pathname); + + // Poor man's version. For better platform handling, use UNICODE and PathCchAppendEx. + if (pathname[0] == '\\' || + (pathname_length > 2 && pathname[1] == ':' && (pathname[2] == '\\' || pathname[2] == '/'))) { + if (GetFullPathName(pathname, MAX_PATH, output, NULL) == 0) WINDOWS_ERROR; + } else { + const char* current_directory = current_dir(process); + if (!current_directory) MALLOC_FAILED; + char temp[MAX_PATH]; + if (snprintf(temp, MAX_PATH, "%s\\%s", current_directory, pathname) >= MAX_PATH) INVALID_ARGUMENT; + if (GetFullPathName(temp, MAX_PATH, output, NULL) == 0) WINDOWS_ERROR; + } + return null; +} + PRIMITIVE(open) { ARGS(cstring, pathname, int, flags, int, mode); + char path[MAX_PATH]; + auto error = get_relative_path(process, pathname, path); + if (error) return error; + int os_flags = _O_BINARY; if ((flags & FILE_RDWR) == FILE_RDONLY) os_flags |= _O_RDONLY; else if ((flags & FILE_RDWR) == FILE_WRONLY) os_flags |= _O_WRONLY; @@ -139,10 +139,10 @@ PRIMITIVE(open) { if ((flags & FILE_APPEND) != 0) os_flags |= _O_APPEND; if ((flags & FILE_CREAT) != 0) os_flags |= _O_CREAT; if ((flags & FILE_TRUNC) != 0) os_flags |= _O_TRUNC; - int fd = _open(pathname, os_flags, mode); + int fd = _open(path, os_flags, mode); AutoCloser closer(fd); if (fd < 0) return return_open_error(process, errno); - struct stat statbuf; + struct stat statbuf{}; int res = fstat(fd, &statbuf); if (res < 0) { if (errno == ENOMEM) MALLOC_FAILED; @@ -154,16 +154,31 @@ PRIMITIVE(open) { // with open (eg a pipe, a socket, a directory). We forbid this because // these file descriptors can block, and this API does not support // blocking. - INVALID_ARGUMENT; + if (strcmpi(R"(\\.\NUL)", pathname) != 0) INVALID_ARGUMENT; } closer.clear(); return Smi::from(fd); } -class Directory { +class Directory : public SimpleResource { public: TAG(Directory); - DIR* dir; + explicit Directory(SimpleResourceGroup* resource_group, const char* path) : SimpleResource(resource_group) { + snprintf(path_, MAX_PATH, "%s\\*", path); + } + + const char* path() { return path_; } + WIN32_FIND_DATA* find_file_data() { return &find_file_data_; } + void set_dir_handle(HANDLE dir_handle) { dir_handle_ = dir_handle; } + HANDLE dir_handle() { return dir_handle_; } + bool done() const { return done_; } + void set_done(bool done) { done_ = done; } + + private: + char path_[MAX_PATH]{}; + WIN32_FIND_DATA find_file_data_{}; + HANDLE dir_handle_ = INVALID_HANDLE_VALUE; + bool done_ = false; }; PRIMITIVE(opendir) { @@ -171,15 +186,81 @@ PRIMITIVE(opendir) { } PRIMITIVE(opendir2) { - UNIMPLEMENTED_PRIMITIVE; + ARGS(SimpleResourceGroup, group, cstring, pathname); + char path[MAX_PATH]; + auto error = get_relative_path(process, pathname, path); + if (error) return error; + + ByteArray* proxy = process->object_heap()->allocate_proxy(); + if (proxy == null) ALLOCATION_FAILED; + + auto directory = _new Directory(group, path); + if (!directory) MALLOC_FAILED; + + HANDLE dir_handle = FindFirstFile(directory->path(), directory->find_file_data()); + if (dir_handle == INVALID_HANDLE_VALUE) { + if (GetLastError() == ERROR_NO_MORE_FILES) { + directory->set_done(true); + } else { + delete directory; + WINDOWS_ERROR; + } + } + + directory->set_dir_handle(dir_handle); + + proxy->set_external_address(directory); + + return proxy; } PRIMITIVE(readdir) { - UNIMPLEMENTED_PRIMITIVE; + ARGS(ByteArray, directory_proxy); + + if (!directory_proxy->has_external_address()) WRONG_TYPE; + + auto directory = directory_proxy->as_external(); + + if (directory->done()) return process->program()->null_object(); + + ByteArray* proxy = process->object_heap()->allocate_proxy(true); + if (proxy == null) { + ALLOCATION_FAILED; + } + + size_t len = strlen(directory->find_file_data()->cFileName); + if (!Utils::is_valid_utf_8(unsigned_cast(directory->find_file_data()->cFileName), static_cast(len))) { + ILLEGAL_UTF_8; + } + + process->register_external_allocation(static_cast(len)); + + auto backing = unvoid_cast(malloc(len)); // Can't fail on non-embedded. + if (!backing) MALLOC_FAILED; + + memcpy(backing, unsigned_cast(directory->find_file_data()->cFileName), len); + proxy->set_external_address(static_cast(len), backing); + + if (FindNextFile(directory->dir_handle(), directory->find_file_data()) == 0) { + if (GetLastError() == ERROR_NO_MORE_FILES) directory->set_done(true); + else WINDOWS_ERROR; + }; + + return proxy; } PRIMITIVE(closedir) { - UNIMPLEMENTED_PRIMITIVE; + ARGS(ByteArray, proxy); + + if (!proxy->has_external_address()) WRONG_TYPE; + auto directory = proxy->as_external(); + + FindClose(directory->dir_handle()); + + directory->resource_group()->unregister_resource(directory); + + proxy->clear_external_address(); + return process->program()->null_object(); } PRIMITIVE(read) { @@ -229,6 +310,9 @@ PRIMITIVE(close) { while (true) { int result = close(fd); if (result < 0) { + if (GetFileType(reinterpret_cast(fd)) == FILE_TYPE_PIPE && errno == EBADF) { + return process->program()->null_object(); // Ignore already closed on PIPEs + } if (errno == EINTR) continue; if (errno == EBADF) ALREADY_CLOSED; if (errno == ENOSPC) QUOTA_EXCEEDED; @@ -247,8 +331,12 @@ Object* time_stamp(Process* process, time_t time) { PRIMITIVE(stat) { ARGS(cstring, pathname, bool, follow_links); USE(follow_links); - struct stat statbuf; - int result = stat(pathname, &statbuf); + char path[MAX_PATH]; + auto error = get_relative_path(process, pathname, path); + if (error) return error; + + struct stat statbuf{}; + int result = stat(path, &statbuf); if (result < 0) { if (errno == ENOENT || errno == ENOTDIR) { return process->program()->null_object(); @@ -297,15 +385,22 @@ PRIMITIVE(stat) { PRIMITIVE(unlink) { ARGS(cstring, pathname); - int result = unlink(pathname); + char path[MAX_PATH]; + auto error = get_relative_path(process, pathname, path); + if (error) return error; + + int result = unlink(path); if (result < 0) return return_open_error(process, errno); return process->program()->null_object(); } PRIMITIVE(rmdir) { ARGS(cstring, pathname); - int result = rmdir(pathname); - if (result < 0) return return_open_error(process, errno); + char path[MAX_PATH]; + auto error = get_relative_path(process, pathname, path); + if (error) return error; + + if (RemoveDirectory(path) == 0) WINDOWS_ERROR; return process->program()->null_object(); } @@ -317,16 +412,32 @@ PRIMITIVE(rename) { } PRIMITIVE(chdir) { - UNIMPLEMENTED_PRIMITIVE; + ARGS(cstring, pathname); + size_t pathname_length = strlen(pathname); + + if (pathname_length == 0) INVALID_ARGUMENT; + + char path[MAX_PATH]; + auto error = get_relative_path(process, pathname, path); + if (error) return error; + + char* copy = strdup(path); + if (!copy) MALLOC_FAILED; + + process->set_current_directory(copy); + + return process->program()->null_object(); } PRIMITIVE(mkdir) { ARGS(cstring, pathname, int, mode); - USE(mode); - int result = mkdir(pathname); - return result < 0 - ? return_open_error(process, errno) - : process->program()->null_object(); + char path[MAX_PATH]; + auto error = get_relative_path(process, pathname, path); + if (error) return error; + + int result = CreateDirectory(path, NULL); + if (result == 0) WINDOWS_ERROR; + return process->program()->null_object(); } PRIMITIVE(mkdtemp) { @@ -352,10 +463,11 @@ PRIMITIVE(mkdtemp) { // Get the location of the Windows temp directory. ret = GetTempPath(MAX_PATH, temp_dir_name); if (ret + 2 > MAX_PATH) INVALID_ARGUMENT; - if (ret == 0) return return_windows_error(process); - strncat(temp_dir_name, "\\", strlen(temp_dir_name) - 1); + if (ret == 0) WINDOWS_ERROR; + if (temp_dir_name[strlen(temp_dir_name)-1] != '\\') { + strncat(temp_dir_name, "\\", strlen(temp_dir_name) - 1); + } } - if (strlen(temp_dir_name) + UUID_TEXT_LENGTH + strlen(prefix) + 1 > MAX_PATH) INVALID_ARGUMENT; UUID uuid; @@ -370,7 +482,7 @@ PRIMITIVE(mkdtemp) { uword total_len = strlen(temp_dir_name); - Object* result = process->allocate_byte_array(total_len); + Object* result = process->allocate_byte_array(static_cast(total_len)); if (result == null) ALLOCATION_FAILED; int posix_result = mkdir(temp_dir_name); @@ -394,24 +506,41 @@ PRIMITIVE(is_open_file) { PRIMITIVE(realpath) { ARGS(cstring, filename); - char* c_result = _fullpath(null, filename, MAXPATHLEN); - if (c_result == null) { - if (errno == ENOMEM) MALLOC_FAILED; - if (errno == ENOENT or errno == ENOTDIR) return process->program()->null_object(); - OTHER_ERROR; + DWORD result_length = GetFullPathName(filename, 0, NULL, NULL); + if (result_length == 0) WINDOWS_ERROR; + + char* c_result = reinterpret_cast(malloc(result_length)); + if (!c_result) MALLOC_FAILED; + + if (GetFullPathName(filename, result_length, c_result, NULL) == 0) { + free(c_result); + WINDOWS_ERROR; } + // The toit package expects a null value when the file does not exist. Win32 does not detect his in GetFile + if (!PathFileExists(c_result)) { + free(c_result); + return process->program()->null_object(); + } + String* result = process->allocate_string(c_result); if (result == null) { free(c_result); ALLOCATION_FAILED; } + return result; } PRIMITIVE(cwd) { - UNIMPLEMENTED_PRIMITIVE; + const char* current_directory = current_dir(process); + if (current_directory == null) MALLOC_FAILED; + + String* result = process->allocate_string(current_directory); + if (result == null) ALLOCATION_FAILED; + + return result; } } -#endif // Linux and BSD. +#endif // TOIT_WINDOWS. diff --git a/src/process.cc b/src/process.cc index f6a364a9c..353a20c5e 100644 --- a/src/process.cc +++ b/src/process.cc @@ -49,7 +49,6 @@ Process::Process(Program* program, ProcessRunner* runner, ProcessGroup* group, S , random_seeded_(false) , random_state0_(1) , random_state1_(2) - , current_directory_(-1) , signals_(0) , state_(IDLE) , scheduler_thread_(null) { @@ -59,6 +58,11 @@ Process::Process(Program* program, ProcessRunner* runner, ProcessGroup* group, S ASSERT(!program || program_heap_size_ > 0); // Link this process to the program heap. group_->add(this); +#if defined(TOIT_WINDOWS) + current_directory_ = null; +#else + current_directory_ = -1; +#endif ASSERT(group_->lookup(id_) == this); } @@ -86,9 +90,13 @@ Process::~Process() { r->tear_down(); // Also removes from linked list. } +#if defined(TOIT_WINDOWS) + free(const_cast(void_cast(current_directory_))); +#else if (current_directory_ >= 0) { OS::close(current_directory_); } +#endif // Use [has_message] to ensure that system_acks are processed and message // budget is returned. @@ -358,4 +366,13 @@ uint8 Process::update_priority() { return priority; } + +#if defined(TOIT_WINDOWS) +const char* Process::current_directory() { return current_directory_; } +void Process::set_current_directory(const char* current_directory) { + free(const_cast(void_cast(current_directory_))); + current_directory_ = current_directory; +} +#endif + } diff --git a/src/process.h b/src/process.h index 30fd2e4ed..94c9fff40 100644 --- a/src/process.h +++ b/src/process.h @@ -167,8 +167,15 @@ class Process : public ProcessListFromProcessGroup::Element, void set_target_priority(uint8 value) { target_priority_ = value; } uint8 update_priority(); + // TODO(mikkel): current_directory could be a union with an int and a char*. The clients of this member would know + // which field to access. +#if defined(TOIT_WINDOWS) + const char* current_directory(); + void set_current_directory(const char* current_directory); +#else int current_directory() { return current_directory_; } void set_current_directory(int fd) { current_directory_ = fd; } +#endif int gc_count(GcType type) { return object_heap_.gc_count(type); } String* allocate_string(const char* content); @@ -265,7 +272,11 @@ class Process : public ProcessListFromProcessGroup::Element, uint64_t random_state0_; uint64_t random_state1_; +#if defined(TOIT_WINDOWS) + const char* current_directory_; +#else int current_directory_; +#endif uint32_t signals_; State state_; diff --git a/src/resources/ble_darwin.mm b/src/resources/ble_darwin.mm index ab6d77311..6382fa5db 100644 --- a/src/resources/ble_darwin.mm +++ b/src/resources/ble_darwin.mm @@ -19,6 +19,7 @@ #include "../objects.h" #include "../objects_inline.h" +#include "../utils.h" #include "../event_sources/ble_host.h" #undef BOOL @@ -707,26 +708,6 @@ - (id)initWithResource:(toit::BLEResource*)resource { return nil; } -class AsyncThread : public Thread { - public: - explicit AsyncThread(std::function func) : Thread("async"), _func(std::move(func)) { - spawn(); - } - - protected: - void entry() override { - _func(); - delete this; - } - - private: - const std::function _func; -}; - -static void run_async(const std::function &func) { - _new AsyncThread(func); -} - MODULE_IMPLEMENTATION(ble, MODULE_BLE) PRIMITIVE(init) { @@ -805,7 +786,7 @@ static void run_async(const std::function &func) { central_manager->set_scan_active(true); [central_manager->central_manager() scanForPeripheralsWithServices:nil options:nil]; - run_async([=]() -> void { + AsyncThread::run_async([=]() -> void { LightLocker locker(central_manager->scan_mutex()); OS::wait_us(central_manager->stop_scan_condition(), duration_us); [central_manager->central_manager() stopScan]; diff --git a/src/resources/pipe.cc b/src/resources/pipe_posix.cc similarity index 100% rename from src/resources/pipe.cc rename to src/resources/pipe_posix.cc diff --git a/src/resources/pipe_win.cc b/src/resources/pipe_win.cc new file mode 100644 index 000000000..908112a19 --- /dev/null +++ b/src/resources/pipe_win.cc @@ -0,0 +1,497 @@ +// Copyright (C) 2018 Toitware ApS. +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 only. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// The license can be found in the file `LICENSE` in the top level +// directory of this repository. + +#include "../top.h" + +#if defined(TOIT_WINDOWS) + +#include +#include +#include + + +#include "../objects.h" +#include "../objects_inline.h" +#include "../vm.h" +#include "../error_win.h" +#include "subprocess.h" +#include "../primitive_file.h" + +namespace toit { + +enum { + PIPE_READ = 1 << 0, + PIPE_WRITE = 1 << 1, + PIPE_CLOSE = 1 << 2, + PIPE_ERROR = 1 << 3, +}; + +static const int READ_BUFFER_SIZE = 1 << 16; + +class PipeResourceGroup : public ResourceGroup { + public: + TAG(PipeResourceGroup); + PipeResourceGroup(Process* process, EventSource* event_source) : ResourceGroup(process, event_source) {} + + bool is_standard_piped(int fd) const { return (standard_pipes_ & ( 1 << fd)) != 0; } + void set_standard_piped(int fd) { standard_pipes_ |= ( 1 << fd); } + volatile long& pipe_serial_number() { return pipe_serial_number_; } + protected: + uint32_t on_event(Resource* resource, word data, uint32_t state) override { + return reinterpret_cast(resource)->on_event( + reinterpret_cast(data), + state); + } + + private: + word standard_pipes_ = 0; + volatile long pipe_serial_number_ = 0; +}; + +class HandlePipeResource : public WindowsResource { + public: + TAG(PipeResource); + HandlePipeResource(ResourceGroup* resource_group, HANDLE handle, HANDLE event) + : WindowsResource(resource_group), handle_(handle) { + overlapped_.hEvent = event; + } + + HANDLE handle() { return handle_; } + + std::vector events() override { + return std::vector( { overlapped_.hEvent } ); + } + + void do_close() override { + CloseHandle(overlapped_.hEvent); + CloseHandle(handle_); + } + + OVERLAPPED* overlapped() { return &overlapped_; } + private: + HANDLE handle_; + OVERLAPPED overlapped_{}; +}; + +class ReadPipeResource : public HandlePipeResource { + public: + ReadPipeResource(ResourceGroup* resource_group, HANDLE handle, HANDLE event) + : HandlePipeResource(resource_group, handle, event) { + issue_read_request(); + } + + uint32_t on_event(HANDLE event, uint32_t state) override { + read_ready_ = true; + return state | PIPE_READ; + } + + bool issue_read_request() { + read_ready_ = false; + read_count_ = 0; + bool success = ReadFile(handle(), read_data_, READ_BUFFER_SIZE, &read_count_, overlapped()); + if (!success && WSAGetLastError() != ERROR_IO_PENDING) { + return false; + } + return true; + } + + bool receive_read_response() { + bool overlapped_result = GetOverlappedResult(handle(), overlapped(), &read_count_, false); + return overlapped_result; + } + + DWORD read_count() const { return read_count_; } + bool read_ready() const { return read_ready_; } + char* read_buffer() { return read_data_; } + void set_pipe_ended(bool pipe_ended) { pipe_ended_ = pipe_ended; } + bool pipe_ended() const { return pipe_ended_; } + private: + char read_data_[READ_BUFFER_SIZE]{}; + DWORD read_count_ = 0; + bool read_ready_ = false; + bool pipe_ended_ = false; +}; + +class WritePipeResource : public HandlePipeResource { + public: + WritePipeResource(ResourceGroup* resource_group, HANDLE handle, HANDLE event) + : HandlePipeResource(resource_group, handle, event) { + set_state(PIPE_WRITE); + //overlapped()->Pointer = this; + } + + ~WritePipeResource() override { + if (write_buffer_ != null) free(write_buffer_); + } + + uint32_t on_event(HANDLE event, uint32_t state) override { + write_ready_ = true; + return state | PIPE_WRITE; + } + + bool ready_for_write() const { return write_ready_; } + + + bool send(const uint8* buffer, int length) { + if (write_buffer_ != null) free(write_buffer_); + + write_ready_ = false; + + // We need to copy the buffer out to a long-lived heap object. + write_buffer_ = static_cast(malloc(length)); + memcpy(write_buffer_, buffer, length); + + DWORD tmp; + bool send_result = WriteFile(handle(), write_buffer_, length, &tmp, overlapped()); + if (!send_result && WSAGetLastError() != ERROR_IO_PENDING) { + return false; + } + return true; + } + + private: + char* write_buffer_ = null; + bool write_ready_ = true; +}; + +MODULE_IMPLEMENTATION(pipe, MODULE_PIPE) + +PRIMITIVE(init) { + ByteArray* proxy = process->object_heap()->allocate_proxy(); + if (proxy == null) ALLOCATION_FAILED; + + auto resource_group = _new PipeResourceGroup(process, WindowsEventSource::instance()); + if (!resource_group) MALLOC_FAILED; + + proxy->set_external_address(resource_group); + return proxy; +} + +PRIMITIVE(close) { + ARGS(Resource, fd_resource, PipeResourceGroup, resource_group); + + resource_group->unregister_resource(fd_resource); + + fd_resource_proxy->clear_external_address(); + + return process->program()->null_object(); +} + +// Create a writable or readable pipe, as used for stdin/stdout/stderr of a child process. +// result[0]: Resource +// result[1]: file descriptor for child process. +PRIMITIVE(create_pipe) { + ARGS(PipeResourceGroup, resource_group, bool, input); + ByteArray* resource_proxy = process->object_heap()->allocate_proxy(); + if (resource_proxy == null) ALLOCATION_FAILED; + Array* array = process->object_heap()->allocate_array(2, Smi::zero()); + if (array == null) ALLOCATION_FAILED; + + HANDLE event = CreateEvent(NULL, true, false, NULL); + if (event == INVALID_HANDLE_VALUE) WINDOWS_ERROR; + + char pipe_name_buffer[MAX_PATH]; + snprintf(pipe_name_buffer, + MAX_PATH, + R"(\\.\Pipe\Toit.%08lx.%08lx)", + GetCurrentProcessId(), + InterlockedIncrement(&resource_group->pipe_serial_number()) + ); + + SECURITY_ATTRIBUTES security_attributes; + + // Set the bInheritHandle flag so pipe handles are inherited. + security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES); + security_attributes.bInheritHandle = input; + security_attributes.lpSecurityDescriptor = NULL; + + HANDLE read = CreateNamedPipe( + pipe_name_buffer, + PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED, + PIPE_TYPE_BYTE | PIPE_WAIT, + 1, // Number of pipes + 8192, // Out buffer size + 8192, // In buffer size + 0, // Default timeout (50 ms) + &security_attributes + ); + + if (read == INVALID_HANDLE_VALUE) { + close_handle_keep_errno(event); + WINDOWS_ERROR; + } + + security_attributes.bInheritHandle = !input; + + HANDLE write = CreateFileA( + pipe_name_buffer, + GENERIC_WRITE, + 0, // No sharing + &security_attributes, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED, + NULL // Template file + ); + + if (write == INVALID_HANDLE_VALUE) { + close_handle_keep_errno(event); + close_handle_keep_errno(read); + WINDOWS_ERROR; + } + + HandlePipeResource* pipe_resource; + if (input) pipe_resource = _new WritePipeResource(resource_group, write, event); + else pipe_resource = _new ReadPipeResource(resource_group, read, event); + + if (!pipe_resource) { + CloseHandle(read); + CloseHandle(write); + CloseHandle(event); + MALLOC_FAILED; + } + + resource_group->register_resource(pipe_resource); + + resource_proxy->set_external_address(pipe_resource); + + array->at_put(0, resource_proxy); + array->at_put(1, Smi::from(reinterpret_cast(input ? read : write))); + + return array; +} + +PRIMITIVE(fd_to_pipe) { + ARGS(PipeResourceGroup, resource_group, int, fd); + + ByteArray* resource_proxy = process->object_heap()->allocate_proxy(); + if (resource_proxy == null) ALLOCATION_FAILED; + + if (fd < 0 || fd > 2) INVALID_ARGUMENT; + + // Check if the standard handle has already been made a pipe. The overlapped + // IO does not support multiple clients. + if (resource_group->is_standard_piped(fd)) INVALID_ARGUMENT; + + HANDLE event = CreateEvent(NULL, true, false, NULL); + if (event == INVALID_HANDLE_VALUE) WINDOWS_ERROR; + HandlePipeResource* pipe_resource; + + switch (fd) { + case 0: { + HANDLE read = GetStdHandle(STD_INPUT_HANDLE); + pipe_resource = _new ReadPipeResource(resource_group, read, event); + break; + } + case 1: { + HANDLE write = GetStdHandle(STD_OUTPUT_HANDLE); + pipe_resource = _new WritePipeResource(resource_group, write, event); + break; + } + case 2: { + HANDLE error = GetStdHandle(STD_ERROR_HANDLE); + pipe_resource = _new WritePipeResource(resource_group, error, event); + break; + } + default: + INVALID_ARGUMENT; + } + + if (!pipe_resource) MALLOC_FAILED; + + resource_group->set_standard_piped(fd); + + resource_proxy->set_external_address(pipe_resource); + resource_group->register_resource(pipe_resource); + + return resource_proxy; +} + +PRIMITIVE(is_a_tty) { + ARGS(Resource, resource); + auto pipe_resource = reinterpret_cast(resource); + + DWORD tmp; + const BOOL success = GetConsoleMode(pipe_resource->handle(), &tmp); + return BOOL(success); +} + +PRIMITIVE(fd) { + ARGS(Resource, resource); + auto handle_resource = reinterpret_cast(resource); + + return Smi::from(reinterpret_cast(handle_resource->handle())); +} + + +PRIMITIVE(write) { + ARGS(WritePipeResource, pipe_resource, Blob, data, int, from, int, to); + + const uint8* tx = data.address(); + if (from < 0 || from > to || to > data.length()) OUT_OF_RANGE; + tx += from; + + if (!pipe_resource->ready_for_write()) return Smi::from(0); + + if (!pipe_resource->send(tx, to - from)) WINDOWS_ERROR; + + return Smi::from(to - from); +} + +PRIMITIVE(read) { + ARGS(ReadPipeResource, read_resource); + + if (read_resource->pipe_ended()) return process->program()->null_object(); + if (!read_resource->read_ready()) return Smi::from(-1); + + ByteArray* array = process->allocate_byte_array(READ_BUFFER_SIZE, true); + if (array == null) ALLOCATION_FAILED; + + if (!read_resource->receive_read_response()) { + if (GetLastError() == ERROR_BROKEN_PIPE) return process->program()->null_object(); + WINDOWS_ERROR; + } + + // A read count of 0 means EOF + if (read_resource->read_count() == 0) return process->program()->null_object(); + + array->resize_external(process, read_resource->read_count()); + + memcpy(ByteArray::Bytes(array).address(), read_resource->read_buffer(), read_resource->read_count()); + + if (!read_resource->issue_read_request()) { + if (GetLastError() != ERROR_BROKEN_PIPE) WINDOWS_ERROR; + read_resource->set_pipe_ended(true); + } + + return array; +} + +HANDLE handle_from_object(Object* object, DWORD std_handle) { + if (is_smi(object)) { + int fd = static_cast(Smi::cast(object)->value()); + if (fd == -1) return GetStdHandle(std_handle); + return reinterpret_cast(fd); + } else if (is_byte_array(object)) { + ByteArray* array = ByteArray::cast(object); + if (!array->has_external_address()) return INVALID_HANDLE_VALUE; + if (array->external_tag() != IntResource::tag) return INVALID_HANDLE_VALUE; + return reinterpret_cast(array->as_external()->id()); + } + return INVALID_HANDLE_VALUE; +} + +bool is_inherited(Object *object) { + return is_smi(object) && static_cast(Smi::cast(object)->value()) == -1; +} +const int MAX_COMMAND_LINE_LENGTH = 32768; + +// Forks and execs a program (optionally found using the PATH environment +// variable. The given file descriptors should be open file descriptors. They +// are attached to the stdin, stdout and stderr of the launched program, and +// are closed in the parent program. If you pass -1 for any of these then the +// forked program inherits the stdin/out/err of this Toit program. +PRIMITIVE(fork) { + ARGS(SubprocessResourceGroup, resource_group, + bool, use_path, + Object, in_object, + Object, out_object, + Object, err_object, + int, fd_3, + int, fd_4, + cstring, command, + Array, arguments); + if (arguments->length() > 1000000) OUT_OF_BOUNDS; + if (strlen(command) > MAX_COMMAND_LINE_LENGTH) OUT_OF_BOUNDS; + + ByteArray* proxy = process->object_heap()->allocate_proxy(); + if (proxy == null) ALLOCATION_FAILED; + + // FD_3 and FD_4 is not supported on Windows. + if (fd_3 != -1 || fd_4 != -1) INVALID_ARGUMENT; + + // Clearing environment not supported n=on windows, yet. + if (!use_path) INVALID_ARGUMENT; + + AllocationManager allocation(process); + char* command_line = reinterpret_cast(allocation.calloc(MAX_COMMAND_LINE_LENGTH, 1)); + if (!command_line) ALLOCATION_FAILED; + + int pos = 0; + for (int i = 0; i < arguments->length(); i++) { + if (!is_string(arguments->at(i))) { + WRONG_TYPE; + } + const char* format; + String* argument = String::cast(arguments->at(i)); + if (strchr(argument->as_cstr(), ' ') != NULL) { + format = (i != arguments->length() - 1) ? "\"%s\" " : "\"%s\""; + } else { + format = (i != arguments->length() - 1) ? "%s " : "%s"; + } + + if (pos + argument->length() + strlen(format) - 2 >= MAX_COMMAND_LINE_LENGTH) OUT_OF_BOUNDS; + pos += snprintf(command_line + pos, MAX_COMMAND_LINE_LENGTH - pos, format, argument->as_cstr()); + } + + // We allocate memory for the SubprocessResource early here so we can handle failure + // and restart the primitive. If we wait until after the fork, the + // subprocess is already running, and it is too late to GC-and-retry. + AllocationManager resource_allocation(process); + if (resource_allocation.alloc(sizeof(SubprocessResource)) == null) { + ALLOCATION_FAILED; + } + + PROCESS_INFORMATION process_information{}; + STARTUPINFO startup_info{}; + + startup_info.cb = sizeof(STARTUPINFO); + startup_info.hStdInput = handle_from_object(in_object, STD_INPUT_HANDLE); + startup_info.hStdOutput = handle_from_object(out_object, STD_OUTPUT_HANDLE); + startup_info.hStdError = handle_from_object(err_object, STD_ERROR_HANDLE); + startup_info.dwFlags |= STARTF_USESTDHANDLES; + + const char* current_directory = current_dir(process); + if (!current_directory) MALLOC_FAILED; + if (!CreateProcess(NULL, + command_line, + NULL, + NULL, + TRUE, // inherit handles. + 0, // creation flags + NULL, // parent's environment + current_directory, + &startup_info, + &process_information)) + WINDOWS_ERROR; + + // Release any handles that are pipes and are parsed down to the child + if (GetFileType(startup_info.hStdInput) == FILE_TYPE_PIPE && !is_inherited(in_object)) + CloseHandle(startup_info.hStdInput); + if (GetFileType(startup_info.hStdOutput) == FILE_TYPE_PIPE && !is_inherited(out_object)) + CloseHandle(startup_info.hStdOutput); + if (GetFileType(startup_info.hStdError) == FILE_TYPE_PIPE && !is_inherited(err_object)) + CloseHandle(startup_info.hStdError); + + auto subprocess = new (resource_allocation.keep_result()) SubprocessResource(resource_group, process_information.hProcess); + proxy->set_external_address(subprocess); + + resource_group->register_resource(subprocess); + + return proxy; +} + +} // namespace toit + +#endif // TOIT_LINUX or TOIT_BSD diff --git a/src/resources/subprocess.h b/src/resources/subprocess.h index 0c415836d..57be2f8c7 100644 --- a/src/resources/subprocess.h +++ b/src/resources/subprocess.h @@ -15,15 +15,47 @@ #pragma once +#include "../top.h" + +#if defined(TOIT_WINDOWS) +#include "../event_sources/event_win.h" +#endif + namespace toit { class SubprocessResourceGroup : public ResourceGroup { public: TAG(SubprocessResourceGroup); SubprocessResourceGroup(Process* process, EventSource* event_source) : ResourceGroup(process, event_source) {} - virtual uint32_t on_event(Resource* resource, word data, uint32_t state); + uint32_t on_event(Resource* resource, word data, uint32_t state) override; + + private: +}; + +#if defined(TOIT_WINDOWS) +class SubprocessResource : public WindowsResource { + public: + TAG(SubprocessResource); + SubprocessResource(ResourceGroup* resource_group, HANDLE handle) + : WindowsResource(resource_group) + , handle_(handle) {} + + std::vector events() override; + + void do_close() override; + + uint32_t on_event(HANDLE event, uint32_t state) override; + + void set_killed() { killed_ = true; } + bool killed() const { return killed_; } + HANDLE handle() const { return handle_; } + bool is_event_enabled(HANDLE event) override { return stopped_state_ == 0; } private: + HANDLE handle_; + bool killed_ = false; + word stopped_state_ = 0; }; +#endif // defined(TOIT_WINDOWS) } // namespace toit diff --git a/src/resources/subprocess.cc b/src/resources/subprocess_posix.cc similarity index 100% rename from src/resources/subprocess.cc rename to src/resources/subprocess_posix.cc diff --git a/src/resources/subprocess_win.cc b/src/resources/subprocess_win.cc new file mode 100644 index 000000000..867ca297d --- /dev/null +++ b/src/resources/subprocess_win.cc @@ -0,0 +1,103 @@ +// Copyright (C) 2022 Toitware ApS. +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 only. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// The license can be found in the file `LICENSE` in the top level +// directory of this repository. + +#include "../top.h" + +#if defined(TOIT_WINDOWS) + +#include +#include "../event_sources/event_win.h" +#include "../objects_inline.h" +#include "../process_group.h" +#include "../vm.h" +#include "subprocess.h" +namespace toit { + +static const int PROCESS_EXITED = 1; +static const int PROCESS_SIGNALLED = 2; +static const int PROCESS_EXIT_CODE_SHIFT = 2; +static const int PROCESS_EXIT_CODE_MASK = 0xff; +static const int PROCESS_SIGNAL_SHIFT = 10; + +uint32_t SubprocessResourceGroup::on_event(Resource* resource, word data, uint32_t state) { + return reinterpret_cast(resource)->on_event( + reinterpret_cast(data), + state); +} + +void SubprocessResource::do_close() { + CloseHandle(handle_); +} + +uint32_t SubprocessResource::on_event(HANDLE event, uint32_t state) { + if (stopped_state_ != 0) return stopped_state_; // This is a one off event. + + DWORD exit_code; + GetExitCodeProcess(handle_, &exit_code); + + if (killed()) state |= PROCESS_SIGNALLED | (9 << PROCESS_SIGNAL_SHIFT); + else state |= PROCESS_EXITED | ((exit_code & PROCESS_EXIT_CODE_MASK) << PROCESS_EXIT_CODE_SHIFT); + + stopped_state_ = state; + return state; +} + +std::vector SubprocessResource::events() { + return std::vector( { handle_ } ); +} + +MODULE_IMPLEMENTATION(subprocess, MODULE_SUBPROCESS) + +PRIMITIVE(init) { + ByteArray* proxy = process->object_heap()->allocate_proxy(); + if (proxy == null) ALLOCATION_FAILED; + + auto resource_group = _new SubprocessResourceGroup(process, WindowsEventSource::instance()); + if (!resource_group) MALLOC_FAILED; + + proxy->set_external_address(resource_group); + return proxy; +} + +PRIMITIVE(wait_for) { + // On Windows we always add an event to get notified when a subprocess ends. So this primitive is intentionally just + // returning null. + return process->program()->null_object(); +} + +PRIMITIVE(dont_wait_for) { + // On Windows we always add an event to get notified when a subprocess ends. So this primitive is intentionally just + // returning null. + return process->program()->null_object(); +} + +PRIMITIVE(kill) { + ARGS(SubprocessResource, subprocess, int, signal); + if (signal != 9) INVALID_ARGUMENT; + + subprocess->set_killed(); + TerminateProcess(subprocess->handle(), signal); + return process->program()->null_object(); +} + +PRIMITIVE(strsignal) { + ARGS(int, signal); + if (signal == 9) return process->allocate_string_or_error("SIGKILL"); + INVALID_ARGUMENT; +} + +} // namespace toit + +#endif // TOIT_WINDOWS diff --git a/src/resources/tcp_win.cc b/src/resources/tcp_win.cc index 31b4a617b..fa9ea42dd 100644 --- a/src/resources/tcp_win.cc +++ b/src/resources/tcp_win.cc @@ -30,7 +30,7 @@ #include "../event_sources/event_win.h" #include "tcp.h" -#include "error_win.h" +#include "../error_win.h" namespace toit { diff --git a/src/resources/uart_win.cc b/src/resources/uart_win.cc index 1718ba73d..7fb1b4abd 100644 --- a/src/resources/uart_win.cc +++ b/src/resources/uart_win.cc @@ -27,7 +27,7 @@ #include "../event_sources/event_win.h" -#include "error_win.h" +#include "../error_win.h" namespace toit { @@ -35,7 +35,7 @@ const int kReadState = 1 << 0; const int kErrorState = 1 << 1; const int kWriteState = 1 << 2; -const int READ_BUFFER_SIZE = 1 << 16; +static const int READ_BUFFER_SIZE = 1 << 16; class UARTResource : public WindowsResource { public: @@ -121,7 +121,7 @@ class UARTResource : public WindowsResource { memcpy(write_buffer_, buffer, length); DWORD tmp; - bool send_result = WriteFile(uart_, buffer, length, &tmp, &write_overlapped_); + bool send_result = WriteFile(uart_, write_buffer_, length, &tmp, &write_overlapped_); if (!send_result && WSAGetLastError() != ERROR_IO_PENDING) { return false; } diff --git a/src/resources/udp_win.cc b/src/resources/udp_win.cc index 61ccd2d67..1bf8af0cb 100644 --- a/src/resources/udp_win.cc +++ b/src/resources/udp_win.cc @@ -17,7 +17,7 @@ #if defined(TOIT_WINDOWS) #include "posix_socket_address.h" -#include "error_win.h" +#include "../error_win.h" #include @@ -229,9 +229,8 @@ PRIMITIVE(bind) { resource_group->register_resource(resource); - AutoUnregisteringResource resource_manager(resource_group, resource); + resource_proxy->set_external_address(resource); - resource_manager.set_external_address(resource_proxy); return resource_proxy; } diff --git a/src/tags.h b/src/tags.h index b31a7a3fd..e7a8942f3 100644 --- a/src/tags.h +++ b/src/tags.h @@ -56,6 +56,8 @@ namespace toit { fn(UDPSocketResource) \ fn(TCPSocketResource) \ fn(TCPServerSocketResource) \ + fn(SubprocessResource) \ + fn(PipeResource) \ #define TLS_CLASSES_DO(fn) \ fn(MbedTLSSocket) \ diff --git a/src/utils.h b/src/utils.h index 292de57ca..690cbf3a4 100644 --- a/src/utils.h +++ b/src/utils.h @@ -19,6 +19,7 @@ #include #include "top.h" +#include "os.h" namespace toit { @@ -443,4 +444,24 @@ class DeferDelete { T* object_; }; +class AsyncThread : public Thread { + public: + static void run_async(const std::function &func) { + _new AsyncThread(func); + } + + protected: + explicit AsyncThread(std::function func) : Thread("async"), _func(std::move(func)) { + spawn(); + } + + void entry() override { + _func(); + delete this; + } + + private: + const std::function _func; +}; + } // namespace toit diff --git a/src/vm_win.cc b/src/vm_win.cc index 56f88a00a..961ec76f1 100644 --- a/src/vm_win.cc +++ b/src/vm_win.cc @@ -27,6 +27,10 @@ namespace toit { void VM::load_platform_event_sources() { + // The Windows host implementation is using stdlib in multiple threads. The standard library collections + // calls new behind the scenes in some cases. The AllowThrowingNew RII is not thread safe, so for this host + // implementation the throwing_new_allowed is enabled globally. + toit::throwing_new_allowed = true; event_manager()->add_event_source(_new TimerEventSource()); event_manager()->add_event_source(_new TLSEventSource()); event_manager()->add_event_source(_new WindowsEventSource()); diff --git a/tests/image/fail.cmake b/tests/image/fail.cmake index d374403ae..dcd39feae 100644 --- a/tests/image/fail.cmake +++ b/tests/image/fail.cmake @@ -16,8 +16,3 @@ set(TOIT_FAILING_TESTS ) -if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Windows" OR "${CMAKE_SYSTEM_NAME}" STREQUAL "MSYS") - list(APPEND TOIT_FAILING_TESTS - tests/image/full_page_test.toit - ) -endif() diff --git a/tests/profiler/fail.cmake b/tests/profiler/fail.cmake index c22f47748..bbc9f06a0 100644 --- a/tests/profiler/fail.cmake +++ b/tests/profiler/fail.cmake @@ -15,10 +15,3 @@ set(TOIT_FAILING_TESTS ) - -if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Windows" OR "${CMAKE_SYSTEM_NAME}" STREQUAL "MSYS") - list(APPEND TOIT_FAILING_TESTS - tests/profiler/basic_test.toit - tests/profiler/lambda_test.toit - ) -endif() diff --git a/tests/profiler/utils.toit b/tests/profiler/utils.toit index 5af242f48..79b310631 100644 --- a/tests/profiler/utils.toit +++ b/tests/profiler/utils.toit @@ -35,5 +35,5 @@ run args -> List: exit_code := pipe.exit_code exit_value if exit_code != 0: throw "Program didn't exit with 0." - lines := output.split "\n" + lines := output.split LINE_TERMINATOR return lines diff --git a/tests/toitp/bytecodes_toitp_test.toit b/tests/toitp/bytecodes_toitp_test.toit index 88d533994..67bc4a10b 100644 --- a/tests/toitp/bytecodes_toitp_test.toit +++ b/tests/toitp/bytecodes_toitp_test.toit @@ -7,7 +7,7 @@ import .utils test args filter: out := run_toitp args ["-bc", filter] - lines := out.split "\n" + lines := out.split LINE_TERMINATOR expect (lines.first.starts_with "Bytecodes for methods") expected_bytecodes := [ diff --git a/tests/toitp/dispatch_toitp_test.toit b/tests/toitp/dispatch_toitp_test.toit index 964a9703b..99f2882ed 100644 --- a/tests/toitp/dispatch_toitp_test.toit +++ b/tests/toitp/dispatch_toitp_test.toit @@ -7,7 +7,7 @@ import .utils main args: out := run_toitp args ["-d"] - lines := out.split "\n" + lines := out.split LINE_TERMINATOR methods := lines.copy 1 methods.filter --in_place: it != "" methods.map --in_place: diff --git a/tests/toitp/fail.cmake b/tests/toitp/fail.cmake index 66ae38660..dcd39feae 100644 --- a/tests/toitp/fail.cmake +++ b/tests/toitp/fail.cmake @@ -16,15 +16,3 @@ set(TOIT_FAILING_TESTS ) -if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Windows" OR "${CMAKE_SYSTEM_NAME}" STREQUAL "MSYS") - list(APPEND TOIT_FAILING_TESTS - tests/toitp/bytecodes_toitp_test.toit - tests/toitp/class_toitp_test.toit - tests/toitp/dispatch_toitp_test.toit - tests/toitp/literal_index_toitp_test.toit - tests/toitp/literal_toitp_test.toit - tests/toitp/method_toitp_test.toit - tests/toitp/senders_toitp_test.toit - tests/toitp/summary_toitp_test.toit - ) -endif() diff --git a/tests/toitp/literal_index_toitp_test.toit b/tests/toitp/literal_index_toitp_test.toit index 1db118ce8..d454d170f 100644 --- a/tests/toitp/literal_index_toitp_test.toit +++ b/tests/toitp/literal_index_toitp_test.toit @@ -13,7 +13,7 @@ main args: // Get the indexes from the program. out /string := pipe.backticks [toit_run, snap] - lines := out.split "\n" + lines := out.split LINE_TERMINATOR lines.do: if it == "": continue.do parts := it.split " - " diff --git a/tests/toitp/literal_toitp_test.toit b/tests/toitp/literal_toitp_test.toit index d34ab5259..01a64f719 100644 --- a/tests/toitp/literal_toitp_test.toit +++ b/tests/toitp/literal_toitp_test.toit @@ -7,7 +7,7 @@ import .utils main args: out := run_toitp args ["-l"] - lines := out.split "\n" + lines := out.split LINE_TERMINATOR found_foo_string := false found_int_literal := false diff --git a/tests/toitp/senders_toitp_test.toit b/tests/toitp/senders_toitp_test.toit index 40b91d7e0..44c9fda4f 100644 --- a/tests/toitp/senders_toitp_test.toit +++ b/tests/toitp/senders_toitp_test.toit @@ -7,7 +7,7 @@ import .utils main args: out := run_toitp args ["--senders", "the_target"] - lines := out.split "\n" + lines := out.split LINE_TERMINATOR expect_equals """Methods with calls to "the_target"[3]:""" lines[0] ["global_lazy_field", "global_fun", "foo"].do: |needle| diff --git a/tests/toitp/utils.toit b/tests/toitp/utils.toit index 7d3b65e58..bf97a187a 100644 --- a/tests/toitp/utils.toit +++ b/tests/toitp/utils.toit @@ -16,7 +16,7 @@ run_toitp test_args/List toitp_args/List -> string: // Extracts the entry names, discarding the index and the location. extract_entries output/string --max_length/int -> List: - lines := output.split "\n" + lines := output.split LINE_TERMINATOR result := lines.copy 1 result.filter --in_place: it != "" result.map --in_place: