Dylan/publish to pypi

.github/workflows/pypi.yml

@@ -0,0 +1,34 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download wheels
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          github_token: ${{secrets.GITHUB_TOKEN}}
+          workflow: wheels.yml
+          workflow_conclusion: ''
+
+      - name: Copy wheels to dist
+        run: |
+          mkdir -p dist;
+          find . -type d -name 'wheels-*' -exec sh -c 'cp -v "$1"/* dist/' _ {} \;
+
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Publish distribution to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/test-pypi.yml

@@ -0,0 +1,41 @@
+name: Test Publish to PyPI
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - v*
+    branches-ignore:
+      - main
+
+jobs:
+  publish:
+    runs-on: ubuntu
+    steps:
+      - name: Download wheels
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          workflow: wheels.yml
+          workflow_conclusion: ''
+
+      - name: Copy wheels to dist
+        run: |
+          mkdir -p dist;
+          find . -type d -name 'wheels-*' -exec sh -c 'cp -v "$1"/* dist/' _ {} \;
+
+      - name: Display output
+        run: |
+          ls -l dist
+
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+          repository-url: https://test.pypi.org/legacy/

.github/workflows/wheels.yml

@@ -0,0 +1,115 @@
+name: Build wheels
+
+on: [workflow_dispatch]
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: [self-hosted, "${{ matrix.os }}"]
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu, macOS]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # Used to host cibuildwheel
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel==2.12.0
+
+      - name: Cache downloads
+        uses: actions/cache@v3
+        with:
+          path: downloads
+          key: ${{ runner.os }}-downloads-boost-xz-zstd
+          restore-keys: |
+            ${{ runner.os }}-downloads
+
+      - name: Download dependencies
+        run: |
+          mkdir -p downloads
+          cd downloads
+          for DEP in \
+            'https://sourceforge.net/projects/boost/files/boost/1.76.0/boost_1_76_0.tar.bz2' \
+            'https://tukaani.org/xz/xz-5.4.1.tar.bz2' \
+            'https://github.com/facebook/zstd/archive/v1.5.2.tar.gz' \
+          ; do
+            if [ ! -e "$(basename "$DEP")" ]; then
+              wget --no-verbose --no-check-certificate "$DEP"
+            fi
+          done
+
+      - name: Build wheels
+        env:
+          CIBW_BUILD: cp38-* cp39-* cp310-*
+          CIBW_SKIP: "*_i686 *-musllinux_*"
+          CIBW_ARCHS_MACOS: x86_64 arm64
+          CIBW_BEFORE_BUILD_LINUX: ./install_boost_linux.sh
+          CIBW_BEFORE_BUILD_MACOS: ./install_deps_macos.sh
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
+            DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} --dylibs-only
+          # CIBW_TEST_COMMAND_MACOS: "python {project}/tests/run_mission.py"
+        run: |
+          cp VERSION scripts/python-wheel/package/
+          VERSION=$(cat VERSION)
+          echo "malmomod.version=${VERSION}" > Minecraft/src/main/resources/version.properties
+          cp -r Malmo/src scripts/python-wheel/package/src
+          cp -r Minecraft scripts/python-wheel/package/malmo/Minecraft
+          cp -r Schemas scripts/python-wheel/package/malmo/Schemas
+          cd scripts/python-wheel/package
+          mkdir deps
+          cd deps
+          tar xf ../../../../downloads/xz-5.4.1.tar.bz2
+          tar xf ../../../../downloads/boost_1_76_0.tar.bz2
+          tar xf ../../../../downloads/v1.5.2.tar.gz
+          cd ..
+          CIBW_ENVIRONMENT="REPAIR_LIBRARY_PATH=$(pwd)/deps/lib" python -m cibuildwheel --output-dir wheelhouse
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels-${{ matrix.os }}
+          path: scripts/python-wheel/package/wheelhouse/*.whl
+
+  test_wheels:
+    name: Test wheels on ${{ matrix.os }} Python ${{ matrix.pyversion }}
+    needs: build_wheels
+    runs-on: [self-hosted, "${{ matrix.os }}"]
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu]
+        pyversion: ['3.8', '3.9', '3.10']
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.pyversion }}
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+
+      - name: Download wheels
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels-${{ matrix.os }}
+          path: wheelhouse
+
+      - name: Install wheel
+        run: |
+          # Kind of janky, but loop through until we find the right wheel.
+          for WHEEL in wheelhouse/*.whl; do pip install "$WHEEL" || true; done
+
+      - name: Run tests
+        uses: GabrielBB/xvfb-action@v1
+        with:
+          run: python scripts/python-wheel/package/tests/run_mission.py

Malmo/src/MissionInitSpec.cpp

@@ -192,4 +192,21 @@ namespace malmo
         this->mission_init.minecraft_server.connection_address = boost::algorithm::trim_copy(address);
         this->mission_init.minecraft_server.connection_port = port;
     }
+
+    boost::optional<int> MissionInitSpec::getMinecraftServerConnectionPort()
+    {
+        return this->mission_init.minecraft_server.connection_port;
+    }
+
+    boost::optional<std::string> MissionInitSpec::getMinecraftServerConnectionAddress()
+    {
+        return this->mission_init.minecraft_server.connection_address;
+    }
+
+    std::ostream& operator<<(std::ostream& os, const MissionInitSpec& mis)
+    {
+        os << "MissionInitSpec:\n";
+        os << mis.getAsXML(true);
+        return os;
+    }
 }

Malmo/src/MissionInitSpec.h

@@ -159,6 +159,11 @@ namespace malmo
         //! \param port The Minecraft server port.
         void setMinecraftServerInformation(const std::string& address, int port);
 
+        boost::optional<int> getMinecraftServerConnectionPort();
+        boost::optional<std::string> getMinecraftServerConnectionAddress();
+
+        friend std::ostream& operator<<(std::ostream& os, const MissionInitSpec& mis);
+
     private:
         MissionInitXML mission_init;
     };

Malmo/src/PosixFrameWriter.h

@@ -23,6 +23,9 @@
 // Local:
 #include "VideoFrameWriter.h"
 
+// STL:
+#include <stack>
+
 namespace malmo
 {
     class PosixFrameWriter : public VideoFrameWriter

Malmo/src/PythonWrapper/python_module.cpp

@@ -18,6 +18,8 @@
 // --------------------------------------------------------------------------------------------------
 
 // Boost:
+#include <boost/noncopyable.hpp>
+#include <boost/optional.hpp>
 #include <boost/python.hpp>
 #include <boost/python/exception_translator.hpp>
 #include <boost/python/register_ptr_to_python.hpp>
@@ -26,6 +28,7 @@
 
 // Malmo:
 #include <AgentHost.h>
+#include <MissionInitSpec.h>
 #include <Logger.h>
 #ifdef WRAP_ALE
     #include <ALEAgentHost.h>
@@ -38,6 +41,7 @@ using namespace malmo;
 // STL:
 #include <sstream>
 #include <cstdint>
+#include <string>
 
 // Python:
 #include <datetime.h>
@@ -65,6 +69,54 @@ void translateXMLStdException(std::exception const& e)
     PyErr_SetString(PyExc_RuntimeError, oss.str().c_str() );
 }
 
+// From https://stackoverflow.com/a/28604575/200508
+template<typename T>
+struct python_optional : private boost::noncopyable {
+    struct conversion : public boost::python::converter::expected_from_python_type<T>
+    {
+        static PyObject* convert(boost::optional<T> const& value)
+        {
+            using namespace boost::python;
+            return incref((value ? object(*value) : object()).ptr());
+        }
+    };
+
+    static void* convertible(PyObject *obj) {
+        using namespace boost::python;
+        return obj == Py_None || extract<T>(obj).check() ? obj : NULL;
+    }
+
+    static void constructor(
+        PyObject *obj,
+        boost::python::converter::rvalue_from_python_stage1_data *data
+    ) {
+        using namespace boost::python;
+        void *const storage =
+        reinterpret_cast<
+            converter::rvalue_from_python_storage<boost::optional<T> >*
+        >(data)->storage.bytes;
+        if(obj == Py_None) {
+            new (storage) boost::optional<T>();
+        } else {
+            new (storage) boost::optional<T>(extract<T>(obj));
+        }
+        data->convertible = storage;
+    }
+
+    explicit python_optional() {
+        using namespace boost::python;
+        if(!extract<boost::optional<T> >(object()).check()) {
+            to_python_converter<boost::optional<T>, conversion, true>();
+            converter::registry::push_back(
+                &convertible,
+                &constructor,
+                type_id<boost::optional<T> >(),
+                &conversion::get_pytype
+            );
+        }
+    }
+};
+
 PyObject* missionExceptionType = NULL;
 
 PyObject* createExceptionClass(const char* name, PyObject* baseTypeObj = PyExc_Exception)
@@ -97,6 +149,9 @@ void (AgentHost::*sendCommandWithKey)(std::string, std::string) = &AgentHost::se
 void (MissionRecordSpec::*recordMP4General)(int, int64_t bit_rate) = &MissionRecordSpec::recordMP4;
 void (MissionRecordSpec::*recordMP4Specific)(TimestampedVideoFrame::FrameType, int, int64_t, bool) = &MissionRecordSpec::recordMP4;
 
+
+boost::shared_ptr<MissionInitSpec> (AgentHost::*getMissionInit)() = &AgentHost::getMissionInit;
+
 #ifdef WRAP_ALE
 void (ALEAgentHost::*startALEMissionSimple)(const MissionSpec&, const MissionRecordSpec&) = &ALEAgentHost::startMission;
 void (ALEAgentHost::*startALEMissionComplex)(const MissionSpec&, const ClientPool&, const MissionRecordSpec&, int, std::string) = &ALEAgentHost::startMission;
@@ -137,6 +192,9 @@ BOOST_PYTHON_MODULE(MalmoPython)
     to_python_converter<boost::posix_time::ptime, ptime_to_python_datetime>();
     to_python_converter<std::vector<unsigned char>, unsigned_char_vec_to_python_array>();
 
+    python_optional<int>();
+    python_optional<std::string>();
+
     enum_< MissionException::MissionErrorCode >("MissionErrorCode")
         .value("MISSION_BAD_ROLE_REQUEST", MissionException::MISSION_BAD_ROLE_REQUEST)
         .value("MISSION_BAD_VIDEO_REQUEST", MissionException::MISSION_BAD_VIDEO_REQUEST)
@@ -216,6 +274,39 @@ BOOST_PYTHON_MODULE(MalmoPython)
         .value( "KEEP_ALL_OBSERVATIONS",    AgentHost::KEEP_ALL_OBSERVATIONS )
     ;
 
+    register_ptr_to_python< boost::shared_ptr< MissionInitSpec > >();
+    class_< MissionInitSpec >("MissionInitSpec", no_init)
+        .def("getAsXML",                       &MissionInitSpec::getAsXML)
+        .def("getExperimentID",                &MissionInitSpec::getExperimentID)
+        .def("getClientAddress",               &MissionInitSpec::getClientAddress)
+        .def("setClientAddress",               &MissionInitSpec::setClientAddress)
+        .def("getClientMissionControlPort",    &MissionInitSpec::getClientMissionControlPort)
+        .def("setClientMissionControlPort",    &MissionInitSpec::setClientMissionControlPort)
+        .def("getClientCommandsPort",          &MissionInitSpec::getClientCommandsPort)
+        .def("setClientCommandsPort",          &MissionInitSpec::setClientCommandsPort)
+        .def("getAgentAddress",                &MissionInitSpec::getAgentAddress)
+        .def("setAgentAddress",                &MissionInitSpec::setAgentAddress)
+        .def("getAgentMissionControlPort",     &MissionInitSpec::getAgentMissionControlPort)
+        .def("setAgentMissionControlPort",     &MissionInitSpec::setAgentMissionControlPort)
+        .def("getAgentVideoPort",              &MissionInitSpec::getAgentVideoPort)
+        .def("getAgentDepthPort",              &MissionInitSpec::getAgentDepthPort)
+        .def("getAgentLuminancePort",          &MissionInitSpec::getAgentLuminancePort)
+        .def("getAgentColourMapPort",          &MissionInitSpec::getAgentColourMapPort)
+        .def("setAgentVideoPort",              &MissionInitSpec::setAgentVideoPort)
+        .def("setAgentDepthPort",              &MissionInitSpec::setAgentDepthPort)
+        .def("setAgentLuminancePort",          &MissionInitSpec::setAgentLuminancePort)
+        .def("setAgentColourMapPort",          &MissionInitSpec::setAgentColourMapPort)
+        .def("getAgentObservationsPort",       &MissionInitSpec::getAgentObservationsPort)
+        .def("setAgentObservationsPort",       &MissionInitSpec::setAgentObservationsPort)
+        .def("getAgentRewardsPort",            &MissionInitSpec::getAgentRewardsPort)
+        .def("setAgentRewardsPort",            &MissionInitSpec::setAgentRewardsPort)
+        .def("hasMinecraftServerInformation",  &MissionInitSpec::hasMinecraftServerInformation)
+        .def("setMinecraftServerInformation",  &MissionInitSpec::setMinecraftServerInformation)
+        .def("getMinecraftServerConnectionPort", &MissionInitSpec::getMinecraftServerConnectionPort)
+        .def("getMinecraftServerConnectionAddress", &MissionInitSpec::getMinecraftServerConnectionAddress)
+        .def(self_ns::str(self_ns::self))
+    ;
+
     class_< AgentHost, bases< ArgumentParser >, boost::noncopyable >("AgentHost", init<>())
         .def( "startMission",                   startMissionSimple )
         .def( "startMission",                   startMissionComplex )
@@ -228,6 +319,7 @@ BOOST_PYTHON_MODULE(MalmoPython)
         .def( "sendCommand",                    sendCommand )
         .def( "sendCommand",                    sendCommandWithKey )
         .def("getRecordingTemporaryDirectory",  &AgentHost::getRecordingTemporaryDirectory)
+        .def( "getMissionInit",                 getMissionInit )
         .def( "setDebugOutput",                 &AgentHost::setDebugOutput )
         .def(self_ns::str(self_ns::self))
     ;

Minecraft/run/options.txt

@@ -83,6 +83,9 @@ key_key.hotbar.8:9
 key_key.hotbar.9:10
 key_key.toggleMalmo:28
 key_key.handyTestHook:22
+key_key.toggleFullBlueprint:19
+key_key.toggleLiquids:38
+key_key.toggleVisible:0
 soundCategory_master:0.0
 soundCategory_music:1.0
 soundCategory_record:1.0