Merge for 2.19.0 release

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
         run: |
           sudo apt-get update -qq
           sudo apt-get install -qq libboost-all-dev swig libhdf5-dev libeigen3-dev \
-            cmake libcgal-dev libcgal-qt5-dev \
+            cmake libcgal-dev libcgal-qt5-dev libcereal-dev \
             libfftw3-dev libopencv-dev libgsl-dev libann-dev \
             libprotobuf-dev protobuf-compiler \
             libopenmpi-dev python3-dev python3-numpy python3-protobuf \

CMakeLists.txt

@@ -232,7 +232,7 @@ include_directories("${CMAKE_BINARY_DIR}/include")
 set(IMP_ALL_DEPENDS_VARS BOOST.FILESYSTEM_LIBRARIES BOOST.SYSTEM_LIBRARIES
     BOOST.THREAD_LIBRARIES BOOST.PROGRAMOPTIONS_LIBRARIES BOOST.REGEX_LIBRARIES
     BOOST.GRAPH_LIBRARIES BOOST.RANDOM_LIBRARIES
-    BOOST.SERIALIZATION_LIBRARIES CACHE INTERNAL "" FORCE)
+    CACHE INTERNAL "" FORCE)
 
 imp_execute_process("setup" ${CMAKE_BINARY_DIR}
                     COMMAND ${PYTHON_EXECUTABLE}
@@ -269,7 +269,7 @@ else()
 endif()
 
 set(Boost_NO_BOOST_CMAKE ON)
-find_package(Boost 1.53.0 COMPONENTS system filesystem thread program_options serialization REQUIRED)
+find_package(Boost 1.53.0 COMPONENTS system filesystem thread program_options REQUIRED)
 if("${Boost_SYSTEM_LIBRARY_RELEASE}" MATCHES ".*NOTFOUND.*")
   message(FATAL_ERROR "Boost is required to build IMP.")
 endif()
@@ -311,13 +311,6 @@ link_directories(${Boost_LIBRARY_DIRS})
 set(BOOST.FILESYSTEM_LIBRARIES ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST.SYSTEM_LIBRARIES ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST.THREAD_LIBRARIES ${Boost_THREAD_LIBRARY_RELEASE})
-if(IMP_STATIC)
-# Boost.Serialization needs pthread, but order is important with static
-# libraries; we must link pthread *after* serialization
-set(BOOST.SERIALIZATION_LIBRARIES ${Boost_SERIALIZATION_LIBRARY_RELEASE} -lpthread)
-else()
-set(BOOST.SERIALIZATION_LIBRARIES ${Boost_SERIALIZATION_LIBRARY_RELEASE})
-endif()
 set(BOOST.PROGRAMOPTIONS_LIBRARIES ${Boost_PROGRAM_OPTIONS_LIBRARY_RELEASE})
 set(BOOST.REGEX_LIBRARIES ${Boost_REGEX_LIBRARY_RELEASE})
 set(BOOST.GRAPH_LIBRARIES ${Boost_GRAPH_LIBRARY_RELEASE})
@@ -340,6 +333,9 @@ find_package(Eigen3 3.0 REQUIRED)
 # b) we're not going to fix, since Eigen isn't our code.
 include_directories(SYSTEM ${EIGEN3_INCLUDE_DIR})
 
+find_package(cereal REQUIRED)
+include_directories(SYSTEM ${cereal_INCLUDE_DIRS})
+
 add_custom_target("IMP-version"
                   COMMAND ${PYTHON_EXECUTABLE}
                           ${CMAKE_SOURCE_DIR}/tools/build/make_version.py
@@ -437,9 +433,9 @@ if(IMP_DOXYGEN_FOUND)
     COMMENT "Building documentation")
 
   add_custom_target(IMP-doc-install
-    COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_BINARY_DIR}/doc/manual" "${CMAKE_INSTALL_FULL_DOCDIR}/manual"
+    COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/build/copy_directory_deref.py "${CMAKE_BINARY_DIR}/doc/manual" "${CMAKE_INSTALL_FULL_DOCDIR}/manual"
     COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_BINARY_DIR}/doxygen/manual-tags.xml" "${CMAKE_INSTALL_FULL_DOCDIR}"
-    COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_BINARY_DIR}/doc/ref" "${CMAKE_INSTALL_FULL_DOCDIR}/ref"
+    COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/build/copy_directory_deref.py "${CMAKE_BINARY_DIR}/doc/ref" "${CMAKE_INSTALL_FULL_DOCDIR}/ref"
     COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_BINARY_DIR}/doxygen/ref-tags.xml" "${CMAKE_INSTALL_FULL_DOCDIR}"
     DEPENDS "IMP-doc"
     COMMENT "Installing documentation to ${CMAKE_INSTALL_FULL_DOCDIR}")
@@ -484,7 +480,7 @@ foreach(cmakefile ${CMAKE_BINARY_DIR}/IMPConfig.cmake
                   ${CMAKE_BINARY_DIR}/cmake/IMPConfig.cmake)
   foreach(var IMP_LIB_TYPE IMP_NO_SWIG_DEPENDENCIES IMP_STATIC IMP_MAX_CHECKS
               IMP_MAX_LOG IMP_SWIG_LIBRARIES EIGEN3_INCLUDE_DIR
-              Boost_INCLUDE_DIR ${IMP_ALL_DEPENDS_VARS})
+              cereal_INCLUDE_DIRS Boost_INCLUDE_DIR ${IMP_ALL_DEPENDS_VARS})
     file(APPEND "${cmakefile}"
          "SET(${var} \"${${var}}\" CACHE INTERNAL \"\" FORCE)\n")
   endforeach()

ChangeLog.md

@@ -1,6 +1,42 @@
 ChangeLog {#changelog}
 =========
 
+# 2.19.0 - 2023-06-22 # {#changelog_2_19_0}
+- Most IMP Value and Object types can now be serialized (or pickled in Python).
+  For example, this can be used to save the current state of an IMP::Model
+  or an IMP.pmi.macros.ReplicaExchange object to be restored later or passed
+  to a separate process. Building IMP from source code now requires the
+  cereal library.
+- Information on most IMP restraints, sampling protocols, input files and
+  analysis/clustering is now stored in the RMF file. Ultimately this can be
+  used to generate an IHM mmCIF file directly from the RMF.
+- IMP.pmi.restraints.crosslinking.CrossLinkingMassSpectrometryRestraint now
+  requires the `linker` argument, which specifies the chemistry of the linker.
+- IMP.pmi.macros.ReplicaExchange0 has been removed.
+  Use IMP.pmi.macros.ReplicaExchange instead.
+- The undocumented IMP::core::ConjugateGradients::set_threshold() method is
+  now deprecated; use set_gradient_threshold() instead.
+- Rigid body members are now handled correctly by IMP::atom::destroy(),
+  and can be removed from their rigid body with a new
+  IMP::core::RigidBody::remove_member() method.
+- The IMP::em2d::CenteredMat and IMP::em2d::Fine2DRegistrationRestraint
+  classes have been moved to the IMP::em2d::internal namespace, as they
+  are implementation details.
+- Windows builds now require MS Visual Studio 2015 or later (for full C++11
+  support). The following macro for pre-C++11 environments is no longer
+  needed and is deprecated: `IMP_NOEXCEPT`.
+- The following pre-C++11 compatibility macros and headers are removed:
+  `IMP_NULLPTR`, `IMP_NULLPTR_T`, `IMP_OVERRIDE`, `IMP_FINAL`, `IMP_UNIQUE_PTR`,
+  `IMP_FOREACH`, `IMP/nullptr.h`, and `IMP/nullptr_macros.h`.
+- The deprecated IMP::em::emreal type has been removed.
+- The deprecated IMP::isd::Weight::get_nstates_key() method has been removed.
+- The deprecated IMP.npc.npc_restraints module has been removed. Use
+  IMP.pmi.restraints.npc or IMP.pmi1.restraints.npc instead.
+- The deprecated IMP::core::{Singleton,Pair,Triplet,Quad}Constraint constructors
+  that take Particle/ParticlePair/ParticleTriplet/ParticleQuad have been
+  removed. Use the constructors that take a Model and ParticleIndex (etc.)
+  instead.
+
 # 2.18.0 - 2022-12-15 # {#changelog_2_18_0}
 - The Windows .exe installer now supports Python 3.11.
 - The IMP::em::emreal type is deprecated; use plain `double` instead.

README.md

@@ -33,7 +33,7 @@ new shell.
 Copyright and License information
 =================================
 
-IMP is Copyright 2007-2022 IMP Inventors. The IMP Inventors are
+IMP is Copyright 2007-2023 IMP Inventors. The IMP Inventors are
 Andrej Sali, Ben Webb, Daniel Russel, Keren Lasker, Dina Schneidman,
 Javier Velázquez-Muriel, Friedrich Förster, Elina Tjioe, Hao Fan,
 Seung Joong Kim, Yannick Spill, Riccardo Pellarin.

VERSION

@@ -1 +1 @@
-2.18.0
+2.19.0

cmake_modules/Findcereal.cmake

@@ -0,0 +1,32 @@
+# Try to find Cereal
+#
+# The following variables are set:
+# cereal_FOUND
+# cereal_INCLUDE_DIRS - directories with Cereal headers
+# cereal_DEFINITIONS - Cereal compiler flags
+
+find_path(cereal_header_paths_tmp
+    NAMES
+    cereal.hpp
+    PATH_SUFFIXES
+    include
+    cereal/include
+    cereal
+	  PATHS
+    ${CMAKE_INSTALL_PREFIX}/include
+    ${CEREAL_ROOT_DIR}
+    ${CEREAL_ROOT_DIR}/include
+    ${CEREAL_ROOT_DIR}/cereal/include
+    $ENV{CEREAL_ROOT_DIR}
+    $ENV{CEREAL_ROOT_DIR}/include
+    $ENV{CEREAL_ROOT_DIR}/cereal
+    )
+
+get_filename_component(cereal_INCLUDE_DIRS ${cereal_header_paths_tmp} PATH)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(cereal
+    REQUIRED_VARS cereal_INCLUDE_DIRS
+    )
+
+mark_as_advanced(cereal_FOUND)

doc/manual/cmake_config.md

@@ -101,7 +101,7 @@ For each dependency CMake will first try to find the header and library
 files for that dependency, reporting success if it finds them. Next, it will
 often try to build a small C or C++ test program that uses those headers
 and libraries. If this fails the dependency cannot be used (and CMake will,
-somewhat confusing, report that the dependency was first found and then not
+somewhat confusingly, report that the dependency was first found and then not
 found). To fix issues like this, check the CMake error log in
 `CMakeFiles/CMakeError.log` to see what failed. In some cases this can be
 fixed by modifying the flags passed to the C or C++ compiler. For example,

doc/manual/code_conventions.md

@@ -122,7 +122,7 @@ As a result, such functions do not need to obey all the coding conventions
 
 # C++ 11 {#codeconv_cxx11}
 Latest %IMP requires a compiler that supports C++ 11, unlike older versions.
-Thus, C++ 11 features, such as the `override`, `final`, `auto` and `nullptr`
-keywords, and range-based for loops, can be used where appropriate.
-The old compatibility macros (such as `IMP_OVERRIDE`, `IMP_FINAL`,
-`IMP_NULLPTR` and `IMP_FOREACH`) should no longer be used.
+Thus, C++ 11 features, such as the `override`, `final`, `auto`, `noexcept`
+and `nullptr` keywords, and range-based for loops, can be used where
+appropriate. The old compatibility macros (such as `IMP_OVERRIDE`, `IMP_FINAL`,
+`IMP_NOEXCEPT`, `IMP_NULLPTR` and `IMP_FOREACH`) should no longer be used.

doc/manual/extdepends.md

@@ -23,6 +23,14 @@ your code in a new module, rather than adding it to an existing module. That
 way, people that elect not to install that dependency will only be deprived
 of your code, not of the existing module.
 
+Try not to require the very latest version of a dependency, as again that may
+make it harder for people to use. (If necessary, you can use preprocessor
+macros so that your code works with both old and new versions of a dependency;
+for examples, look in the C++ code for `BOOST_VERSION`.) We try to make %IMP
+work with the versions of packages available in the oldest supported versions
+of RedHat Enterprise Linux (RHEL), Ubuntu LTS, and macOS. For example, RHEL 7
+ships with Boost 1.53 and Python 2.7, so %IMP works with both of those.
+
 # Simple dependencies # {#extdep_simple}
 
 The simplest way to add a C/C++ dependency `foo` is to create a file

doc/manual/install_windows.md

@@ -3,11 +3,9 @@ Building from source code on Windows {#install_windows}
 
 We build and test %IMP on
 Windows, built with the Microsoft Visual Studio compilers (we use Visual Studio
-Express 2010 SP1 for the 32-bit Windows `.exe` installer, and
-VS Express 2012 for the 64-bit `.exe` installer; for the
+2015 for both the 32-bit and 64-bit Windows `.exe` installers; for the
 [Anaconda packages](https://integrativemodeling.org/download-anaconda.html),
-both 32-bit and 64-bit,
-we use VS 2008 SP1 for Python 2.7 and VS 2015 for Python 3.5).
+which are built by [conda-forge](https://conda-forge.org/), VS 2019 is used).
 One complication is that different packages are compiled
 with different versions of Visual Studio, and mixing the different runtimes
 (`msvc*.dll`) can cause odd behavior; therefore, we recommend building most
@@ -18,11 +16,10 @@ We recommend building within the Anaconda environment, since many of the
 dependencies are already built, and the procedure is scripted so it is more
 easily reproducible:
 
-  - Install the right version of Microsoft Visual Studio Express (it is free,
+  - Install the right version of Microsoft Visual Studio (it is free,
     but registration with Microsoft is required). Current Anaconda policy is
     to build packages using the same version of Visual Studio that was used
-    to build Python. This means VS 2008 for Python 2.7 and VS 2015 for
-    Python 3.5, for example.
+    to build Python.
   - Get and install [Miniconda](https://conda.io/miniconda.html) or the
     full Anaconda environment.
   - Install necessary conda packages for development:
@@ -31,7 +28,7 @@ easily reproducible:
     `cd` to the directory above the `foo` directory containing the
     [conda build recipe](https://docs.conda.io/projects/conda-build/en/latest/concepts/recipe.html)
     then build it with
-    `conda build --python=2.7 foo`
+    `conda build --python=3.9 foo`
   - Feel free to refer to
     [our conda recipes](https://github.com/salilab/conda-recipes)
     for IMP and all needed dependencies. In particular, each recipe contains
@@ -41,14 +38,14 @@ easily reproducible:
 If you want to build outside of the Anaconda environment, the basic procedure
 we employed is as follows:
 
-  - Install Microsoft Visual Studio Express (it is free, but registration with
+  - Install Microsoft Visual Studio (it is free, but registration with
     Microsoft is required).
   - Get and install [cmake](https://cmake.org).
   - Get [Python](https://www.python.org)
     (make sure you get the
     32-bit version if you're going to build %IMP for 32-bit Windows).
   - Get and install the
-    [zlib package](http://gnuwin32.sourceforge.net/packages/zlib.htm)
+    [zlib package](https://gnuwin32.sourceforge.net/packages/zlib.htm)
     (both the "complete package, except sources" and the "sources" installers).
      - The package without sources can be installed anywhere; we chose the
        default location of `C:\Program Files\GnuWin32`. The sources, however,
@@ -57,29 +54,29 @@ we employed is as follows:
      - We found that the zconf.h header included with zlib erroneously includes
        unistd.h, which doesn't exist on Windows, so we commented out that line
        (in both packages).
-  - Download the [Boost source code](http://www.boost.org)
+  - Download the [Boost source code](https://www.boost.org)
     (we extracted it into `C:\Program Files\boost_1_53_0`), then
      - Open a Visual Studio Command Prompt, and cd into the directory where
        Boost was extracted
      - Run bootstrap.bat
      - You may need to help the compiler find the zlib header file with
        `set INCLUDE=C:\Program Files\GnuWin32\include`
      - Run `bjam link=shared runtime-link=shared -sNO_ZLIB=0 -sZLIB_SOURCE=C:\zlib\1.2.3\zlib-1.2.3`
-  - Get and install [SWIG for Windows](http://www.swig.org)
+  - Get and install [SWIG for Windows](https://www.swig.org)
   - Get the [HDF5 source code](https://www.hdfgroup.org/downloads/hdf5/)
      - Make a 'build' subdirectory, then run from a command prompt in
        that subdirectory something similar to
        `cmake.exe -G "Visual Studio 10" -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF -DHDF5_ENABLE_Z_LIB_SUPPORT:BOOL=ON -DHDF5_BUILD_HL_LIB:BOOL=ON -DZLIB_INCLUDE_DIR="C:\Program Files\GnuWin32\include" -DZLIB_LIBRARY="C:\Program Files\GnuWin32\lib\zlib.lib" -DBUILD_SHARED_LIBS:BOOL=ON ..`
      - Open the resulting HDF5 solution file in Visual Studio, change to
        Release configuration, then build the hdf5 project.
-  - (Optional) [Build CGAL from source code](http://www.cgal.org/windows_installation.html).
+  - (Optional) [Build CGAL from source code](https://www.cgal.org/download/windows.html).
   - (Optional) Download the
     [FFTW DLLs](http://www.fftw.org/install/windows.html) and follow the
     instructions at that website to make .lib import libraries needed for
     Visual Studio.
      - Copy `libfftw3-3.lib` to `fftw3.lib` and `libfftw3-3.dll` to `fftw3.dll` to help cmake find it
   - (Optional) Get the
-    [GSL source code](http://gnuwin32.sourceforge.net/packages/gsl.htm)
+    [GSL source code](https://gnuwin32.sourceforge.net/packages/gsl.htm)
     and build it:
      - Open the libgsl project file in the `src\gsl\1.8\gsl-1.8\VC8`
        subdirectory
@@ -90,12 +87,12 @@ we employed is as follows:
        (we recommend removing the _dll suffix and the lib prefix when you do
        this so that cmake has an easier time finding them, i.e. call them
        gsl.lib and gslcblas.lib).
-  - (Optional) Get [numpy and scipy](http://www.scipy.org) to match your
+  - (Optional) Get [numpy and scipy](https://scipy.org) to match your
     Python version.
   - (Optional) Get and install
     [libTAU](https://integrativemodeling.org/libTAU.html)
      - Copy `libTAU.lib` to `TAU.lib` to help cmake find it.
-  - (Optional) Get the [OpenCV source code](http://opencv.org/)
+  - (Optional) Get the [OpenCV source code](https://opencv.org/)
     and build it by [following these instructions](https://docs.opencv.org/3.4.15/d3/d52/tutorial_windows_install.html#tutorial_windows_install_build)
      - Copy each `opencv_*.lib` to a similar file without the version extension
        (e.g. copy `opencv_ml244.lib` to `opencv_ml.lib`) to help cmake find it