Merge for 2.18.0 release

CMakeLists.txt

@@ -231,7 +231,8 @@ include_directories("${CMAKE_BINARY_DIR}/include")
 # Keep track of all variables relating to dependencies
 set(IMP_ALL_DEPENDS_VARS BOOST.FILESYSTEM_LIBRARIES BOOST.SYSTEM_LIBRARIES
     BOOST.THREAD_LIBRARIES BOOST.PROGRAMOPTIONS_LIBRARIES BOOST.REGEX_LIBRARIES
-    BOOST.GRAPH_LIBRARIES BOOST.RANDOM_LIBRARIES CACHE INTERNAL "" FORCE)
+    BOOST.GRAPH_LIBRARIES BOOST.RANDOM_LIBRARIES
+    BOOST.SERIALIZATION_LIBRARIES CACHE INTERNAL "" FORCE)
 
 imp_execute_process("setup" ${CMAKE_BINARY_DIR}
                     COMMAND ${PYTHON_EXECUTABLE}
@@ -268,7 +269,7 @@ else()
 endif()
 
 set(Boost_NO_BOOST_CMAKE ON)
-find_package(Boost 1.53.0 COMPONENTS system filesystem thread program_options REQUIRED)
+find_package(Boost 1.53.0 COMPONENTS system filesystem thread program_options serialization REQUIRED)
 if("${Boost_SYSTEM_LIBRARY_RELEASE}" MATCHES ".*NOTFOUND.*")
   message(FATAL_ERROR "Boost is required to build IMP.")
 endif()
@@ -310,13 +311,21 @@ link_directories(${Boost_LIBRARY_DIRS})
 set(BOOST.FILESYSTEM_LIBRARIES ${Boost_FILESYSTEM_LIBRARY_RELEASE})
 set(BOOST.SYSTEM_LIBRARIES ${Boost_SYSTEM_LIBRARY_RELEASE})
 set(BOOST.THREAD_LIBRARIES ${Boost_THREAD_LIBRARY_RELEASE})
+if(IMP_STATIC)
+# Boost.Serialization needs pthread, but order is important with static
+# libraries; we must link pthread *after* serialization
+set(BOOST.SERIALIZATION_LIBRARIES ${Boost_SERIALIZATION_LIBRARY_RELEASE} -lpthread)
+else()
+set(BOOST.SERIALIZATION_LIBRARIES ${Boost_SERIALIZATION_LIBRARY_RELEASE})
+endif()
 set(BOOST.PROGRAMOPTIONS_LIBRARIES ${Boost_PROGRAM_OPTIONS_LIBRARY_RELEASE})
 set(BOOST.REGEX_LIBRARIES ${Boost_REGEX_LIBRARY_RELEASE})
 set(BOOST.GRAPH_LIBRARIES ${Boost_GRAPH_LIBRARY_RELEASE})
 set(BOOST.RANDOM_LIBRARIES ${Boost_RANDOM_LIBRARY_RELEASE})
 
 file(WRITE "${CMAKE_BINARY_DIR}/build_info/Boost.FileSystem" "ok=True")
 file(WRITE "${CMAKE_BINARY_DIR}/build_info/Boost.ProgramOptions" "ok=True")
+file(WRITE "${CMAKE_BINARY_DIR}/build_info/Boost.Serialization" "ok=True")
 file(WRITE "${CMAKE_BINARY_DIR}/build_info/Boost.System" "ok=True")
 file(WRITE "${CMAKE_BINARY_DIR}/build_info/Boost.Thread" "ok=True")
 

ChangeLog.md

@@ -1,6 +1,42 @@
 ChangeLog {#changelog}
 =========
 
+# 2.18.0 - 2022-12-15 # {#changelog_2_18_0}
+- The Windows .exe installer now supports Python 3.11.
+- The IMP::em::emreal type is deprecated; use plain `double` instead.
+- Most IMP Value types (e.g. IMP::algebra::Vector3D) now support serialization,
+  so can be written to or read from a stream in C++ using the
+  Boost.Serialization library, or pickled in Python.
+- IMP::atom::CHARMMTopology::add_sequence() now allows for full residue names
+  to be specified, to simplify construction of structures of DNA or RNA.
+- IMP::atom::CHARMMTopology::create_hierarchy() now supports construction of
+  more than 26 chains, assigning them multi-character chain IDs.
+- Objects that contain lists of other objects now provide a list-like object
+  in Python to simplify management of the list (e.g. IMP::RestraintSet now
+  contains a `restraints` member in Python which is a list of the IMP::Restraint
+  objects in the set).
+- The NPC-specific PMI restraints in the IMP.npc.npc_restraints module (which
+  only work with PMI1) are now provided in IMP.pmi.restraints.npc (for current
+  PMI) and IMP.pmi1.restraints.npc (for the old PMI1). IMP.npc.npc_restraints
+  itself is deprecated.
+- IMP.pmi.macros.ReplicaExchange0 has been renamed to
+  IMP.pmi.macros.ReplicaExchange; the old name is still present but deprecated.
+- The obsolete `sample_objects` and `crosslink_restraints` arguments to
+  IMP.pmi.macros.ReplicaExchange have been removed.
+- Passing anything other than Monte Carlo movers in the
+  `monte_carlo_sample_objects` argument to IMP.pmi.macros.ReplicaExchange
+  is now deprecated.
+- The IMP.sampcon module can now provide inputs for the PrISM method, and has
+  improved support for symmetry groups (requires pyRMSD 4.3.2 or later).
+- Bugfix: move-aware scoring (see IMP::core::MonteCarlo::set_score_moved())
+  should now work correctly with nested RestraintSets and/or non-default
+  weights.
+- Bugfix: IMP::em::DensityMap::add() now invalidates any previously-calculated
+  RMS, so that followup calculations (e.g. of cross correlation) now work
+  correctly.
+- Bugfix: all C++ code now uses the same IMP random number generator, so
+  that resetting the seed now yields a consistent sequence of random numbers.
+
 # 2.17.0 - 2022-06-23 # {#changelog_2_17_0}
 - IMP::atom::Selection no longer checks that the passed hierarchies are valid,
   as this can be quite computationally expensive. If in doubt, call

VERSION

@@ -1 +1 @@
-2.17.0
+2.18.0

doc/manual/cmake_config.md

@@ -89,6 +89,12 @@ by telling CMake where the Homebrew Python headers and library
 are, by adding to your CMake invocation something like
 `-DPYTHON_LIBRARY=/usr/local/opt/python@2/Frameworks/Python.framework/Versions/Current/lib/libpython2.7.dylib -DPYTHON_INCLUDE_DIR=/usr/local/opt/python@2/Frameworks/Python.framework/Versions/Current/Headers`
 
+## Wrong version of Python found {#cmake_pyver}
+
+CMake will attempt to use the newest version of Python available on your
+machine. If you want to use an older version, one way to do this is to set
+the Python executable you want to use, e.g. `-DPython3_EXECUTABLE=/usr/local/bin/python3`.
+
 ## CMake reports that it found a dependency but then reports failed {#cmake_compile}
 
 For each dependency CMake will first try to find the header and library
@@ -100,9 +106,10 @@ found). To fix issues like this, check the CMake error log in
 `CMakeFiles/CMakeError.log` to see what failed. In some cases this can be
 fixed by modifying the flags passed to the C or C++ compiler. For example,
 recent versions of [Protobuf](https://developers.google.com/protocol-buffers/)
-fail on some systems because they require C++11 support, and this can be
-fixed by adding to your CMake invocation
-`-DCMAKE_CXX_FLAGS="-std=c++11"`
+or [CGAL](https://www.cgal.org/)
+fail on some systems because they require C++17 support, and this
+can be fixed by adding to your CMake invocation
+`-DCMAKE_CXX_FLAGS="-std=c++17"`
 
 ## Wrong version of helper binaries found {#cmake_path}
 

doc/manual/conventions.md

@@ -21,7 +21,7 @@ and interface conventions.
    - `load_` and `save_` or `read_` and `write_` move data between files and memory
    - `link_` create a connection between something and an IMP::Object
    - `update_` change the internal state of an IMP::Object
-   - `do_` is a virtual method as part of a \external{http://en.wikipedia.org/wiki/Non-virtual_interface_pattern,non-virtual interface pattern}
+   - `do_` is a virtual method as part of a \external{https://en.wikipedia.org/wiki/Non-virtual_interface_pattern,non-virtual interface pattern}
    - `handle_` take action when an event occurs
    - `validate_` check the state of data and print messages and throw exceptions if something is corrupted
    - `setup_` and `teardown_` create or destroy some type of invariant (e.g. the constraints for a rigid body)
@@ -42,9 +42,9 @@ The Boost.Graph interface cannot be easily exported to Python so we instead prov
 As is conventional in C++, IMP classes are divided into broad, exclusive types
 - *Object classes*: They inherit from IMP::Object and are always passed by pointer. They are reference counted and so should only be stored using IMP::Pointer in C++ (in Python everything is reference counted). Never allocate these on the stack as very bad things can happen. Objects cannot be duplicated. Equality on objects is defined as identity (e.g. two different objects are different even if the data they contain is identical).
 
-- *Value classes* which are normal data types. They are passed by value (or `const&`), never by pointer. Equality is defined based on the data stored in the value. Most value types in IMP are always valid, but a few, mostly geometric types (IMP::algebra::Vector3D) are designed for fast, low-level use and are left in an uninitialized state by their default constructor.
+- *Value classes* which are normal data types. They are passed by value (or `const&`), never by pointer. Equality is defined based on the data stored in the value. Most value types in IMP are always valid, but a few, mostly geometric types (IMP::algebra::Vector3D) are designed for fast, low-level use and are left in an uninitialized state by their default constructor. Most Values can also be serialized using the [Boost.Serialization](https://www.boost.org/doc/libs/1_78_0/libs/serialization/doc/) library in C++, or [pickled](https://docs.python.org/3/library/pickle.html) in Python.
 
-- *RAII classes* control some particular resource using the [RAII idiom](http://en.wikipedia.org/wiki/Resource_Acquisition_Is_Initialization).
+- *RAII classes* control some particular resource using the [RAII idiom](https://en.wikipedia.org/wiki/Resource_acquisition_is_initialization).
 They grab control of a resource when created and then free it when they are destroyed. As a result, they cannot be copied. Non-IMP examples include things like files in Python, which are automatically closed when the file object is deleted.
 
 All types in %IMP, with a few documented exceptions, can be

doc/manual/installation.md

@@ -76,6 +76,13 @@ requested otherwise (see [CMake](@ref cmake_config) for more information):
 - [python-ihm](https://github.com/ihmwg/python-ihm) for handling mmCIF and
   BinaryCIF files.
 
+(Note that if you build a stable release of %IMP from source code, using
+versions of dependencies that were released _after_ that %IMP release
+(e.g. a brand new version of Python), you may run into build issues.
+Either use older versions of the dependencies, or look at the
+[patches we've applied to the conda package](https://github.com/conda-forge/imp-feedstock/blob/main/recipe/meta.yaml)
+and apply them to your source code checkout.)
+
 ### Getting prerequisites on Linux {#installation_prereqs_linux}
 All of the prerequisites should be available as pre-built packages for
 your Linux distribution of choice. For example, on a Fedora system the

doc/ref/mainpage.md

@@ -37,6 +37,6 @@ dependencies. See the doc page for each module for details.
 | IMP::domino | | | IMP::domino::DominoSampler, IMP::domino::BranchAndBoundSampler | |
 | IMP::modeller | | Access to the Modeller scoring functions | | |
 | IMP::isd | | IMP::isd::GaussianEMRestraint, IMP::isd::AtomicCrossLinkMSRestraint | | |
-| IMP::pmi | IMP::pmi::topology, IMP::pmi::dof | IMP::pmi::restraints | IMP::pmi::macros::ReplicaExchange0 | IMP::pmi::analysis |
+| IMP::pmi | IMP::pmi::topology, IMP::pmi::dof | IMP::pmi::restraints | IMP::pmi::macros::ReplicaExchange | IMP::pmi::analysis |
 
 

modules/EMageFit/pyext/src/monte_carlo.py

@@ -36,7 +36,7 @@ def set_random_seed(index):
         log.debug("Seed for the random number generator: %s", index)
         IMP.random_number_generator.seed(index)
     else:
-        IMP.random_number_generator.seed(time.time())
+        IMP.random_number_generator.seed(int(time.time()))
 
 
 class MonteCarloRelativeMoves:

modules/EMageFit/test/expensive_test_3sfd_example.py

@@ -11,13 +11,13 @@ def make_dummy_hex(self, input_file_dir):
            install HEXDOCK."""
         os.environ['PATH'] = os.getcwd() + os.pathsep + os.environ['PATH']
         with open('hex', 'w') as fh:
-            fh.write("""#!/usr/bin/env python
+            fh.write("""#!%s
 import sys, shutil
 for line in sys.stdin.readlines():
     spl = line.rstrip('\\r\\n').split()
     if len(spl) == 2 and spl[0] == 'save_transform':
         shutil.copy('%s/' + spl[1], spl[1])
-""" % input_file_dir)
+""" % (sys.executable, input_file_dir))
         os.chmod('hex', 0o755)
 
     def test_3sfd(self):

modules/algebra/dependencies.py

@@ -1,4 +1,3 @@
 required_modules = 'kernel'
 optional_modules = 'cgal'
-required_dependencies = ''
 optional_dependencies = 'ANN'

modules/algebra/include/BoundingBoxD.h

@@ -13,6 +13,7 @@
 #include "internal/utility.h"
 #include "algebra_macros.h"
 #include <IMP/exception.h>
+#include <boost/serialization/access.hpp>
 
 IMPALGEBRA_BEGIN_NAMESPACE
 
@@ -157,6 +158,12 @@ class BoundingBoxD {
 
  private:
   VectorD<D> b_[2];
+
+  friend class boost::serialization::access;
+
+  template<class Archive> void serialize(Archive &ar, const unsigned int) {
+    ar & b_[0] & b_[1];
+  }
 };
 //! See BoundingBoxD
 template <int D>

modules/algebra/include/Cone3D.h

@@ -16,6 +16,7 @@
 #include "GeometricPrimitiveD.h"
 #include <IMP/showable_macros.h>
 #include <IMP/warning_macros.h>
+#include <boost/serialization/access.hpp>
 #include <iostream>
 #include "constants.h"
 
@@ -56,6 +57,12 @@ class IMPALGEBRAEXPORT Cone3D : public GeometricPrimitiveD<3> {
  private:
   Segment3D seg_;
   double radius_;
+
+  friend class boost::serialization::access;
+
+  template<class Archive> void serialize(Archive &ar, const unsigned int) {
+    ar & seg_ & radius_;
+  }
 };
 
 IMP_VOLUME_GEOMETRY_METHODS(Cone3D, cone_3d, IMP_UNUSED(g);

modules/algebra/include/Cylinder3D.h

@@ -14,6 +14,7 @@
 #include "Segment3D.h"
 #include "GeometricPrimitiveD.h"
 #include <IMP/showable_macros.h>
+#include <boost/serialization/access.hpp>
 #include <iostream>
 #include "constants.h"
 
@@ -66,6 +67,12 @@ class IMPALGEBRAEXPORT Cylinder3D : public GeometricPrimitiveD<3> {
  private:
   Segment3D s_;
   double radius_;
+
+  friend class boost::serialization::access;
+
+  template<class Archive> void serialize(Archive &ar, const unsigned int) {
+    ar & s_ & radius_;
+  }
 };
 
 IMP_VOLUME_GEOMETRY_METHODS(Cylinder3D, cylinder_3d,

modules/algebra/include/Ellipsoid3D.h

@@ -14,6 +14,7 @@
 #include "Transformation3D.h"
 #include "ReferenceFrame3D.h"
 #include "GeometricPrimitiveD.h"
+#include <boost/serialization/access.hpp>
 
 IMPALGEBRA_BEGIN_NAMESPACE
 
@@ -33,6 +34,12 @@ class IMPALGEBRAEXPORT Ellipsoid3D : public GeometricPrimitiveD<3> {
  private:
   ReferenceFrame3D rf_;
   Vector3D radii_;
+
+  friend class boost::serialization::access;
+
+  template<class Archive> void serialize(Archive &ar, const unsigned int) {
+    ar & rf_ & radii_;
+  }
 };
 
 IMP_VOLUME_GEOMETRY_METHODS(Ellipsoid3D, ellipsoid_3d, IMP_UNUSED(g);

modules/algebra/include/Gaussian3D.h

@@ -14,6 +14,7 @@
 #include "ReferenceFrame3D.h"
 #include <IMP/algebra/VectorD.h>
 #include <IMP/algebra/standard_grids.h>
+#include <boost/serialization/access.hpp>
 #include <Eigen/Dense>
 
 IMPALGEBRA_BEGIN_NAMESPACE
@@ -25,6 +26,12 @@ class Gaussian3D : public GeometricPrimitiveD<3> {
   ReferenceFrame3D tr_;
   Vector3D variances_;
 
+  friend class boost::serialization::access;
+
+  template<class Archive> void serialize(Archive &ar, const unsigned int) {
+    ar & tr_ & variances_;
+  }
+
 public:
   Gaussian3D() {
     tr_ = ReferenceFrame3D();

modules/algebra/include/Line3D.h

@@ -14,6 +14,7 @@
 #include "BoundingBoxD.h"
 #include "algebra_macros.h"
 #include "GeometricPrimitiveD.h"
+#include <boost/serialization/access.hpp>
 
 IMPALGEBRA_BEGIN_NAMESPACE
 //! Simple implementation of lines in 3D
@@ -28,6 +29,12 @@ IMPALGEBRA_BEGIN_NAMESPACE
 class IMPALGEBRAEXPORT Line3D : public GeometricPrimitiveD<3> {
   Vector3D l_, m_;
 
+  friend class boost::serialization::access;
+
+  template<class Archive> void serialize(Archive &ar, const unsigned int) {
+    ar & l_ & m_;
+  }
+
   public:
     Line3D() {}
     Line3D(const Vector3D &direction, const Vector3D &point_on_line);

modules/algebra/include/LinearFit.h

@@ -11,13 +11,16 @@
 
 #include "Vector2D.h"
 #include "GeometricPrimitiveD.h"
+#include <boost/serialization/access.hpp>
 #include <vector>
 
 IMPALGEBRA_BEGIN_NAMESPACE
 
 //! Calculate line that fits best the input data points (Linear least squares)
 class IMPALGEBRAEXPORT LinearFit2D : public GeometricPrimitiveD<2> {
  public:
+  LinearFit2D() {}
+
   //! Constructor
   /**
      \param[in] data vector of pairs (VectorD<2>) with x and their
@@ -47,6 +50,12 @@ class IMPALGEBRAEXPORT LinearFit2D : public GeometricPrimitiveD<2> {
   void evaluate_error(const Vector2Ds& data, const Floats& errors);
   double a_, b_;
   double error_;
+
+  friend class boost::serialization::access;
+
+  template<class Archive> void serialize(Archive &ar, const unsigned int) {
+    ar & a_ & b_ & error_;
+  }
 };
 
 IMP_VALUES(LinearFit2D, LinearFit2Ds);

modules/algebra/include/ParabolicFit.h

@@ -11,13 +11,16 @@
 
 #include "Vector2D.h"
 #include "GeometricPrimitiveD.h"
+#include <boost/serialization/access.hpp>
 #include <vector>
 
 IMPALGEBRA_BEGIN_NAMESPACE
 
 //! Calculate parabola that fits best the input data points
 class IMPALGEBRAEXPORT ParabolicFit2D : public GeometricPrimitiveD<2> {
  public:
+  ParabolicFit2D() {}
+
   //! Constructor
   /**
      \param[in] data vector of pairs (VectorD<2>) with x and their
@@ -49,6 +52,12 @@ class IMPALGEBRAEXPORT ParabolicFit2D : public GeometricPrimitiveD<2> {
  private:
   double a_, b_, c_;
   double error_;
+
+  friend class boost::serialization::access;
+
+  template<class Archive> void serialize(Archive &ar, const unsigned int) {
+    ar & a_ & b_ & c_ & error_;
+  }
 };
 
 IMP_VALUES(ParabolicFit2D, ParabolicFit2Ds);

modules/algebra/include/Plane3D.h

@@ -12,6 +12,7 @@
 #include "Vector3D.h"
 #include "BoundingBoxD.h"
 #include "GeometricPrimitiveD.h"
+#include <boost/serialization/access.hpp>
 
 IMPALGEBRA_BEGIN_NAMESPACE
 
@@ -53,9 +54,16 @@ class Plane3D : public GeometricPrimitiveD<3> {
   Plane3D get_opposite() const { return Plane3D(-distance_, -normal_); }
   double get_distance_from_origin() const { return distance_; }
 
- private:
+private:
   double distance_;
   Vector3D normal_;  // normal to plane
+
+  friend class boost::serialization::access;
+
+  template<class Archive> void serialize(Archive &ar, const unsigned int) {
+    ar & distance_ & normal_;
+  }
+
 };
 
 //! Return the distance between a plane and a point in 3D