diff --git a/.gitsubprojects b/.gitsubprojects index 0398609c..aa67aee3 100644 --- a/.gitsubprojects +++ b/.gitsubprojects @@ -1,4 +1,5 @@ # -*- mode: cmake -*- +git_subproject(libsonata https://github.com/BlueBrain/libsonata.git ec58d16) git_subproject(Servus https://github.com/HBPVIS/Servus.git 170bd93) git_subproject(Lunchbox https://github.com/Eyescale/Lunchbox.git 9bd6a7d) git_subproject(Keyv https://github.com/BlueBrain/Keyv.git 8340fa2) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b4f89a2..9699fa72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ # cmake_minimum_required(VERSION 3.1 FATAL_ERROR) -project(Brion VERSION 3.0.0) +project(Brion VERSION 3.1.0) set(Brion_VERSION_ABI 10) list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/CMake @@ -43,6 +43,7 @@ common_find_package(PythonLibs) common_find_package(Servus REQUIRED) common_find_package(Sphinx 1.3) common_find_package(vmmlib REQUIRED) +common_find_package(sonata REQUIRED) option(BRION_USE_ZEROEQ "Use ZeroEQ for plugin backend and morphologyServer" OFF) if(BRION_USE_ZEROEQ) git_subproject(ZeroEQ https://github.com/HBPVIS/ZeroEQ.git 1e66ee3) diff --git a/brain/CMakeLists.txt b/brain/CMakeLists.txt index e5dc9ec1..d7f31e87 100644 --- a/brain/CMakeLists.txt +++ b/brain/CMakeLists.txt @@ -51,7 +51,7 @@ set(BRAIN_SOURCES set(BRAIN_PUBLIC_INCLUDE_DIRECTORIES ${Boost_INCLUDE_DIRS}) set(BRAIN_LINK_LIBRARIES PUBLIC Brion vmmlib - PRIVATE Lunchbox ${Boost_FILESYSTEM_LIBRARIES} + PRIVATE sonata::sonata_shared Lunchbox ${Boost_FILESYSTEM_LIBRARIES} ) if(TARGET Keyv) @@ -65,6 +65,8 @@ if(TARGET MVDTool) list(APPEND BRAIN_LINK_LIBRARIES PRIVATE MVDTool) endif() +#set(BRAIN_OMIT_EXPORT ON) + common_library(Brain) if(TARGET MVDTool) target_compile_definitions(Brain PUBLIC BRAIN_USE_MVD3) diff --git a/brain/circuit.cpp b/brain/circuit.cpp index 535894ff..5fc1cc2b 100644 --- a/brain/circuit.cpp +++ b/brain/circuit.cpp @@ -23,8 +23,11 @@ #include "circuit.h" #include "detail/circuit.h" +#include "synapses.h" #include "synapsesStream.h" +#include + #include #include @@ -525,4 +528,44 @@ SynapsesStream Circuit::getProjectedSynapses( { return SynapsesStream(*this, preGIDs, postGIDs, prefetch); } + +uint32_ts Circuit::getProjectedEfferentGIDs( + const GIDSet& preGIDs, const std::string& projection) const +{ + const std::string projPath = _impl->getSynapseProjectionSource(projection); + if(projPath.find("sonata") == std::string::npos) + { + SynapsesStream ss = getExternalAfferentSynapses(preGIDs, projection); + auto future = ss.read(ss.getRemaining()); + future.wait(); + Synapses syn = future.get(); + uint32_ts result (syn.preGIDs(), syn.preGIDs() + syn.size()); + return result; + } + else + { + std::set uniqueIds; + // Input ids must be decreased by 1 + for(uint32_t oldId : preGIDs) + { + if(oldId > 0) + uniqueIds.insert(oldId - 1); + } + + const size_ts nodeIds (uniqueIds.begin(), uniqueIds.end()); + uint32_ts result; + const bbp::sonata::EdgeStorage edgeStorage (projPath); + for(const auto& name : edgeStorage.populationNames()) + { + const bbp::sonata::EdgePopulation edges (projPath, "", name); + const bbp::sonata::Selection s = edges.efferentEdges(nodeIds); + + for(auto gid : edges.targetNodeIDs(s)) + { + result.push_back(static_cast(gid) + 1); + } + } + return result; + } +} } diff --git a/brain/circuit.h b/brain/circuit.h index bbb00847..6e75d723 100644 --- a/brain/circuit.h +++ b/brain/circuit.h @@ -244,6 +244,16 @@ class Circuit const GIDSet& preGIDs, const GIDSet& postGIDs, SynapsePrefetch prefetch = SynapsePrefetch::none) const; + /** + * Return a list of efferent cells GIDs from a given projection + * + * @param preGIDs projection source cell IDs + * @param projection projection name + * @return list of efferent GIDs from inside the circuit + */ + BRAIN_API uint32_ts getProjectedEfferentGIDs( + const GIDSet& preGIDs, const std::string& projection) const; + class Impl; //!< @internal, public for inheritance MVD2/3 impls private: diff --git a/brain/detail/circuit.h b/brain/detail/circuit.h index 3ce8f2b9..d087a0f3 100644 --- a/brain/detail/circuit.h +++ b/brain/detail/circuit.h @@ -414,6 +414,8 @@ class Circuit::Impl virtual URI getMorphologySource() const = 0; virtual MorphologyCache* getMorphologyCache() const { return nullptr; } + virtual std::string getSynapseSource() const = 0; + virtual std::string getSynapseProjectionSource(const std::string& name) const = 0; virtual SynapseCache* getSynapseCache() const { return nullptr; } virtual const brion::SynapseSummary& getSynapseSummary() const = 0; virtual const brion::Synapse& getSynapseAttributes( @@ -679,6 +681,15 @@ class SonataCircuit : public Circuit::Impl } URI getMorphologySource() const final { return morphologySource; } + std::string getSynapseSource() const + { + return synapseSource.getPath(); + } + std::string getSynapseProjectionSource(const std::string& name) const + { + (void)name; + LBTHROW(std::runtime_error("Unimplemented")); + } const brion::SynapseSummary& getSynapseSummary() const final { LBTHROW(std::runtime_error("Unimplemented")); @@ -768,6 +779,20 @@ class BBPCircuit : public Circuit::Impl return nullptr; } + std::string getSynapseSource() const + { + return _synapseSource.getPath(); + } + + std::string getSynapseProjectionSource(const std::string& name) const + { + auto it = _afferentProjectionSources.find(name); + if(it == _afferentProjectionSources.end()) + LBTHROW(std::runtime_error("Projection " + name + " not found")) + + return it->second.getPath(); + } + SynapseCache* getSynapseCache() const final { if (_synapseCache) diff --git a/brain/synapses.cpp b/brain/synapses.cpp index b4399a9e..87ef140c 100644 --- a/brain/synapses.cpp +++ b/brain/synapses.cpp @@ -31,6 +31,8 @@ #include #include +#include + #include #include @@ -69,15 +71,162 @@ bool _hasSurfacePositions(const Circuit::Impl& circuit) #endif } -struct Synapses::Impl : public Synapses::BaseImpl +struct Synapses::InternalBaseImpl : public Synapses::BaseImpl { - Impl(const Circuit& circuit, const GIDSet& gids, const GIDSet& filterGIDs, + InternalBaseImpl(const Circuit& circuit, const GIDSet& gids, const GIDSet& filterGIDs, const bool afferent, const SynapsePrefetch prefetch) : _circuit(circuit._impl) , _gids(prefetch != SynapsePrefetch::all ? gids : GIDSet()) , _filterGIDs(prefetch != SynapsePrefetch::all ? filterGIDs : GIDSet()) , _afferent(afferent) , _size(0) + { + } + + InternalBaseImpl(const Circuit& circuit, const GIDSet& gids, const std::string& source, + const SynapsePrefetch prefetch) + : _circuit(circuit._impl) + , _gids(prefetch != SynapsePrefetch::all ? gids : GIDSet()) + , _afferent(true) + , _externalSource(source) + , _size(0) + { + } + +#define FILTER(gid) \ + if (!filterGIDs->empty() && filterGIDs->find(gid) == filterGIDs->end()) \ + continue; + + virtual ~InternalBaseImpl() {} + + virtual void _loadConnectivity(const GIDSet* gids, const GIDSet* filterGIDs) const = 0; + virtual void _loadAttributes(const GIDSet* gids, const GIDSet* filterGIDs) const = 0; + virtual void _loadPositions(const GIDSet* gids, const GIDSet* filterGIDs) const = 0; + virtual void _ensureGIDs() const = 0; + virtual void _ensureAttributes() const = 0; + virtual void _ensurePositions() const = 0; + + void _allocateAttributes(const size_t size, const bool allocateIndex) const + { + if (allocateIndex) + _allocate(_index, size); + + _allocate(_preSectionID, size); + _allocate(_preSegmentID, size); + _allocate(_preDistance, size); + + _allocate(_postSectionID, size); + _allocate(_postSegmentID, size); + _allocate(_postDistance, size); + + _allocate(_delay, size); + _allocate(_conductance, size); + _allocate(_utilization, size); + _allocate(_depression, size); + _allocate(_facilitation, size); + _allocate(_decay, size); + _allocate(_efficacy, size); + } + + void _allocatePositions(const size_t size) const + { + _allocate(_preSurfacePositionX, size); + _allocate(_preSurfacePositionY, size); + _allocate(_preSurfacePositionZ, size); + _allocate(_preCenterPositionX, size); + _allocate(_preCenterPositionY, size); + _allocate(_preCenterPositionZ, size); + + _allocate(_postSurfacePositionX, size); + _allocate(_postSurfacePositionY, size); + _allocate(_postSurfacePositionZ, size); + _allocate(_postCenterPositionX, size); + _allocate(_postCenterPositionY, size); + _allocate(_postCenterPositionZ, size); + } + + void _clearPositions() const + { + _preSurfacePositionX.reset(); + _preSurfacePositionY.reset(); + _preSurfacePositionZ.reset(); + _preCenterPositionX.reset(); + _preCenterPositionY.reset(); + _preCenterPositionZ.reset(); + _postSurfacePositionX.reset(); + _postSurfacePositionY.reset(); + _postSurfacePositionZ.reset(); + _postCenterPositionX.reset(); + _postCenterPositionY.reset(); + _postCenterPositionZ.reset(); + } + + size_t _getSize() const { return _size; } + + const std::shared_ptr _circuit; + const GIDSet _gids; + const GIDSet _filterGIDs; + const bool _afferent; + std::string _externalSource; + + template + struct FreeDeleter + { + void operator()(T* ptr) { free(ptr); } + }; + + typedef std::unique_ptr> UIntPtr; + typedef std::unique_ptr> IntPtr; + typedef std::unique_ptr> size_tPtr; + typedef std::unique_ptr> floatPtr; + + mutable size_t _size; + + mutable size_tPtr _index; + + mutable UIntPtr _preGID; + mutable UIntPtr _preSectionID; + mutable UIntPtr _preSegmentID; + mutable floatPtr _preDistance; + mutable floatPtr _preSurfacePositionX; + mutable floatPtr _preSurfacePositionY; + mutable floatPtr _preSurfacePositionZ; + mutable floatPtr _preCenterPositionX; + mutable floatPtr _preCenterPositionY; + mutable floatPtr _preCenterPositionZ; + + mutable UIntPtr _postGID; + mutable UIntPtr _postSectionID; + mutable UIntPtr _postSegmentID; + mutable floatPtr _postDistance; + mutable floatPtr _postSurfacePositionX; + mutable floatPtr _postSurfacePositionY; + mutable floatPtr _postSurfacePositionZ; + mutable floatPtr _postCenterPositionX; + mutable floatPtr _postCenterPositionY; + mutable floatPtr _postCenterPositionZ; + + mutable floatPtr _delay; + mutable floatPtr _conductance; + mutable floatPtr _utilization; + mutable floatPtr _depression; + mutable floatPtr _facilitation; + mutable floatPtr _decay; + mutable IntPtr _efficacy; + + mutable std::once_flag _attributeFlag; + mutable std::once_flag _positionFlag; + mutable std::once_flag _indexFlag; + // Besides the call_once flags, we still need to ensure exclusive access + // to state. + mutable std::mutex _mutex; +}; + +struct Synapses::Impl : public Synapses::InternalBaseImpl +{ + Impl(const Circuit& circuit, const GIDSet& gids, const GIDSet& filterGIDs, + const bool afferent, const SynapsePrefetch prefetch) + : Synapses::InternalBaseImpl(circuit, gids, filterGIDs, afferent, prefetch) { _loadConnectivity(&gids, &filterGIDs); @@ -91,11 +240,7 @@ struct Synapses::Impl : public Synapses::BaseImpl Impl(const Circuit& circuit, const GIDSet& gids, const std::string& source, const SynapsePrefetch prefetch) - : _circuit(circuit._impl) - , _gids(prefetch != SynapsePrefetch::all ? gids : GIDSet()) - , _afferent(true) - , _externalSource(source) - , _size(0) + : Synapses::InternalBaseImpl(circuit, gids, source, prefetch) { // We don't have a summary file for projected afferent synapses. // But at least we have to figure out the size of the container. @@ -111,10 +256,6 @@ struct Synapses::Impl : public Synapses::BaseImpl } } -#define FILTER(gid) \ - if (!filterGIDs->empty() && filterGIDs->find(gid) == filterGIDs->end()) \ - continue; - void _loadConnectivity(const GIDSet* gids, const GIDSet* filterGIDs) const { const brion::SynapseSummary& synapseSummary = @@ -345,61 +486,6 @@ struct Synapses::Impl : public Synapses::BaseImpl } } - void _allocateAttributes(const size_t size, const bool allocateIndex) const - { - if (allocateIndex) - _allocate(_index, size); - - _allocate(_preSectionID, size); - _allocate(_preSegmentID, size); - _allocate(_preDistance, size); - - _allocate(_postSectionID, size); - _allocate(_postSegmentID, size); - _allocate(_postDistance, size); - - _allocate(_delay, size); - _allocate(_conductance, size); - _allocate(_utilization, size); - _allocate(_depression, size); - _allocate(_facilitation, size); - _allocate(_decay, size); - _allocate(_efficacy, size); - } - - void _allocatePositions(const size_t size) const - { - _allocate(_preSurfacePositionX, size); - _allocate(_preSurfacePositionY, size); - _allocate(_preSurfacePositionZ, size); - _allocate(_preCenterPositionX, size); - _allocate(_preCenterPositionY, size); - _allocate(_preCenterPositionZ, size); - - _allocate(_postSurfacePositionX, size); - _allocate(_postSurfacePositionY, size); - _allocate(_postSurfacePositionZ, size); - _allocate(_postCenterPositionX, size); - _allocate(_postCenterPositionY, size); - _allocate(_postCenterPositionZ, size); - } - - void _clearPositions() const - { - _preSurfacePositionX.reset(); - _preSurfacePositionY.reset(); - _preSurfacePositionZ.reset(); - _preCenterPositionX.reset(); - _preCenterPositionY.reset(); - _preCenterPositionZ.reset(); - _postSurfacePositionX.reset(); - _postSurfacePositionY.reset(); - _postSurfacePositionZ.reset(); - _postCenterPositionX.reset(); - _postCenterPositionY.reset(); - _postCenterPositionZ.reset(); - } - void _ensureGIDs() const { if (_externalSource.empty()) @@ -422,79 +508,326 @@ struct Synapses::Impl : public Synapses::BaseImpl // Until C++17 call_once is using decay_copy std::call_once(_positionFlag, &Impl::_loadPositions, this, &_gids, &_filterGIDs); + } +}; + +struct Synapses::SonataImpl : public Synapses::InternalBaseImpl +{ + +#define TRY_GET_ATTRIBUTE(buffer, source, type, name, selection) \ + try { \ + buffer = source.getAttribute(name, selection); \ + } catch(...) { } + +#define ADD_TO_STORAGE(storage, storageIndex, source, sourceIndex) \ + if(!source.empty()) \ + storage.get()[storageIndex] = source[sourceIndex]; + + SonataImpl(const Circuit& circuit, const GIDSet& gids, const GIDSet& filterGIDs, + const bool afferent, const SynapsePrefetch prefetch) + : Synapses::InternalBaseImpl(circuit, gids, filterGIDs, afferent, prefetch) + { + _loadConnectivity(&gids, &filterGIDs); + + if (int(prefetch) & int(SynapsePrefetch::attributes)) + std::call_once(_attributeFlag, &SonataImpl::_loadAttributes, this, &gids, + &filterGIDs); + if (int(prefetch) & int(SynapsePrefetch::positions)) + std::call_once(_positionFlag, &SonataImpl::_loadPositions, this, &gids, + &filterGIDs); } - size_t _getSize() const { return _size; } - const std::shared_ptr _circuit; - const GIDSet _gids; - const GIDSet _filterGIDs; - const bool _afferent; - std::string _externalSource; + SonataImpl(const Circuit& circuit, const GIDSet& gids, const std::string& source, + const SynapsePrefetch prefetch) + : Synapses::InternalBaseImpl(circuit, gids, source, prefetch) + { + const std::string projSourceFile = _circuit->getSynapseProjectionSource(source); + // We don't have a summary file for projected afferent synapses. + // But at least we have to figure out the size of the container. + const bbp::sonata::EdgeStorage edgeStorage (projSourceFile); + for(const auto& popName : edgeStorage.populationNames()) + { + const bbp::sonata::EdgePopulation edges (projSourceFile, "", popName); + _size += edges.size(); + } - template - struct FreeDeleter + if (int(prefetch) & int(SynapsePrefetch::attributes)) + { + GIDSet empty; + std::call_once(_attributeFlag, &SonataImpl::_loadAttributes, this, &gids, + &empty); + } + } + + void _loadConnectivity(const GIDSet* gids, const GIDSet* filterGIDs) const { - void operator()(T* ptr) { free(ptr); } - }; + const std::string synapseFilePath = _circuit->getSynapseSource(); - typedef std::unique_ptr> UIntPtr; - typedef std::unique_ptr> IntPtr; - typedef std::unique_ptr> size_tPtr; - typedef std::unique_ptr> floatPtr; + std::set uniqueIds; + // Input ids must be decreased by 1 + for(uint32_t oldId : *gids) + { + if(oldId > 0) + uniqueIds.insert(oldId - 1); + } - mutable size_t _size; + uint32_ts filteredPreGids, filteredPostGids; + const std::vector nodeIds (uniqueIds.begin(), uniqueIds.end()); + const bbp::sonata::EdgeStorage edgeStorage (synapseFilePath); + for(const auto& name : edgeStorage.populationNames()) + { + const bbp::sonata::EdgePopulation edges (synapseFilePath, "", name); + const bbp::sonata::Selection s = _afferent? edges.afferentEdges(nodeIds) + : edges.efferentEdges(nodeIds); - mutable size_tPtr _index; + const size_ts preGidsTemp = edges.sourceNodeIDs(s); + const size_ts postGidsTemp = edges.targetNodeIDs(s); - mutable UIntPtr _preGID; - mutable UIntPtr _preSectionID; - mutable UIntPtr _preSegmentID; - mutable floatPtr _preDistance; - mutable floatPtr _preSurfacePositionX; - mutable floatPtr _preSurfacePositionY; - mutable floatPtr _preSurfacePositionZ; - mutable floatPtr _preCenterPositionX; - mutable floatPtr _preCenterPositionY; - mutable floatPtr _preCenterPositionZ; + // Result ids must be incremented by one to return in the same + // range as the input source GIDs + for(size_t i = 0; i < preGidsTemp.size(); i++) + { + uint32_t preN = static_cast(preGidsTemp[i]); + FILTER(preN) - mutable UIntPtr _postGID; - mutable UIntPtr _postSectionID; - mutable UIntPtr _postSegmentID; - mutable floatPtr _postDistance; - mutable floatPtr _postSurfacePositionX; - mutable floatPtr _postSurfacePositionY; - mutable floatPtr _postSurfacePositionZ; - mutable floatPtr _postCenterPositionX; - mutable floatPtr _postCenterPositionY; - mutable floatPtr _postCenterPositionZ; + filteredPreGids.push_back(preN + 1); + filteredPostGids.push_back( + static_cast(postGidsTemp[i]) + 1); + } + } - mutable floatPtr _delay; - mutable floatPtr _conductance; - mutable floatPtr _utilization; - mutable floatPtr _depression; - mutable floatPtr _facilitation; - mutable floatPtr _decay; - mutable IntPtr _efficacy; + _size = filteredPreGids.size(); + _allocate(_preGID, _size); + _allocate(_postGID, _size); + memcpy(_preGID.get(), filteredPreGids.data(), _size * sizeof(uint32_t)); + memcpy(_postGID.get(), filteredPostGids.data(), _size * sizeof(uint32_t)); + } - mutable std::once_flag _attributeFlag; - mutable std::once_flag _positionFlag; - mutable std::once_flag _indexFlag; - // Besides the call_once flags, we still need to ensure exclusive access - // to state. - mutable std::mutex _mutex; + void _loadAttributes(const GIDSet* gids, const GIDSet* filterGIDs) const + { + std::lock_guard lock(_mutex); + + if (_efficacy) + return; + + std::string synapseFilePath; + if(_externalSource.empty()) + synapseFilePath = _circuit->getSynapseSource(); + else + synapseFilePath = _circuit->getSynapseProjectionSource(_externalSource); + + std::set uniqueIds; + // Input ids must be decreased by 1 + for(uint32_t oldId : *gids) + { + if(oldId > 0) + uniqueIds.insert(oldId - 1); + } + const std::vector nodeIds (uniqueIds.begin(), uniqueIds.end()); + const bbp::sonata::EdgeStorage edgeStorage (synapseFilePath); + + // For external afferent projections we haven't had the chance to + // get the connectivity. + const bool haveGIDs = _externalSource.empty(); + if (!haveGIDs) + { + _allocate(_preGID, _size); + _allocate(_postGID, _size); + } + + _allocateAttributes(_size, _afferent); + + size_t i = 0; + for(const auto& name : edgeStorage.populationNames()) + { + const bbp::sonata::EdgePopulation edges (synapseFilePath, "", name); + const bbp::sonata::Selection s = _afferent? edges.afferentEdges(nodeIds) + : edges.efferentEdges(nodeIds); + + const auto preGIDs = edges.sourceNodeIDs(s); + const auto postGIDs = edges.targetNodeIDs(s); + + std::vector delays; + std::vector decayTimes; + std::vector preSegmentIds; + std::vector preSectionIds; + std::vector postSegmentIds; + std::vector postSectionIds; + std::vector conductances; + std::vector facilitations; + std::vector depressions; + TRY_GET_ATTRIBUTE(delays, edges, float, "delay", s) + TRY_GET_ATTRIBUTE(decayTimes, edges, float, "decay_time", s) + TRY_GET_ATTRIBUTE(preSegmentIds, edges, uint32_t, "afferent_segment_id", s) + TRY_GET_ATTRIBUTE(preSectionIds, edges, uint32_t, "afferent_section_id", s) + TRY_GET_ATTRIBUTE(postSegmentIds, edges, uint32_t, "efferent_segment_id", s) + TRY_GET_ATTRIBUTE(postSectionIds, edges, uint32_t, "efferent_section_id", s) + TRY_GET_ATTRIBUTE(conductances, edges, float, "conductance", s) + TRY_GET_ATTRIBUTE(facilitations, edges, float, "facilitation_time", s) + TRY_GET_ATTRIBUTE(depressions, edges, float, "depression_time", s) + + + for(size_t j = 0; j < edges.size(); j++) + { + FILTER(static_cast(preGIDs[j] + 1)); + if(!haveGIDs) + { + if(preGIDs.size() > j) + _preGID.get()[i] = static_cast(preGIDs[j] + 1); + if(postGIDs.size() > j) + _postGID.get()[i] = static_cast(postGIDs[j] + 1); + } + + ADD_TO_STORAGE(_delay, i, delays, j) + ADD_TO_STORAGE(_decay, i, decayTimes, j) + ADD_TO_STORAGE(_preSectionID, i, preSectionIds, j) + ADD_TO_STORAGE(_preSegmentID, i, preSegmentIds, j) + ADD_TO_STORAGE(_postSectionID, i, postSectionIds, j) + ADD_TO_STORAGE(_postSegmentID, i, postSectionIds, j) + ADD_TO_STORAGE(_conductance, i, conductances, j) + ADD_TO_STORAGE(_facilitation, i, facilitations, j) + ADD_TO_STORAGE(_depression, i, depressions, j) + i++; + } + } + + if(!_afferent) + { + swap(_preGID, _postGID); + } + } + + void _loadPositions(const GIDSet* gids, const GIDSet* filterGIDs) const + { + std::lock_guard lock(_mutex); + + if (!_externalSource.empty()) + { + LBTHROW( + std::runtime_error("Synapse positions are not available " + "for external projection synapses")); + } + + if (_efficacy) + return; + + std::string synapseFilePath = _circuit->getSynapseSource(); + + std::set uniqueIds; + // Input ids must be decreased by 1 + for(uint32_t oldId : *gids) + { + if(oldId > 0) + uniqueIds.insert(oldId - 1); + } + const std::vector nodeIds (uniqueIds.begin(), uniqueIds.end()); + const bbp::sonata::EdgeStorage edgeStorage (synapseFilePath); + + if (_preCenterPositionX) + return; + + _allocatePositions(_size); + + size_t i = 0; + + for(const auto& name : edgeStorage.populationNames()) + { + const bbp::sonata::EdgePopulation edges (synapseFilePath, "", name); + const bbp::sonata::Selection s = _afferent? edges.afferentEdges(nodeIds) + : edges.efferentEdges(nodeIds); + + const auto preGids = edges.sourceNodeIDs(s); + const auto postGids = edges.targetNodeIDs(s); + + const auto preCenterXs = edges.getAttribute("afferent_center_x", s); + const auto preCenterYs = edges.getAttribute("afferent_center_y", s); + const auto preCenterZs = edges.getAttribute("afferent_center_z", s); + const auto preSurfaceXs = edges.getAttribute("afferent_surface_x", s); + const auto preSurfaceYs = edges.getAttribute("afferent_surface_y", s); + const auto preSurfaceZs = edges.getAttribute("afferent_surface_z", s); + const auto postCenterXs = edges.getAttribute("efferent_center_x", s); + const auto postCenterYs = edges.getAttribute("efferent_center_y", s); + const auto postCenterZs = edges.getAttribute("efferent_center_z", s); + const auto postSurfaceXs = edges.getAttribute("efferent_surface_x", s); + const auto postSurfaceYs = edges.getAttribute("efferent_surface_y", s); + const auto postSurfaceZs = edges.getAttribute("efferent_surface_z", s); + + for(size_t j = 0; j < edges.size(); j++) + { + FILTER(static_cast(preGids[j] + 1)) + + ADD_TO_STORAGE(_preCenterPositionX, i, preCenterXs, j) + ADD_TO_STORAGE(_preCenterPositionY, i, preCenterYs, j) + ADD_TO_STORAGE(_preCenterPositionZ, i, preCenterZs, j) + ADD_TO_STORAGE(_preSurfacePositionX, i, preSurfaceXs, j) + ADD_TO_STORAGE(_preSurfacePositionY, i, preSurfaceYs, j) + ADD_TO_STORAGE(_preSurfacePositionZ, i, preSurfaceZs, j) + ADD_TO_STORAGE(_postCenterPositionX, i, postCenterXs, j) + ADD_TO_STORAGE(_postCenterPositionY, i, postCenterYs, j) + ADD_TO_STORAGE(_postCenterPositionZ, i, postCenterZs, j) + ADD_TO_STORAGE(_postSurfacePositionX, i, postSurfaceXs, j) + ADD_TO_STORAGE(_postSurfacePositionY, i, postSurfaceYs, j) + ADD_TO_STORAGE(_postSurfacePositionZ, i, postSurfaceZs, j) + + i++; + } + } + + if(!_afferent) + { + swap(_preCenterPositionX, _postCenterPositionX); + swap(_preCenterPositionY, _postCenterPositionY); + swap(_preCenterPositionZ, _postCenterPositionZ); + swap(_preSurfacePositionX, _postSurfacePositionX); + swap(_preSurfacePositionY, _postSurfacePositionY); + swap(_preSurfacePositionZ, _postSurfacePositionZ); + } + } + + void _ensureGIDs() const + { + if (_externalSource.empty()) + return; + + // Until C++17 call_once is using decay_copy + std::call_once(_attributeFlag, &SonataImpl::_loadAttributes, this, &_gids, + &_filterGIDs); + } + + void _ensureAttributes() const + { + // Until C++17 call_once is using decay_copy + std::call_once(_attributeFlag, &SonataImpl::_loadAttributes, this, &_gids, + &_filterGIDs); + } + + void _ensurePositions() const + { + // Until C++17 call_once is using decay_copy + std::call_once(_positionFlag, &SonataImpl::_loadPositions, this, &_gids, + &_filterGIDs); + } }; Synapses::Synapses(const Circuit& circuit, const GIDSet& gids, const GIDSet& filterGIDs, const bool afferent, const SynapsePrefetch prefetch) - : _impl(new Impl(circuit, gids, filterGIDs, afferent, prefetch)) +// : _impl(new Impl(circuit, gids, filterGIDs, afferent, prefetch)) { + if(circuit._impl->getSynapseSource().find("sonata") != std::string::npos) + _impl = std::shared_ptr(new SonataImpl(circuit, gids, filterGIDs, afferent, prefetch)); + else + _impl = std::shared_ptr(new Impl(circuit, gids, filterGIDs, afferent, prefetch)); } Synapses::Synapses(const Circuit& circuit, const GIDSet& gids, const std::string& source, const SynapsePrefetch prefetch) - : _impl(new Impl(circuit, gids, source, prefetch)) +// : _impl(new Impl(circuit, gids, source, prefetch)) { + if(circuit._impl->getSynapseSource().find("sonata") != std::string::npos) + _impl = std::shared_ptr(new SonataImpl(circuit, gids, source, prefetch)); + else + _impl = std::shared_ptr(new Impl(circuit, gids, source, prefetch)); } Synapses::~Synapses() @@ -510,6 +843,36 @@ Synapses::Synapses(const SynapsesStream& stream) stream._impl->_externalSource, stream._impl->_prefetch)) { + if(stream._impl->_externalSource.empty()) + { + if(stream._impl->_circuit._impl->getSynapseSource().find("sonata") != std::string::npos) + _impl = std::shared_ptr( + new SonataImpl( + stream._impl->_circuit, stream._impl->_gids, + stream._impl->_filterGIDs, stream._impl->_afferent, + stream._impl->_prefetch)); + else + _impl = std::shared_ptr( + new Impl( + stream._impl->_circuit, stream._impl->_gids, + stream._impl->_filterGIDs, stream._impl->_afferent, + stream._impl->_prefetch)); + } + else + { + if(stream._impl->_circuit._impl->getSynapseSource().find("sonata") != std::string::npos) + _impl = std::shared_ptr( + new SonataImpl( + stream._impl->_circuit, stream._impl->_gids, + stream._impl->_externalSource, + stream._impl->_prefetch)); + else + _impl = std::shared_ptr( + new Impl( + stream._impl->_circuit, stream._impl->_gids, + stream._impl->_externalSource, + stream._impl->_prefetch)); + } } Synapses::Synapses(const Synapses& rhs) noexcept : _impl(rhs._impl) diff --git a/brain/synapses.h b/brain/synapses.h index 187a4539..596ab1ce 100644 --- a/brain/synapses.h +++ b/brain/synapses.h @@ -219,13 +219,15 @@ class Synapses /** @internal */ struct BaseImpl { - virtual ~BaseImpl(){}; + virtual ~BaseImpl(){} }; /** @internal */ std::shared_ptr _impl; private: + struct InternalBaseImpl; struct Impl; + struct SonataImpl; friend struct detail::SynapsesStream; Synapses(const Circuit& circuit, const GIDSet& gids, diff --git a/brion/blueConfig.cpp b/brion/blueConfig.cpp index b3aece49..4c9843cd 100644 --- a/brion/blueConfig.cpp +++ b/brion/blueConfig.cpp @@ -356,27 +356,37 @@ brion::URIs BlueConfig::getTargetSources() const const std::string& run = _impl->getRun(); URIs uris; - const auto nrnPath = + auto nrnPath = get(brion::CONFIGSECTION_RUN, run, BLUECONFIG_NRN_PATH_KEY); + boost::trim_right_if(nrnPath, [](auto c) { return c == '/'; }); - const auto circuitPath = + auto circuitPath = get(brion::CONFIGSECTION_RUN, run, BLUECONFIG_CIRCUIT_PATH_KEY); + boost::trim_right_if(circuitPath, [](auto c) { return c == '/'; }); - if (!nrnPath.empty()) + // fs::is_directory may wrongly return true for symlinks + // (even if these are pointing to files) + + // Check if valid nrnPath (on sonata build circuit config files, + // nrnPath is a symlink to highfive edge file) + if (!nrnPath.empty() && fs::exists(nrnPath + "/" + CIRCUIT_TARGET_FILE)) { URI uri; uri.setScheme("file"); - - // If nrnPath is a directory we look for 'start.target' there, otherwise - // we fallback to circuitPath - auto basePath = fs::is_directory(nrnPath) ? nrnPath : circuitPath; - // Trim trailing slashes - boost::trim_right_if(basePath, [](auto c) { return c == '/'; }); - - uri.setPath(basePath + "/" + CIRCUIT_TARGET_FILE); + uri.setPath(nrnPath + "/" + CIRCUIT_TARGET_FILE); + uris.push_back(uri); + } + // Otherwise try to fetch start.target from circuit path + else if(!circuitPath.empty() + && fs::exists(circuitPath + "/" + CIRCUIT_TARGET_FILE)) + { + URI uri; + uri.setScheme("file"); + uri.setPath(circuitPath + "/" + CIRCUIT_TARGET_FILE); uris.push_back(uri); } + // Finally, fetch any user defined target file const std::string& targetPath = get(brion::CONFIGSECTION_RUN, run, BLUECONFIG_TARGET_FILE_KEY); if (!targetPath.empty()) diff --git a/brion/plugin/compartmentReportLegacyHDF5.cpp b/brion/plugin/compartmentReportLegacyHDF5.cpp index a2f2ac24..2cd7b490 100644 --- a/brion/plugin/compartmentReportLegacyHDF5.cpp +++ b/brion/plugin/compartmentReportLegacyHDF5.cpp @@ -274,7 +274,7 @@ bool CompartmentReportLegacyHDF5::writeCompartments(const uint32_t gid, for (size_t k = 0; k < counts[j]; ++k) mapping[0][i++] = j; - dataset.write(mapping); + dataset.write(mapping.data()); return true; } catch (const HighFive::Exception& e) diff --git a/brion/synapse.cpp b/brion/synapse.cpp index bab0fbfc..d5e0aaeb 100644 --- a/brion/synapse.cpp +++ b/brion/synapse.cpp @@ -141,7 +141,7 @@ class SynapseFile : public boost::noncopyable const auto selection = dataset.dataset->select(columns); SynapseMatrix values(boost::extents[dataset.dims.first][bits.count()]); - selection.read(values); + selection.read(values.data()); return values; } diff --git a/brion/synapseSummary.cpp b/brion/synapseSummary.cpp index 168fdada..ab60d8dc 100644 --- a/brion/synapseSummary.cpp +++ b/brion/synapseSummary.cpp @@ -91,7 +91,7 @@ class SynapseSummary return SynapseSummaryMatrix(); SynapseSummaryMatrix values(boost::extents[_dims[0]][_dims[1]]); - _dataset->read(values); + _dataset->read(values.data()); return values; } diff --git a/doc/Changelog.md b/doc/Changelog.md index 031e98e0..96d8b662 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -1,6 +1,15 @@ Changelog {#Changelog} ========= +# Release 3.1.0 (07-01-2020) + +* [269](https://github.com/BlueBrain/Brion/pull/269): + Introduce SONATA support +* [263](https://github.com/BlueBrain/Brion/pull/263): + Detect start.target in CircuitPath +* [255](https://github.com/BlueBrain/Brion/pull/255): + Adding Python 3.7 and removing 3.4 from wheels + # Release 3.0.0 (20-03-2019) * [250](https://github.com/BlueBrain/Brion/pull/250):