ParallelGravity.ci

//ParallelGravity.ci

mainmodule ParallelGravity {

  extern module Reductions;

  include "keytype.h";

  readonly CProxy_Main mainChare;
  readonly int verbosity;
  readonly CProxy_TreePiece treeProxy;
#ifdef REDUCTION_HELPER
  readonly CProxy_ReductionHelper reductionHelperProxy;
#endif
  readonly CProxy_LvArray lvProxy;
  readonly CProxy_LvArray smoothProxy;
  readonly CProxy_LvArray gravityProxy;
  readonly CProxy_TreePiece streamingProxy;
  readonly CProxy_CkCacheManager<KeyType> cacheNode;
  readonly CProxy_CkCacheManager<KeyType> cacheGravPart;
  readonly CProxy_CkCacheManager<KeyType> cacheSmoothPart;
  readonly CProxy_DataManager dMProxy;
  readonly CProxy_PETreeMerger peTreeMergerProxy;
  readonly CProxy_DumpFrameData dfDataProxy;
  readonly CProxy_IntraNodeLBManager nodeLBMgrProxy;

  readonly bool _cache;
  readonly int _nocache;
  readonly int _cacheLineDepth;
  readonly unsigned int _yieldPeriod;
  readonly DomainsDec domainDecomposition;
  readonly double dExtraStore;
  readonly double dMaxBalance;
  readonly double dFracLoadBalance;
  readonly double dGlassDamper;
  readonly int bUseCkLoopPar;
  readonly int peanoKey;
  readonly GenericTrees useTree;
  readonly int _prefetch;
  readonly int _randChunks;
  readonly int _numChunks;
  readonly CkArrayID treePieceID;
  readonly CkGroupID dataManagerID;
  readonly unsigned int numTreePieces;
  readonly unsigned int particlesPerChare;
  readonly unsigned int bucketSize;
  readonly int nIOProcessor;
  // jetley
  readonly int localNodesPerReq;
  readonly int remoteNodesPerReq;
  readonly int remoteResumeNodesPerReq;
  readonly int localPartsPerReq;
  readonly int remotePartsPerReq;
  readonly int remoteResumePartsPerReq;
  readonly double largePhaseThreshold;

  readonly int START_REG;
  readonly int START_IB;
  readonly int START_PW;

  readonly int boundaryEvaluationUE;
  readonly int weightBalanceUE;
  readonly int networkProgressUE;
  readonly int nodeForceUE;
  readonly int partForceUE;

  readonly int tbFlushRequestsUE;
  readonly int prefetchDoneUE;

#ifdef PUSH_GRAVITY
  readonly CkGroupID ckMulticastGrpId;
#endif

  readonly CProxy_ProjectionsControl prjgrp;

  // number of bins to use for the first
  // iteration of every decomposition step
  readonly int numInitDecompBins;

  // Specifies the number of sub-bins a bin is split into
  //  for Oct decomposition
  readonly int octRefineLevel;
  readonly int doDumpLB;
  readonly int lbDumpIteration;
  readonly int doSimulateLB;

  message dummyMsg;
  message ComputeChunkMsg;

  message EwaldMsg{
    bool fromInit;
  };

  message ORBSplittersMsg{
    double pos[];
    char dim[];
  };

  message ParticleShuffleMsg {
    double loads[];
    unsigned int parts_per_phase[];
    GravityParticle particles[];
    extraSPHData pGas[];
    extraStarData pStar[];
  };

#ifdef PUSH_GRAVITY
  message BucketMsg {
    GenericTreeNode buckets[];
    ExternalGravityParticle particles[];
  };
#endif

  PUPable DensitySmoothParams;
  PUPable DenDvDxSmoothParams;
  PUPable DenDvDxNeighborSmParams;
  PUPable MarkSmoothParams;
  PUPable PressureSmoothParams;
  PUPable DistDeletedGasSmoothParams;
  PUPable AGORApreCheckSmoothParams;
  PUPable DistStellarFeedbackSmoothParams;
  PUPable SinkFormTestSmoothParams;
  PUPable SinkFormSmoothParams;
  PUPable BHDensitySmoothParams;
  PUPable BHAccreteSmoothParams;
  PUPable BHIdentifySmoothParams;
  PUPable BHSinkMergeSmoothParams;
  PUPable SinkAccreteTestSmoothParams;
  PUPable SinkingAverageSmoothParams;
  PUPable SinkAccreteSmoothParams;
  PUPable Fdbk;
  PUPable StarLog;
  PUPable Chabrier;
  PUPable MillerScalo;
  PUPable Kroupa93;
  PUPable Kroupa01;

#ifdef SUPERBUBBLE
  PUPable PromoteToHotGasSmoothParams;
  PUPable ShareWithHotGasSmoothParams;
  PUPable uHotOutputParams;
  PUPable uOutputParams;
  PUPable MassHotOutputParams;
  PUPable TempEffOutputParams;
#endif
  //SIDM
  PUPable SIDMSmoothParams;
#ifdef COLLISION
  PUPable CollisionSmoothParams;
#endif

  PUPable MassOutputParams;
  PUPable PosOutputParams;
  PUPable VelOutputParams;
  PUPable PotOutputParams;
#ifdef COLLISION
  PUPable SpinOutputParams;
#endif
  PUPable GasDenOutputParams;
  PUPable TempOutputParams;
  PUPable AccOutputParams;
  PUPable DenOutputParams;
  PUPable HsmOutputParams;
  PUPable SoftOutputParams;
  PUPable PresOutputParams;
  PUPable DivVOutputParams;
  PUPable PDVOutputParams;
  PUPable MuMaxOutputParams;
  PUPable BSwOutputParams;
  PUPable CsOutputParams;
  PUPable EDotOutputParams;
  PUPable Cool0OutputParams;
  PUPable Cool1OutputParams;
  PUPable Cool2OutputParams;
  PUPable Cool3OutputParams;
#ifdef COOLING_MOLECULARH
  PUPable LWOutputParams;
#endif /*COOLING_MOLECULARH*/
  PUPable OxOutputParams;
  PUPable FeOutputParams;
  PUPable MetalsOutputParams;
  PUPable MetalsDotOutputParams;
  PUPable OxygenMassFracDotOutputParams;
  PUPable IronMassFracDotOutputParams;
  PUPable MFormOutputParams;
  PUPable TimeFormOutputParams;
  PUPable AgeOutputParams;
  PUPable coolontimeOutputParams;
  PUPable ESNRateOutputParams;
  PUPable AlphaOutputParams;
  PUPable DvDsOutputParams;
  PUPable DtOutputParams;
  PUPable KeyOutputParams;
  PUPable DomainOutputParams;
  PUPable RungOutputParams;
  PUPable IOrderOutputParams;
  PUPable IGasOrderOutputParams;
  PUPable iNSIDMOutputParams;


  mainchare [migratable] Main {
    entry Main(CkArgMsg*);
    entry [threaded] void setupICs();
    entry [threaded] void initialForces();
    entry [threaded] void doSimulation();
    entry [threaded] void restart(CkCheckpointStatusMsg *msg);
    entry [threaded] void cbOpen(Ck::IO::FileReadyMsg *msg);
    entry void cbIOReady(Ck::IO::SessionReadyMsg *msg);
    entry void cbIOComplete(CkMessage *msg);
    entry [threaded] void cbIOClosed(CkMessage *msg);
    entry void niceExit();
    entry void liveVizImagePrep(liveVizRequestMsg *msg);
  };

  group [migratable] ProjectionsControl {
    entry ProjectionsControl();
    entry [notrace] void on(CkCallback);
    entry [notrace] void off(CkCallback);                     
  };                                                          

#ifdef REDUCTION_HELPER
  group [migratable] ReductionHelper {
    entry ReductionHelper();
    entry void countTreePieces(const CkCallback &cb);
    entry [nokeep] void evaluateBoundaries(SFC::Key keys[n], const int n, int isRefine, const CkCallback& cb);

    entry void evaluateBoundaries(const CkBitVector &binsToSplit, const CkCallback& cb);
  };
#endif

  chare [migratable] Sorter {
    entry Sorter();

    entry [threaded]
	void startSorting(const CkGroupID& dataManagerID,
        		const double toler,
        		const CkCallback& cb, bool decompose);
    entry void collectEvaluations(CkReductionMsg* m);
    entry void collectORBCounts(CkReductionMsg* m);
    entry void finishPhase(CkReductionMsg* m);
    entry void doORBDecomposition(CkReductionMsg* m);
    entry void readytoSendORB(CkReductionMsg* m);
  };

  nodegroup [migratable] DataManager {
    entry DataManager(const CkArrayID& treePieceID);

    entry void acceptResponsibleIndex(const int responsible[n], const int n,
				      const CkCallback& cb);
    entry void acceptFinalKeys(const SFC::Key keys[n],
        const int responsible[n - 1],
        uint64_t bins[n - 1], const int n,
        const CkCallback& cb);
    entry void clearRegisteredPieces(const CkCallback& cb);
    entry void combineLocalTrees(CkReductionMsg *m);
#ifdef CUDA
    entry void startLocalWalk();
    entry void resumeRemoteChunk();
    entry void createStreams(int _numStreams, const CkCallback& cb);
    entry void unmarkTreePiecesForCleanup(const CkCallback& cb);
#endif
    entry void initCooling(double dGmPerCcUnit, double dComovingGmPerCcUnit,
		       double dErgPerGmUnit, double dSecUnit, double dKpcUnit,
			 COOLPARAM inParam, const CkCallback& cb);
    entry void dmCoolTableRead(double dTableData[nData], int nData, const CkCallback& cb);
    entry void CoolingSetTime(double z, double dTime, const CkCallback& cb);
    entry void SetStarCM(double dCenterOfMass[4], const CkCallback& cb);
    entry void memoryStats(const CkCallback& cb);
    entry void resetReadOnly(Parameters param, const CkCallback &cb);
    entry void initStarLog(std::string _fileName, const CkCallback &cb);
  };

  array [1D] TreePiece {
    entry TreePiece();

    // DEBUGGING
    entry void quiescence();
    entry void memCacheStats(const CkCallback &cb);

    // entry void report();

    entry void nextBucket(dummyMsg *m);	
    entry void nextBucketSmooth(dummyMsg *msg);
    entry void nextBucketReSmooth(dummyMsg *msg);
    entry void nextBucketMarkSmooth(dummyMsg *msg);

    entry void resetObjectLoad(const CkCallback& cb);
    entry void setPeriodic(int nReplicas, Vector3D<cosmoType> fPeriod, int bEwald,
                           double fEwCut, double fEwhCut, int bPeriod,
                           int bComove, double dRhoFac);
    entry [notrace] void EwaldInit();
    entry [notrace] void initCoolingData(const CkCallback& cb);
    entry void calculateEwald(EwaldMsg *m);
    entry [notrace] void EwaldGPUComplete(); 
    entry void EwaldGPU();
    entry void velScale(double dScale, const CkCallback& cb);
    //entry void markPresence(const CkCallback& cb);
    entry void loadNChilada(const std::string& filename, const double dTuFac,
                            const CkCallback& cb);
    entry void readFloatBinary(CkReference<OutputParams> params, int bParaRead,
                               const CkCallback& cb);
    entry void loadTipsy(const std::string& filename,
        const double dTuFac,
        const bool bDoublePos,
        const bool bDoubleVel,
        const CkCallback& cb);
    entry void readTipsyArray(CkReference<OutputParams> params, const CkCallback& cb);
    entry void resetMetals(const CkCallback& cb);
    entry void getMaxIOrds(const CkCallback& cb);
    entry void RestartEnergy(double dTuFac, const CkCallback& cb);
    entry void findTotalMass(const CkCallback &cb);
    entry void recvTotalMass(CkReductionMsg *msg);
    entry void setupWrite(int iStage, u_int64_t iPrevOffset,
        const std::string& filename,
        const double dTime, const double dvFac,
        const double duTfac,
        const bool bDoublePos,
        const bool bDoubleVel,
        const int bCool,
        const CkCallback& cb);
    entry void parallelWrite(int iPass, const CkCallback& cb,
			   const std::string& filename, const double dTime,
			   const double dvFac, // scale velocities
			   const double duTFac, // convert temperature
                           const bool bDoublePos,
                           const bool bDoubleVel,
			   const int bCool);
    entry void serialWrite(u_int64_t iPrevOffset, const std::string& filename,
        const double dTime, const double dvFac,
        const double duTfac,
        const bool bDoublePos,
        const bool bDoubleVel,
        const int bCool,
        const CkCallback& cb);
    entry void oneNodeWrite(int iIndex,
			       int iOutParticles,
			       int iOutSPH,
			       int iOutStar,
			       GravityParticle particles[iOutParticles+2], // particles to
						     // write
			       extraSPHData pGas[iOutSPH], // SPH data
			       extraStarData pStar[iOutStar], // Star data
			       int piSPH[iOutSPH], // SPH data offsets
			       int piStar[iOutStar], // Star data offsets
			       const u_int64_t iPrevOffset,
			       const std::string& filename,  // output file
			       const double dTime,      // time or expansion
			       const double dvFac,  // velocity conversion
			     const double duTFac, // temperature conversion
                             const bool bDoublePos,
                             const bool bDoubleVel,
			     const int bCool, const CkCallback& cb);
    entry void reOrder(int64_t nMaxOrder, const CkCallback& cb);
    entry void ioShuffle(CkReductionMsg *msg);
    entry void ioAcceptSortedParticles(ParticleShuffleMsg *);
    entry void assignKeys(CkReductionMsg* m);
    entry [nokeep] void evaluateBoundaries(SFC::Key keys[n], const int n, int isRefine, const CkCallback& cb);
    entry void unshuffleParticles(CkReductionMsg* m);
    entry void acceptSortedParticles(ParticleShuffleMsg *);
    entry void unshuffleParticlesWoDD(const CkCallback& cb);
    entry void shuffleAfterQD();
    entry void acceptSortedParticlesFromOther(ParticleShuffleMsg *);

    //ORB
    entry void initORBPieces(const CkCallback& cb);
    entry void initBeforeORBSend(unsigned int myCount, unsigned int myCountGas,
			     unsigned int myCountStar,
			     const CkCallback& cb, const CkCallback& cback);
    entry void sendORBParticles();
    entry void acceptORBParticles(const GravityParticle particles[n],
				  const int n,
			  	  const extraSPHData pGas[nGasIn],
				  const int nGasIn,
			  	  const extraStarData pStar[nStarIn],
				  const int nStarIn);
    entry void finalizeBoundaries(ORBSplittersMsg *splittersMsg);
    entry void evaluateParticleCounts(ORBSplittersMsg *splittersMsg);

    entry void kick(int iKickRung, double dDelta[MAXRUNG+1], int bClosing,
	    int bNeedVPred, int bGasIsothermal, double dMaxEnergy, double duDelta[MAXRUNG+1],
        double gammam1, double dThermalCondSatCoeff,
        double dMultiPhaseMaxTime, double dMultiPhaseMinTemp, double dEvapCoeff, const CkCallback& cb);
    entry void initAccel(int iKickRung, const CkCallback& cb);
    entry void applyFrameAcc(int iKickRung, Vector3D<double> frameAcc, const CkCallback& cb);
    entry void externalForce(int activeRung, const ExternalForce& exForce, int bKepStep,
                       const CkCallback& cb);
#ifdef COOLING_MOLECULARH
    entry void distribLymanWerner(const CkCallback& cb);
#endif
#ifdef COLLISION
    entry void adjust(int iKickRung, int bCollStep, int bEpsAccStep,
           int bGravStep, int bKepStep, int bSphStep,
           int bViscosityLimitdt, double dEta, double dEtaCourant,
           double dEtauDot, double dDiffCoeff, double dEtaDiffusion,
           double dDelta, double dAccFac,
           double dCosmoFac, double dhMinOverSoft,
                   double dResolveJeans,
	      int bDoGas,
	      const CkCallback& cb);
#else
    entry void adjust(int iKickRung, int bEpsAccStep,
           int bGravStep, int bSphStep,
           int bViscosityLimitdt, double dEta, double dEtaCourant,
           double dEtauDot, double dDiffCoeff, double dEtaDiffusion,
           double dDelta, double dAccFac,
           double dCosmoFac, double dhMinOverSoft,
                   double dResolveJeans,
	      int bDoGas,
	      const CkCallback& cb);
#endif
    entry void truncateRung(int iCurrMaxRung, const CkCallback& cb);
    entry void rungStats(const CkCallback& cb);
    entry void countActive(int activeRung, const CkCallback& cb);
    entry void countType(int iType, const CkCallback& cb);
    entry void outputBlackHoles(const std::string& pszFileName, double dvFac,
                                long lFPos, const CkCallback &cb);
    entry void SetSink(double dSinkMassMin, const CkCallback &cb);
    entry void SinkStep(int iCurrSinkRung, int iKickRung, const CkCallback &cb);
    entry void formSinks(int bJeans, double dJConst2, int bDensity,
		 double dDensityCut, double dTime, int iKickRung, int bSimple,
		 const CkCallback &cb);
    entry void emergencyAdjust(int iRung, double dDelta, double dDeltaThresh,
		               const CkCallback &cb);
    entry void assignDomain(const CkCallback &cb);
    entry void drift(double dDelta, int bNeedVPred, int bGasIsoThermal,
		     double dvDelta, double duDelta, int nGrowMass,
		     bool buildTree, double dMaxEnergy, const CkCallback& cb);
    entry void starCenterOfMass(const CkCallback& cb);
    entry void calcEnergy(const CkCallback& cb);
    entry void colNParts(const CkCallback &cb);
    entry [nokeep] void newOrder(const NewMaxOrder nStarts[n], const int n,
			  const CkCallback &cb);
    entry void setNParts(int64_t _nTotalSPH, int64_t _nTotalDark,
		 int64_t _nTotalStar, const CkCallback &cb);
    entry void setSoft(const double dSoft, const CkCallback &cb);
    entry void physicalSoft(const double dSoftMax, const double dFac,
        const int bSoftMaxMul, const CkCallback& cb);
    entry void growMass(int nGrowMass, double dDeltaM, const CkCallback& cb);
    entry void InitEnergy(double dTuFac, double z, double dTime, double gammam1,
			const CkCallback& cb);
    entry void updateuDot(int activeRung, double duDelta[MAXRUNG+1],
    	       		double dStartTime[MAXRUNG+1], int bCool, int bAll,
			int bUpdateState, double dResolveJeans, double gammam1, const CkCallback& cb);
    entry void ballMax(int activeRung, double dFac, const CkCallback& cb);
    entry void sphViscosityLimiter(int bOn, int activeRung,
	const CkCallback& cb);
    entry void getAdiabaticGasPressure(double gamma, double gammam1, double dTuFac, double dThermalCondCoeff,
        double dThermalCond2Coeff, double dThermalCondSatCoeff, double dThermalCond2SatCoeff,
        double dEvapMinTemp, double dDtCourantFac, double dResolveJeans, const CkCallback &cb);
    entry void getCoolingGasPressure(double gamma, double gammam1, double dThermalCondCoeff,
        double dThermalCond2Coeff, double dThermalCondSatCoeff, double dThermalCond2SatCoeff,
        double dEvapMinTemp, double dDtCourantFac, double dResolveJeans, const CkCallback &cb);
#ifdef SPLITGAS
    entry void SplitGas(double dInitGasMass,const CkCallback& cb);
#endif
    entry void initRand(int iRand, const CkCallback &cb);
    entry void FormStars(Stfm param, double dTime, double dDelta, 
    	       		 double dCosmoFac, const CkCallback& cb);
    entry void flushStarLog(const CkCallback& cb);
    entry void Feedback(Fdbk fb, double dTime, double dDelta,
    	       		const CkCallback& cb);
    entry void massMetalsEnergyCheck(int bPreDist, const CkCallback& cb);
#ifdef COLLISION
    entry void delEjected(double dDelDist, const CkCallback& cb);
    entry void getNearCollPartners(const CkCallback& cb);
    entry void logOverlaps(const CkCallback& cb);
    entry void getCollInfo(const CkCallback& cb);
    entry void getCollInfo(int64_t iOrder, const CkCallback& cb);
    entry void resolveCollision(Collision coll, const ColliderInfo &c1, const ColliderInfo &c2,
        double baseStep, double timeNow, double dCentMass, const CkCallback& cb);
    entry void sameHigherRung(int64_t iord1, int rung1, int64_t iord2, int rung2, const CkCallback& cb);
    entry void resolveWallCollision(Collision coll, const ColliderInfo &c1, const CkCallback& cb);
    entry void unKickCollStep(int iKickRung, double dDeltaBase, const CkCallback& cb);
    entry void placeOnCollRung(int64_t iOrder, int collStepRung, const CkCallback& cb);
    entry void resetRungs(const CkCallback& cb);
    entry void getNeedCollStep(int collStepRung, const CkCallback& cb);
#endif
    entry void setTypeFromFile(int iSetMask, char file[256],
        const CkCallback& cb);
    entry void getCOM(const CkCallback& cb, int bLiveViz);
    entry void getCOMByType(int iType, const CkCallback& cb, int bLiveViz);
    entry void DumpFrame(InDumpFrame in, const CkCallback& cb, int liveVizDump) ;
    entry void liveVizDumpFrameInit(liveVizRequestMsg * msg);
    entry void setProjections(int bOn);
#ifdef PUSH_GRAVITY
    entry void buildTree(int bucketSize, const CkCallback& cb, bool merge);
#else
    entry void buildTree(int bucketSize, const CkCallback& cb);
#endif

    entry void startOctTreeBuild(CkReductionMsg* m);
    entry void recvBoundary(SFC::Key key, NborDir dir);
    entry void recvdBoundaries(CkReductionMsg* m);

    entry void startORBTreeBuild(CkReductionMsg* m);

    entry void startGravity(int activeRung, int bUseCpu_, double myTheta, const CkCallback &cb);

#ifdef PUSH_GRAVITY
    entry void startPushGravity(int am, double myTheta);
    entry void recvPushBuckets(BucketMsg *);
    entry void recvPushAccelerations(CkReductionMsg *);
#endif

    entry void startSmooth(CkPointer<SmoothParams>, int iLowhFix, int nSmooth,
				    double dfBall2OverSoft2, const CkCallback &cb);
    entry void startReSmooth(CkPointer<SmoothParams>, const CkCallback &cb);
    entry void startMarkSmooth(CkPointer<SmoothParams>, const CkCallback &cb);
    entry void finishNodeCache(const CkCallback &cb);

    entry
#ifndef MERGE_REMOTE_REQUESTS
          [inline]
#endif
          void requestRemoteMoments(const Tree::NodeKey key, int sender);
    entry void receiveRemoteMoments(Tree::NodeKey key, Tree::NodeType type,
      int firstParticle, int numParticles, int remIdx, MultipoleMoments& moments,
      OrientedBox<double>& box, OrientedBox<double>& boxBall,
      unsigned int iParticleTypes, int64_t nSPH);

#ifdef CUDA
    entry void commenceCalculateGravityLocal(intptr_t d_localMoments, 
                                             intptr_t d_localParts, 
                                             intptr_t d_localVars,
                                             intptr_t streams, int numStreams,
                                             size_t sMoments, size_t sCompactParts, size_t sVarParts);
#else
    entry void commenceCalculateGravityLocal();
#endif
    entry void calculateGravityRemote(ComputeChunkMsg *msg);

    entry [notrace] void calculateReSmoothLocal();
    entry [notrace] void calculateMarkSmoothLocal();

    entry void finishWalk();
    entry void finishSmoothWalk();

    entry [expedited] void fillRequestNode(CkCacheRequestMsg<KeyType> *msg);
    entry [local] void receiveNodeCallback(GenericTreeNode *node, int chunk, int reqID, int awi, void *source);
    //entry void receiveNode(GenericTreeNode node[1],
    //	       unsigned int reqID);
    //entry void receiveParticle(GravityParticle part,
    //			   BucketGravityRequest &req);
    entry [expedited] void fillRequestParticles(CkCacheRequestMsg<KeyType> *msg);
    entry [expedited] void fillRequestSmoothParticles(CkCacheRequestMsg<KeyType> *msg);
    entry void flushSmoothParticles(CkCacheFillMsg<KeyType> *msg);
    entry [local] void receiveParticlesCallback(ExternalGravityParticle *egp, int num, int chunk, int reqID, Tree::NodeKey &remoteBucket, int awi, void *source);
    entry [local] void receiveParticlesFullCallback(GravityParticle *egp, int num, int chunk, int reqID, Tree::NodeKey &remoteBucket, int awi, void *source);

    // jetley
    entry void startlb(const CkCallback &cb, int activeRung);
    entry void ResumeFromSync();
    entry [reductiontarget] void getParticleInfoForLB(int64_t active_part,
        int64_t total_part);
    //jetley
    // entry void receiveProxy(CkGroupID);
    entry void doAtSync();

    entry void outputASCII(CkReference<OutputParams>, int bParaWrite,
			   const CkCallback& cb);
    entry void oneNodeOutVec(CkReference<OutputParams>, Vector3D<double> avOut[nPart],
			   int nPart, int iIndex, int bDone,
			   const CkCallback& cb) ;
    entry void oneNodeOutArr(CkReference<OutputParams>, double adOut[nPart],
			   int nPart, int iIndex, int bDone,
			   const CkCallback& cb) ;
    entry void outputBinary(Ck::IO::Session, CkReference<OutputParams>);
    entry void minmaxNCOut(CkReference<OutputParams> params, const CkCallback& cb);
    entry void outputBinaryStart(CkReference<OutputParams> params,
                                 int64_t nStart, const CkCallback& cb);
    entry void oneNodeOutIntArr(CkReference<OutputParams>, int aiOut[nPart],
                                int nPart, int iIndex, const CkCallback& cb);
    entry void outputStatistics(const CkCallback& cb);
    entry void collectStatistics(const CkCallback &cb);

    // jetley
#ifdef CUDA
    entry void continueStartRemoteChunk(int chunk, intptr_t d_remoteMoments, intptr_t d_remoteParts);
    entry void fillGPUBuffer(intptr_t bufLocalParts,
                             intptr_t bufLocalMoments,
                             intptr_t pLocalMoments, int partIndex, int nParts, intptr_t node);
    entry void updateParticles(intptr_t data, int partIndex);
#else
    entry void continueStartRemoteChunk(int chunk);
#endif
    entry void continueWrapUp();

    // jetley - balancing before initial forces
    entry void balanceBeforeInitialForces(const CkCallback &cb);

  };

  array [1D] LvArray {
      entry LvArray();
  };

  group [migratable] PETreeMerger {
    entry PETreeMerger();
  };

  group [migratable] DumpFrameData {
    entry DumpFrameData();
    entry void clearFrame(InDumpFrame in, const CkCallback& cb);
    entry void combineFrame(InDumpFrame in, const CkCallback& cb);
  };

  group [migratable] IntraNodeLBManager {
    entry IntraNodeLBManager(int dummy, CkGroupID gid);
    entry [local] void registerTP();
    entry [local] void finishedTPWork();
  };



  initproc void registerStatistics();
};