diff --git a/AmpTools/GPUManager/GPUManager.cc b/AmpTools/GPUManager/GPUManager.cc index 2e858bef..05dc7009 100644 --- a/AmpTools/GPUManager/GPUManager.cc +++ b/AmpTools/GPUManager/GPUManager.cc @@ -754,7 +754,7 @@ GPUManager::clearData(){ } void -GPUManager::clearTerms(){ +GPUManager::clearTerms( bool clearUserVars ){ m_iNAmps=0; m_iNUserVars=0; @@ -779,10 +779,12 @@ GPUManager::clearTerms(){ //Device Memory - if(m_pfDevUserVars) - cudaFree(m_pfDevUserVars); - m_pfDevUserVars=0; - + if( clearUserVars ) { + if(m_pfDevUserVars) + cudaFree(m_pfDevUserVars); + m_pfDevUserVars=0; + } + if(m_pcDevAmpFact) cudaFree(m_pcDevAmpFact); m_pcDevAmpFact=0; diff --git a/AmpTools/GPUManager/GPUManager.h b/AmpTools/GPUManager/GPUManager.h index fa7505a4..c5d64934 100644 --- a/AmpTools/GPUManager/GPUManager.h +++ b/AmpTools/GPUManager/GPUManager.h @@ -70,7 +70,7 @@ class GPUManager void clearAll(); void clearData(); - void clearTerms(); + void clearTerms( bool clearUserVars = true ); void init( const AmpVecs& a, bool use4Vectors = true ); diff --git a/AmpTools/IUAmpTools/AmpToolsInterface.cc b/AmpTools/IUAmpTools/AmpToolsInterface.cc index 4ae72a43..764c5d21 100644 --- a/AmpTools/IUAmpTools/AmpToolsInterface.cc +++ b/AmpTools/IUAmpTools/AmpToolsInterface.cc @@ -212,6 +212,9 @@ AmpToolsInterface::resetConfigurationInfo(ConfigurationInfo* configurationInfo){ if (genMCRdr && accMCRdr && intenMan && !(reaction->normIntFileInput())){ normInt = new NormIntInterface(genMCRdr, accMCRdr, *intenMan); + // doing this explicitly here allows linking of shared data sets + // (otherwise loadMC will be triggered by the first cache update) + normInt->loadMC(); m_normIntMap[reactionName] = normInt; if (reaction->normIntFile() == "") report( WARNING, kModule ) << "no name given to NormInt file for reaction " << reactionName << endl; diff --git a/AmpTools/IUAmpTools/AmpVecs.cc b/AmpTools/IUAmpTools/AmpVecs.cc index c5a797a1..4ea5c40a 100644 --- a/AmpTools/IUAmpTools/AmpVecs.cc +++ b/AmpTools/IUAmpTools/AmpVecs.cc @@ -70,6 +70,7 @@ AmpVecs::AmpVecs(){ m_pdIntegralMatrix = 0 ; m_termsValid = false ; + m_userVarsValid = false ; m_integralValid = false ; m_dataLoaded = false; m_usesSharedData = false; @@ -98,13 +99,13 @@ AmpVecs::deallocAmpVecs() // this handles the case that // other classes may be looking // to this class for the data - clearFourVecs(); + clearFourVecs( true ); if(m_pdWeights) delete[] m_pdWeights; m_pdWeights=0; - deallocTerms(); + deallocTerms( true ); #ifdef GPU_ACCELERATION //Deallocate "pinned memory" @@ -122,7 +123,7 @@ AmpVecs::deallocAmpVecs() } void -AmpVecs::clearFourVecs(){ +AmpVecs::clearFourVecs( bool destruct ){ if( m_usesSharedData ){ @@ -132,9 +133,10 @@ AmpVecs::clearFourVecs(){ // should do the sharing assert( m_sharedDataFriends.empty() ); - m_sharedDataHost->removeFriend( this ); - m_sharedDataHost = NULL; - m_usesSharedData = false; + // if this is called from the destructor then + // we can't leave a bad pointer in the host's + // list of friends + if( destruct ) m_sharedDataHost->removeFriend( this ); // set the pointer to zero to avoid deleting data // that others need in the steps below @@ -150,7 +152,16 @@ AmpVecs::clearFourVecs(){ std::set::iterator avItr = m_sharedDataFriends.begin(); AmpVecs* newDataOwner = *avItr; + + // remove the new owner from the set of friends m_sharedDataFriends.erase( avItr ); + + // the current class becomes a friend of the new owner + // unless we are destroying the class + // this is critical because the new owner will fetch + // cached user variables from the current class if it + // needs them for subsequent amplitude calculations + if( !destruct ) m_sharedDataFriends.insert( this ); newDataOwner->claimDataOwnership( m_sharedDataFriends ); @@ -203,6 +214,7 @@ AmpVecs::loadEvent( const Kinematics* pKinematics, size_t iEvent, m_integralValid = false; m_dataLoaded = true; m_userVarsOffset.clear(); + m_userVarsValid = false; } void @@ -308,6 +320,7 @@ AmpVecs::loadData( DataReader* pDataReader, bool needsUserVarsOnly, size_t chunk m_integralValid = false; m_dataLoaded = true; m_userVarsOffset.clear(); + m_userVarsValid = false; } void @@ -329,13 +342,20 @@ AmpVecs::loadDataArrayElement( const Kinematics* pKinematics, size_t iEvent ){ void AmpVecs::allocateTerms( const IntensityManager& intenMan, bool bAllocIntensity, size_t chunkSize ){ + // if one deallocates and then allocates terms for the same AmpVecs object + // one needs to be aware that the user variables might not have been cleared + // during the deallocation -- if this is the case, then this assertion should + // be correct: + + if( m_pdUserVars != 0 ) assert( m_userVarsPerEvent == intenMan.userVarsPerEvent() ); + size_t ampsEvents = ( chunkSize == 0 ? m_iNEvents : chunkSize ); m_iNTerms = intenMan.getTermNames().size(); m_maxFactPerEvent = intenMan.maxFactorStoragePerEvent(); // in units of doubles; includes factor of 2 for complex numbers m_userVarsPerEvent = intenMan.userVarsPerEvent(); - if( m_pdAmps!=0 || m_pdAmpFactors!=0 || m_pdUserVars!=0 || m_pdIntensity!=0 ) + if( m_pdAmps!=0 || m_pdAmpFactors!=0 || m_pdIntensity!=0 || m_pdIntegralMatrix!=0 ) { report( ERROR, kModule ) << "ERROR: trying to reallocate terms in AmpVecs after\n" << flush; report( ERROR, kModule ) << " they have already been allocated. Please\n" << flush; @@ -357,13 +377,14 @@ AmpVecs::allocateTerms( const IntensityManager& intenMan, bool bAllocIntensity, m_pdIntensity = new GDouble[m_iNEvents]; } - if( m_userVarsPerEvent > 0 ){ + if( m_userVarsPerEvent > 0 && m_pdUserVars == 0 ){ // if there is no user data, we need pdUserVars to be NULL // in order to ensure backwards compatibility with older // amplitude definitions m_pdUserVars = new GDouble[m_iNEvents * m_userVarsPerEvent]; + m_userVarsValid = false; } #ifndef GPU_ACCELERATION @@ -384,15 +405,14 @@ AmpVecs::allocateTerms( const IntensityManager& intenMan, bool bAllocIntensity, } void -AmpVecs::deallocTerms(){ +AmpVecs::deallocTerms( bool clearUserVars ){ m_iNTerms = 0; m_maxFactPerEvent = 0; - m_userVarsPerEvent = 0; m_termsValid = false; m_integralValid = false; - + if( m_pdIntegralMatrix ) delete[] m_pdIntegralMatrix; m_pdIntegralMatrix = 0; @@ -401,11 +421,17 @@ AmpVecs::deallocTerms(){ delete[] m_pdIntensity; m_pdIntensity = 0; - if( m_pdUserVars ) - delete[] m_pdUserVars; - m_pdUserVars = 0; + if( clearUserVars ){ - m_userVarsOffset.clear(); + m_userVarsPerEvent = 0; + m_userVarsValid = false; + + if( m_pdUserVars ) + delete[] m_pdUserVars; + m_pdUserVars = 0; + + m_userVarsOffset.clear(); + } #ifndef GPU_ACCELERATION @@ -418,7 +444,7 @@ if( m_pdAmps ) #else - m_gpuMan.clearTerms(); + m_gpuMan.clearTerms( clearUserVars ); #endif // GPU_ACCELERATION @@ -531,6 +557,7 @@ AmpVecs::claimDataOwnership( set< AmpVecs* > sharedFriends ){ avItr != sharedFriends.end(); ++avItr ){ (*avItr)->m_sharedDataHost = this; + (*avItr)->m_usesSharedData = true; } #ifdef GPU_ACCELERATION @@ -538,3 +565,42 @@ AmpVecs::claimDataOwnership( set< AmpVecs* > sharedFriends ){ #endif } + +GDouble* +AmpVecs::findSharedUserVars( const string& ampIdentifier ){ + + // needs to use the host class because the host has + // the complete set of shared data friends + if( m_usesSharedData && m_sharedDataHost != NULL ){ + + return m_sharedDataHost->findSharedUserVars( ampIdentifier ); + } + + // check to see if this class has already calculated + // user vars for the requested amplitude identifier + map< string, size_t >::iterator offsetItr = m_userVarsOffset.find( ampIdentifier ); + + if( offsetItr != m_userVarsOffset.end() ){ + + size_t offset = offsetItr->second; + + return m_pdUserVars + offset; + } + + // check to see if any of the friends have already calculated + // user vars for the requested amplitude identifier + for( set< AmpVecs* >::iterator avItr = m_sharedDataFriends.begin(); + avItr != m_sharedDataFriends.end(); ++avItr ){ + + map< string, size_t >::iterator offsetItr = (*avItr)->m_userVarsOffset.find( ampIdentifier ); + if( offsetItr != (*avItr)->m_userVarsOffset.end() ){ + + size_t offset = offsetItr->second; + + return (*avItr)->m_pdUserVars + offset; + } + } + + return NULL; +} + diff --git a/AmpTools/IUAmpTools/AmpVecs.h b/AmpTools/IUAmpTools/AmpVecs.h index b91950d7..d78a5cd3 100644 --- a/AmpTools/IUAmpTools/AmpVecs.h +++ b/AmpTools/IUAmpTools/AmpVecs.h @@ -176,6 +176,11 @@ struct AmpVecs */ bool m_dataLoaded; + /** + * A boolean that tracks whether user data has been calculated and stored. + */ + bool m_userVarsValid; + /** * This is a map from amplitude identifer to the location in memory * where user data for that amplitude exists. It is @@ -239,8 +244,11 @@ struct AmpVecs * This routine deallocates the arrays that hold the calculated terms and * (optionally) the calculated intensities. It should be called before * the AmpVecs object is destroyed or before reallocation of the arrays. + * + * \param[in] clearUserVars if set to true, also release cached user + * variables and their offsets. */ - void deallocTerms(); + void deallocTerms( bool clearUserVars = true ); #ifdef GPU_ACCELERATION /** @@ -308,7 +316,7 @@ struct AmpVecs * This clears only the four vectors from memory. It can be used * in the case all amplitudes depend only on user data. */ - void clearFourVecs(); + void clearFourVecs( bool destruct = false ); /** * This function will share this classes data four vectors with the @@ -328,6 +336,14 @@ struct AmpVecs * which is necessary, e.g., if the friend goes out of scope. */ void removeFriend( AmpVecs* dataFriend ); + + /** + * This function will look through shared data friends to find + * if user vars have been computed for a particular amplitude + * identifier already and if so, return a pointer to the location in memory + * where the user vars are stored. + */ + GDouble* findSharedUserVars( const string& ampIdentifier ); bool m_usesSharedData; AmpVecs* m_sharedDataHost; diff --git a/AmpTools/IUAmpTools/AmplitudeManager.cc b/AmpTools/IUAmpTools/AmplitudeManager.cc index 1f9ca722..c5522235 100644 --- a/AmpTools/IUAmpTools/AmplitudeManager.cc +++ b/AmpTools/IUAmpTools/AmplitudeManager.cc @@ -184,8 +184,7 @@ AmplitudeManager::termStoragePerEvent() const { size_t AmplitudeManager::userVarsPerEvent() const { - set< string > countedStaticAmps; - set< string > countedUniqueAmps; + set< string > countedAmpIds; vector< string > ampNames = getTermNames(); @@ -194,32 +193,21 @@ AmplitudeManager::userVarsPerEvent() const { for( int i = 0; i < getTermNames().size(); i++ ) { size_t iNPermutations = getPermutations( ampNames[i] ).size(); + string permTag = getPermutationTag( getPermutations( ampNames[i] ) ); vector< const Amplitude* > factorVec = m_mapNameToAmps.find( ampNames[i] )->second; for( int j = 0; j < factorVec.size(); ++j ){ - if( factorVec[j]->areUserVarsStatic() ){ - - // for factors that have static data, we only want to - // count the allocation once -- if we have counted - // it already then skip to the next factor - if( countedStaticAmps.find( factorVec[j]->name() ) == - countedStaticAmps.end() ) - countedStaticAmps.insert( factorVec[j]->name() ); - else continue; - } - else{ - // user data is not static, so - // check to see if we have seen an instance of this - // same amplitude that would behave in the same way - // (i.e., has the same arguments) and if it exists, we - // will use user data block from it instead - if( countedUniqueAmps.find( factorVec[j]->identifier() ) == - countedUniqueAmps.end() ) - countedUniqueAmps.insert( factorVec[j]->identifier() ); - else continue; - } + // This key must match calcUserVars/calcTerms exactly to avoid + // under-allocating user-variable storage. + string ampId = ( factorVec[j]->areUserVarsStatic() ? + factorVec[j]->name() : factorVec[j]->identifier() ); + ampId += permTag; + + if( countedAmpIds.find( ampId ) == countedAmpIds.end() ) + countedAmpIds.insert( ampId ); + else continue; userStorage += iNPermutations * factorVec[j]->numUserVars(); } @@ -279,10 +267,11 @@ SCOREP_USER_REGION_BEGIN( calcUserVars, "calcUserVars", SCOREP_USER_REGION_TYPE_ { map< string, vector< vector< int > > >::const_iterator permItr = - m_ampPermutations.find( ampNames[iAmpIndex] ); + m_ampPermutations.find( ampNames[iAmpIndex] ); assert( permItr != m_ampPermutations.end() ); const vector< vector< int > >& vvPermuations = permItr->second; int iNPerms = vvPermuations.size(); + string permTag = getPermutationTag( vvPermuations ); vector< const Amplitude* > vAmps = m_mapNameToAmps.find(ampNames.at(iAmpIndex))->second; @@ -298,61 +287,29 @@ SCOREP_USER_REGION_BEGIN( calcUserVars, "calcUserVars", SCOREP_USER_REGION_TYPE_ // this is the number of variables for the data set int iNVars = pCurrAmp->numUserVars(); - int iNData = iNVars * a.m_iNEvents * iNPerms; + size_t iNData = iNVars * a.m_iNEvents * iNPerms; // we will set this based on the algorithm below size_t thisOffset = 0; - if( pCurrAmp->areUserVarsStatic() ){ - - // the user variables are static, so let's look at the - // list of data pointers stored in the relevant AmpVecs - // object and see if there is one associated with this - // amplitude name - - map< string, size_t >::const_iterator offsetItr = - a.m_userVarsOffset.find( pCurrAmp->name() ); - - if( offsetItr == a.m_userVarsOffset.end() ){ - - // set the offset to where the calculation will end up - a.m_userVarsOffset[pCurrAmp->name()] = iUserVarsOffset; + string ampId = ( pCurrAmp->areUserVarsStatic() ? + pCurrAmp->name() : pCurrAmp->identifier() ); + + ampId += permTag; + + map< string, size_t >::const_iterator offsetItr = + a.m_userVarsOffset.find( ampId ); + + if( offsetItr == a.m_userVarsOffset.end() ){ // record it to use in the lines below thisOffset = iUserVarsOffset; // increment iUserVarsOffset += iNData; - } - else // we have done the calculation already - if( m_forceUserVarRecalculation ){ // but should redo it - - thisOffset = offsetItr->second; - } - else{ // and don't want to repeat it - - continue; - } } - else{ - // the variables are not static, repeat the algorithm - // above but search based on identifier of the amplitude - - map< string, size_t >::const_iterator offsetItr = - a.m_userVarsOffset.find( pCurrAmp->identifier() ); - - if( offsetItr == a.m_userVarsOffset.end() ){ - - // set the offset to where the calculation will end up - a.m_userVarsOffset[pCurrAmp->identifier()] = iUserVarsOffset; - - // record it to use in the lines below - thisOffset = iUserVarsOffset; + else{ // we have done the calculation already - // increment -- can only happen at most once either above or here - iUserVarsOffset += iNData; - } - else // we have done the calculation already if( m_forceUserVarRecalculation ){ // but should redo it thisOffset = offsetItr->second; @@ -360,18 +317,26 @@ SCOREP_USER_REGION_BEGIN( calcUserVars, "calcUserVars", SCOREP_USER_REGION_TYPE_ else{ // and don't want to repeat it continue; - } + } } + + GDouble* sharedUserVars = a.findSharedUserVars( ampId ); + + // set the offset to where the calculation will end up + a.m_userVarsOffset[ampId] = thisOffset; - // calculation of user-defined kinematics data - // is something that should only be done once - // per fit, so do it on the CPU no matter what - - pCurrAmp-> - calcUserVarsAll( a.m_pdData, - a.m_pdUserVars + thisOffset, - a.m_iNEvents, &vvPermuations ); - + if( sharedUserVars == NULL || m_forceUserVarRecalculation ){ + + report( DEBUG, kModule ) << "Calculating userVars for factor: " << ampId << endl; + + // if we are not sharing data, or we are forcing a recalculation + // then we will do the calculation here and store it in the + // userVars block for this data set + pCurrAmp-> + calcUserVarsAll( a.m_pdData, + a.m_pdUserVars + thisOffset, + a.m_iNEvents, &vvPermuations ); + #ifdef GPU_ACCELERATION // we want to reorder the userVars if we are working on the @@ -399,8 +364,19 @@ SCOREP_USER_REGION_BEGIN( calcUserVars, "calcUserVars", SCOREP_USER_REGION_TYPE_ iNData*sizeof(GDouble) ); delete[] tmpVarStorage; -#endif //GPU_ACCELERATION - +#endif //GPU_ACCELERATION + } + else{ + + // GPU user vars are already in the correct order, so we can just copy them + + report( DEBUG, kModule ) << "Reusing userVars for factor: " << ampId << endl; + + // if we are sharing data, then copy the user vars from the + // shared location to the location for this data set + memcpy( a.m_pdUserVars + thisOffset, sharedUserVars, + iNData*sizeof(GDouble) ); + } } } @@ -415,6 +391,7 @@ SCOREP_USER_REGION_BEGIN( calcUserVars, "calcUserVars", SCOREP_USER_REGION_TYPE_ SCOREP_USER_REGION_END( calcUserVars ) #endif + a.m_userVarsValid = true; return; } @@ -439,14 +416,15 @@ SCOREP_USER_REGION_BEGIN( calcTerms, "calcTerms", SCOREP_USER_REGION_TYPE_COMMON size_t nEvents = ( chunkSize == 0 ? a.m_iNEvents : chunkSize ); - report( DEBUG, kModule ) << "Calculating terms... termsValid = " - << a.m_termsValid << endl; + report( DEBUG, kModule ) << "Calculating terms... userVarsValid = " + << a.m_userVarsValid << ", termsValid = " + << a.m_termsValid << endl; // on the first pass through this data set be sure to calculate // the user data first, if needed, before doing term calculations // the last criteria will make sure that userData is only computed // once for the first chunk in sequential calls of calcTerms - if( !a.m_termsValid && a.m_userVarsPerEvent > 0 && startEvent == 0 ){ + if( !a.m_userVarsValid && a.m_userVarsPerEvent > 0 && startEvent == 0 ){ calcUserVars( a ); if( m_needsUserVarsOnly && !m_forceUserVarRecalculation @@ -474,6 +452,7 @@ SCOREP_USER_REGION_BEGIN( calcTerms, "calcTerms", SCOREP_USER_REGION_TYPE_COMMON assert( permItr != m_ampPermutations.end() ); const vector< vector< int > >& vvPermuations = permItr->second; int iNPermutations = vvPermuations.size(); + string permTag = getPermutationTag( vvPermuations ); vector< const Amplitude* > vAmps = m_mapNameToAmps.find(ampNames.at(iAmpIndex))->second; @@ -528,20 +507,30 @@ SCOREP_USER_REGION_BEGIN( calcTerms, "calcTerms", SCOREP_USER_REGION_TYPE_COMMON iFactor++, uAmpFactOffset += 2 * nEvents * iNPermutations ){ pCurrAmp = vAmps.at( iFactor ); + + GDouble* userVars = 0; + size_t userVarsOffset = 0; + + if( pCurrAmp->numUserVars() > 0 ){ - // if we have static user data, look up the location in the data array - // if not, then look up by identifier - size_t userVarsOffset = - ( pCurrAmp->areUserVarsStatic() ? - a.m_userVarsOffset[pCurrAmp->name()] : - a.m_userVarsOffset[pCurrAmp->identifier()] ); + string ampId = ( pCurrAmp->areUserVarsStatic() ? + pCurrAmp->name() : pCurrAmp->identifier() ); + ampId += permTag; + + map< string, size_t >::const_iterator offsetItr = + a.m_userVarsOffset.find( ampId ); + assert( offsetItr != a.m_userVarsOffset.end() ); + + userVarsOffset = offsetItr->second; + userVars = a.m_pdUserVars + userVarsOffset; + } #ifndef GPU_ACCELERATION pCurrAmp-> calcAmplitudeAll( a.m_pdData, a.m_pdAmpFactors + uAmpFactOffset, a.m_iNEvents, &vvPermuations, - a.m_pdUserVars + userVarsOffset, + userVars, startEvent, nEvents ); #else a.m_gpuMan.calcAmplitudeAll( pCurrAmp, uAmpFactOffset, &vvPermuations, @@ -1343,3 +1332,27 @@ AmplitudeManager::generateSymmetricCombos( const vector< pair< int, int > >& pre } } +string +AmplitudeManager::getPermutationTag( const vector< vector< int > >& vvPerm ) const { + + stringstream sig; + + sig << "|"; + + for( vector< vector< int > >::const_iterator vecItr = vvPerm.begin(); + vecItr != vvPerm.end(); ++vecItr ){ + + sig << "|"; + for( vector< int >::const_iterator itr = vecItr->begin(); + itr != vecItr->end(); ++itr ){ + + sig << *itr; + } + sig << "|"; + } + + sig << "|"; + + return sig.str(); +} + diff --git a/AmpTools/IUAmpTools/AmplitudeManager.h b/AmpTools/IUAmpTools/AmplitudeManager.h index e281129d..5ce96828 100644 --- a/AmpTools/IUAmpTools/AmplitudeManager.h +++ b/AmpTools/IUAmpTools/AmplitudeManager.h @@ -465,6 +465,9 @@ class AmplitudeManager : public IntensityManager vector< vector< pair< int, int > > > remainingSwaps, const vector< int >& defaultOrder ); + // generates a string for a set of permutations + string getPermutationTag( const vector< vector< int > >& vvPerm ) const; + // amplitude name -> vector of amplitude factors map< string, vector< const Amplitude* > > m_mapNameToAmps; @@ -494,7 +497,6 @@ class AmplitudeManager : public IntensityManager mutable map< const Amplitude*, int > m_ampIteration; mutable map< AmpVecs*, map< const Amplitude*, int > > m_dataAmpIteration; - mutable map< string, size_t > m_staticUserVarsOffset; static const char* kModule; }; diff --git a/AmpTools/IUAmpTools/NormIntInterface.cc b/AmpTools/IUAmpTools/NormIntInterface.cc index c04ba1a0..8e5f53f7 100644 --- a/AmpTools/IUAmpTools/NormIntInterface.cc +++ b/AmpTools/IUAmpTools/NormIntInterface.cc @@ -103,6 +103,10 @@ m_termNames( intenManager.getTermNames() ) m_termNames = intenManager.getTermNames(); initializeCache(); + + // class relies on an external call to loadMC() + // after construction -- it can't go here because the + // MPI implementation must be different } NormIntInterface::~NormIntInterface(){ @@ -355,14 +359,7 @@ NormIntInterface::forceCacheUpdate( bool normIntOnly ) const << normIntOnly << ", emptyNormIntCache = " << m_emptyNormIntCache << ", emptyAmpIntCache = " << m_emptyAmpIntCache << endl; - // do "lazy" allocation of memory here -- this is important for MPI jobs - // where forceCacheUpdate is only called on follower nodes, as it - // avoids big memory allocations on the lead nodes - - // this will load both the accepted and generated MC into memory - // (For MPI jobs, loadMC is called explicitly on the follower nodes - // during the steup step, so this will not be called again on those nodes - // and forceCacheUpdate is never called on the MPI lead node) + // trigger loading if the data has not been loaded yet if( !m_accMCVecs.m_dataLoaded ) loadMC(); // allocate the space for calculating the amplitudes @@ -410,8 +407,10 @@ NormIntInterface::forceCacheUpdate( bool normIntOnly ) const setAmpIntMatrix( m_genMCVecs.m_pdIntegralMatrix ); // try to keep memory usage down by freeing the memory needed to calculate the - // integrals -- the data will remain in memory - m_genMCVecs.deallocTerms(); + // integrals -- the data will remain in memory -- the "false" argument tells + // the function to not deallocate the user variables, which may be needed + // for subsequent amplitude calculations + m_genMCVecs.deallocTerms( false ); m_emptyAmpIntCache = false; } diff --git a/AmpTools/IUAmpTools/NormIntInterface.h b/AmpTools/IUAmpTools/NormIntInterface.h index 624ce4a9..d702f155 100644 --- a/AmpTools/IUAmpTools/NormIntInterface.h +++ b/AmpTools/IUAmpTools/NormIntInterface.h @@ -91,6 +91,9 @@ class NormIntInterface void invalidateTerms(); + // needs different implementations in MPI so make virtual + virtual void loadMC() const; + #endif void exportNormIntCache( const string& fileName ) const; @@ -116,8 +119,6 @@ class NormIntInterface void setAmpIntMatrix( const double* input ) const; void setNormIntMatrix( const double* input ) const; - - void loadMC() const; private: diff --git a/AmpTools/IUAmpToolsMPI/NormIntInterfaceMPI.cc b/AmpTools/IUAmpToolsMPI/NormIntInterfaceMPI.cc index 01ef82de..a998f1b5 100644 --- a/AmpTools/IUAmpToolsMPI/NormIntInterfaceMPI.cc +++ b/AmpTools/IUAmpToolsMPI/NormIntInterfaceMPI.cc @@ -201,3 +201,11 @@ NormIntInterfaceMPI::sumIntegrals( IntType type ) const delete[] result; } + +void +NormIntInterfaceMPI::loadMC() const +{ + // avoids large memory allocations on the lead node + if( !m_isLeader ) NormIntInterface::loadMC(); +} + diff --git a/AmpTools/IUAmpToolsMPI/NormIntInterfaceMPI.h b/AmpTools/IUAmpToolsMPI/NormIntInterfaceMPI.h index b4ef51b3..3f450586 100644 --- a/AmpTools/IUAmpToolsMPI/NormIntInterfaceMPI.h +++ b/AmpTools/IUAmpToolsMPI/NormIntInterfaceMPI.h @@ -57,6 +57,8 @@ class NormIntInterfaceMPI : public NormIntInterface complex< double > normInt( string amp, string conjAmp, bool forceUseCache = false ) const; void forceCacheUpdate( bool normIntOnly = false ) const; + + void loadMC() const; private: