Skip to content

Commit

Permalink
Support LLAMA frames in IO
Browse files Browse the repository at this point in the history
  • Loading branch information
@bernhardmgruber
bernhardmgruber committed Dec 1, 2022
1 parent 8cfdc7b commit 9f510dc
Show file tree
Hide file tree
Showing 5 changed files with 58 additions and 87 deletions.
15 changes: 5 additions & 10 deletions include/picongpu/plugins/openPMD/WriteSpecies.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,15 +113,13 @@ namespace picongpu
{
/* malloc host memory */
log<picLog::INPUT_OUTPUT>("openPMD: (begin) malloc host memory: %1%") % name;
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, MallocHostMemory<bmpl::_1>> mallocMem;
mallocMem(hostFrame, myNumParticles);
mallocFrameMemory(hostFrame);
log<picLog::INPUT_OUTPUT>("openPMD: ( end ) malloc host memory: %1%") % name;
}

void free(openPMDFrameType& hostFrame) override
{
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, FreeHostMemory<bmpl::_1>> freeMem;
freeMem(hostFrame);
freeFrameMemory(hostFrame);
}


Expand Down Expand Up @@ -182,16 +180,13 @@ namespace picongpu
void malloc(std::string name, openPMDFrameType& mappedFrame, uint64_cu const myNumParticles) override
{
log<picLog::INPUT_OUTPUT>("openPMD: (begin) malloc mapped memory: %1%") % name;
/*malloc mapped memory*/
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, MallocMappedMemory<bmpl::_1>> mallocMem;
mallocMem(mappedFrame, myNumParticles);
mallocMappedFrameMemory(mappedFrame);
log<picLog::INPUT_OUTPUT>("openPMD: ( end ) malloc mapped memory: %1%") % name;
}

void free(openPMDFrameType& mappedFrame) override
{
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, FreeMappedMemory<bmpl::_1>> freeMem;
freeMem(mappedFrame);
freeMappedFrameMemory(mappedFrame);
}

void prepare(std::string name, openPMDFrameType& mappedFrame, RunParameters rp) override
Expand Down Expand Up @@ -247,7 +242,7 @@ namespace picongpu
using NewParticleDescription =
typename ReplaceValueTypeSeq<ParticleDescription, ParticleNewAttributeList>::type;

using openPMDFrameType = Frame<OperatorCreateVectorBox, NewParticleDescription>;
using openPMDFrameType = Frame<llama::dyn, NewParticleDescription>;

void setParticleAttributes(
::openPMD::ParticleSpecies& record,
Expand Down
13 changes: 8 additions & 5 deletions include/picongpu/plugins/openPMD/restart/LoadParticleAttributesFromOpenPMD.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,15 @@ namespace picongpu
{
using Identifier = T_Identifier;
using ValueType = typename pmacc::traits::Resolve<Identifier>::type::type;
const uint32_t components = GetNComponents<ValueType>::value;
constexpr uint32_t components = GetNComponents<ValueType>::value;
using ComponentType = typename GetComponentsType<ValueType>::type;
OpenPMDName<Identifier> openPMDName;

log<picLog::INPUT_OUTPUT>("openPMD: ( begin ) load species attribute: %1%") % openPMDName();

const std::string name_lookup[] = {"x", "y", "z"};

// TODO(bgruber): make this a std::shared_ptr<ComponentType[]> with openPMD 0.15
std::shared_ptr<ComponentType> loadBfr;
if(elements > 0)
{
Expand All @@ -90,7 +91,6 @@ namespace picongpu
::openPMD::RecordComponent rc
= components > 1 ? record[name_lookup[n]] : record[::openPMD::RecordComponent::SCALAR];

ValueType* dataPtr = frame.getIdentifier(Identifier()).getPointer();

if(elements > 0)
{
Expand Down Expand Up @@ -119,12 +119,15 @@ namespace picongpu
"%3%")
% elements % globalNumElements % openPMDName();

/* copy component from temporary array to array of structs */
/* copy component from temporary array to array of structs */
#pragma omp parallel for simd
for(size_t i = 0; i < elements; ++i)
{
ComponentType* ref = &reinterpret_cast<ComponentType*>(dataPtr)[i * components + n];
*ref = loadBfr.get()[i];
auto& attrib = frame[i][Identifier{}];
if constexpr(components == 1)
attrib = loadBfr.get()[i];
else
reinterpret_cast<ComponentType*>(&attrib)[n] = loadBfr.get()[i];
}
}

Expand Down
11 changes: 4 additions & 7 deletions include/picongpu/plugins/openPMD/restart/LoadSpecies.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ namespace picongpu
using NewParticleDescription =
typename ReplaceValueTypeSeq<ParticleDescription, ParticleNewAttributeList>::type;

using openPMDFrameType = Frame<OperatorCreateVectorBox, NewParticleDescription>;
using openPMDFrameType = Frame<llama::dyn, NewParticleDescription>;

/** Load species from openPMD checkpoint storage
*
Expand Down Expand Up @@ -138,9 +138,8 @@ namespace picongpu
// memory is visible on host and device
openPMDFrameType mappedFrame;
log<picLog::INPUT_OUTPUT>("openPMD: malloc mapped memory: %1%") % speciesName;
/*malloc mapped memory*/
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, MallocMappedMemory<bmpl::_1>> mallocMem;
mallocMem(mappedFrame, totalNumParticles);

mallocMappedFrameMemory(mappedFrame);

meta::ForEach<typename openPMDFrameType::ValueTypeSeq, LoadParticleAttributesFromOpenPMD<bmpl::_1>>
loadAttributes;
Expand All @@ -158,9 +157,7 @@ namespace picongpu
*(params->cellDescription),
picLog::INPUT_OUTPUT());

/*free host memory*/
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, FreeMappedMemory<bmpl::_1>> freeMem;
freeMem(mappedFrame);
freeMappedFrameMemory(mappedFrame);
}
log<picLog::INPUT_OUTPUT>("openPMD: ( end ) load species: %1%") % speciesName;
}
Expand Down
11 changes: 7 additions & 4 deletions include/picongpu/plugins/openPMD/writer/ParticleAttribute.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ namespace picongpu
{
using Identifier = T_Identifier;
using ValueType = typename pmacc::traits::Resolve<Identifier>::type::type;
const uint32_t components = GetNComponents<ValueType>::value;
constexpr uint32_t components = GetNComponents<ValueType>::value;
using ComponentType = typename GetComponentsType<ValueType>::type;

OpenPMDName<T_Identifier> openPMDName;
Expand Down Expand Up @@ -93,7 +93,7 @@ namespace picongpu
::openPMD::RecordComponent recordComponent
= components > 1 ? record[name_lookup[d]] : record[::openPMD::MeshRecordComponent::SCALAR];

std::string datasetName = components > 1 ? baseName + "/" + name_lookup[d] : baseName;
const std::string datasetName = components > 1 ? baseName + "/" + name_lookup[d] : baseName;
params->initDataset<DIM1>(recordComponent, openPMDType, {globalElements}, datasetName);

if(unit.size() >= (d + 1))
Expand All @@ -107,7 +107,6 @@ namespace picongpu
continue;
}

ValueType* dataPtr = frame.getIdentifier(Identifier()).getPointer(); // can be moved up?
// ask openPMD to create a buffer for us
// in some backends (ADIOS2), this allows avoiding memcopies
auto span = storeChunkSpan<ComponentType>(
Expand All @@ -132,7 +131,11 @@ namespace picongpu
#pragma omp parallel for simd
for(size_t i = 0; i < elements; ++i)
{
span[i] = reinterpret_cast<ComponentType*>(dataPtr)[d + i * components];
const auto attrib = frame[i][Identifier{}];
if constexpr(components == 1)
span[i] = attrib;
else
span[i] = reinterpret_cast<const ComponentType*>(&attrib)[d];
}

flushSeries(*params->openPMDSeries, PreferredFlushTarget::Disk);
Expand Down
95 changes: 34 additions & 61 deletions include/picongpu/plugins/output/WriteSpeciesCommon.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,51 +31,48 @@ namespace picongpu
{
using namespace pmacc;


template<typename T_Type>
struct MallocMappedMemory
template<typename Frame>
void mallocMappedFrameMemory(Frame& frame)
{
template<typename ValueType>
HINLINE void operator()(ValueType& v1, const size_t size) const
{
using type = typename pmacc::traits::Resolve<T_Type>::type::type;

bool isMappedMemorySupported = alpaka::hasMappedBufSupport<::alpaka::Pltf<cupla::AccDev>>;

PMACC_VERIFY_MSG(isMappedMemorySupported, "Device must support mapped memory!");
constexpr bool isMappedMemorySupported = alpaka::hasMappedBufSupport<::alpaka::Pltf<cupla::AccDev>>;
PMACC_VERIFY_MSG(isMappedMemorySupported, "Device must support mapped memory!");

type* ptr = nullptr;
int i = 0;
for(std::byte*& ptr : frame.blobs())
{
const auto size = frame.blobSize(i);
if(size != 0)
{
// Memory is automatically mapped to the device if supported.
CUDA_CHECK(cuplaMallocHost((void**) &ptr, size * sizeof(type)));
CUDA_CHECK(cuplaMallocHost((void**) &ptr, size));
}
v1.getIdentifier(T_Type()) = VectorDataBox<type>(ptr);
else
ptr = nullptr;
i++;
}
};
}

/** allocate memory on host
*
* This functor use `new[]` to allocate memory
*/
template<typename T_Attribute>
struct MallocHostMemory
template<typename Frame>
void mallocFrameMemory(Frame& frame)
{
template<typename ValueType>
HINLINE void operator()(ValueType& v1, const size_t size) const
{
using Attribute = T_Attribute;
using type = typename pmacc::traits::Resolve<Attribute>::type::type;
constexpr bool isMappedMemorySupported = alpaka::hasMappedBufSupport<::alpaka::Pltf<cupla::AccDev>>;
PMACC_VERIFY_MSG(isMappedMemorySupported, "Device must support mapped memory!");

type* ptr = nullptr;
int i = 0;
for(std::byte*& ptr : frame.blobs())
{
const auto size = frame.blobSize(i);
if(size != 0)
{
ptr = new type[size];
}
v1.getIdentifier(Attribute()) = VectorDataBox<type>(ptr);
ptr = new std::byte[size];
else
ptr = nullptr;
i++;
}
};

}

/** copy species to host memory
*
Expand All @@ -94,42 +91,18 @@ namespace picongpu
}
};

template<typename T_Type>
struct FreeMappedMemory
template<typename Frame>
void freeMappedFrameMemory(Frame& frame)
{
template<typename ValueType>
HINLINE void operator()(ValueType& value) const
{
auto* ptr = value.getIdentifier(T_Type()).getPointer();
for(auto* ptr : frame.blobs())
if(ptr != nullptr)
{
CUDA_CHECK(cuplaFreeHost(ptr));
}
}
};
}

//! Free memory
template<typename T_Attribute>
struct FreeHostMemory
template<typename Frame>
void freeFrameMemory(Frame& frame)
{
template<typename ValueType>
HINLINE void operator()(ValueType& value) const
{
using Attribute = T_Attribute;

auto* ptr = value.getIdentifier(Attribute()).getPointer();
for(auto* ptr : frame.blobs())
delete[] ptr;
}
};

/*functor to create a pair for a MapTuple map*/
struct OperatorCreateVectorBox
{
template<typename InType>
struct apply
{
typedef bmpl::pair<InType, pmacc::VectorDataBox<typename pmacc::traits::Resolve<InType>::type::type>> type;
};
};

}
} // namespace picongpu

0 comments on commit 9f510dc

Please sign in to comment.