Skip to content

Commit

Permalink
Merge pull request vsg-dev#1033 from vsg-dev/TransferTaskSynchronizat…
Browse files Browse the repository at this point in the history 
…ionFix

Refactored TransferTask and RecordAndSubmitTask to use a Semaphore to signal that the consumer of TransferTask data is complete and a new TransferTask submission can run.
  • Loading branch information
@robertosfield
robertosfield committed Nov 22, 2023
2 parents a9dd8af + 470727e commit e200820
Show file tree
Hide file tree
Showing 8 changed files with 83 additions and 44 deletions.
10 changes: 7 additions & 3 deletions include/vsg/app/RecordAndSubmitTask.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,14 @@ namespace vsg
ref_ptr<Device> device;
Windows windows;
Semaphores waitSemaphores;
CommandGraphs commandGraphs; // assign in application setup
Semaphores signalSemaphores; // connect to Presentation.waitSemaphores
CommandGraphs commandGraphs; // assign in application setup
Semaphores signalSemaphores; // connect to Presentation.waitSemaphores

ref_ptr<TransferTask> earlyTransferTask; // data is updated prior to record traversal so can be transferred before/in parallel to record traversal
ref_ptr<TransferTask> lateTransferTask; // data is updated during the record traversal so has to be transferred after record traversal
ref_ptr<Semaphore> earlyTransferTaskConsumerCompletedSemaphore;

ref_ptr<TransferTask> lateTransferTask; // data is updated during the record traversal so has to be transferred after record traversal
ref_ptr<Semaphore> lateTransferTaskConsumerCompletedSemaphore;

/// advance the currentFrameIndex
void advance();
Expand Down
9 changes: 2 additions & 7 deletions include/vsg/app/TransferTask.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,6 @@ namespace vsg
/// advance the currentFrameIndex
void advance();

/// return the fence index value for relativeFrameIndex where 0 is current frame, 1 is previous frame etc.
size_t index(size_t relativeFrameIndex = 0) const;

/// fence() and fence(0) return the Fence for the frame currently being rendered, fence(1) returns the previous frame's Fence etc.
Fence* fence(size_t relativeFrameIndex = 0);

void assign(const ResourceRequirements::DynamicData& dynamicData);
void assign(const BufferInfoList& bufferInfoList);
void assign(const ImageInfoList& imageInfoList);
Expand All @@ -59,6 +53,8 @@ namespace vsg
using OffsetBufferInfoMap = std::map<VkDeviceSize, ref_ptr<BufferInfo>>;
using BufferMap = std::map<ref_ptr<Buffer>, OffsetBufferInfoMap>;

size_t index(size_t relativeFrameIndex = 0) const;

VkDeviceSize _dynamicDataTotalRegions = 0;
VkDeviceSize _dynamicDataTotalSize = 0;
VkDeviceSize _dynamicImageTotalSize = 0;
Expand All @@ -70,7 +66,6 @@ namespace vsg

struct Frame
{
ref_ptr<Fence> fence;
ref_ptr<CommandBuffer> transferCommandBuffer;
ref_ptr<Semaphore> transferCompleteSemaphore;
ref_ptr<Buffer> staging;
Expand Down
14 changes: 11 additions & 3 deletions include/vsg/io/AsciiOutput.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
</editor-fold> */

#include <vsg/io/Logger.h>
#include <vsg/io/Options.h>
#include <vsg/io/Output.h>
#include <vsg/io/Logger.h>

#include <algorithm>
#include <fstream>
Expand Down Expand Up @@ -126,8 +126,16 @@ namespace vsg
void write(size_t num, const uint32_t* value) override { _write(num, value); }
void write(size_t num, const int64_t* value) override { _write(num, value); }
void write(size_t num, const uint64_t* value) override { _write(num, value); }
void write(size_t num, const float* value) override { _output.precision(float_precision); _write_real(num, value); }
void write(size_t num, const double* value) override { _output.precision(double_precision); _write_real(num, value); }
void write(size_t num, const float* value) override
{
_output.precision(float_precision);
_write_real(num, value);
}
void write(size_t num, const double* value) override
{
_output.precision(double_precision);
_write_real(num, value);
}

void _write(const std::string& str)
{
Expand Down
1 change: 1 addition & 0 deletions include/vsg/text/Font.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ namespace vsg
return 0;
}
void createFontImages();

protected:
};
VSG_type_name(vsg::Font);
Expand Down
6 changes: 4 additions & 2 deletions include/vsg/vk/State.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,8 +302,10 @@ namespace vsg
inline void pushFrustum()
{
_frustumStack.push(Frustum(_frustumProjected, modelviewMatrixStack.top()));
if (inheritViewForLODScaling) _frustumStack.top().computeLodScale(inheritedProjectionMatrix, inheritedViewTransform * modelviewMatrixStack.top());
else _frustumStack.top().computeLodScale(projectionMatrixStack.top(), modelviewMatrixStack.top());
if (inheritViewForLODScaling)
_frustumStack.top().computeLodScale(inheritedProjectionMatrix, inheritedViewTransform * modelviewMatrixStack.top());
else
_frustumStack.top().computeLodScale(projectionMatrixStack.top(), modelviewMatrixStack.top());
}

inline void applyFrustum()
Expand Down
29 changes: 19 additions & 10 deletions src/vsg/app/RecordAndSubmitTask.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,14 @@ RecordAndSubmitTask::RecordAndSubmitTask(Device* in_device, uint32_t numBuffers)
_fences.resize(numBuffers);
for (uint32_t i = 0; i < numBuffers; ++i)
{
_fences[i] = vsg::Fence::create(device);
_fences[i] = Fence::create(device);
}

earlyTransferTask = vsg::TransferTask::create(in_device, numBuffers);
lateTransferTask = vsg::TransferTask::create(in_device, numBuffers);
earlyTransferTask = TransferTask::create(in_device, numBuffers);
earlyTransferTaskConsumerCompletedSemaphore = Semaphore::create(in_device);

lateTransferTask = TransferTask::create(in_device, numBuffers);
lateTransferTaskConsumerCompletedSemaphore = Semaphore::create(in_device);
}

void RecordAndSubmitTask::advance()
Expand Down Expand Up @@ -77,15 +80,15 @@ Fence* RecordAndSubmitTask::fence(size_t relativeFrameIndex)

VkResult RecordAndSubmitTask::submit(ref_ptr<FrameStamp> frameStamp)
{
auto recordedCommandBuffers = RecordedCommandBuffers::create();

if (VkResult result = start(); result != VK_SUCCESS) return result;

if (earlyTransferTask)
{
if (VkResult result = earlyTransferTask->transferDynamicData(); result != VK_SUCCESS) return result;
}

auto recordedCommandBuffers = RecordedCommandBuffers::create();

if (VkResult result = record(recordedCommandBuffers, frameStamp); result != VK_SUCCESS) return result;

return finish(recordedCommandBuffers);
Expand Down Expand Up @@ -154,12 +157,18 @@ VkResult RecordAndSubmitTask::finish(ref_ptr<RecordedCommandBuffers> recordedCom
{
vk_waitSemaphores.emplace_back(*earlyTransferTask->currentTransferCompletedSemaphore);
vk_waitStages.emplace_back(earlyTransferTask->currentTransferCompletedSemaphore->pipelineStageFlags());

earlyTransferTask->waitSemaphores.push_back(earlyTransferTaskConsumerCompletedSemaphore);
vk_signalSemaphores.emplace_back(*earlyTransferTaskConsumerCompletedSemaphore);
}

if (lateTransferTask && lateTransferTask->currentTransferCompletedSemaphore)
{
vk_waitSemaphores.emplace_back(*lateTransferTask->currentTransferCompletedSemaphore);
vk_waitStages.emplace_back(lateTransferTask->currentTransferCompletedSemaphore->pipelineStageFlags());

lateTransferTask->waitSemaphores.push_back(lateTransferTaskConsumerCompletedSemaphore);
vk_signalSemaphores.emplace_back(*lateTransferTaskConsumerCompletedSemaphore);
}

for (auto& window : windows)
Expand Down Expand Up @@ -234,15 +243,15 @@ void vsg::updateTasks(RecordAndSubmitTasks& tasks, ref_ptr<CompileManager> compi
// assign database pager if required
if (compileResult.containsPagedLOD)
{
vsg::ref_ptr<vsg::DatabasePager> databasePager;
ref_ptr<DatabasePager> databasePager;
for (auto& task : tasks)
{
if (task->databasePager && !databasePager) databasePager = task->databasePager;
}

if (!databasePager)
{
databasePager = vsg::DatabasePager::create();
databasePager = DatabasePager::create();
for (auto& task : tasks)
{
if (!task->databasePager)
Expand All @@ -259,7 +268,7 @@ void vsg::updateTasks(RecordAndSubmitTasks& tasks, ref_ptr<CompileManager> compi
/// handle any new Bin needs
for (auto& [const_view, binDetails] : compileResult.views)
{
auto view = const_cast<vsg::View*>(const_view);
auto view = const_cast<View*>(const_view);
for (auto& binNumber : binDetails.indices)
{
bool binNumberMatched = false;
Expand All @@ -272,8 +281,8 @@ void vsg::updateTasks(RecordAndSubmitTasks& tasks, ref_ptr<CompileManager> compi
}
if (!binNumberMatched)
{
vsg::Bin::SortOrder sortOrder = (binNumber < 0) ? vsg::Bin::ASCENDING : ((binNumber == 0) ? vsg::Bin::NO_SORT : vsg::Bin::DESCENDING);
view->bins.push_back(vsg::Bin::create(binNumber, sortOrder));
Bin::SortOrder sortOrder = (binNumber < 0) ? Bin::ASCENDING : ((binNumber == 0) ? Bin::NO_SORT : Bin::DESCENDING);
view->bins.push_back(Bin::create(binNumber, sortOrder));
}
}
}
Expand Down
1 change: 0 additions & 1 deletion src/vsg/app/RecordTraversal.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,6 @@ void RecordTraversal::apply(const Command& command)
command.record(*(_state->_commandBuffer));
}


void RecordTraversal::apply(const View& view)
{
// note, View::accept() updates the RecordTraversal's traversalMask
Expand Down
57 changes: 39 additions & 18 deletions src/vsg/app/TransferTask.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,6 @@ TransferTask::TransferTask(Device* in_device, uint32_t numBuffers) :
}

_frames.resize(numBuffers);
for (uint32_t i = 0; i < numBuffers; ++i)
{
_frames[i].fence = vsg::Fence::create(device);
}
}

void TransferTask::advance()
Expand Down Expand Up @@ -62,13 +58,6 @@ size_t TransferTask::index(size_t relativeFrameIndex) const
return relativeFrameIndex < _indices.size() ? _indices[relativeFrameIndex] : _indices.size();
}

/// fence() and fence(0) return the Fence for the frame currently being rendered, fence(1) returns the previous frame's Fence etc.
Fence* TransferTask::fence(size_t relativeFrameIndex)
{
size_t i = index(relativeFrameIndex);
return i < _frames.size() ? _frames[i].fence.get() : nullptr;
}

bool TransferTask::containsDataToTransfer() const
{
return !_dynamicDataMap.empty() || !_dynamicImageInfoSet.empty();
Expand Down Expand Up @@ -360,6 +349,7 @@ VkResult TransferTask::transferDynamicData()

if (!semaphore)
{
// signal transfer submission has completed
semaphore = Semaphore::create(device, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
}

Expand Down Expand Up @@ -402,25 +392,56 @@ VkResult TransferTask::transferDynamicData()
VkSubmitInfo submitInfo = {};
submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;

submitInfo.waitSemaphoreCount = 0;
submitInfo.pWaitSemaphores = nullptr;
submitInfo.pWaitDstStageMask = nullptr;
// set up the vulkan wait sempahore
std::vector<VkSemaphore> vk_waitSemaphores;
std::vector<VkPipelineStageFlags> vk_waitStages;
if (waitSemaphores.empty())
{
submitInfo.waitSemaphoreCount = 0;
submitInfo.pWaitSemaphores = nullptr;
submitInfo.pWaitDstStageMask = nullptr;
// info("TransferTask::transferDynamicData() ", this, ", _currentFrameIndex = ", _currentFrameIndex);
}
else
{
for (auto& waitSemaphore : waitSemaphores)
{
vk_waitSemaphores.emplace_back(*(waitSemaphore));
vk_waitStages.emplace_back(waitSemaphore->pipelineStageFlags());
}

submitInfo.waitSemaphoreCount = static_cast<uint32_t>(vk_waitSemaphores.size());
submitInfo.pWaitSemaphores = vk_waitSemaphores.data();
submitInfo.pWaitDstStageMask = vk_waitStages.data();
}

// set up the vulkan signal sempahore
std::vector<VkSemaphore> vk_signalSemaphores;
vk_signalSemaphores.push_back(*semaphore);
for (auto& ss : signalSemaphores)
{
vk_signalSemaphores.push_back(*ss);
}

submitInfo.signalSemaphoreCount = static_cast<uint32_t>(vk_signalSemaphores.size());
submitInfo.pSignalSemaphores = vk_signalSemaphores.data();

submitInfo.commandBufferCount = 1;
submitInfo.pCommandBuffers = &vk_commandBuffer;

submitInfo.signalSemaphoreCount = 1;
VkSemaphore vk_transferCompletedSemaphore = *semaphore;
submitInfo.pSignalSemaphores = &vk_transferCompletedSemaphore;

result = transferQueue->submit(submitInfo);

waitSemaphores.clear();

if (result != VK_SUCCESS) return result;

currentTransferCompletedSemaphore = semaphore;
}
else
{
log(level, "Nothing to submit");

waitSemaphores.clear();
}

return VK_SUCCESS;
Expand Down

0 comments on commit e200820

Please sign in to comment.