diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index 774589865..2c1cf01b6 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -18,6 +18,7 @@ #include "common/l2_swimlane_profiling.h" #include "common/memory_barrier.h" #include "common/platform_config.h" +#include "device_time_fast.h" #include "pto_runtime2.h" #include "runtime.h" #include "spin_hint.h" @@ -506,7 +507,7 @@ void SchedulerContext::drain_worker_dispatch(int32_t block_num) { uint64_t dispatch_ts = 0; #if PTO2_PROFILING if (l2_swimlane_level_ >= L2SwimlaneLevel::AICPU_TIMING) { - dispatch_ts = get_sys_cnt_aicpu(); + dispatch_ts = fast_sys_cnt_aicpu(); } #endif for (int i = 0; i < handle_count; i++) { diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index c4a10369d..28b185f9b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -24,6 +24,7 @@ #include "common/l2_swimlane_profiling.h" #include "common/memory_barrier.h" #include "common/platform_config.h" +#include "device_time_fast.h" #include "pto_runtime2.h" #include "runtime.h" #include "spin_hint.h" @@ -161,6 +162,10 @@ SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( uint32_t buf_idx = reg_task_id & 1u; PTO2DispatchPayload &payload = payload_per_core_[core_id][buf_idx]; DeferredCompletionSlab *deferred_slab = &deferred_slab_per_core_[core_id][buf_idx]; + // Pull ownership of the per-core dispatch buffers before the hot stores below. + __builtin_prefetch(reinterpret_cast(&payload), 1, 3); + __builtin_prefetch(reinterpret_cast(&payload) + 64, 1, 3); + __builtin_prefetch(reinterpret_cast(deferred_slab), 1, 3); deferred_slab->count = 0; deferred_slab->error_code = PTO2_ERROR_NONE; AsyncCtx async_ctx = AsyncCtx::make(slot_state.task->task_id, deferred_slab); @@ -346,7 +351,7 @@ void SchedulerContext::dispatch_shape( uint64_t dispatch_ts = 0; #if PTO2_PROFILING if (l2_swimlane_level_ >= L2SwimlaneLevel::AICPU_TIMING) { - dispatch_ts = get_sys_cnt_aicpu(); + dispatch_ts = fast_sys_cnt_aicpu(); } #endif for (int i = 0; i < handle_count; i++) { @@ -643,7 +648,7 @@ int32_t SchedulerContext::stage_consumer_blocks( CoreTracker &tracker = core_trackers_[thread_idx]; // Stamp the real pre-stage time (NOT 0) so the swimlane shows these blocks // dispatched during the producer's run, not at trace start. - uint64_t early_dispatch_ts = get_sys_cnt_aicpu(); + uint64_t early_dispatch_ts = fast_sys_cnt_aicpu(); uint64_t my_cores[PTO2_SPEC_CORE_MASK_WORDS] = {0}; // cores this thread gated (for self-ring) int32_t staged = 0; int32_t block = start; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index beea5ada4..1d62229cf 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -23,6 +23,7 @@ #include "common/l2_swimlane_profiling.h" #include "common/memory_barrier.h" #include "common/platform_config.h" +#include "device_time_fast.h" #include "pto_runtime2.h" #include "runtime.h" #include "spin_hint.h" @@ -145,6 +146,10 @@ void SchedulerContext::dispatch_subtask_to_core( uint32_t buf_idx = reg_task_id & 1u; PTO2DispatchPayload &payload = payload_per_core_[core_id][buf_idx]; DeferredCompletionSlab *deferred_slab = &deferred_slab_per_core_[core_id][buf_idx]; + // Pull ownership of the per-core dispatch buffers before the hot stores below. + __builtin_prefetch(reinterpret_cast(&payload), 1, 3); + __builtin_prefetch(reinterpret_cast(&payload) + 64, 1, 3); + __builtin_prefetch(reinterpret_cast(deferred_slab), 1, 3); deferred_slab->count = 0; deferred_slab->error_code = PTO2_ERROR_NONE; AsyncCtx async_ctx = AsyncCtx::make(slot_state.task->task_id, deferred_slab); @@ -189,7 +194,7 @@ void SchedulerContext::dispatch_subtask_to_core( // immediately before the DATA_MAIN_BASE write. #if PTO2_PROFILING if (l2_swimlane_level_ >= L2SwimlaneLevel::AICPU_TIMING) { - uint64_t dispatch_ts = get_sys_cnt_aicpu(); + uint64_t dispatch_ts = fast_sys_cnt_aicpu(); if (to_pending) { core_exec_state.pending_dispatch_timestamp = dispatch_ts; } else { diff --git a/src/common/platform/onboard/aicpu/device_time_fast.h b/src/common/platform/onboard/aicpu/device_time_fast.h new file mode 100644 index 000000000..979382a29 --- /dev/null +++ b/src/common/platform/onboard/aicpu/device_time_fast.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef SRC_COMMON_PLATFORM_ONBOARD_AICPU_DEVICE_TIME_FAST_H_ +#define SRC_COMMON_PLATFORM_ONBOARD_AICPU_DEVICE_TIME_FAST_H_ + +#include + +static inline __attribute__((always_inline)) uint64_t fast_sys_cnt_aicpu() { + uint64_t ticks; + asm volatile("mrs %0, cntvct_el0" : "=r"(ticks)); + return ticks; +} + +#endif // SRC_COMMON_PLATFORM_ONBOARD_AICPU_DEVICE_TIME_FAST_H_ diff --git a/src/common/platform/sim/aicpu/device_time_fast.h b/src/common/platform/sim/aicpu/device_time_fast.h new file mode 100644 index 000000000..b55de11c4 --- /dev/null +++ b/src/common/platform/sim/aicpu/device_time_fast.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef SRC_COMMON_PLATFORM_SIM_AICPU_DEVICE_TIME_FAST_H_ +#define SRC_COMMON_PLATFORM_SIM_AICPU_DEVICE_TIME_FAST_H_ + +#include + +#include "aicpu/device_time.h" + +static inline uint64_t fast_sys_cnt_aicpu() { return get_sys_cnt_aicpu(); } + +#endif // SRC_COMMON_PLATFORM_SIM_AICPU_DEVICE_TIME_FAST_H_