[libspirv] refactor generic atomic builtin to use CLC functions

libclc/libspirv/include/libspirv/atomic/atomic_helper.h

@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_LIBSPIRV_ATOMIC_ATOMIC_HELPER_H__
+#define __CLC_LIBSPIRV_ATOMIC_ATOMIC_HELPER_H__
+
+#include <clc/clcfunc.h>
+#include <libspirv/spirv_types.h>
+
+static _CLC_INLINE int __spirv_get_clang_memory_scope(int Scope) {
+  switch (Scope) {
+  case CrossDevice:
+    return __MEMORY_SCOPE_SYSTEM;
+  case Device:
+    return __MEMORY_SCOPE_DEVICE;
+  case Workgroup:
+    return __MEMORY_SCOPE_WRKGRP;
+  case Subgroup:
+    return __MEMORY_SCOPE_WVFRNT;
+  case Invocation:
+    return __MEMORY_SCOPE_SINGLE;
+  default:
+    __builtin_unreachable();
+  }
+}
+
+static _CLC_INLINE int __spirv_get_clang_memory_order(int Semantics) {
+  switch (Semantics & 0x1F) {
+  case None:
+    return __ATOMIC_RELAXED;
+  case Acquire:
+    return __ATOMIC_ACQUIRE;
+  case Release:
+    return __ATOMIC_RELEASE;
+  case (Acquire | Release):
+  case AcquireRelease:
+    return __ATOMIC_ACQ_REL;
+  case SequentiallyConsistent:
+    // FIXME use __ATOMIC_SEQ_CST
+    return __ATOMIC_ACQ_REL;
+  default:
+    __builtin_unreachable();
+  }
+}
+
+#endif // __CLC_LIBSPIRV_ATOMIC_ATOMIC_HELPER_H__

libclc/libspirv/lib/generic/SOURCES

@@ -1,7 +1,3 @@
-atomic/loadstore_helpers_unordered.ll
-atomic/loadstore_helpers_release.ll
-atomic/loadstore_helpers_acquire.ll
-atomic/loadstore_helpers_seq_cst.ll
 subnormal_config.cl
 subnormal_helper_func.ll
 async/async_work_group_strided_copy.cl
@@ -11,15 +7,17 @@ atomic/atomic_add.cl
 atomic/atomic_and.cl
 atomic/atomic_cmpxchg.cl
 atomic/atomic_dec.cl
+atomic/atomic_flag_clear.cl
+atomic/atomic_flag_test_and_set.cl
 atomic/atomic_inc.cl
+atomic/atomic_load.cl
 atomic/atomic_max.cl
 atomic/atomic_min.cl
 atomic/atomic_or.cl
+atomic/atomic_store.cl
 atomic/atomic_sub.cl
 atomic/atomic_xchg.cl
 atomic/atomic_xor.cl
-atomic/atomic_load.cl
-atomic/atomic_store.cl
 common/degrees.cl
 common/fclamp.cl
 common/mix.cl

libclc/libspirv/lib/generic/atomic/atomic_add.cl

@@ -6,38 +6,35 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <libspirv/spirv.h>
+/*========================== begin_copyright_notice ============================
+
+INTEL CONFIDENTIAL
 
-#define IMPL(TYPE, AS, FN_NAME)                                                \
-  _CLC_OVERLOAD _CLC_DEF TYPE __spirv_AtomicIAdd(AS TYPE *p, int scope,        \
-                                                 int semantics, TYPE val) {    \
-    return FN_NAME(p, val);                                                    \
-  }
+Copyright (C) 2025 Intel Corporation
 
-IMPL(int, global, __sync_fetch_and_add)
-IMPL(unsigned int, global, __sync_fetch_and_add)
-IMPL(int, local, __sync_fetch_and_add)
-IMPL(unsigned int, local, __sync_fetch_and_add)
+This software and the related documents are Intel copyrighted materials,
+and your use of them is governed by the express license under which they were
+provided to you ("License"). Unless the License provides otherwise,
+you may not use, modify, copy, publish, distribute, disclose or transmit this
+software or the related documents without Intel's prior written permission.
 
-#ifdef cl_khr_int64_base_atomics
-IMPL(long, global, __sync_fetch_and_add_8)
-IMPL(unsigned long, global, __sync_fetch_and_add_8)
-IMPL(long, local, __sync_fetch_and_add_8)
-IMPL(unsigned long, local, __sync_fetch_and_add_8)
-#endif
+This software and the related documents are provided as is, with no express or
+implied warranties, other than those that are expressly stated in the License.
 
-#if _CLC_GENERIC_AS_SUPPORTED
+============================= end_copyright_notice ===========================*/
 
-#define IMPL_GENERIC(TYPE, FN_NAME) IMPL(TYPE, , FN_NAME)
+#include <clc/atomic/clc_atomic_fetch_add.h>
+#include <libspirv/atomic/atomic_helper.h>
+#include <libspirv/spirv.h>
 
-IMPL_GENERIC(int, __sync_fetch_and_add)
-IMPL_GENERIC(unsigned int, __sync_fetch_and_add)
+#define __CLC_FUNCTION __spirv_AtomicIAdd
+#define __CLC_IMPL_FUNCTION __clc_atomic_fetch_add
 
-#ifdef cl_khr_int64_base_atomics
-IMPL_GENERIC(long, __sync_fetch_and_add_8)
-IMPL_GENERIC(unsigned long, __sync_fetch_and_add_8)
-#endif
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
 
-#endif //_CLC_GENERIC_AS_SUPPORTED
+#undef __CLC_FUNCTION
+#define __CLC_FUNCTION __spirv_AtomicFAddEXT
 
-#undef IMPL
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/math/gentype.inc>

libclc/libspirv/lib/generic/atomic/atomic_and.cl

@@ -6,38 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <clc/atomic/clc_atomic_fetch_and.h>
+#include <libspirv/atomic/atomic_helper.h>
 #include <libspirv/spirv.h>
 
-#define IMPL(TYPE, AS, FN_NAME)                                                \
-  _CLC_OVERLOAD _CLC_DEF TYPE __spirv_AtomicAnd(AS TYPE *p, int scope,         \
-                                                int semantics, TYPE val) {     \
-    return FN_NAME(p, val);                                                    \
-  }
+#define __CLC_FUNCTION __spirv_AtomicAnd
+#define __CLC_IMPL_FUNCTION __clc_atomic_fetch_and
 
-IMPL(int, global, __sync_fetch_and_and)
-IMPL(unsigned int, global, __sync_fetch_and_and)
-IMPL(int, local, __sync_fetch_and_and)
-IMPL(unsigned int, local, __sync_fetch_and_and)
-
-#ifdef cl_khr_int64_extended_atomics
-IMPL(long, global, __sync_fetch_and_and_8)
-IMPL(unsigned long, global, __sync_fetch_and_and_8)
-IMPL(long, local, __sync_fetch_and_and_8)
-IMPL(unsigned long, local, __sync_fetch_and_and_8)
-#endif
-
-#if _CLC_GENERIC_AS_SUPPORTED
-
-#define IMPL_GENERIC(TYPE, FN_NAME) IMPL(TYPE, , FN_NAME)
-
-IMPL_GENERIC(int, __sync_fetch_and_and)
-IMPL_GENERIC(unsigned int, __sync_fetch_and_and)
-
-#ifdef cl_khr_int64_base_atomics
-IMPL_GENERIC(long, __sync_fetch_and_and_8)
-IMPL_GENERIC(unsigned long, __sync_fetch_and_and_8)
-#endif
-
-#endif //_CLC_GENERIC_AS_SUPPORTED
-
-#undef IMPL
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>

libclc/libspirv/lib/generic/atomic/atomic_cmpxchg.cl

@@ -6,96 +6,23 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <clc/atomic/clc_atomic_compare_exchange.h>
+#include <libspirv/atomic/atomic_helper.h>
 #include <libspirv/spirv.h>
 
-_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicCompareExchange(local int *p,
-                                                         int scope, int eq,
-                                                         int neq, int val,
-                                                         int cmp) {
-  return __sync_val_compare_and_swap(p, cmp, val);
-}
+#define __CLC_FUNCTION __spirv_AtomicCompareExchange
+#define __CLC_IMPL_FUNCTION __clc_atomic_compare_exchange
+#define __CLC_COMPARE_EXCHANGE
 
-_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicCompareExchange(global int *p,
-                                                         int scope, int eq,
-                                                         int neq, int val,
-                                                         int cmp) {
-  return __sync_val_compare_and_swap(p, cmp, val);
-}
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
 
-_CLC_OVERLOAD _CLC_DEF uint __spirv_AtomicCompareExchange(local uint *p,
-                                                          int scope, int eq,
-                                                          int neq, uint val,
-                                                          uint cmp) {
-  return __sync_val_compare_and_swap(p, cmp, val);
-}
+#define __CLC_FLOAT_ONLY
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/math/gentype.inc>
 
-_CLC_OVERLOAD _CLC_DEF uint __spirv_AtomicCompareExchange(global uint *p,
-                                                          int scope, int eq,
-                                                          int neq, uint val,
-                                                          uint cmp) {
-  return __sync_val_compare_and_swap(p, cmp, val);
-}
+#undef __CLC_FUNCTION
+#define __CLC_FUNCTION __spirv_AtomicCompareExchangeWeak
 
-#ifdef cl_khr_int64_base_atomics
-_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicCompareExchange(local long *p,
-                                                          int scope, int eq,
-                                                          int neq, long val,
-                                                          long cmp) {
-  return __sync_val_compare_and_swap_8(p, cmp, val);
-}
-
-_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicCompareExchange(global long *p,
-                                                          int scope, int eq,
-                                                          int neq, long val,
-                                                          long cmp) {
-  return __sync_val_compare_and_swap_8(p, cmp, val);
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong __spirv_AtomicCompareExchange(local ulong *p,
-                                                           int scope, int eq,
-                                                           int neq, ulong val,
-                                                           ulong cmp) {
-  return __sync_val_compare_and_swap_8(p, cmp, val);
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong __spirv_AtomicCompareExchange(global ulong *p,
-                                                           int scope, int eq,
-                                                           int neq, ulong val,
-                                                           ulong cmp) {
-  return __sync_val_compare_and_swap_8(p, cmp, val);
-}
-
-#endif
-
-#if _CLC_GENERIC_AS_SUPPORTED
-
-_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicCompareExchange(int *p, int scope,
-                                                         int eq, int neq,
-                                                         int val, int cmp) {
-  return __sync_val_compare_and_swap(p, cmp, val);
-}
-
-_CLC_OVERLOAD _CLC_DEF uint __spirv_AtomicCompareExchange(uint *p, int scope,
-                                                          int eq, int neq,
-                                                          uint val, uint cmp) {
-  return __sync_val_compare_and_swap(p, cmp, val);
-}
-
-#ifdef cl_khr_int64_base_atomics
-
-_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicCompareExchange(long *p, int scope,
-                                                          int eq, int neq,
-                                                          long val, long cmp) {
-  return __sync_val_compare_and_swap_8(p, cmp, val);
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong __spirv_AtomicCompareExchange(ulong *p, int scope,
-                                                           int eq, int neq,
-                                                           ulong val,
-                                                           ulong cmp) {
-  return __sync_val_compare_and_swap_8(p, cmp, val);
-}
-
-#endif
-
-#endif //_CLC_GENERIC_AS_SUPPORTED
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>

libclc/libspirv/lib/generic/atomic/atomic_dec.cl

@@ -6,47 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <clc/atomic/clc_atomic_dec.h>
+#include <libspirv/atomic/atomic_helper.h>
 #include <libspirv/spirv.h>
 
-_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicIDecrement(local int *p, int scope,
-                                                    int semantics) {
-  return __sync_fetch_and_sub(p, (int)1);
-}
+#define __CLC_FUNCTION __spirv_AtomicIDecrement
+#define __CLC_IMPL_FUNCTION __clc_atomic_dec
+#define __CLC_NO_VALUE_ARG
 
-_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicIDecrement(global int *p, int scope,
-                                                    int semantics) {
-  return __sync_fetch_and_sub(p, (int)1);
-}
-
-_CLC_OVERLOAD _CLC_DEF uint __spirv_AtomicIDecrement(local uint *p, int scope,
-                                                     int semantics) {
-  return __sync_fetch_and_sub(p, (uint)1);
-}
-
-_CLC_OVERLOAD _CLC_DEF uint __spirv_AtomicIDecrement(global uint *p, int scope,
-                                                     int semantics) {
-  return __sync_fetch_and_sub(p, (uint)1);
-}
-
-#ifdef cl_khr_int64_base_atomics
-_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicIDecrement(local long *p, int scope,
-                                                     int semantics) {
-  return __sync_fetch_and_sub(p, (long)1);
-}
-
-_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicIDecrement(global long *p, int scope,
-                                                     int semantics) {
-  return __sync_fetch_and_sub(p, (long)1);
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong __spirv_AtomicIDecrement(local ulong *p, int scope,
-                                                      int semantics) {
-  return __sync_fetch_and_sub(p, (ulong)1);
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong __spirv_AtomicIDecrement(global ulong *p,
-                                                      int scope,
-                                                      int semantics) {
-  return __sync_fetch_and_sub(p, (ulong)1);
-}
-#endif
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>