176 lines
4.7 KiB
C
176 lines
4.7 KiB
C
|
/* SPDX-License-Identifier: GPL-2.0 */
|
||
|
|
|
||
|
|
#include <asm/spr-regs.h>
|
||
|
|
|
||
|
|
#ifdef __ATOMIC_LIB__
|
||
|
|
|
||
|
|
#ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS
|
||
|
|
|
||
|
|
#define ATOMIC_QUALS
|
||
|
|
#define ATOMIC_EXPORT(x) EXPORT_SYMBOL(x)
|
||
|
|
|
||
|
|
#else /* !OUTOFLINE && LIB */
|
||
|
|
|
||
|
|
#define ATOMIC_OP_RETURN(op)
|
||
|
|
#define ATOMIC_FETCH_OP(op)
|
||
|
|
|
||
|
|
#endif /* OUTOFLINE */
|
||
|
|
|
||
|
|
#else /* !__ATOMIC_LIB__ */
|
||
|
|
|
||
|
|
#define ATOMIC_EXPORT(x)
|
||
|
|
|
||
|
|
#ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS
|
||
|
|
|
||
|
|
#define ATOMIC_OP_RETURN(op) \
|
||
|
|
extern int __atomic_##op##_return(int i, int *v); \
|
||
|
|
extern long long __atomic64_##op##_return(long long i, long long *v);
|
||
|
|
|
||
|
|
#define ATOMIC_FETCH_OP(op) \
|
||
|
|
extern int __atomic32_fetch_##op(int i, int *v); \
|
||
|
|
extern long long __atomic64_fetch_##op(long long i, long long *v);
|
||
|
|
|
||
|
|
#else /* !OUTOFLINE && !LIB */
|
||
|
|
|
||
|
|
#define ATOMIC_QUALS static inline
|
||
|
|
|
||
|
|
#endif /* OUTOFLINE */
|
||
|
|
#endif /* __ATOMIC_LIB__ */
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Note on the 64 bit inline asm variants...
|
||
|
|
*
|
||
|
|
* CSTD is a conditional instruction and needs a constrained memory reference.
|
||
|
|
* Normally 'U' provides the correct constraints for conditional instructions
|
||
|
|
* and this is used for the 32 bit version, however 'U' does not appear to work
|
||
|
|
* for 64 bit values (gcc-4.9)
|
||
|
|
*
|
||
|
|
* The exact constraint is that conditional instructions cannot deal with an
|
||
|
|
* immediate displacement in the memory reference, so what we do is we read the
|
||
|
|
* address through a volatile cast into a local variable in order to insure we
|
||
|
|
* _have_ to compute the correct address without displacement. This allows us
|
||
|
|
* to use the regular 'm' for the memory address.
|
||
|
|
*
|
||
|
|
* Furthermore, the %Ln operand, which prints the low word register (r+1),
|
||
|
|
* really only works for registers, this means we cannot allow immediate values
|
||
|
|
* for the 64 bit versions -- like we do for the 32 bit ones.
|
||
|
|
*
|
||
|
|
*/
|
||
|
|
|
||
|
|
#ifndef ATOMIC_OP_RETURN
|
||
|
|
#define ATOMIC_OP_RETURN(op) \
|
||
|
|
ATOMIC_QUALS int __atomic_##op##_return(int i, int *v) \
|
||
|
|
{ \
|
||
|
|
int val; \
|
||
|
|
\
|
||
|
|
asm volatile( \
|
||
|
|
"0: \n" \
|
||
|
|
" orcc gr0,gr0,gr0,icc3 \n" \
|
||
|
|
" ckeq icc3,cc7 \n" \
|
||
|
|
" ld.p %M0,%1 \n" \
|
||
|
|
" orcr cc7,cc7,cc3 \n" \
|
||
|
|
" "#op"%I2 %1,%2,%1 \n" \
|
||
|
|
" cst.p %1,%M0 ,cc3,#1 \n" \
|
||
|
|
" corcc gr29,gr29,gr0 ,cc3,#1 \n" \
|
||
|
|
" beq icc3,#0,0b \n" \
|
||
|
|
: "+U"(*v), "=&r"(val) \
|
||
|
|
: "NPr"(i) \
|
||
|
|
: "memory", "cc7", "cc3", "icc3" \
|
||
|
|
); \
|
||
|
|
\
|
||
|
|
return val; \
|
||
|
|
} \
|
||
|
|
ATOMIC_EXPORT(__atomic_##op##_return); \
|
||
|
|
\
|
||
|
|
ATOMIC_QUALS long long __atomic64_##op##_return(long long i, long long *v) \
|
||
|
|
{ \
|
||
|
|
long long *__v = READ_ONCE(v); \
|
||
|
|
long long val; \
|
||
|
|
\
|
||
|
|
asm volatile( \
|
||
|
|
"0: \n" \
|
||
|
|
" orcc gr0,gr0,gr0,icc3 \n" \
|
||
|
|
" ckeq icc3,cc7 \n" \
|
||
|
|
" ldd.p %M0,%1 \n" \
|
||
|
|
" orcr cc7,cc7,cc3 \n" \
|
||
|
|
" "#op"cc %L1,%L2,%L1,icc0 \n" \
|
||
|
|
" "#op"x %1,%2,%1,icc0 \n" \
|
||
|
|
" cstd.p %1,%M0 ,cc3,#1 \n" \
|
||
|
|
" corcc gr29,gr29,gr0 ,cc3,#1 \n" \
|
||
|
|
" beq icc3,#0,0b \n" \
|
||
|
|
: "+m"(*__v), "=&e"(val) \
|
||
|
|
: "e"(i) \
|
||
|
|
: "memory", "cc7", "cc3", "icc0", "icc3" \
|
||
|
|
); \
|
||
|
|
\
|
||
|
|
return val; \
|
||
|
|
} \
|
||
|
|
ATOMIC_EXPORT(__atomic64_##op##_return);
|
||
|
|
#endif
|
||
|
|
|
||
|
|
#ifndef ATOMIC_FETCH_OP
|
||
|
|
#define ATOMIC_FETCH_OP(op) \
|
||
|
|
ATOMIC_QUALS int __atomic32_fetch_##op(int i, int *v) \
|
||
|
|
{ \
|
||
|
|
int old, tmp; \
|
||
|
|
\
|
||
|
|
asm volatile( \
|
||
|
|
"0: \n" \
|
||
|
|
" orcc gr0,gr0,gr0,icc3 \n" \
|
||
|
|
" ckeq icc3,cc7 \n" \
|
||
|
|
" ld.p %M0,%1 \n" \
|
||
|
|
" orcr cc7,cc7,cc3 \n" \
|
||
|
|
" "#op"%I3 %1,%3,%2 \n" \
|
||
|
|
" cst.p %2,%M0 ,cc3,#1 \n" \
|
||
|
|
" corcc gr29,gr29,gr0 ,cc3,#1 \n" \
|
||
|
|
" beq icc3,#0,0b \n" \
|
||
|
|
: "+U"(*v), "=&r"(old), "=r"(tmp) \
|
||
|
|
: "NPr"(i) \
|
||
|
|
: "memory", "cc7", "cc3", "icc3" \
|
||
|
|
); \
|
||
|
|
\
|
||
|
|
return old; \
|
||
|
|
} \
|
||
|
|
ATOMIC_EXPORT(__atomic32_fetch_##op); \
|
||
|
|
\
|
||
|
|
ATOMIC_QUALS long long __atomic64_fetch_##op(long long i, long long *v) \
|
||
|
|
{ \
|
||
|
|
long long *__v = READ_ONCE(v); \
|
||
|
|
long long old, tmp; \
|
||
|
|
\
|
||
|
|
asm volatile( \
|
||
|
|
"0: \n" \
|
||
|
|
" orcc gr0,gr0,gr0,icc3 \n" \
|
||
|
|
" ckeq icc3,cc7 \n" \
|
||
|
|
" ldd.p %M0,%1 \n" \
|
||
|
|
" orcr cc7,cc7,cc3 \n" \
|
||
|
|
" "#op" %L1,%L3,%L2 \n" \
|
||
|
|
" "#op" %1,%3,%2 \n" \
|
||
|
|
" cstd.p %2,%M0 ,cc3,#1 \n" \
|
||
|
|
" corcc gr29,gr29,gr0 ,cc3,#1 \n" \
|
||
|
|
" beq icc3,#0,0b \n" \
|
||
|
|
: "+m"(*__v), "=&e"(old), "=e"(tmp) \
|
||
|
|
: "e"(i) \
|
||
|
|
: "memory", "cc7", "cc3", "icc3" \
|
||
|
|
); \
|
||
|
|
\
|
||
|
|
return old; \
|
||
|
|
} \
|
||
|
|
ATOMIC_EXPORT(__atomic64_fetch_##op);
|
||
|
|
#endif
|
||
|
|
|
||
|
|
ATOMIC_FETCH_OP(or)
|
||
|
|
ATOMIC_FETCH_OP(and)
|
||
|
|
ATOMIC_FETCH_OP(xor)
|
||
|
|
ATOMIC_FETCH_OP(add)
|
||
|
|
ATOMIC_FETCH_OP(sub)
|
||
|
|
|
||
|
|
ATOMIC_OP_RETURN(add)
|
||
|
|
ATOMIC_OP_RETURN(sub)
|
||
|
|
|
||
|
|
#undef ATOMIC_FETCH_OP
|
||
|
|
#undef ATOMIC_OP_RETURN
|
||
|
|
#undef ATOMIC_QUALS
|
||
|
|
#undef ATOMIC_EXPORT
|