[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Guile-commits] 03/09: Add atomic operations
From: |
Andy Wingo |
Subject: |
[Guile-commits] 03/09: Add atomic operations |
Date: |
Mon, 27 May 2019 08:27:04 -0400 (EDT) |
wingo pushed a commit to branch master
in repository guile.
commit bcdde6656b33077171b04d7c5ab26d933970b308
Author: Andy Wingo <address@hidden>
Date: Mon May 27 11:32:18 2019 +0200
Add atomic operations
These operations emit the same code that GCC does for corresponding
operations under the sequential consistency memory model. It would be
possible to relax to acquire/release or something in the future.
---
lightening.h | 5 ++
lightening/aarch64-cpu.c | 75 +++++++++++++++++++++++++++++
lightening/arm-cpu.c | 80 ++++++++++++++++++++++++++++++-
lightening/x86-cpu.c | 119 +++++++++++++++++++++++++++++++++++++++++++++++
tests/Makefile | 11 ++---
5 files changed, 283 insertions(+), 7 deletions(-)
diff --git a/lightening.h b/lightening.h
index 32e3e3a..d9e8e1f 100644
--- a/lightening.h
+++ b/lightening.h
@@ -477,6 +477,11 @@ jit_load_args_3(jit_state_t *_jit, jit_operand_t a,
jit_operand_t b,
M(_FGo_, ldxi_f) \
M(_FGG_, ldxr_d) \
M(_FGo_, ldxi_d) \
+ \
+ M(_GG__, ldr_atomic) \
+ M(_GG__, str_atomic) \
+ M(_GGG_, swap_atomic) \
+ M(_GGGG, cas_atomic) \
\
M(_GG__, str_c) \
M(_pG__, sti_c) \
diff --git a/lightening/aarch64-cpu.c b/lightening/aarch64-cpu.c
index d242620..39ca06a 100644
--- a/lightening/aarch64-cpu.c
+++ b/lightening/aarch64-cpu.c
@@ -221,6 +221,10 @@ oxxrs(jit_state_t *_jit, int32_t Op,
#define A64_MUL 0x1b007c00
#define A64_SMULH 0x9b407c00
#define A64_UMULH 0x9bc07c00
+#define A64_LDAR 0xc8dffc00
+#define A64_STLR 0xc89ffc00
+#define A64_LDAXR 0xc85ffc00
+#define A64_STLXR 0xc800fc00
#define A64_STRBI 0x39000000
#define A64_LDRBI 0x39400000
#define A64_LDRSBI 0x39800000
@@ -647,6 +651,30 @@ REV(jit_state_t *_jit, int32_t Rd, int32_t Rn)
}
static void
+LDAR(jit_state_t *_jit, int32_t Rt, int32_t Rn)
+{
+ return o_xx(_jit, A64_LDAR, Rt, Rn);
+}
+
+static void
+STLR(jit_state_t *_jit, int32_t Rt, int32_t Rn)
+{
+ return o_xx(_jit, A64_STLR, Rt, Rn);
+}
+
+static void
+LDAXR(jit_state_t *_jit, int32_t Rt, int32_t Rn)
+{
+ return o_xx(_jit, A64_LDAXR, Rt, Rn);
+}
+
+static void
+STLXR(jit_state_t *_jit, int32_t Rt, int32_t Rn, int32_t Rm)
+{
+ return oxxx(_jit, A64_STLXR, Rt, Rn, Rm);
+}
+
+static void
LDRSB(jit_state_t *_jit, int32_t Rt, int32_t Rn, int32_t Rm)
{
return oxxx(_jit, A64_LDRSB,Rt,Rn,Rm);
@@ -2465,3 +2493,50 @@ patch_jmp_without_veneer(jit_state_t *_jit, uint32_t
*loc)
{
patch_jmp_offset(loc, _jit->pc.ui - loc);
}
+
+static void
+ldr_atomic(jit_state_t *_jit, int32_t dst, int32_t loc)
+{
+ LDAR(_jit, dst, loc);
+}
+
+static void
+str_atomic(jit_state_t *_jit, int32_t loc, int32_t val)
+{
+ STLR(_jit, val, loc);
+}
+
+static void
+swap_atomic(jit_state_t *_jit, int32_t dst, int32_t loc, int32_t val)
+{
+ void *retry = jit_address(_jit);
+ int32_t result = jit_gpr_regno(get_temp_gpr(_jit));
+ int32_t val_or_tmp = dst == val ? jit_gpr_regno(get_temp_gpr(_jit)) : val;
+ movr(_jit, val_or_tmp, val);
+ LDAXR(_jit, dst, loc);
+ STLXR(_jit, val_or_tmp, loc, result);
+ jit_patch_there(_jit, bnei(_jit, result, 0), retry);
+ if (dst == val) unget_temp_gpr(_jit);
+ unget_temp_gpr(_jit);
+}
+
+static void
+cas_atomic(jit_state_t *_jit, int32_t dst, int32_t loc, int32_t expected,
+ int32_t desired)
+{
+ int32_t dst_or_tmp;
+ if (dst == loc || dst == expected || dst == expected)
+ dst_or_tmp = jit_gpr_regno(get_temp_gpr(_jit));
+ else
+ dst_or_tmp = dst;
+ void *retry = jit_address(_jit);
+ LDAXR(_jit, dst_or_tmp, loc);
+ jit_reloc_t bad = bner(_jit, dst_or_tmp, expected);
+ int result = jit_gpr_regno(get_temp_gpr(_jit));
+ STLXR(_jit, desired, loc, result);
+ jit_patch_there(_jit, bnei(_jit, result, 0), retry);
+ unget_temp_gpr(_jit);
+ jit_patch_here(_jit, bad);
+ movr(_jit, dst, dst_or_tmp);
+ unget_temp_gpr(_jit);
+}
diff --git a/lightening/arm-cpu.c b/lightening/arm-cpu.c
index 6d91f2f..72f5d28 100644
--- a/lightening/arm-cpu.c
+++ b/lightening/arm-cpu.c
@@ -179,6 +179,9 @@
#define THUMB2_STRI 0xf8400c00
#define THUMB2_LDM_W 0x00200000
#define THUMB2_PUSH 0xe92d0000
+#define THUMB_DMB 0xf3bf8f50
+#define THUMB_LDREX 0xe8500f00
+#define THUMB_STREX 0xe8400000
#define _NOREG (jit_gpr_regno(_PC))
@@ -1030,6 +1033,26 @@ T2_BLXI(jit_state_t *_jit)
return tb(_jit, THUMB2_BLXI);
}
+enum dmb_option { DMB_ISH = 0xb };
+static void
+T1_DMB(jit_state_t *_jit, enum dmb_option option)
+{
+ emit_wide_thumb(_jit, THUMB_DMB|_u4(option));
+}
+
+static void
+T1_LDREX(jit_state_t *_jit, int32_t rt, int32_t rn, int8_t offset)
+{
+ emit_wide_thumb(_jit, THUMB_LDREX|(_u4(rn)<<16)|(_u4(rt)<<12)|_u8(offset));
+}
+
+static void
+T1_STREX(jit_state_t *_jit, int32_t rd, int32_t rt, int32_t rn, int8_t offset)
+{
+ emit_wide_thumb
+ (_jit, THUMB_STREX|(_u4(rn)<<16)|(_u4(rt)<<12)|(_u4(rd)<<8)|_u8(offset));
+}
+
static void
T1_LDRSB(jit_state_t *_jit, int32_t rt, int32_t rn, int32_t rm)
{
@@ -1171,7 +1194,7 @@ T1_LDR(jit_state_t *_jit, int32_t rt, int32_t rn, int32_t
rm)
static void
T2_LDR(jit_state_t *_jit, int32_t rt, int32_t rn, int32_t rm)
{
- return torxr(_jit, THUMB2_LDR,rn,rt,rm);
+ emit_u16_with_pool(_jit, THUMB_LDR|(_u3(rm)<<6)|(_u3(rn)<<3)|_u3(rt));
}
static void
@@ -2975,3 +2998,58 @@ emit_veneer(jit_state_t *_jit, jit_pointer_t target)
emit_u16(_jit, THUMB_MOV|((_u4(rd)&8)<<4)|(_u4(tmp)<<3)|(rd&7));
emit_u32(_jit, (uint32_t) target);
}
+
+static void
+ldr_atomic(jit_state_t *_jit, int32_t dst, int32_t loc)
+{
+ T1_DMB(_jit, DMB_ISH);
+ ldr_i(_jit, dst, loc);
+ T1_DMB(_jit, DMB_ISH);
+}
+
+static void
+str_atomic(jit_state_t *_jit, int32_t loc, int32_t val)
+{
+ T1_DMB(_jit, DMB_ISH);
+ str_i(_jit, loc, val);
+ T1_DMB(_jit, DMB_ISH);
+}
+
+static void
+swap_atomic(jit_state_t *_jit, int32_t dst, int32_t loc, int32_t val)
+{
+ int32_t result = jit_gpr_regno(get_temp_gpr(_jit));
+ int32_t val_or_tmp = dst == val ? jit_gpr_regno(get_temp_gpr(_jit)) : val;
+ movr(_jit, val_or_tmp, val);
+ T1_DMB(_jit, DMB_ISH);
+ void *retry = jit_address(_jit);
+ T1_LDREX(_jit, dst, loc, 0);
+ T1_STREX(_jit, result, val_or_tmp, loc, 0);
+ jit_patch_there(_jit, bnei(_jit, result, 0), retry);
+ T1_DMB(_jit, DMB_ISH);
+ if (dst == val) unget_temp_gpr(_jit);
+ unget_temp_gpr(_jit);
+}
+
+static void
+cas_atomic(jit_state_t *_jit, int32_t dst, int32_t loc, int32_t expected,
+ int32_t desired)
+{
+ int32_t dst_or_tmp;
+ if (dst == loc || dst == expected || dst == expected)
+ dst_or_tmp = jit_gpr_regno(get_temp_gpr(_jit));
+ else
+ dst_or_tmp = dst;
+ T1_DMB(_jit, DMB_ISH);
+ void *retry = jit_address(_jit);
+ T1_LDREX(_jit, dst_or_tmp, loc, 0);
+ jit_reloc_t bad = bner(_jit, dst_or_tmp, expected);
+ int result = jit_gpr_regno(get_temp_gpr(_jit));
+ T1_STREX(_jit, result, desired, loc, 0);
+ jit_patch_there(_jit, bnei(_jit, result, 0), retry);
+ unget_temp_gpr(_jit);
+ jit_patch_here(_jit, bad);
+ T1_DMB(_jit, DMB_ISH);
+ movr(_jit, dst, dst_or_tmp);
+ unget_temp_gpr(_jit);
+}
diff --git a/lightening/x86-cpu.c b/lightening/x86-cpu.c
index ab0da68..28ec738 100644
--- a/lightening/x86-cpu.c
+++ b/lightening/x86-cpu.c
@@ -610,6 +610,30 @@ xchgr(jit_state_t *_jit, int32_t r0, int32_t r1)
}
static void
+xchgrm(jit_state_t *_jit, int32_t val_and_dst, int32_t loc)
+{
+ rex(_jit, 0, WIDE, val_and_dst, _NOREG, loc);
+ ic(_jit, 0x87);
+ rx(_jit, val_and_dst, 0, loc, _NOREG, _SCL1);
+}
+
+static void
+lock(jit_state_t *_jit)
+{
+ ic(_jit, 0xf0);
+}
+
+static void
+cmpxchgmr(jit_state_t *_jit, int32_t loc, int32_t desired)
+{
+ lock(_jit);
+ rex(_jit, 0, WIDE, desired, _NOREG, loc);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xb1);
+ rx(_jit, desired, 0, loc, _NOREG, _SCL1);
+}
+
+static void
testr(jit_state_t *_jit, int32_t r0, int32_t r1)
{
rex(_jit, 0, WIDE, r1, _NOREG, r0);
@@ -2634,3 +2658,98 @@ retval_l(jit_state_t *_jit, int32_t r0)
movr(_jit, r0, _RAX_REGNO);
}
#endif
+
+static void
+mfence(jit_state_t *_jit)
+{
+ ic(_jit, 0x0f);
+ ic(_jit, 0xae);
+ ic(_jit, 0xf0);
+}
+
+static void
+ldr_atomic(jit_state_t *_jit, int32_t dst, int32_t loc)
+{
+#if __X64
+ ldr_l(_jit, dst, loc);
+#else
+ ldr_i(_jit, dst, loc);
+#endif
+}
+
+static void
+str_atomic(jit_state_t *_jit, int32_t loc, int32_t val)
+{
+#if __X64
+ str_l(_jit, loc, val);
+#else
+ str_i(_jit, loc, val);
+#endif
+ mfence(_jit);
+}
+
+static void
+swap_atomic(jit_state_t *_jit, int32_t dst, int32_t loc, int32_t val)
+{
+ if (dst == val) {
+ xchgrm(_jit, dst, loc);
+ } else {
+ int32_t tmp = jit_gpr_regno(get_temp_gpr(_jit));
+ movr(_jit, tmp, val);
+ xchgrm(_jit, tmp, loc);
+ movr(_jit, dst, tmp);
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+cas_atomic(jit_state_t *_jit, int32_t dst, int32_t loc, int32_t expected,
+ int32_t desired)
+{
+ ASSERT(loc != expected);
+ ASSERT(loc != desired);
+
+ if (dst == jit_gpr_regno(_RAX)) {
+ if (loc == dst) {
+ int32_t tmp = jit_gpr_regno(get_temp_gpr(_jit));
+ movr(_jit, tmp ,loc);
+ movr(_jit, dst, expected);
+ cmpxchgmr(_jit, tmp, desired);
+ unget_temp_gpr(_jit);
+ } else {
+ movr(_jit, dst, expected);
+ cmpxchgmr(_jit, loc, desired);
+ }
+ } else if (loc == jit_gpr_regno(_RAX)) {
+ int32_t tmp = jit_gpr_regno(get_temp_gpr(_jit));
+ movr(_jit, tmp, loc);
+ movr(_jit, jit_gpr_regno(_RAX), expected);
+ cmpxchgmr(_jit, tmp, desired);
+ movr(_jit, dst, jit_gpr_regno(_RAX));
+ movr(_jit, loc, tmp);
+ unget_temp_gpr(_jit);
+ } else if (expected == jit_gpr_regno(_RAX)) {
+ int32_t tmp = jit_gpr_regno(get_temp_gpr(_jit));
+ movr(_jit, tmp, expected);
+ cmpxchgmr(_jit, loc, desired);
+ movr(_jit, dst, jit_gpr_regno(_RAX));
+ movr(_jit, expected, tmp);
+ unget_temp_gpr(_jit);
+ } else if (desired == jit_gpr_regno(_RAX)) {
+ int32_t tmp = jit_gpr_regno(get_temp_gpr(_jit));
+ movr(_jit, tmp, desired);
+ movr(_jit, jit_gpr_regno(_RAX), expected);
+ cmpxchgmr(_jit, loc, tmp);
+ movr(_jit, dst, jit_gpr_regno(_RAX));
+ movr(_jit, desired, tmp);
+ unget_temp_gpr(_jit);
+ } else {
+ int32_t tmp = jit_gpr_regno(get_temp_gpr(_jit));
+ movr(_jit, tmp, jit_gpr_regno(_RAX));
+ movr(_jit, jit_gpr_regno(_RAX), expected);
+ cmpxchgmr(_jit, loc, desired);
+ movr(_jit, dst, jit_gpr_regno(_RAX));
+ movr(_jit, jit_gpr_regno(_RAX), tmp);
+ unget_temp_gpr(_jit);
+ }
+}
diff --git a/tests/Makefile b/tests/Makefile
index 2a04850..8127972 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,6 +1,5 @@
TESTS=$(sort $(basename $(wildcard *.c)))
-TARGETS=native ia32 aarch64
-ALL_TARGETS=$(TARGETS) armv7
+TARGETS=native ia32 aarch64 armv7
# Suitable values of cross-compiler variables for Debian:
#
@@ -55,9 +54,9 @@ test-armv7-%: CC = $(CC_ARMv7)
test-armv7-%: %.c lightening-armv7.o test.h
$(CC) $(CFLAGS) $(CPPFLAGS) -I.. -o $@ lightening-armv7.o $<
-.PRECIOUS: $(foreach TARGET,$(ALL_TARGETS),$(addprefix
test-$(TARGET)-,$(TESTS)))
-.PRECIOUS: $(foreach TARGET,$(ALL_TARGETS),lightening-$(TARGET).o)
+.PRECIOUS: $(foreach TARGET,$(TARGETS),$(addprefix test-$(TARGET)-,$(TESTS)))
+.PRECIOUS: $(foreach TARGET,$(TARGETS),lightening-$(TARGET).o)
clean:
- rm -f $(foreach TARGET,$(ALL_TARGETS),$(addprefix
test-$(TARGET)-,$(TESTS)))
- rm -f $(foreach TARGET,$(ALL_TARGETS),lightening-$(TARGET).o)
+ rm -f $(foreach TARGET,$(TARGETS),$(addprefix test-$(TARGET)-,$(TESTS)))
+ rm -f $(foreach TARGET,$(TARGETS),lightening-$(TARGET).o)
- [Guile-commits] branch master updated (8b8ce79 -> 09e4fc4), Andy Wingo, 2019/05/27
- [Guile-commits] 01/09: Switch to use atomic_compare_exchange_strong, Andy Wingo, 2019/05/27
- [Guile-commits] 06/09: Inline the atomic intrinsics, Andy Wingo, 2019/05/27
- [Guile-commits] 08/09: Fix accidental change to ARMv7 ldr, Andy Wingo, 2019/05/27
- [Guile-commits] 02/09: Correctly handle overflow when emitting literal pools, Andy Wingo, 2019/05/27
- [Guile-commits] 04/09: Merge remote-tracking branch 'lightening/master', Andy Wingo, 2019/05/27
- [Guile-commits] 07/09: Fix atomics usage in handle-interrupts JIT., Andy Wingo, 2019/05/27
- [Guile-commits] 05/09: Use new atomic instructions from lightening, Andy Wingo, 2019/05/27
- [Guile-commits] 09/09: Merge remote-tracking branch 'lightening/master', Andy Wingo, 2019/05/27
- [Guile-commits] 03/09: Add atomic operations,
Andy Wingo <=