rds/ib: Fix lfstack routines

After updating lfstack-code to use try_cmpxchg128() as a replacement
for cmpxchg_double(), we introduced errors caused by variables being
read twice and possibly being changed in between the reads causing
wrong values to be inserted in the list-structure.

The following crash is hit when attempting a basic rds-stress

[  398.146678] kernel BUG at net/rds/recv.c:97!
[  398.197773] invalid opcode: 0000 [#1] PREEMPT SMP PTI
[  398.258218] CPU: 28 PID: 0 Comm: swapper/28 Kdump: loaded Not tainted 6.5.0-2135.20230914050031.el8uek.rc1.x86_64 #2
[  398.384196] Hardware name: Oracle Corporation ORACLE SERVER X6-2/ASM,MOTHERBOARD,1U, BIOS 38320100 04/15/2020
[  398.502889] RIP: 0010:rds_inc_put+0x3d/0x40 [rds]
[  398.559219] Code: 08 48 8d 47 08 48 39 c2 75 20 48 8b 47 18 48 8b 40 50 48 8b 80 e0 00 00 00 e9 ff c4 e4 f3 cc 31 c0 89 c2 89 c7 c3 cc cc cc cc <0f> 0b 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 0f 1f 44
[  398.784014] RSP: 0018:ffffa8b68ccd4e70 EFLAGS: 00010206
[  398.846541] RAX: ffff920363351120 RBX: ffff92034792a000 RCX: 0000000000000000
[  398.931948] RDX: ffff9222e14df270 RSI: 0000000000000000 RDI: ffff920363351118
[  399.017355] RBP: ffffa8b6aa41bc40 R08: 0000000000000000 R09: 0000000000000000
[  399.102763] R10: 0000000000000000 R11: 0000000000000000 R12: 00000000000362c0
[  399.188169] R13: ffff9203c0435a40 R14: ffffa8b68ccd4f00 R15: ffff92034792a980
[  399.273576] FS:  0000000000000000(0000) GS:ffff92223f980000(0000) knlGS:0000000000000000
[  399.370426] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  399.439191] CR2: 00007fbe536030c0 CR3: 0000002bdcc36002 CR4: 00000000003706e0
[  399.524599] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[  399.610013] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[  399.695420] Call Trace:
[  399.724660]  <IRQ>
[  399.748700]  ? die+0x37/0x38
[  399.783143]  ? do_trap+0xf7/0x38
[  399.821756]  ? rds_inc_put+0x3d/0x40 [rds]
[  399.870780]  ? do_error_trap+0x6a/0x38
[  399.915616]  ? rds_inc_put+0x3d/0x40 [rds]
[  399.964639]  ? exc_invalid_op+0x52/0x2c
[  400.010522]  ? rds_inc_put+0x3d/0x40 [rds]
[  400.059542]  ? asm_exc_invalid_op+0x1a/0x2c
[  400.109592]  ? rds_inc_put+0x3d/0x40 [rds]
[  400.158618]  rds_ib_recv_cqe_handler+0xe4/0x390 [rds_rdma]
[  400.224283]  poll_rcq+0x84/0xc0 [rds_rdma]
[  400.273295]  rds_ib_rx+0xad/0x260 [rds_rdma]
[  400.324386]  rds_ib_tasklet_fn_recv+0x30/0x40 [rds_rdma]
[  400.387957]  tasklet_action_common.constprop.0+0x140/0xf0
[  400.452568]  __do_softirq+0xd4/0x2c
[  400.494289]  __irq_exit_rcu+0xc8/0x1b8
[  400.539131]  common_interrupt+0x84/0x2c
[  400.585011]  </IRQ>
[  400.610085]  <TASK>
[  400.635158]  asm_common_interrupt+0x26/0x2c
[  400.685202] RIP: 0010:cpuidle_enter_state+0xcc/0x2c
[  400.743567] Code: 6a 3a 2c ff e8 f5 f2 ff ff 8b 53 04 49 89 c5 0f 1f 44 00 00 31 ff e8 33 13 2b ff 45 84 ff 0f 85 4d 02 00 00 fb 0f 1f 44 00 00 <45> 85 f6 0f 88 7c 01 00 00 49 63 d6 4c 2b 2c 24 48 8d 04 52 48 8d
[  400.968362] RSP: 0018:ffffa8b688237e80 EFLAGS: 00000246
[  401.030885] RAX: 0000000000000000 RBX: ffffc8967f982128 RCX: 0000000000000000
[  401.116291] RDX: 000000000000001c RSI: 0000000000000000 RDI: 0000000000000000
[  401.201699] RBP: 0000000000000002 R08: 0000000000000000 R09: 0000000000000000
[  401.287108] R10: 0000000000000000 R11: 0000000000000000 R12: ffffffffb68c4dc0
[  401.372512] R13: 0000005cb363ad51 R14: 0000000000000002 R15: 0000000000000000
[  401.457925]  cpuidle_enter+0x2d/0x10
[  401.500686]  cpuidle_idle_call+0x108/0x7
[  401.547606]  do_idle+0x80/0x8
[  401.583085]  cpu_startup_entry+0x1d/0xb
[  401.628968]  start_secondary+0x11e/0x38
[  401.674848]  secondary_startup_64_no_verify+0x17e/0x8
[  401.735294]  </TASK>

Orabug: 35836155

Fixes: 606495f9b842 ("rds/ib: Replace cmpxchg_double with try_cmpxchg128")
Suggested-by: Gerd Rausch <gerd.rausch@oracle.com>
Signed-off-by: Hans Westgaard Ry <hans.westgaard.ry@oracle.com>
Reviewed-by: Håkon Bugge <haakon.bugge@oracle.com>

Author: Hans Westgaard Ry

net/rds/lfstack.h

@@ -54,11 +54,11 @@ static inline void lfstack_push(union lfstack *stack, struct lfstack_el *el)
 	union lfstack old_v, new_v;
 
 	while (true) {
-		el->next = stack->first;
 		old_v.first = READ_ONCE(stack->first);
 		old_v.seq   = READ_ONCE(stack->seq);
+		el->next = old_v.first;
 		new_v.first = el;
-		new_v.seq   = stack->seq + 1;
+		new_v.seq   = old_v.seq + 1;
 		if (try_cmpxchg128(&stack->full, &old_v.full, new_v.full))
 			break;
 	}
@@ -73,11 +73,11 @@ static inline void lfstack_push_many(union lfstack *stack, struct lfstack_el *el
 	union lfstack old_v, new_v;
 
 	while (true) {
-		el_last->next = stack->first;
 		old_v.first = READ_ONCE(stack->first);
 		old_v.seq   = READ_ONCE(stack->seq);
+		el_last->next = old_v.first;
 		new_v.first = el_first;
-		new_v.seq = stack->seq + 1;
+		new_v.seq = old_v.seq + 1;
 		if (try_cmpxchg128(&stack->full, &old_v.full, new_v.full))
 			break;
 	}
@@ -90,20 +90,18 @@ static inline struct lfstack_el *lfstack_pop(union lfstack *stack)
 {
 #ifdef LFSTACK_LOCKFREE
 	union lfstack old_v, new_v;
-	struct lfstack_el *first;
 
 	while (true) {
-		first = READ_ONCE(stack->first);
-		if (!first)
+		old_v.first = READ_ONCE(stack->first);
+		if (!old_v.first)
 			break;
-		old_v.first = stack->first;
 		old_v.seq   = READ_ONCE(stack->seq);
-		new_v.first = first->next;
-		new_v.seq   = stack->seq + 1;
+		new_v.first = old_v.first->next;
+		new_v.seq   = old_v.seq + 1;
 		if (try_cmpxchg128(&stack->full, &old_v.full, new_v.full))
 			break;
 	}
-	return first;
+	return old_v.first;
 #else
 	struct lfstack_el *el;
 	unsigned long flags;