@@ -679,6 +679,7 @@ struct xfrm_state *xfrm_state_alloc(struct net *net)
679
679
x -> lft .hard_packet_limit = XFRM_INF ;
680
680
x -> replay_maxage = 0 ;
681
681
x -> replay_maxdiff = 0 ;
682
+ x -> pcpu_num = UINT_MAX ;
682
683
spin_lock_init (& x -> lock );
683
684
}
684
685
return x ;
@@ -1155,6 +1156,12 @@ static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x,
1155
1156
struct xfrm_state * * best , int * acq_in_progress ,
1156
1157
int * error )
1157
1158
{
1159
+ /* We need the cpu id just as a lookup key,
1160
+ * we don't require it to be stable.
1161
+ */
1162
+ unsigned int pcpu_id = get_cpu ();
1163
+ put_cpu ();
1164
+
1158
1165
/* Resolution logic:
1159
1166
* 1. There is a valid state with matching selector. Done.
1160
1167
* 2. Valid state with inappropriate selector. Skip.
@@ -1174,13 +1181,18 @@ static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x,
1174
1181
& fl -> u .__fl_common ))
1175
1182
return ;
1176
1183
1184
+ if (x -> pcpu_num != UINT_MAX && x -> pcpu_num != pcpu_id )
1185
+ return ;
1186
+
1177
1187
if (!* best ||
1188
+ ((* best )-> pcpu_num == UINT_MAX && x -> pcpu_num == pcpu_id ) ||
1178
1189
(* best )-> km .dying > x -> km .dying ||
1179
1190
((* best )-> km .dying == x -> km .dying &&
1180
1191
(* best )-> curlft .add_time < x -> curlft .add_time ))
1181
1192
* best = x ;
1182
1193
} else if (x -> km .state == XFRM_STATE_ACQ ) {
1183
- * acq_in_progress = 1 ;
1194
+ if (!* best || x -> pcpu_num == pcpu_id )
1195
+ * acq_in_progress = 1 ;
1184
1196
} else if (x -> km .state == XFRM_STATE_ERROR ||
1185
1197
x -> km .state == XFRM_STATE_EXPIRED ) {
1186
1198
if ((!x -> sel .family ||
@@ -1209,6 +1221,13 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1209
1221
unsigned short encap_family = tmpl -> encap_family ;
1210
1222
unsigned int sequence ;
1211
1223
struct km_event c ;
1224
+ unsigned int pcpu_id ;
1225
+
1226
+ /* We need the cpu id just as a lookup key,
1227
+ * we don't require it to be stable.
1228
+ */
1229
+ pcpu_id = get_cpu ();
1230
+ put_cpu ();
1212
1231
1213
1232
to_put = NULL ;
1214
1233
@@ -1282,7 +1301,10 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1282
1301
}
1283
1302
1284
1303
found :
1285
- x = best ;
1304
+ if (!(pol -> flags & XFRM_POLICY_CPU_ACQUIRE ) ||
1305
+ (best && (best -> pcpu_num == pcpu_id )))
1306
+ x = best ;
1307
+
1286
1308
if (!x && !error && !acquire_in_progress ) {
1287
1309
if (tmpl -> id .spi &&
1288
1310
(x0 = __xfrm_state_lookup_all (net , mark , daddr ,
@@ -1314,6 +1336,8 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1314
1336
xfrm_init_tempstate (x , fl , tmpl , daddr , saddr , family );
1315
1337
memcpy (& x -> mark , & pol -> mark , sizeof (x -> mark ));
1316
1338
x -> if_id = if_id ;
1339
+ if ((pol -> flags & XFRM_POLICY_CPU_ACQUIRE ) && best )
1340
+ x -> pcpu_num = pcpu_id ;
1317
1341
1318
1342
error = security_xfrm_state_alloc_acquire (x , pol -> security , fl -> flowi_secid );
1319
1343
if (error ) {
@@ -1392,6 +1416,11 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1392
1416
x = NULL ;
1393
1417
error = - ESRCH ;
1394
1418
}
1419
+
1420
+ /* Use the already installed 'fallback' while the CPU-specific
1421
+ * SA acquire is handled*/
1422
+ if (best )
1423
+ x = best ;
1395
1424
}
1396
1425
out :
1397
1426
if (x ) {
@@ -1524,12 +1553,14 @@ static void __xfrm_state_bump_genids(struct xfrm_state *xnew)
1524
1553
unsigned int h ;
1525
1554
u32 mark = xnew -> mark .v & xnew -> mark .m ;
1526
1555
u32 if_id = xnew -> if_id ;
1556
+ u32 cpu_id = xnew -> pcpu_num ;
1527
1557
1528
1558
h = xfrm_dst_hash (net , & xnew -> id .daddr , & xnew -> props .saddr , reqid , family );
1529
1559
hlist_for_each_entry (x , net -> xfrm .state_bydst + h , bydst ) {
1530
1560
if (x -> props .family == family &&
1531
1561
x -> props .reqid == reqid &&
1532
1562
x -> if_id == if_id &&
1563
+ x -> pcpu_num == cpu_id &&
1533
1564
(mark & x -> mark .m ) == x -> mark .v &&
1534
1565
xfrm_addr_equal (& x -> id .daddr , & xnew -> id .daddr , family ) &&
1535
1566
xfrm_addr_equal (& x -> props .saddr , & xnew -> props .saddr , family ))
@@ -1552,7 +1583,7 @@ EXPORT_SYMBOL(xfrm_state_insert);
1552
1583
static struct xfrm_state * __find_acq_core (struct net * net ,
1553
1584
const struct xfrm_mark * m ,
1554
1585
unsigned short family , u8 mode ,
1555
- u32 reqid , u32 if_id , u8 proto ,
1586
+ u32 reqid , u32 if_id , u32 pcpu_num , u8 proto ,
1556
1587
const xfrm_address_t * daddr ,
1557
1588
const xfrm_address_t * saddr ,
1558
1589
int create )
@@ -1569,6 +1600,7 @@ static struct xfrm_state *__find_acq_core(struct net *net,
1569
1600
x -> id .spi != 0 ||
1570
1601
x -> id .proto != proto ||
1571
1602
(mark & x -> mark .m ) != x -> mark .v ||
1603
+ x -> pcpu_num != pcpu_num ||
1572
1604
!xfrm_addr_equal (& x -> id .daddr , daddr , family ) ||
1573
1605
!xfrm_addr_equal (& x -> props .saddr , saddr , family ))
1574
1606
continue ;
@@ -1602,6 +1634,7 @@ static struct xfrm_state *__find_acq_core(struct net *net,
1602
1634
break ;
1603
1635
}
1604
1636
1637
+ x -> pcpu_num = pcpu_num ;
1605
1638
x -> km .state = XFRM_STATE_ACQ ;
1606
1639
x -> id .proto = proto ;
1607
1640
x -> props .family = family ;
@@ -1630,7 +1663,7 @@ static struct xfrm_state *__find_acq_core(struct net *net,
1630
1663
return x ;
1631
1664
}
1632
1665
1633
- static struct xfrm_state * __xfrm_find_acq_byseq (struct net * net , u32 mark , u32 seq );
1666
+ static struct xfrm_state * __xfrm_find_acq_byseq (struct net * net , u32 mark , u32 seq , u32 pcpu_num );
1634
1667
1635
1668
int xfrm_state_add (struct xfrm_state * x )
1636
1669
{
@@ -1656,7 +1689,7 @@ int xfrm_state_add(struct xfrm_state *x)
1656
1689
}
1657
1690
1658
1691
if (use_spi && x -> km .seq ) {
1659
- x1 = __xfrm_find_acq_byseq (net , mark , x -> km .seq );
1692
+ x1 = __xfrm_find_acq_byseq (net , mark , x -> km .seq , x -> pcpu_num );
1660
1693
if (x1 && ((x1 -> id .proto != x -> id .proto ) ||
1661
1694
!xfrm_addr_equal (& x1 -> id .daddr , & x -> id .daddr , family ))) {
1662
1695
to_put = x1 ;
@@ -1666,7 +1699,7 @@ int xfrm_state_add(struct xfrm_state *x)
1666
1699
1667
1700
if (use_spi && !x1 )
1668
1701
x1 = __find_acq_core (net , & x -> mark , family , x -> props .mode ,
1669
- x -> props .reqid , x -> if_id , x -> id .proto ,
1702
+ x -> props .reqid , x -> if_id , x -> pcpu_num , x -> id .proto ,
1670
1703
& x -> id .daddr , & x -> props .saddr , 0 );
1671
1704
1672
1705
__xfrm_state_bump_genids (x );
@@ -1791,6 +1824,7 @@ static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig,
1791
1824
x -> props .flags = orig -> props .flags ;
1792
1825
x -> props .extra_flags = orig -> props .extra_flags ;
1793
1826
1827
+ x -> pcpu_num = orig -> pcpu_num ;
1794
1828
x -> if_id = orig -> if_id ;
1795
1829
x -> tfcpad = orig -> tfcpad ;
1796
1830
x -> replay_maxdiff = orig -> replay_maxdiff ;
@@ -2066,13 +2100,14 @@ EXPORT_SYMBOL(xfrm_state_lookup_byaddr);
2066
2100
2067
2101
struct xfrm_state *
2068
2102
xfrm_find_acq (struct net * net , const struct xfrm_mark * mark , u8 mode , u32 reqid ,
2069
- u32 if_id , u8 proto , const xfrm_address_t * daddr ,
2103
+ u32 if_id , u32 pcpu_num , u8 proto , const xfrm_address_t * daddr ,
2070
2104
const xfrm_address_t * saddr , int create , unsigned short family )
2071
2105
{
2072
2106
struct xfrm_state * x ;
2073
2107
2074
2108
spin_lock_bh (& net -> xfrm .xfrm_state_lock );
2075
- x = __find_acq_core (net , mark , family , mode , reqid , if_id , proto , daddr , saddr , create );
2109
+ x = __find_acq_core (net , mark , family , mode , reqid , if_id , pcpu_num ,
2110
+ proto , daddr , saddr , create );
2076
2111
spin_unlock_bh (& net -> xfrm .xfrm_state_lock );
2077
2112
2078
2113
return x ;
@@ -2207,14 +2242,15 @@ xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n,
2207
2242
2208
2243
/* Silly enough, but I'm lazy to build resolution list */
2209
2244
2210
- static struct xfrm_state * __xfrm_find_acq_byseq (struct net * net , u32 mark , u32 seq )
2245
+ static struct xfrm_state * __xfrm_find_acq_byseq (struct net * net , u32 mark , u32 seq , u32 pcpu_num )
2211
2246
{
2212
2247
unsigned int h = xfrm_seq_hash (net , seq );
2213
2248
struct xfrm_state * x ;
2214
2249
2215
2250
hlist_for_each_entry_rcu (x , net -> xfrm .state_byseq + h , byseq ) {
2216
2251
if (x -> km .seq == seq &&
2217
2252
(mark & x -> mark .m ) == x -> mark .v &&
2253
+ x -> pcpu_num == pcpu_num &&
2218
2254
x -> km .state == XFRM_STATE_ACQ ) {
2219
2255
xfrm_state_hold (x );
2220
2256
return x ;
@@ -2224,12 +2260,12 @@ static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 s
2224
2260
return NULL ;
2225
2261
}
2226
2262
2227
- struct xfrm_state * xfrm_find_acq_byseq (struct net * net , u32 mark , u32 seq )
2263
+ struct xfrm_state * xfrm_find_acq_byseq (struct net * net , u32 mark , u32 seq , u32 pcpu_num )
2228
2264
{
2229
2265
struct xfrm_state * x ;
2230
2266
2231
2267
spin_lock_bh (& net -> xfrm .xfrm_state_lock );
2232
- x = __xfrm_find_acq_byseq (net , mark , seq );
2268
+ x = __xfrm_find_acq_byseq (net , mark , seq , pcpu_num );
2233
2269
spin_unlock_bh (& net -> xfrm .xfrm_state_lock );
2234
2270
return x ;
2235
2271
}
0 commit comments