Skip to content

Commit cb80329

Browse files
Jack Wangjgunthorpe
Jack Wang
authored and
jgunthorpe
committed
RDMA/rtrs: client: private header with client structs and functions
This header describes main structs and functions used by rtrs-client module, mainly for managing rtrs sessions, creating/destroying sysfs entries, accounting statistics on client side. Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Danil Kipnis <[email protected]> Signed-off-by: Jack Wang <[email protected]> Signed-off-by: Jason Gunthorpe <[email protected]>
1 parent c0894b3 commit cb80329

File tree

1 file changed

+251
-0
lines changed

1 file changed

+251
-0
lines changed

‎drivers/infiniband/ulp/rtrs/rtrs-clt.h

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
/* SPDX-License-Identifier: GPL-2.0-or-later */
2+
/*
3+
* RDMA Transport Layer
4+
*
5+
* Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
6+
* Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
7+
* Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
8+
*/
9+
10+
#ifndef RTRS_CLT_H
11+
#define RTRS_CLT_H
12+
13+
#include <linux/device.h>
14+
#include "rtrs-pri.h"
15+
16+
/**
17+
* enum rtrs_clt_state - Client states.
18+
*/
19+
enum rtrs_clt_state {
20+
RTRS_CLT_CONNECTING,
21+
RTRS_CLT_CONNECTING_ERR,
22+
RTRS_CLT_RECONNECTING,
23+
RTRS_CLT_CONNECTED,
24+
RTRS_CLT_CLOSING,
25+
RTRS_CLT_CLOSED,
26+
RTRS_CLT_DEAD,
27+
};
28+
29+
enum rtrs_mp_policy {
30+
MP_POLICY_RR,
31+
MP_POLICY_MIN_INFLIGHT,
32+
};
33+
34+
/* see Documentation/ABI/testing/sysfs-class-rtrs-client for details */
35+
struct rtrs_clt_stats_reconnects {
36+
int successful_cnt;
37+
int fail_cnt;
38+
};
39+
40+
/* see Documentation/ABI/testing/sysfs-class-rtrs-client for details */
41+
struct rtrs_clt_stats_cpu_migr {
42+
atomic_t from;
43+
int to;
44+
};
45+
46+
/* stats for Read and write operation.
47+
* see Documentation/ABI/testing/sysfs-class-rtrs-client for details
48+
*/
49+
struct rtrs_clt_stats_rdma {
50+
struct {
51+
u64 cnt;
52+
u64 size_total;
53+
} dir[2];
54+
55+
u64 failover_cnt;
56+
};
57+
58+
struct rtrs_clt_stats_pcpu {
59+
struct rtrs_clt_stats_cpu_migr cpu_migr;
60+
struct rtrs_clt_stats_rdma rdma;
61+
};
62+
63+
struct rtrs_clt_stats {
64+
struct kobject kobj_stats;
65+
struct rtrs_clt_stats_pcpu __percpu *pcpu_stats;
66+
struct rtrs_clt_stats_reconnects reconnects;
67+
atomic_t inflight;
68+
};
69+
70+
struct rtrs_clt_con {
71+
struct rtrs_con c;
72+
struct rtrs_iu *rsp_ius;
73+
u32 queue_size;
74+
unsigned int cpu;
75+
atomic_t io_cnt;
76+
int cm_err;
77+
};
78+
79+
/**
80+
* rtrs_permit - permits the memory allocation for future RDMA operation.
81+
* Combine with irq pinning to keep IO on same CPU.
82+
*/
83+
struct rtrs_permit {
84+
enum rtrs_clt_con_type con_type;
85+
unsigned int cpu_id;
86+
unsigned int mem_id;
87+
unsigned int mem_off;
88+
};
89+
90+
/**
91+
* rtrs_clt_io_req - describes one inflight IO request
92+
*/
93+
struct rtrs_clt_io_req {
94+
struct list_head list;
95+
struct rtrs_iu *iu;
96+
struct scatterlist *sglist; /* list holding user data */
97+
unsigned int sg_cnt;
98+
unsigned int sg_size;
99+
unsigned int data_len;
100+
unsigned int usr_len;
101+
void *priv;
102+
bool in_use;
103+
struct rtrs_clt_con *con;
104+
struct rtrs_sg_desc *desc;
105+
struct ib_sge *sge;
106+
struct rtrs_permit *permit;
107+
enum dma_data_direction dir;
108+
void (*conf)(void *priv, int errno);
109+
unsigned long start_jiffies;
110+
111+
struct ib_mr *mr;
112+
struct ib_cqe inv_cqe;
113+
struct completion inv_comp;
114+
int inv_errno;
115+
bool need_inv_comp;
116+
bool need_inv;
117+
};
118+
119+
struct rtrs_rbuf {
120+
u64 addr;
121+
u32 rkey;
122+
};
123+
124+
struct rtrs_clt_sess {
125+
struct rtrs_sess s;
126+
struct rtrs_clt *clt;
127+
wait_queue_head_t state_wq;
128+
enum rtrs_clt_state state;
129+
atomic_t connected_cnt;
130+
struct mutex init_mutex;
131+
struct rtrs_clt_io_req *reqs;
132+
struct delayed_work reconnect_dwork;
133+
struct work_struct close_work;
134+
unsigned int reconnect_attempts;
135+
bool established;
136+
struct rtrs_rbuf *rbufs;
137+
size_t max_io_size;
138+
u32 max_hdr_size;
139+
u32 chunk_size;
140+
size_t queue_depth;
141+
u32 max_pages_per_mr;
142+
int max_send_sge;
143+
u32 flags;
144+
struct kobject kobj;
145+
struct rtrs_clt_stats *stats;
146+
/* cache hca_port and hca_name to display in sysfs */
147+
u8 hca_port;
148+
char hca_name[IB_DEVICE_NAME_MAX];
149+
struct list_head __percpu
150+
*mp_skip_entry;
151+
};
152+
153+
struct rtrs_clt {
154+
struct list_head paths_list; /* rcu protected list */
155+
size_t paths_num;
156+
struct rtrs_clt_sess
157+
__rcu * __percpu *pcpu_path;
158+
uuid_t paths_uuid;
159+
int paths_up;
160+
struct mutex paths_mutex;
161+
struct mutex paths_ev_mutex;
162+
char sessname[NAME_MAX];
163+
u16 port;
164+
unsigned int max_reconnect_attempts;
165+
unsigned int reconnect_delay_sec;
166+
unsigned int max_segments;
167+
void *permits;
168+
unsigned long *permits_map;
169+
size_t queue_depth;
170+
size_t max_io_size;
171+
wait_queue_head_t permits_wait;
172+
size_t pdu_sz;
173+
void *priv;
174+
void (*link_ev)(void *priv,
175+
enum rtrs_clt_link_ev ev);
176+
struct device dev;
177+
struct kobject *kobj_paths;
178+
enum rtrs_mp_policy mp_policy;
179+
};
180+
181+
static inline struct rtrs_clt_con *to_clt_con(struct rtrs_con *c)
182+
{
183+
return container_of(c, struct rtrs_clt_con, c);
184+
}
185+
186+
static inline struct rtrs_clt_sess *to_clt_sess(struct rtrs_sess *s)
187+
{
188+
return container_of(s, struct rtrs_clt_sess, s);
189+
}
190+
191+
static inline int permit_size(struct rtrs_clt *clt)
192+
{
193+
return sizeof(struct rtrs_permit) + clt->pdu_sz;
194+
}
195+
196+
static inline struct rtrs_permit *get_permit(struct rtrs_clt *clt, int idx)
197+
{
198+
return (struct rtrs_permit *)(clt->permits + permit_size(clt) * idx);
199+
}
200+
201+
int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_sess *sess);
202+
int rtrs_clt_disconnect_from_sysfs(struct rtrs_clt_sess *sess);
203+
int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt,
204+
struct rtrs_addr *addr);
205+
int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess,
206+
const struct attribute *sysfs_self);
207+
208+
void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt *clt, int value);
209+
int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt *clt);
210+
void free_sess(struct rtrs_clt_sess *sess);
211+
212+
/* rtrs-clt-stats.c */
213+
214+
int rtrs_clt_init_stats(struct rtrs_clt_stats *stats);
215+
216+
void rtrs_clt_inc_failover_cnt(struct rtrs_clt_stats *s);
217+
218+
void rtrs_clt_update_wc_stats(struct rtrs_clt_con *con);
219+
void rtrs_clt_update_all_stats(struct rtrs_clt_io_req *req, int dir);
220+
221+
int rtrs_clt_reset_rdma_lat_distr_stats(struct rtrs_clt_stats *stats,
222+
bool enable);
223+
ssize_t rtrs_clt_stats_rdma_lat_distr_to_str(struct rtrs_clt_stats *stats,
224+
char *page, size_t len);
225+
int rtrs_clt_reset_cpu_migr_stats(struct rtrs_clt_stats *stats, bool enable);
226+
int rtrs_clt_stats_migration_cnt_to_str(struct rtrs_clt_stats *stats, char *buf,
227+
size_t len);
228+
int rtrs_clt_reset_reconnects_stat(struct rtrs_clt_stats *stats, bool enable);
229+
int rtrs_clt_stats_reconnects_to_str(struct rtrs_clt_stats *stats, char *buf,
230+
size_t len);
231+
int rtrs_clt_reset_wc_comp_stats(struct rtrs_clt_stats *stats, bool enable);
232+
int rtrs_clt_stats_wc_completion_to_str(struct rtrs_clt_stats *stats, char *buf,
233+
size_t len);
234+
int rtrs_clt_reset_rdma_stats(struct rtrs_clt_stats *stats, bool enable);
235+
ssize_t rtrs_clt_stats_rdma_to_str(struct rtrs_clt_stats *stats,
236+
char *page, size_t len);
237+
int rtrs_clt_reset_all_stats(struct rtrs_clt_stats *stats, bool enable);
238+
ssize_t rtrs_clt_reset_all_help(struct rtrs_clt_stats *stats,
239+
char *page, size_t len);
240+
241+
/* rtrs-clt-sysfs.c */
242+
243+
int rtrs_clt_create_sysfs_root_files(struct rtrs_clt *clt);
244+
void rtrs_clt_destroy_sysfs_root_folders(struct rtrs_clt *clt);
245+
void rtrs_clt_destroy_sysfs_root_files(struct rtrs_clt *clt);
246+
247+
int rtrs_clt_create_sess_files(struct rtrs_clt_sess *sess);
248+
void rtrs_clt_destroy_sess_files(struct rtrs_clt_sess *sess,
249+
const struct attribute *sysfs_self);
250+
251+
#endif /* RTRS_CLT_H */

0 commit comments

Comments
 (0)