Skip to content

Commit b1e9a46

Browse files
adamdmossRyan Moeller
adamdmoss
authored and
Ryan Moeller
committed
Microoptimizations for VERIFY() and friends
Add branch hints and constify the intermediate evaluations of left/right params in VERIFY3*(). Reviewed-by: Ryan Moeller <[email protected]> Reviewed-by: Brian Behlendorf <[email protected]> Signed-off-by: Adam Moss <[email protected]> Closes openzfs#11708
1 parent 4f8ed09 commit b1e9a46

File tree

2 files changed

+79
-79
lines changed

2 files changed

+79
-79
lines changed

‎include/os/freebsd/spl/sys/debug.h

Lines changed: 40 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -68,65 +68,65 @@ void spl_dumpstack(void);
6868
#define PANIC(fmt, a...) \
6969
spl_panic(__FILE__, __FUNCTION__, __LINE__, fmt, ## a)
7070

71-
#define VERIFY(cond) \
72-
(void) (unlikely(!(cond)) && \
71+
#define VERIFY(cond) \
72+
(void) (unlikely(!(cond)) && \
7373
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
7474
"%s", "VERIFY(" #cond ") failed\n"))
7575

76-
#define VERIFY3B(LEFT, OP, RIGHT) do { \
77-
boolean_t _verify3_left = (boolean_t)(LEFT); \
78-
boolean_t _verify3_right = (boolean_t)(RIGHT); \
79-
if (!(_verify3_left OP _verify3_right)) \
76+
#define VERIFY3B(LEFT, OP, RIGHT) do { \
77+
const boolean_t _verify3_left = (boolean_t)(LEFT); \
78+
const boolean_t _verify3_right = (boolean_t)(RIGHT);\
79+
if (unlikely(!(_verify3_left OP _verify3_right))) \
8080
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
8181
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
82-
"failed (%d " #OP " %d)\n", \
83-
(boolean_t) (_verify3_left), \
84-
(boolean_t) (_verify3_right)); \
82+
"failed (%d " #OP " %d)\n", \
83+
(boolean_t) (_verify3_left), \
84+
(boolean_t) (_verify3_right)); \
8585
} while (0)
8686

87-
#define VERIFY3S(LEFT, OP, RIGHT) do { \
88-
int64_t _verify3_left = (int64_t)(LEFT); \
89-
int64_t _verify3_right = (int64_t)(RIGHT); \
90-
if (!(_verify3_left OP _verify3_right)) \
87+
#define VERIFY3S(LEFT, OP, RIGHT) do { \
88+
const int64_t _verify3_left = (int64_t)(LEFT); \
89+
const int64_t _verify3_right = (int64_t)(RIGHT); \
90+
if (unlikely(!(_verify3_left OP _verify3_right))) \
9191
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
9292
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
93-
"failed (%lld " #OP " %lld)\n", \
94-
(long long) (_verify3_left), \
95-
(long long) (_verify3_right)); \
93+
"failed (%lld " #OP " %lld)\n", \
94+
(long long) (_verify3_left), \
95+
(long long) (_verify3_right)); \
9696
} while (0)
9797

98-
#define VERIFY3U(LEFT, OP, RIGHT) do { \
99-
uint64_t _verify3_left = (uint64_t)(LEFT); \
100-
uint64_t _verify3_right = (uint64_t)(RIGHT); \
101-
if (!(_verify3_left OP _verify3_right)) \
98+
#define VERIFY3U(LEFT, OP, RIGHT) do { \
99+
const uint64_t _verify3_left = (uint64_t)(LEFT); \
100+
const uint64_t _verify3_right = (uint64_t)(RIGHT); \
101+
if (unlikely(!(_verify3_left OP _verify3_right))) \
102102
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
103103
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
104-
"failed (%llu " #OP " %llu)\n", \
105-
(unsigned long long) (_verify3_left), \
106-
(unsigned long long) (_verify3_right)); \
104+
"failed (%llu " #OP " %llu)\n", \
105+
(unsigned long long) (_verify3_left), \
106+
(unsigned long long) (_verify3_right)); \
107107
} while (0)
108108

109-
#define VERIFY3P(LEFT, OP, RIGHT) do { \
110-
uintptr_t _verify3_left = (uintptr_t)(LEFT); \
111-
uintptr_t _verify3_right = (uintptr_t)(RIGHT); \
112-
if (!(_verify3_left OP _verify3_right)) \
109+
#define VERIFY3P(LEFT, OP, RIGHT) do { \
110+
const uintptr_t _verify3_left = (uintptr_t)(LEFT); \
111+
const uintptr_t _verify3_right = (uintptr_t)(RIGHT);\
112+
if (unlikely(!(_verify3_left OP _verify3_right))) \
113113
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
114114
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
115-
"failed (%px " #OP " %px)\n", \
116-
(void *) (_verify3_left), \
117-
(void *) (_verify3_right)); \
115+
"failed (%px " #OP " %px)\n", \
116+
(void *) (_verify3_left), \
117+
(void *) (_verify3_right)); \
118118
} while (0)
119119

120-
#define VERIFY0(RIGHT) do { \
121-
int64_t _verify3_left = (int64_t)(0); \
122-
int64_t _verify3_right = (int64_t)(RIGHT); \
123-
if (!(_verify3_left == _verify3_right)) \
120+
#define VERIFY0(RIGHT) do { \
121+
const int64_t _verify3_left = (int64_t)(0); \
122+
const int64_t _verify3_right = (int64_t)(RIGHT); \
123+
if (unlikely(!(_verify3_left == _verify3_right))) \
124124
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
125-
"VERIFY3(0 == " #RIGHT ") " \
126-
"failed (0 == %lld)\n", \
127-
(long long) (_verify3_right)); \
125+
"VERIFY3(0 == " #RIGHT ") " \
126+
"failed (0 == %lld)\n", \
127+
(long long) (_verify3_right)); \
128128
} while (0)
129-
#define CTASSERT_GLOBAL(x) CTASSERT(x)
129+
#define CTASSERT_GLOBAL(x) CTASSERT(x)
130130

131131
/*
132132
* Debugging disabled (--disable-debug)
@@ -154,11 +154,11 @@ void spl_dumpstack(void);
154154
#define ASSERT0 VERIFY0
155155
#define ASSERT VERIFY
156156
#define IMPLY(A, B) \
157-
((void)(((!(A)) || (B)) || \
157+
((void)(likely((!(A)) || (B)) || \
158158
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
159159
"(" #A ") implies (" #B ")")))
160160
#define EQUIV(A, B) \
161-
((void)((!!(A) == !!(B)) || \
161+
((void)(likely(!!(A) == !!(B)) || \
162162
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
163163
"(" #A ") is equivalent to (" #B ")")))
164164
/* END CSTYLED */

‎include/os/linux/spl/sys/debug.h

Lines changed: 39 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -62,63 +62,63 @@ void spl_dumpstack(void);
6262
#define PANIC(fmt, a...) \
6363
spl_panic(__FILE__, __FUNCTION__, __LINE__, fmt, ## a)
6464

65-
#define VERIFY(cond) \
66-
(void) (unlikely(!(cond)) && \
65+
#define VERIFY(cond) \
66+
(void) (unlikely(!(cond)) && \
6767
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
6868
"%s", "VERIFY(" #cond ") failed\n"))
6969

70-
#define VERIFY3B(LEFT, OP, RIGHT) do { \
71-
boolean_t _verify3_left = (boolean_t)(LEFT); \
72-
boolean_t _verify3_right = (boolean_t)(RIGHT); \
73-
if (!(_verify3_left OP _verify3_right)) \
70+
#define VERIFY3B(LEFT, OP, RIGHT) do { \
71+
const boolean_t _verify3_left = (boolean_t)(LEFT); \
72+
const boolean_t _verify3_right = (boolean_t)(RIGHT);\
73+
if (unlikely(!(_verify3_left OP _verify3_right))) \
7474
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
7575
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
76-
"failed (%d " #OP " %d)\n", \
77-
(boolean_t) (_verify3_left), \
78-
(boolean_t) (_verify3_right)); \
76+
"failed (%d " #OP " %d)\n", \
77+
(boolean_t) (_verify3_left), \
78+
(boolean_t) (_verify3_right)); \
7979
} while (0)
8080

81-
#define VERIFY3S(LEFT, OP, RIGHT) do { \
82-
int64_t _verify3_left = (int64_t)(LEFT); \
83-
int64_t _verify3_right = (int64_t)(RIGHT); \
84-
if (!(_verify3_left OP _verify3_right)) \
81+
#define VERIFY3S(LEFT, OP, RIGHT) do { \
82+
const int64_t _verify3_left = (int64_t)(LEFT); \
83+
const int64_t _verify3_right = (int64_t)(RIGHT); \
84+
if (unlikely(!(_verify3_left OP _verify3_right))) \
8585
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
8686
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
87-
"failed (%lld " #OP " %lld)\n", \
88-
(long long) (_verify3_left), \
89-
(long long) (_verify3_right)); \
87+
"failed (%lld " #OP " %lld)\n", \
88+
(long long) (_verify3_left), \
89+
(long long) (_verify3_right)); \
9090
} while (0)
9191

92-
#define VERIFY3U(LEFT, OP, RIGHT) do { \
93-
uint64_t _verify3_left = (uint64_t)(LEFT); \
94-
uint64_t _verify3_right = (uint64_t)(RIGHT); \
95-
if (!(_verify3_left OP _verify3_right)) \
92+
#define VERIFY3U(LEFT, OP, RIGHT) do { \
93+
const uint64_t _verify3_left = (uint64_t)(LEFT); \
94+
const uint64_t _verify3_right = (uint64_t)(RIGHT); \
95+
if (unlikely(!(_verify3_left OP _verify3_right))) \
9696
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
9797
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
98-
"failed (%llu " #OP " %llu)\n", \
99-
(unsigned long long) (_verify3_left), \
100-
(unsigned long long) (_verify3_right)); \
98+
"failed (%llu " #OP " %llu)\n", \
99+
(unsigned long long) (_verify3_left), \
100+
(unsigned long long) (_verify3_right)); \
101101
} while (0)
102102

103-
#define VERIFY3P(LEFT, OP, RIGHT) do { \
104-
uintptr_t _verify3_left = (uintptr_t)(LEFT); \
105-
uintptr_t _verify3_right = (uintptr_t)(RIGHT); \
106-
if (!(_verify3_left OP _verify3_right)) \
103+
#define VERIFY3P(LEFT, OP, RIGHT) do { \
104+
const uintptr_t _verify3_left = (uintptr_t)(LEFT); \
105+
const uintptr_t _verify3_right = (uintptr_t)(RIGHT);\
106+
if (unlikely(!(_verify3_left OP _verify3_right))) \
107107
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
108108
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
109-
"failed (%px " #OP " %px)\n", \
110-
(void *) (_verify3_left), \
111-
(void *) (_verify3_right)); \
109+
"failed (%px " #OP " %px)\n", \
110+
(void *) (_verify3_left), \
111+
(void *) (_verify3_right)); \
112112
} while (0)
113113

114-
#define VERIFY0(RIGHT) do { \
115-
int64_t _verify3_left = (int64_t)(0); \
116-
int64_t _verify3_right = (int64_t)(RIGHT); \
117-
if (!(_verify3_left == _verify3_right)) \
114+
#define VERIFY0(RIGHT) do { \
115+
const int64_t _verify3_left = (int64_t)(0); \
116+
const int64_t _verify3_right = (int64_t)(RIGHT); \
117+
if (unlikely(!(_verify3_left == _verify3_right))) \
118118
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
119-
"VERIFY3(0 == " #RIGHT ") " \
120-
"failed (0 == %lld)\n", \
121-
(long long) (_verify3_right)); \
119+
"VERIFY3(0 == " #RIGHT ") " \
120+
"failed (0 == %lld)\n", \
121+
(long long) (_verify3_right)); \
122122
} while (0)
123123

124124
#define CTASSERT_GLOBAL(x) _CTASSERT(x, __LINE__)
@@ -154,11 +154,11 @@ void spl_dumpstack(void);
154154
#define ASSERT0 VERIFY0
155155
#define ASSERT VERIFY
156156
#define IMPLY(A, B) \
157-
((void)(((!(A)) || (B)) || \
157+
((void)(likely((!(A)) || (B)) || \
158158
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
159159
"(" #A ") implies (" #B ")")))
160160
#define EQUIV(A, B) \
161-
((void)((!!(A) == !!(B)) || \
161+
((void)(likely(!!(A) == !!(B)) || \
162162
spl_panic(__FILE__, __FUNCTION__, __LINE__, \
163163
"(" #A ") is equivalent to (" #B ")")))
164164
/* END CSTYLED */

0 commit comments

Comments
 (0)