Skip to content

Commit bdbc03f

Browse files
vrutkovsvrutkovs
committed
bootstrap: update openshift-gcp-routes
Sync with openshift-gcp-routes.* from MCO master
1 parent b747757 commit bdbc03f

File tree

2 files changed

+115
-101
lines changed

2 files changed

+115
-101
lines changed

‎bootstrap/overlay/etc/systemd/system/openshift-gcp-routes.service

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,15 @@
22
Description=Update GCP routes for forwarded IPs.
33
ConditionKernelCommandLine=|ignition.platform.id=gce
44
ConditionKernelCommandLine=|ignition.platform.id=gcp
5-
6-
Wants=network-online.target
7-
After=network-online.target
5+
Before=network-online.target
86
[Service]
97
Type=simple
108
ExecStart=/bin/bash /usr/local/bin/openshift-gcp-routes.sh start
119
ExecStopPost=/bin/bash /usr/local/bin/openshift-gcp-routes.sh cleanup
1210
User=root
1311
RestartSec=30
1412
Restart=always
15-
1613
[Install]
1714
WantedBy=multi-user.target
15+
# Ensure that network-online.target will not complete until the node has working external LBs.
16+
RequiredBy=network-online.target

‎bootstrap/overlay/usr/local/bin/openshift-gcp-routes.sh

Lines changed: 112 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -23,138 +23,153 @@ set -e
2323
declare -A vips
2424

2525
curler() {
26-
curl --silent -L -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/${1}"
26+
curl --silent -L -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/${1}"
2727
}
2828

2929
CHAIN_NAME="gcp-vips"
30-
3130
RUN_DIR="/run/cloud-routes"
31+
3232
# Create a chan if it doesn't exist
3333
ensure_chain() {
34-
local table="${1}"
35-
local chain="${2}"
34+
local table="${1}"
35+
local chain="${2}"
3636

37-
if ! iptables -w -t "${table}" -S "${chain}" &> /dev/null ; then
38-
iptables -w -t "${table}" -N "${chain}";
39-
fi;
37+
if ! iptables -w -t "${table}" -S "${chain}" &> /dev/null ; then
38+
iptables -w -t "${table}" -N "${chain}";
39+
fi;
4040
}
4141

4242
ensure_rule() {
43-
local table="${1}"
44-
local chain="${2}"
45-
shift 2
43+
local table="${1}"
44+
local chain="${2}"
45+
shift 2
4646

47-
if ! iptables -w -t "${table}" -C "${chain}" "$@" &> /dev/null; then
48-
iptables -w -t "${table}" -A "${chain}" "$@"
49-
fi
47+
if ! iptables -w -t "${table}" -C "${chain}" "$@" &> /dev/null; then
48+
iptables -w -t "${table}" -A "${chain}" "$@"
49+
fi
5050
}
5151

5252
# set the chain, ensure entry rules, ensure ESTABLISHED rule
5353
initialize() {
54-
ensure_chain nat "${CHAIN_NAME}"
55-
ensure_chain nat "${CHAIN_NAME}-local"
56-
ensure_rule nat PREROUTING -m comment --comment 'gcp LB vip DNAT' -j ${CHAIN_NAME}
57-
ensure_rule nat OUTPUT -m comment --comment 'gcp LB vip DNAT for local clients' -j ${CHAIN_NAME}-local
58-
59-
# Need this so that existing flows (with an entry in conntrack) continue to be
60-
# balanced, even if the DNAT entry is removed
61-
ensure_rule filter INPUT -m comment --comment 'gcp LB vip existing' -m addrtype ! --dst-type LOCAL -m state --state ESTABLISHED,RELATED -j ACCEPT
62-
63-
mkdir -p "${RUN_DIR}"
54+
ensure_chain nat "${CHAIN_NAME}"
55+
ensure_chain nat "${CHAIN_NAME}-local"
56+
ensure_rule nat PREROUTING -m comment --comment 'gcp LB vip DNAT' -j ${CHAIN_NAME}
57+
ensure_rule nat OUTPUT -m comment --comment 'gcp LB vip DNAT for local clients' -j ${CHAIN_NAME}-local
58+
59+
# Need this so that existing flows (with an entry in conntrack) continue to be
60+
# balanced, even if the DNAT entry is removed
61+
ensure_rule filter INPUT -m comment --comment 'gcp LB vip existing' -m addrtype ! --dst-type LOCAL -m state --state ESTABLISHED,RELATED -j ACCEPT
62+
63+
# bz1925698: GCP LBs can create stale entries causing apiservers disruption
64+
# The GCP LB Health Check polls continuously the LB VIP assigned to the VM
65+
# but, if the LB VIP is down, we are not handling the VIP traffic and
66+
# the traffic can hairpin, leaving stale conntrack entries on the host.
67+
# xref: https://bugzilla.redhat.com/show_bug.cgi?id=1925698#c29
68+
# Deleting conntrack entries solves the problem, but it also affects other connections
69+
# directed to the LB vips, causing unexpected networks disruptions.
70+
# The solution is to not FORWARD GCP HealthCheckers traffic, because that traffic
71+
# is only directed to the host, and it must go to the INPUT chain.
72+
# xref: https://bugzilla.redhat.com/show_bug.cgi?id=1930457#c8
73+
# The HealthCheck origin ip-ranges are documented: 130.211.0.0/22 and 35.191.0.0/16
74+
# xref: https://cloud.google.com/load-balancing/docs/health-check-concepts#ip-ranges
75+
ensure_rule filter FORWARD -m comment --comment 'gcp HealthCheck traffic' -s 35.191.0.0/16 -j DROP
76+
ensure_rule filter FORWARD -m comment --comment 'gcp HealthCheck traffic' -s 130.211.0.0/22 -j DROP
77+
78+
mkdir -p "${RUN_DIR}"
6479
}
6580

6681
remove_stale() {
67-
## find extra iptables rules
68-
for ipt_vip in $(iptables -w -t nat -S "${CHAIN_NAME}" | awk '$4{print $4}' | awk -F/ '{print $1}'); do
69-
if [[ -z "${vips[${ipt_vip}]}" ]]; then
70-
echo removing stale vip "${ipt_vip}" for external clients
71-
iptables -w -t nat -D "${CHAIN_NAME}" --dst "${ipt_vip}" -j REDIRECT
72-
fi
73-
done
74-
for ipt_vip in $(iptables -w -t nat -S "${CHAIN_NAME}-local" | awk '$4{print $4}' | awk -F/ '{print $1}'); do
75-
if [[ -z "${vips[${ipt_vip}]}" ]] || [[ "${vips[${ipt_vip}]}" = down ]]; then
76-
echo removing stale vip "${ipt_vip}" for local clients
77-
iptables -w -t nat -D "${CHAIN_NAME}-local" --dst "${ipt_vip}" -j REDIRECT
78-
fi
79-
done
82+
## find extra iptables rules
83+
for ipt_vip in $(iptables -w -t nat -S "${CHAIN_NAME}" | awk '$4{print $4}' | awk -F/ '{print $1}'); do
84+
if [[ -z "${vips[${ipt_vip}]}" ]]; then
85+
echo removing stale vip "${ipt_vip}" for external clients
86+
iptables -w -t nat -D "${CHAIN_NAME}" --dst "${ipt_vip}" -j REDIRECT
87+
fi
88+
done
89+
for ipt_vip in $(iptables -w -t nat -S "${CHAIN_NAME}-local" | awk '$4{print $4}' | awk -F/ '{print $1}'); do
90+
if [[ -z "${vips[${ipt_vip}]}" ]] || [[ "${vips[${ipt_vip}]}" = down ]]; then
91+
echo removing stale vip "${ipt_vip}" for local clients
92+
iptables -w -t nat -D "${CHAIN_NAME}-local" --dst "${ipt_vip}" -j REDIRECT
93+
fi
94+
done
8095
}
8196

8297
add_rules() {
83-
for vip in "${!vips[@]}"; do
84-
echo "ensuring rule for ${vip} for external clients"
85-
ensure_rule nat "${CHAIN_NAME}" --dst "${vip}" -j REDIRECT
86-
87-
if [[ "${vips[${vip}]}" != down ]]; then
88-
echo "ensuring rule for ${vip} for internal clients"
89-
ensure_rule nat "${CHAIN_NAME}-local" --dst "${vip}" -j REDIRECT
90-
fi
91-
done
98+
for vip in "${!vips[@]}"; do
99+
echo "ensuring rule for ${vip} for external clients"
100+
ensure_rule nat "${CHAIN_NAME}" --dst "${vip}" -j REDIRECT
101+
102+
if [[ "${vips[${vip}]}" != down ]]; then
103+
echo "ensuring rule for ${vip} for internal clients"
104+
ensure_rule nat "${CHAIN_NAME}-local" --dst "${vip}" -j REDIRECT
105+
fi
106+
done
92107
}
93108

94109
clear_rules() {
95-
iptables -t nat -F "${CHAIN_NAME}" || true
96-
iptables -t nat -F "${CHAIN_NAME}-local" || true
110+
iptables -t nat -F "${CHAIN_NAME}" || true
111+
iptables -t nat -F "${CHAIN_NAME}-local" || true
97112
}
98113

99114
# out paramater: vips
100115
list_lb_ips() {
101-
for k in "${!vips[@]}"; do
102-
unset vips["${k}"]
103-
done
104-
105-
local net_path="network-interfaces/"
106-
for vif in $(curler ${net_path}); do
107-
local hw_addr; hw_addr=$(curler "${net_path}${vif}mac")
108-
local fwip_path; fwip_path="${net_path}${vif}forwarded-ips/"
109-
for level in $(curler "${fwip_path}"); do
110-
for fwip in $(curler "${fwip_path}${level}"); do
111-
if [[ -e "${RUN_DIR}/${fwip}.down" ]]; then
112-
echo "${fwip} is manually marked as down, skipping for internal clients..."
113-
vips[${fwip}]="down"
114-
else
115-
echo "Processing route for NIC ${vif}${hw_addr} for ${fwip}"
116-
vips[${fwip}]="${fwip}"
117-
fi
118-
done
116+
for k in "${!vips[@]}"; do
117+
unset vips["${k}"]
118+
done
119+
120+
local net_path="network-interfaces/"
121+
for vif in $(curler ${net_path}); do
122+
local hw_addr; hw_addr=$(curler "${net_path}${vif}mac")
123+
local fwip_path; fwip_path="${net_path}${vif}forwarded-ips/"
124+
for level in $(curler "${fwip_path}"); do
125+
for fwip in $(curler "${fwip_path}${level}"); do
126+
if [[ -e "${RUN_DIR}/${fwip}.down" ]]; then
127+
echo "${fwip} is manually marked as down, skipping for internal clients..."
128+
vips[${fwip}]="down"
129+
else
130+
echo "Processing route for NIC ${vif}${hw_addr} for ${fwip}"
131+
vips[${fwip}]="${fwip}"
132+
fi
119133
done
120134
done
135+
done
121136
}
122137

123138
sleep_or_watch() {
124-
if hash inotifywait &> /dev/null; then
125-
inotifywait -t 30 -r "${RUN_DIR}" &> /dev/null || true
126-
else
127-
# no inotify, need to manually poll
128-
for i in {0..5}; do
129-
for vip in "${!vips[@]}"; do
130-
if [[ "${vips[${vip}]}" != down ]] && [[ -e "${RUN_DIR}/${vip}.down" ]]; then
131-
echo "new downfile detected"
132-
break 2
133-
elif [[ "${vips[${vip}]}" = down ]] && ! [[ -e "${RUN_DIR}/${vip}.down" ]]; then
134-
echo "downfile disappeared"
135-
break 2
136-
fi
137-
done
138-
sleep 1 # keep this small enough to not make gcp-routes slower than LBs on recovery
139+
if hash inotifywait &> /dev/null; then
140+
inotifywait -t 30 -r "${RUN_DIR}" &> /dev/null || true
141+
else
142+
# no inotify, need to manually poll
143+
for i in {0..5}; do
144+
for vip in "${!vips[@]}"; do
145+
if [[ "${vips[${vip}]}" != down ]] && [[ -e "${RUN_DIR}/${vip}.down" ]]; then
146+
echo "new downfile detected"
147+
break 2
148+
elif [[ "${vips[${vip}]}" = down ]] && ! [[ -e "${RUN_DIR}/${vip}.down" ]]; then
149+
echo "downfile disappeared"
150+
break 2
151+
fi
139152
done
140-
fi
153+
sleep 1 # keep this small enough to not make gcp-routes slower than LBs on recovery
154+
done
155+
fi
141156
}
142157

143158
case "$1" in
144-
start)
145-
initialize
146-
while :; do
147-
list_lb_ips
148-
remove_stale
149-
add_rules
150-
echo "done applying vip rules"
151-
sleep_or_watch
152-
done
153-
;;
154-
cleanup)
155-
clear_rules
156-
;;
157-
*)
158-
echo $"Usage: $0 {start|cleanup}"
159-
exit 1
159+
start)
160+
initialize
161+
while :; do
162+
list_lb_ips
163+
remove_stale
164+
add_rules
165+
echo "done applying vip rules"
166+
sleep_or_watch
167+
done
168+
;;
169+
cleanup)
170+
clear_rules
171+
;;
172+
*)
173+
echo $"Usage: $0 {start|cleanup}"
174+
exit 1
160175
esac

0 commit comments

Comments
 (0)