这是一个创建于 121 天前的主题，其中的信息可能已经有所发展或是发生改变。
scclb event 概括：Back-off restarting failed container lb-tcp-443 in pod svclb-traefik-e91c51d8-5vt7w_kube-system(2a842b27-2c1f-42d7-b24a-2504dfdde6a7)
使用kubectl logs svclb-traefik-e91c51d8-5vt7w -n kube-system -c lb-tcp-443，输出如下：
+ trap exit TERM INT
+ BIN_DIR=/sbin
+ check_iptables_mode
+ set +e
+ lsmod
+ grep nf_tables
+ '[' 1 '=' 0 ]
+ mode=legacy
+ set -e
+ info 'legacy mode detected'
+ echo '[INFO] ' 'legacy mode detected'
[INFO] legacy mode detected
+ set_legacy
+ ln -sf /sbin/xtables-legacy-multi /sbin/iptables
+ ln -sf /sbin/xtables-legacy-multi /sbin/iptables-save
+ ln -sf /sbin/xtables-legacy-multi /sbin/iptables-restore
+ ln -sf /sbin/xtables-legacy-multi /sbin/ip6tables
+ start_proxy
+ echo 0.0.0.0/0
+ grep -Eq :
+ iptables -t filter -I FORWARD -s 0.0.0.0/0 -p TCP --dport 443 -j ACCEPT
/usr/bin/entry: line 46: iptables: not found
系统日志如下：
Jul 07 09:45:10 ArchVF2 k3s[505]: I0707 09:45:10.681870 505 scope.go:115] "RemoveContainer" containerID="3571d6d9336cdea67e4d6b4da173b5f519e5c1e48aa141d6cf40489eb8a6fc0e"
Jul 07 09:45:10 ArchVF2 k3s[505]: I0707 09:45:10.682112 505 scope.go:115] "RemoveContainer" containerID="df5231ece32ced969d38c5687d3559384c6791c65a2f484e1e84863e6d15efa9"
Jul 07 09:45:10 ArchVF2 k3s[505]: E0707 09:45:10.687671 505 pod_workers.go:1294] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"lb-tcp-80\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=lb-tcp-80 pod=svclb-traefik-e91c51d8-5vt7w_kube-system(2a842b27-2c1f-42d7-b24a-2504dfdde6a7)\", failed to \"StartContainer\" for \"lb-tcp-443\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=lb-tcp-443 pod=svclb-traefik-e91c51d8-5vt7w_kube-system(2a842b27-2c1f-42d7-b24a-2504dfdde6a7)\"]" pod="kube-system/svclb-traefik-e91c51d8-5vt7w" podUID=2a842b27-2c1f-42d7-b24a-2504dfdde6a7
Jul 07 09:45:14 ArchVF2 k3s[505]: I0707 09:45:14.448587 505 handler.go:232] Adding GroupVersion traefik.containo.us v1alpha1 to ResourceManager
Jul 07 09:45:14 ArchVF2 k3s[505]: I0707 09:45:14.448982 505 handler.go:232] Adding GroupVersion traefik.containo.us v1alpha1 to ResourceManager
Jul 07 09:45:14 ArchVF2 k3s[505]: I0707 09:45:14.451023 505 handler.go:232] Adding GroupVersion k3s.cattle.io v1 to ResourceManager
Jul 07 09:45:14 ArchVF2 k3s[505]: I0707 09:45:14.452731 505 handler.go:232] Adding GroupVersion traefik.containo.us v1alpha1 to ResourceManager
Jul 07 09:45:14 ArchVF2 k3s[505]: I0707 09:45:14.454400 505 handler.go:232] Adding GroupVersion traefik.containo.us v1alpha1 to ResourceManager
Jul 07 09:45:14 ArchVF2 k3s[505]: I0707 09:45:14.454759 505 handler.go:232] Adding GroupVersion traefik.containo.us v1alpha1 to ResourceManager
Jul 07 09:45:14 ArchVF2 k3s[505]: I0707 09:45:14.454992 505 handler.go:232] Adding GroupVersion helm.cattle.io v1 to ResourceManager
Jul 07 09:45:14 ArchVF2 k3s[505]: I0707 09:45:14.455554 505 handler.go:232] Adding GroupVersion helm.cattle.io v1 to ResourceManager
Jul 07 09:45:14 ArchVF2 k3s[505]: I0707 09:45:14.455838 505 handler.go:232] Adding GroupVersion traefik.containo.us v1alpha1 to ResourceManager
Jul 07 09:45:14 ArchVF2 k3s[505]: I0707 09:45:14.456597 505 handler.go:232] Adding GroupVersion traefik.containo.us v1alpha1 to ResourceManager
Jul 07 09:45:22 ArchVF2 k3s[505]: W0707 09:45:22.768545 505 sysinfo.go:203] Nodes topology is not available, providing CPU topology
Jul 07 09:45:22 ArchVF2 k3s[505]: W0707 09:45:22.770967 505 machine.go:65] Cannot read vendor id correctly, set empty.
Jul 07 09:45:23 ArchVF2 k3s[505]: I0707 09:45:23.681533 505 scope.go:115] "RemoveContainer" containerID="3571d6d9336cdea67e4d6b4da173b5f519e5c1e48aa141d6cf40489eb8a6fc0e"
Jul 07 09:45:23 ArchVF2 k3s[505]: I0707 09:45:23.681782 505 scope.go:115] "RemoveContainer" containerID="df5231ece32ced969d38c5687d3559384c6791c65a2f484e1e84863e6d15efa9"
Jul 07 09:45:23 ArchVF2 k3s[505]: E0707 09:45:23.687267 505 pod_workers.go:1294] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"lb-tcp-80\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=lb-tcp-80 pod=svclb-traefik-e91c51d8-5vt7w_kube-system(2a842b27-2c1f-42d7-b24a-2504dfdde6a7)\", failed to \"StartContainer\" for \"lb-tcp-443\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=lb-tcp-443 pod=svclb-traefik-e91c51d8-5vt7w_kube-system(2a842b27-2c1f-42d7-b24a-2504dfdde6a7)\"]" pod="kube-system/svclb-traefik-e91c51d8-5vt7w" podUID=2a842b27-2c1f-42d7-b24a-2504dfdde6a7
补充，查找资料后提示似乎是宿主机iptables与iptables-nft内核模块问题，lsmod输出如下：
Module Size Used by
veth 45056 0
nft_log 12288 5
nft_limit 16384 5
xt_limit 12288 0
xt_NFLOG 12288 0
nfnetlink_log 28672 5
xt_physdev 12288 10
xt_comment 12288 172
ip_set 61440 0
vxlan 167936 0
nft_chain_nat 12288 6
nf_conntrack_netlink 65536 0
xt_addrtype 12288 13
tun 86016 4
overlay 217088 9
8188eu 1658880 0
rtl8xxxu 270336 0
mac80211 1032192 1 rtl8xxxu
dwmac_starfive 12288 0
cfg80211 638976 2 mac80211,rtl8xxxu
stmmac_platform 28672 1 dwmac_starfive
stmmac 421888 4 dwmac_starfive,stmmac_platform
rfkill 40960 3 cfg80211
vfat 28672 1
pcs_xpcs 24576 1 stmmac
evdev 32768 1
libarc4 12288 1 mac80211
fat 122880 1 vfat
phylink 69632 2 stmmac,pcs_xpcs
ptp 36864 1 stmmac
wave5 147456 0
pps_core 24576 1 ptp
v4l2_mem2mem 36864 1 wave5
axp20x_pek 16384 0
imx708 28672 0
goodix_ts 40960 0
designware_i2s 36864 2
sch_fq_codel 24576 5
br_netfilter 40960 0
bridge 442368 1 br_netfilter
stp 12288 1 bridge
llc 16384 2 bridge,stp
dm_mod 212992 0
zram 53248 2
3 条回复 • 2024-07-07 16:40:08 +08:00
 |
1
julyclyde 121 天前
不是缺命令吗？你咋会联想到模块去的？
|
 |
2
guanzhangzhang 121 天前
1. 手动 docker run 下 image 看下有没有 iptables 命令，还是 PATH env 问题
2. 你自己做的镜像还是官方的，skopeo list-tags 看看新版本 |
 |
3
dxdydz OP @guanzhangzhang 看到了：
# cat /usr/bin/entry #!/bin/sh set -ex trap exit TERM INT BIN_DIR="/sbin" info() { echo '[INFO] ' "$@" } fatal() { echo '[ERROR] ' "$@" >&2 exit 1 } check_iptables_mode() { set +e lsmod | grep "nf_tables" 2> /dev/null if [ $? = 0 ]; then mode=nft else mode=legacy fi set -e } set_nft() { for i in iptables iptables-save iptables-restore ip6tables; do ln -sf /sbin/xtables-nft-multi "$BIN_DIR/$i"; done } set_legacy() { for i in iptables iptables-save iptables-restore ip6tables; do ln -sf /sbin/xtables-legacy-multi "$BIN_DIR/$i"; done } start_proxy() { for src_range in ${SRC_RANGES}; do if echo ${src_range} | grep -Eq ":"; then ip6tables -t filter -I FORWARD -s ${src_range} -p ${DEST_PROTO} --dport ${DEST_PORT} -j ACCEPT else iptables -t filter -I FORWARD -s ${src_range} -p ${DEST_PROTO} --dport ${DEST_PORT} -j ACCEPT fi done for dest_ip in ${DEST_IPS}; do if echo ${dest_ip} | grep -Eq ":"; then [ $(cat /proc/sys/net/ipv6/conf/all/forwarding) == 1 ] || exit 1 ip6tables -t filter -A FORWARD -d ${dest_ip}/128 -p ${DEST_PROTO} --dport ${DEST_PORT} -j DROP ip6tables -t nat -I PREROUTING -p ${DEST_PROTO} --dport ${SRC_PORT} -j DNAT --to [${dest_ip}]:${DEST_PORT} ip6tables -t nat -I POSTROUTING -d ${dest_ip}/128 -p ${DEST_PROTO} -j MASQUERADE else [ $(cat /proc/sys/net/ipv4/ip_forward) == 1 ] || exit 1 iptables -t filter -A FORWARD -d ${dest_ip}/32 -p ${DEST_PROTO} --dport ${DEST_PORT} -j DROP iptables -t nat -I PREROUTING -p ${DEST_PROTO} --dport ${SRC_PORT} -j DNAT --to ${dest_ip}:${DEST_PORT} iptables -t nat -I POSTROUTING -d ${dest_ip}/32 -p ${DEST_PROTO} -j MASQUERADE fi done } check_iptables_mode case $mode in nft) info "nft mode detected" set_nft ;; legacy) info "legacy mode detected" set_legacy ;; *) fatal "invalid iptables mode" ;; esac start_proxy if [ ! -e /pause ]; then mkfifo /pause fi </pause / # cat /usr/bin/entry #!/bin/sh set -ex trap exit TERM INT BIN_DIR="/sbin" info() { echo '[INFO] ' "$@" } fatal() { echo '[ERROR] ' "$@" >&2 exit 1 } check_iptables_mode() { set +e lsmod | grep "nf_tables" 2> /dev/null if [ $? = 0 ]; then mode=nft else mode=legacy fi set -e } set_nft() { for i in iptables iptables-save iptables-restore ip6tables; do ln -sf /sbin/xtables-nft-multi "$BIN_DIR/$i"; done } set_legacy() { for i in iptables iptables-save iptables-restore ip6tables; do ln -sf /sbin/xtables-legacy-multi "$BIN_DIR/$i"; done } start_proxy() { for src_range in ${SRC_RANGES}; do if echo ${src_range} | grep -Eq ":"; then ip6tables -t filter -I FORWARD -s ${src_range} -p ${DEST_PROTO} --dport ${DEST_PORT} -j ACCEPT else iptables -t filter -I FORWARD -s ${src_range} -p ${DEST_PROTO} --dport ${DEST_PORT} -j ACCEPT fi done for dest_ip in ${DEST_IPS}; do if echo ${dest_ip} | grep -Eq ":"; then [ $(cat /proc/sys/net/ipv6/conf/all/forwarding) == 1 ] || exit 1 ip6tables -t filter -A FORWARD -d ${dest_ip}/128 -p ${DEST_PROTO} --dport ${DEST_PORT} -j DROP ip6tables -t nat -I PREROUTING -p ${DEST_PROTO} --dport ${SRC_PORT} -j DNAT --to [${dest_ip}]:${DEST_PORT} ip6tables -t nat -I POSTROUTING -d ${dest_ip}/128 -p ${DEST_PROTO} -j MASQUERADE else [ $(cat /proc/sys/net/ipv4/ip_forward) == 1 ] || exit 1 iptables -t filter -A FORWARD -d ${dest_ip}/32 -p ${DEST_PROTO} --dport ${DEST_PORT} -j DROP iptables -t nat -I PREROUTING -p ${DEST_PROTO} --dport ${SRC_PORT} -j DNAT --to ${dest_ip}:${DEST_PORT} iptables -t nat -I POSTROUTING -d ${dest_ip}/32 -p ${DEST_PROTO} -j MASQUERADE fi done } check_iptables_mode case $mode in nft) info "nft mode detected" set_nft ;; legacy) info "legacy mode detected" set_legacy ;; *) fatal "invalid iptables mode" ;; esac start_proxy if [ ! -e /pause ]; then mkfifo /pause fi </pause / # ls /sbin/xtables-nft-multi /sbin/xtables-nft-multi / # ls /sbin/iptables /sbin/iptables / # ls /sbin/xtables-legacy-multi ls: /sbin/xtables-legacy-multi: No such file or directory 我大概知道怎么改了，感谢你 |
Select Language