统一收集 kubernetes worker 节点上所产生的 coredump 文件并报警

yum install -y openssl-devel fuse fuse-devel
wget https://obs-community.obs.cn-north-1.myhuaweicloud.com/obsfs/current/obsfs_CentOS7.6_amd64.tar.gz
tar zxf obsfs_CentOS7.6_amd64.tar.gz

cd obsfs_CentOS7.6_amd64/
bash install_obsfs.sh 

echo xxx:xxxx > /etc/passwd-obsfs # obs 认证信息
chmod 600 /etc/passwd-obsfs

mkdir /var/coredump

/usr/local/bin/obsfs coredump /var/coredump -o url=obs.cn-east-3.myhuaweicloud.com -o passwd_file=/etc/passwd-obsfs -o big_writes -o max_write=131072 -o allow_other -o use_ino

echo "/usr/local/bin/obsfs coredump /var/coredump -o url=obs.cn-east-3.myhuaweicloud.com -o passwd_file=/etc/passwd-obsfs -o big_writes -o max_write=131072 -o allow_other -o use_ino" >> /etc/rc.d/rc.local

chmod +x /etc/rc.d/rc.local

cat <<EOF > /etc/sysctl.d/coredump.conf 
kernel.core_pattern=|/usr/bin/coredump_helper.sh core_%h_exe_%e_tid_%I_pid_%p_sig_%s_time_%t.gz
kernel.core_uses_pid=1
EOF
sysctl --system

cp /etc/security/limits.conf  /etc/security/limits.conf.bak
cat << EOF >> /etc/security/limits.conf
root soft nofile 165535
root hard nofile 165535
root soft nproc 165535
root hard nproc 165535
root soft core unlimited
root hard core unlimited

* soft nofile 165535
* hard nofile 165535
* soft nproc 165535
* hard nproc 165535
* soft core unlimited
* hard core unlimited
EOF

sed -i 's#4096#165535#g' /etc/security/limits.d/20-nproc.conf

echo "DefaultLimitCORE=infinity"  >> /etc/systemd/system.conf
echo "DefaultLimitNOFILE=165535" >> /etc/systemd/system.conf
echo "DefaultLimitNPROC=165535" >>/etc/systemd/system.conf

sed -i "s#LimitCORE=.*#LimitCORE=unlimited#g" /usr/lib/systemd/system/docker.service

systemctl daemon-reload
systemctl restart docker

#!/bin/bash

function log {
    echo "[$(date --rfc-3339=seconds)]: $*" >> "/tmp/coredump_$(date +"%Y-%m").log"
}

coredump_file="${1}"
coredump_path="/var/coredump"
host_ip="$(/usr/sbin/ip -4 route get 8.8.8.8 | head -1 | awk '{print $7}')"
pod_name="$(echo ${coredump_file} | awk -F'_' '{print $2}')"
exe_name="$(echo ${coredump_file} | awk -F'_exe_|_tid_' '{print $2}')"
file_time="$(date +'%Y-%m-%d %T' -d @$(echo ${coredump_file} | awk -F'_time_|.gz' '{print $2}'))"
file_url="https://xxx.com/" # coredump 文件的下载路径
notify_url="https://open.feishu.cn/open-apis/bot/v2/hook/XXXX"  # 飞书的通知API
lock_file="${coredump_path}/${exe_name}:-ABC123456.lock"

log "start."
log "host_ip: ${host_ip}"
log "pod_name: ${pod_name}"
log "exe_name: ${exe_name}"
log "file_time: ${file_time}"

[ ! -d "${coredump_path}" ] &&  mkdir -p "${coredump_path}"

# 加锁，避免多实例重复生成coredump
find "${coredump_path}" -type f -mmin +5 -name "${lock_file}" -exec rm -fv {} \;
[ -f "${lock_file}"] && {log "lock file exits: ${lock_file}"; exit 0}
touch "${lock_file}"

gzip > "/tmp/${coredump_file}" && log "coredump: /tmp/${coredump_file}"


if [ -f "/tmp/${coredump_file}" ];then

  find /var/lib/docker -type f -executable -size +1M -name "${exe_name:-ABC123456}*" -exec cp -fv {} /tmp/ \;

  tar zcvf "/tmp/${coredump_file}.tgz" -C /tmp "${coredump_file}" $(find /tmp/ -type f -executable -size +1k -name "${exe_name:-ABC123456}*")
  log "compressed file: /tmp/${coredump_file}.tgz"

  rm -fv "/tmp/${coredump_file}" "${coredump_file}" $(find /tmp/ -type f -executable -size +1k -name "${exe_name:-ABC123456}*")

  core_size="$(du -sh /tmp/${coredump_file}.tgz 2>/dev/null | awk '{print $1}')"
  log "core_size: ${core_size}"

  mv -fv "/tmp/${coredump_file}.tgz" "${coredump_path}/${coredump_file}.tgz"

  curl -X POST -H "Content-Type: application/json" \
     -d "{\"msg_type\":\"interactive\",\"card\":{\"config\":{\"wide_screen_mode\":true,\"enable_forward\":true},\"elements\":[{\"tag\":\"div\",\"text\":{\"content\":\"**Time:** ${file_time}\\n**Host:** ${host_ip}\\n**POD:** ${pod_name}\\n**EXE:** ${exe_name}\\n**Size:** ${core_size}\",\"tag\":\"lark_md\"}},{\"actions\":[{\"tag\":\"button\",\"text\":{\"content\":\"点我下载 coredump 文件\",\"tag\":\"lark_md\"},\"url\":\"${file_url}${coredump_file}.tgz\",\"type\":\"default\",\"value\":{}}],\"tag\":\"action\"}],\"header\":{\"template\":\"turquoise\",\"title\":{\"content\":\"coredump 文件生成\",\"tag\":\"plain_text\"}}}}" \
     ${notify_url}
  log "success."
else
  curl -X POST -H "Content-Type: application/json" \
     -d "{\"msg_type\":\"interactive\",\"card\":{\"config\":{\"wide_screen_mode\":true,\"enable_forward\":true},\"elements\":[{\"tag\":\"div\",\"text\":{\"content\":\"**Host:** ${host_ip}\\n**POD:** ${pod_name}\\n**File:** 未生成文件\\n**Size:** ${core_size}\",\"tag\":\"lark_md\"}}],\"header\":{\"template\": \"red\",\"title\":{\"content\":\"coredump 文件生成\",\"tag\":\"plain_text\"}}}}" \
     ${notify_url}
  log "error."
fi

find "${coredump_path:-/tmp/123}" -type f -mtime +1 -exec rm -rf {} \;
log "done."

chmod +x /usr/bin/coredump_helper.sh

docker run -ti --rm  centos bash
yum install -y gcc 

cat <<EOF > a.c
#include<stdio.h>
int main(){
  char *p = NULL;
  *p = 0;
  return 0;
}
EOF

gcc a.c
./a.out