12

Linux soft lockup时远程调试的可能性

 3 years ago
source link: https://blog.csdn.net/dog250/article/details/111400078
Go to the source link to view the article. You can view the picture content, updated content and better typesetting reading experience. If the link is broken, please click the button below to view the snapshot at that time.

Linux soft lockup时远程调试的可能性

dog250 2020-12-19 11:08:14 1700
文章标签: soft lockup

曾经写过一个模块,当运行Linux内核的机器死机时,SSH肯定无法登录了,但只要它还响应中断,就尽力让它可以通过网络带回一些信息。陈年的事了:
https://blog.csdn.net/dog250/article/details/43370611

今日重提这件事,不陌生,但纠结。

本文不谈sysrq,也不谈别的。


Linux内核在发生soft lockup的时候,是可以ping通的,只要没有关中断,ping通一般没有问题。既然可以ping通,何必不带回一些真正重要的信息而不仅仅是echo的reply?

且慢,你可能会觉得这一切没有意义,懂kdump的人都会这么抬杠,毕竟如果这个时候让内核panic掉,保留一个vmcore,事后便可以随便分析了。

哈哈,我也不是不懂kdump,我当然懂得如何分析vmcore,我只是不信任它而已,我不觉得它保留有足够的信息,相比之下,我只想知道在事故发生的当时,到底发生了什么,因此,我需要尽可能的去debug将死未死的系统,也就是说,我想要获取已经soft lockup的内核的信息。

如果你重启了内核,保留了一具vmcore尸体,如果是攻击的情况,很可能在系统重启的过程中,攻击者就发觉了,暂停了攻击或者更改了方式…

不要在既有的框架内就事论事,找些没文化的流氓一起讨论会比和经理讨论可能更有收获。有的时候我不想争论,不是说我不善于争论,而是我觉得和我争论的人根本不知道我在说什么,唉。

SSH已经不能指望了,怎么办?

想法简单,不足道,仅仅是个POC,也希望能有人一起讨论:

  • 注册一个新的四层协议,除了TCP/UDP/ICMP等熟知协议之外的新协议,这是为了避免每一个数据包都要经过过滤,避免影响性能。
  • 事先分配skb,避免当事故发生时回送信息时分配skb失败。

好了,看代码,先给出载入内核的代码,这个代码的大部分都是我从网上抄来的,并不是自己写的,我只是重组了逻辑:

#include <net/protocol.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/udp.h>

#define IPPROTO_MYPROTO  123
#define QUOTA	30

struct sk_buff *eskb[QUOTA];

static int quota = QUOTA - 1;
//module_param(quota, int, 0644);
//MODULE_PARM_DESC(quota, "soft_lockup");

unsigned short _csum(unsigned short* data, int len)
{
	int pad = 0;
	int i = 0;
	unsigned short ret = 0;
	unsigned int sum = 0;
	if (len % 2 != 0)
		pad = 1;
	len /= 2;
	for ( i = 0; i < len; i++) {
		sum += data[i];
	}
	if (pad == 1)
		sum += ((unsigned char*)(data + len))[0] ;
	sum = (sum & 0xffff) + (sum >> 16);
	sum += (sum >> 16);
	ret = ~sum;
	return ret;
}

int myproto_rcv(struct sk_buff *skb)
{
	struct udphdr *uh, *euh;
	struct iphdr *iph, *eiph;
	struct ethhdr *eh, *ethh;
	char esaddr[6];
	unsigned char *pos;

	if (quota < 0) {
		goto end;
	}
	iph = ip_hdr(skb);
	uh = udp_hdr(skb);
	eh = (struct ethhdr *)(((unsigned char *)iph) - sizeof(struct ethhdr));

	// 出事的时候,直接构造已经分配的skb
	eskb[quota]->ip_summed = CHECKSUM_NONE;
	eskb[quota]->protocol = htons(ETH_P_IP);
	eskb[quota]->priority = 0;
	eskb[quota]->dev = skb->dev;
	eskb[quota]->pkt_type = PACKET_OTHERHOST;
	skb_reserve(eskb[quota], 1300 + sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct udphdr));

	pos = skb_push(eskb[quota], 1300);
	strcpy(pos, "abcdefghijk123456789");
	pos = skb_push(eskb[quota], sizeof(struct udphdr));
	skb_reset_transport_header(eskb[quota]);
	euh = (struct udphdr *)pos;
	euh->source = uh->dest;
	euh->dest = uh->source;
	euh->len = htons(1300 + sizeof(struct udphdr));
	euh->check = 0;

	memcpy(pos - 12, &iph->daddr, 4);
	memcpy(pos - 8, &iph->saddr, 4);
	((unsigned short *)(pos - 4))[0] = 0x1100;
	memcpy(pos - 2, &euh->len, sizeof(euh->len));
	euh->check = _csum((unsigned short*)(pos - 12), 12 + 1300 + sizeof(struct udphdr));

	pos = skb_push(eskb[quota], sizeof(struct iphdr));
	skb_reset_network_header(eskb[quota]);
	eiph = (struct iphdr *)pos;
	eiph->version = 4;
	eiph->ihl = 5;
	eiph->tos = 0;
	eiph->tot_len = htons(1300 + sizeof(struct udphdr) + sizeof(struct iphdr));
	eiph->id = 0x1122;
	eiph->frag_off = 0;
	eiph->ttl = 64;
	eiph->protocol = 0x11;
	eiph->check = 0;
	eiph->saddr = iph->daddr;
	eiph->daddr = iph->saddr;
	eiph->check = _csum((unsigned short*)pos, sizeof(struct iphdr));

	pos = skb_push(eskb[quota], sizeof(struct ethhdr));
	skb_reset_mac_header(eskb[quota]);

	ethh = (struct ethhdr *)pos;

	memcpy(esaddr, eh->h_dest, 6);
	memcpy(ethh->h_dest, eh->h_source, ETH_ALEN);
	memcpy(ethh->h_source, eh->h_dest, ETH_ALEN);
	ethh->h_proto = htons(ETH_P_IP);

	printk("myproto_rcv is called, length:%d  %x %x\n", skb->len, esaddr[2], esaddr[3]);

	dev_queue_xmit(eskb[quota]);

	quota --;
end:
	kfree_skb(skb);
	return 0;
}

int myproto_rcv_err(struct sk_buff *skb, unsigned int err)
{
	printk("myproto_rcv is called:%d\n", err);
	kfree_skb(skb);
	return 0;
}

static const struct net_protocol myproto_protocol = {
	.handler = myproto_rcv,
	.err_handler = myproto_rcv_err,
	.no_policy = 1,
	.netns_ok = 1,
};

int init_module(void)
{
	int ret = 0, i;

	// 事先分配skb
	for (i = 0; i < QUOTA; i++) {
		eskb[i] = alloc_skb(1300 + sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct udphdr), GFP_ATOMIC);
		if (eskb[i] == NULL) {
			//int j;
			//for () {
			//}
			printk("alloc failed\n");
			return -1;
		}
	}

	// 注册123协议,它不是TCP,UDP,ICMP
	ret = inet_add_protocol(&myproto_protocol, IPPROTO_MYPROTO);
	if (ret) {
		printk("failed\n");
		return ret;
	}
	printk("successful\n");
	return 0;
}

void cleanup_module(void)
{
	int rc = 0;
	inet_del_protocol(&myproto_protocol, IPPROTO_MYPROTO);
	//for (i = quota; i >=0; i--) {
		//kfree_skb(eskb[i]);
	//}
	return;
}

int init_module(void);
void cleanup_module(void);
MODULE_LICENSE("GPL");

来来来,看一下客户端的代码。

客户端需要通过raw套接字发送一个“请求”,它的传输层协议是123,然而回复的却是一个标准的UDP报文,所以客户端需要在该UDP上接收。

代码如下:

#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <linux/ip.h>
#include <linux/udp.h>

#define PCKT_LEN 8192

unsigned short csum(unsigned short *buf, int nwords)
{
  unsigned long sum;
  for(sum=0; nwords>0; nwords--)
    sum += *buf++;
  sum = (sum >> 16) + (sum &0xffff);
  sum += (sum >> 16);
  return (unsigned short)(~sum);
}

int main(int argc, char const *argv[])
{
	int sd, usd;
	struct iphdr *ip;
	struct udphdr *udp;
	struct sockaddr_in sin, usin, csin;
	u_int16_t src_port, dst_port;
	u_int32_t src_addr, dst_addr;
	int one = 1;
	const int *val = &one;
	int dlen, rlen, clen = sizeof(csin);
	char *data;
	char buf[1300];

	if (argc != 6) {
		printf("Usage: %s <source hostname/IP> <source port> <target hostname/IP> <target port>\n", argv[0]);
		exit(1);
	}

	src_addr = inet_addr(argv[1]);
	dst_addr = inet_addr(argv[3]);
	src_port = atoi(argv[2]);
	dst_port = atoi(argv[4]);
	dlen = atoi(argv[5]);

	data = malloc(sizeof(struct iphdr) + sizeof(struct udphdr) + dlen);

	ip = (struct iphdr *)data;
	udp = (struct udphdr *)(data + sizeof(struct iphdr));

	sd = socket(PF_INET, SOCK_RAW, IPPROTO_UDP);
	if (sd < 0) {
		perror("raw error");
		exit(2);
	}


	if(setsockopt(sd, IPPROTO_IP, IP_HDRINCL, val, sizeof(one)) < 0) {
		perror("setsockopt() error");
		exit(2);
	}
	sin.sin_family = AF_INET;
	sin.sin_port = htons(dst_port);

	sin.sin_addr.s_addr = inet_addr(argv[3]);

	ip->ihl = 5;
	ip->version = 4;
	ip->tos = 16; // low delay
	ip->tot_len = sizeof(struct iphdr) + sizeof(struct udphdr) + dlen;
	ip->id = htons(54321);
	ip->ttl = 64; // hops
	ip->protocol = 123; // UDP
	ip->saddr = src_addr;
	ip->daddr = dst_addr;

	udp->source = htons(src_port);
	udp->dest = htons(dst_port);
	udp->len = htons(sizeof(struct udphdr) + dlen);

	ip->check = csum((unsigned short *)data, sizeof(struct iphdr) + sizeof(struct udphdr) + dlen);

	usd = socket(AF_INET, SOCK_DGRAM, 0);
	if (usd < 0) {
		perror("usd error");
		exit(2);
	}

	bzero(&usin, sizeof(usin));
	usin.sin_family = AF_INET;
	usin.sin_port   = htons(src_port);
	usin.sin_addr.s_addr = inet_addr(argv[1]);

	if (bind(usd, (struct sockaddr *)&usin, sizeof(usin))) {
		perror("bind error");
		exit(2);
	}


	if (sendto(sd, data, ip->tot_len, 0, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
		perror("sendto()");
		exit(3);
	}
	rlen = recvfrom(usd, buf, sizeof(buf), 0, (struct sockaddr*)&csin, &clen);
	printf("recv:%s\n", buf);
	close(sd);
	return 0;
}


好了,我们在服务端加载内核模块,制造一个死锁或者玩一个fork炸弹,SSH已经无法登录但是能ping通的情况下,执行我们的客户端程序,可以完美给出结果。

我们只需要把“abcdefghijk123456789”改成当前内核能取到的信息即可,没意思也不好玩了。

哦,对了,必须补充一段。这个代码有很多不可行的情况,比如你用了_irq前缀把硬中断禁用了,比如你的网络拓扑不是直来直往的,比如你有更好的带外系统,比如各种其它的不适用。但是至少,在直连的情况下,你SSH都登录不上了,我这个破烂玩意儿可以带回一些信息,哪怕只是一双皮鞋👞。


浙江温州皮鞋湿,下雨进水不会胖。


About Joyk


Aggregate valuable and interesting links.
Joyk means Joy of geeK