linux kernel 网络协议栈之GRO(Generic receive offload)

gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
{
	skb_gro_reset_offset(skb);
 
	return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
}

gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
{
	switch (ret) {
	case GRO_NORMAL:
		//将数据包送进协议栈
		if (netif_receive_skb(skb))
			ret = GRO_DROP;
		break;
	//表示skb可以被free，因为gro已经将skb合并并保存起来。
	case GRO_DROP:
	case GRO_MERGED_FREE:
		//free skb
		kfree_skb(skb);
		break;
	//这个表示当前数据已经被gro保存起来，但是并没有进行合并，因此skb还需要保存。
	case GRO_HELD:
	case GRO_MERGED:
		break;
	}
 
	return ret;
}

struct napi_struct {
................................................
	//个数
	unsigned int        gro_count;
......................................
	//积攒的数据包
	struct sk_buff      *gro_list;
	struct sk_buff      *skb;
};

struct napi_gro_cb {
	/* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
	void *frag0;
 
	/* Length of frag0. */
	unsigned int frag0_len;
 
	/* This indicates where we are processing relative to skb->data. */
	int data_offset;
 
	/* This is non-zero if the packet may be of the same flow. */
	int same_flow;
 
	/* This is non-zero if the packet cannot be merged with the new skb. */
	int flush;
 
	/* Number of segments aggregated. */
	int count;
 
	/* Free the skb? */
	int free;
};

static const struct net_protocol tcp_protocol = {
	.handler =  tcp_v4_rcv,
	.err_handler =  tcp_v4_err,
	.gso_send_check = tcp_v4_gso_send_check,
	.gso_segment =  tcp_tso_segment,
	//gso回调
	.gro_receive =  tcp4_gro_receive,
	.gro_complete = tcp4_gro_complete,
	.no_policy =    1,
	.netns_ok = 1,
};
 
static struct packet_type ip_packet_type __read_mostly = {
	.type = cpu_to_be16(ETH_P_IP),
	.func = ip_rcv,
	.gso_send_check = inet_gso_send_check,
	.gso_segment = inet_gso_segment,
	//gso回调
	.gro_receive = inet_gro_receive,
	.gro_complete = inet_gro_complete,
};

gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
{
	//reset gro对应的域
	skb_gro_reset_offset(skb);
 
	return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
}

void skb_gro_reset_offset(struct sk_buff *skb)
{
	NAPI_GRO_CB(skb)->data_offset = 0;
	NAPI_GRO_CB(skb)->frag0 = NULL;
	NAPI_GRO_CB(skb)->frag0_len = 0;
	//如果mac_header和skb->tail相等并且地址不在高端内存，则说明包头保存在skb_shinfo中，所以我们需要从frags中取得对应的数据包
	if (skb->mac_header == skb->tail &&
		!PageHighMem(skb_shinfo(skb)->frags[0].page)) {
		// 可以看到frag0保存的就是对应的skb的frags的第一个元素的地址
		// frag0的作用是: 有些包的包头会存在skb->frag[0]里面，gro合并时会调用skb_gro_header_slow将包头拉到线性空间中，那么在非线性skb->frag[0]中的包头部分就应该删掉。
			NAPI_GRO_CB(skb)->frag0 =
				page_address(skb_shinfo(skb)->frags[0].page) +
				skb_shinfo(skb)->frags[0].page_offset;
		//然后保存对应的大小。
		NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;
	}
}

static gro_result_t
__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
{
	struct sk_buff *p;
 
	if (netpoll_rx_on(skb))
		return GRO_NORMAL;
	//遍历gro_list,然后判断是否有可能两个skb 相似。
	for (p = napi->gro_list; p; p = p->next) {
		//给same_flow赋值
		NAPI_GRO_CB(p)->same_flow =
			(p->dev == skb->dev) &&
			!compare_ether_header(skb_mac_header(p),
				skb_gro_mac_header(skb));
		NAPI_GRO_CB(p)->flush = 0;
	}
	//调用dev_gro_receiv
	return dev_gro_receive(napi, skb);
}

enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
{
	struct sk_buff **pp = NULL;
	struct packet_type *ptype;
	__be16 type = skb->protocol;
	struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
	int same_flow;
	int mac_len;
	enum gro_result ret;
	//判断是否支持gro
	if (!(skb->dev->features & NETIF_F_GRO))
		goto normal;
	//判断是否为切片的ip包
	if (skb_is_gso(skb) || skb_has_frags(skb))
		goto normal;

	rcu_read_lock();
	//开始遍历对应的协议表
	list_for_each_entry_rcu(ptype, head, list) {
		if (ptype->type != type || ptype->dev || !ptype->gro_receive)
			continue;

		skb_set_network_header(skb, skb_gro_offset(skb));
		mac_len = skb->network_header - skb->mac_header;
		skb->mac_len = mac_len;
		NAPI_GRO_CB(skb)->same_flow = 0;
		NAPI_GRO_CB(skb)->flush = 0;
		NAPI_GRO_CB(skb)->free = 0;
		//调用对应的gro接收函数
		pp = ptype->gro_receive(&napi->gro_list, skb);
		break;
	}
	rcu_read_unlock();
	//如果是没有实现gro的协议则也直接调到normal处理
	if (&ptype->list == head)
		goto normal;
 
	//到达这里，则说明gro_receive已经调用过了，因此进行后续的处理
 
	//得到same_flow
	same_flow = NAPI_GRO_CB(skb)->same_flow;
	//看是否有需要free对应的skb
	ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
	//如果返回值pp部位空，则说明pp需要马上被feed进协议栈
	if (pp) {
		struct sk_buff *nskb = *pp;
 
		*pp = nskb->next;
		nskb->next = NULL;
		//调用napi_gro_complete 将pp刷进协议栈
		napi_gro_complete(nskb);
		napi->gro_count--;
	}
	//如果same_flow有设置，则说明skb已经被正确的合并，因此直接返回。
	if (same_flow)
		goto ok;
	//查看是否有设置flush和gro list的个数是否已经超过限制
	// BUG: 这里是有点不对的，因为这时的skb是比gro_list中的skb更晚到的，但是却被先feed进了协议栈
	if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
		goto normal;
 
	//到达这里说明skb对应gro list来说是一个新的skb，也就是说当前的gro list并不存在可以和skb合并的数据包，因此此时将这个skb插入到gro_list的头。
	napi->gro_count++;
	NAPI_GRO_CB(skb)->count = 1;
	skb_shinfo(skb)->gso_size = skb_gro_len(skb);
	//将skb插入到gro list的头
	skb->next = napi->gro_list;
	napi->gro_list = skb;
	//设置返回值
	ret = GRO_HELD;

pull:
	//是否需要拷贝头
	if (skb_headlen(skb) < skb_gro_offset(skb)) {
		//得到对应的头的大小
		int grow = skb_gro_offset(skb) - skb_headlen(skb);
 
		BUG_ON(skb->end - skb->tail < grow);
		//开始拷贝
		memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
 
		skb->tail += grow;
		skb->data_len -= grow;
		//更新对应的frags[0]
		skb_shinfo(skb)->frags[0].page_offset += grow;
		skb_shinfo(skb)->frags[0].size -= grow;
		//如果size为0了，则说明第一个页全部包含头，因此需要将后面的页全部移动到前面。
		if (unlikely(!skb_shinfo(skb)->frags[0].size)) {
			put_page(skb_shinfo(skb)->frags[0].page);
			//开始移动。
			memmove(skb_shinfo(skb)->frags,
				skb_shinfo(skb)->frags + 1,
				--skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
		}
	}
 
ok:
	return ret;
 
normal:
	ret = GRO_NORMAL;
	goto pull;
}

//得到偏移
off = skb_gro_offset(skb);
//得到头的整个长度(mac+ip)
hlen = off + sizeof(*iph);
//得到ip头
iph = skb_gro_header_fast(skb, off);
//是否需要复制
if (skb_gro_header_hard(skb, hlen)) {
	iph = skb_gro_header_slow(skb, hlen, off);
	if (unlikely(!iph))
		goto out;
}

proto = iph->protocol & (MAX_INET_PROTOS - 1);
 
rcu_read_lock();
ops = rcu_dereference(inet_protos[proto]);
//是否支持gro
if (!ops || !ops->gro_receive)
	goto out_unlock;
//ip头是否合法, iph->version = 4, iph->ipl = 5
if (*(u8 *)iph != 0x45)
	goto out_unlock;
//ip头教研
if (unlikely(ip_fast_csum((u8 *)iph, iph->ihl)))
	goto out_unlock;

	//取出id
	id = ntohl(*(__be32 *)&iph->id);
	//判断是否需要切片
	flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb)) | (id ^ IP_DF));
	id >>= 16;
	//开始遍历gro list
	for (p = *head; p; p = p->next) {
		struct iphdr *iph2;
		//如果上一层已经不可能same flow则直接继续下一个
		if (!NAPI_GRO_CB(p)->same_flow)
			continue;
		//取出ip头
		iph2 = ip_hdr(p);
		//开始same flow的判断
		if ((iph->protocol ^ iph2->protocol) |
			(iph->tos ^ iph2->tos) |
			((__force u32)iph->saddr ^ (__force u32)iph2->saddr) |
			((__force u32)iph->daddr ^ (__force u32)iph2->daddr)) {
			NAPI_GRO_CB(p)->same_flow = 0;
			continue;
		}
		//开始flush的判断。这里注意如果不是same_flow的话，就没必要进行flush的判断。
		/* All fields must match except length and checksum. */
		NAPI_GRO_CB(p)->flush |=
			(iph->ttl ^ iph2->ttl) |
			((u16)(ntohs(iph2->id) + NAPI_GRO_CB(p)->count) ^ id);
 
		NAPI_GRO_CB(p)->flush |= flush;
	}
 
	NAPI_GRO_CB(skb)->flush |= flush;
	//pull ip头进gro，这里更新data_offset
	skb_gro_pull(skb, sizeof(*iph));
	//设置传输层的头的位置
	skb_set_transport_header(skb, skb_gro_offset(skb));
	//调用传输层的reveive方法。
	pp = ops->gro_receive(head, skb);
 
out_unlock:
	rcu_read_unlock();
 
out:
	NAPI_GRO_CB(skb)->flush |= flush;
 
}

//遍历gro list
for (; (p = *head); head = &p->next) {
	//如果ip层已经不可能same flow则直接进行下一次匹配
	if (!NAPI_GRO_CB(p)->same_flow)
		continue;
 
	th2 = tcp_hdr(p);
	//判断源地址
	if (*(u32 *)&th->source ^ *(u32 *)&th2->source) {
		NAPI_GRO_CB(p)->same_flow = 0;
		continue;
	}
 
	goto found;
}

found:
	flush = NAPI_GRO_CB(p)->flush;
	//如果设置拥塞，则肯定需要刷出skb到协议栈
	flush |= (__force int)(flags & TCP_FLAG_CWR);
	//如果相差的域是除了这3个中的，就需要flush出skb
	flush |= (__force int)((flags ^ tcp_flag_word(th2)) &
		  ~(TCP_FLAG_CWR | TCP_FLAG_FIN | TCP_FLAG_PSH));
	//ack的序列号必须一致
	flush |= (__force int)(th->ack_seq ^ th2->ack_seq);
	//tcp的option头必须一致
	for (i = sizeof(*th); i < thlen; i += 4)
		flush |= *(u32 *)((u8 *)th + i) ^
			 *(u32 *)((u8 *)th2 + i);
 
	mss = skb_shinfo(p)->gso_size;
	// 0-1 = 0xFFFFFFFF, 所以skb的数据部分长度为0的包是不会被合并的
	flush |= (len - 1) >= mss;
	flush |= (ntohl(th2->seq) + skb_gro_len(p)) ^ ntohl(th->seq);
	//如果flush有设置则不会调用 skb_gro_receive，也就是不需要进行合并，否则调用skb_gro_receive进行数据包合并
	if (flush || skb_gro_receive(head, skb)) {
		mss = 1;
		goto out_check_final;
	}
 
	p = *head;
	th2 = tcp_hdr(p);
	//更新p的头。到达这里说明合并完毕，因此需要更新合并完的新包的头。
	tcp_flag_word(th2) |= flags & (TCP_FLAG_FIN | TCP_FLAG_PSH);

out_check_final:
	flush = len < mss;
	//根据flag得到flush
	flush |= (__force int)(flags & (TCP_FLAG_URG | TCP_FLAG_PSH |
					TCP_FLAG_RST | TCP_FLAG_SYN |
					TCP_FLAG_FIN));
 
	if (p && (!NAPI_GRO_CB(skb)->same_flow || flush))
		pp = head;
 
out:
	NAPI_GRO_CB(skb)->flush |= flush;

//一些需要用到的变量
struct sk_buff *p = *head;
struct sk_buff *nskb;
//当前的skb的 share_ino
struct skb_shared_info *skbinfo = skb_shinfo(skb);
//当前的gro list中的要合并的skb的share_info
struct skb_shared_info *pinfo = skb_shinfo(p);
unsigned int headroom;
unsigned int len = skb_gro_len(skb);
unsigned int offset = skb_gro_offset(skb);
unsigned int headlen = skb_headlen(skb);
//如果有frag_list的话，则直接去非Scatter-Gather I/O部分处理，也就是合并到frag_list.
if (pinfo->frag_list)
	goto merge;
else if (headlen <= offset) {
	//支持Scatter-Gather I/O的处理
	skb_frag_t *frag;
	skb_frag_t *frag2;
	int i = skbinfo->nr_frags;
	//这里遍历是从后向前。
	int nr_frags = pinfo->nr_frags + i;
 
	offset -= headlen;
 
	if (nr_frags > MAX_SKB_FRAGS)
		return -E2BIG;
	//设置pinfo的frags的大小，可以看到就是加上skb的frags的大小
	pinfo->nr_frags = nr_frags;
	skbinfo->nr_frags = 0;
 
	frag = pinfo->frags + nr_frags;
	frag2 = skbinfo->frags + i;
	//遍历赋值，其实就是地址赋值，这里就是将skb的frag加到pinfo的frgas后面。
	do {
		*--frag = *--frag2;
	} while (--i);
	//更改page_offet的值
	frag->page_offset += offset;
	//修改size大小
	frag->size -= offset;
	//更新skb的相关值
	skb->truesize -= skb->data_len;
	skb->len -= skb->data_len;
	skb->data_len = 0;
 
	NAPI_GRO_CB(skb)->free = 1;
	//最终完成
	goto done;
} else if (skb_gro_len(p) != pinfo->gso_size)
	return -E2BIG;

	headroom = skb_headroom(p);
	nskb = alloc_skb(headroom + skb_gro_offset(p), GFP_ATOMIC);
	if (unlikely(!nskb))
		return -ENOMEM;
	//复制头
	__copy_skb_header(nskb, p);
	nskb->mac_len = p->mac_len;
 
	skb_reserve(nskb, headroom);
	__skb_put(nskb, skb_gro_offset(p));
	//设置各层的头
	skb_set_mac_header(nskb, skb_mac_header(p) - p->data);
	skb_set_network_header(nskb, skb_network_offset(p));
	skb_set_transport_header(nskb, skb_transport_offset(p));
 
	__skb_pull(p, skb_gro_offset(p));
	//复制数据
	memcpy(skb_mac_header(nskb), skb_mac_header(p),
		   p->data - skb_mac_header(p));
	//对应的gro 域的赋值
	*NAPI_GRO_CB(nskb) = *NAPI_GRO_CB(p);
	//可以看到frag_list被赋值
	skb_shinfo(nskb)->frag_list = p;
	skb_shinfo(nskb)->gso_size = pinfo->gso_size;
	pinfo->gso_size = 0;
	skb_header_release(p);
	nskb->prev = p;
	//更新新的skb的数据段
	nskb->data_len += p->len;
	nskb->truesize += p->len;  // 应该改成 nskb->truesize += p->truesize; 更准确
	nskb->len += p->len;
	//将新的skb插入到gro list中
	*head = nskb;
	nskb->next = p->next;
	p->next = NULL;
 
	p = nskb;
 
merge:
	if (offset > headlen) {
		skbinfo->frags[0].page_offset += offset - headlen;
		skbinfo->frags[0].size -= offset - headlen;
		offset = headlen;
	}
 
	__skb_pull(skb, offset);
	//将skb插入新的skb的(或者老的skb，当frag list本身存在)fraglist
	// 这里是用p->prev来记录了p->fraglist的最后一个包，所以在gro向协议栈提交时最好加一句skb->prev = NULL;
	p->prev->next = skb;
	p->prev = skb;
	skb_header_release(skb);