//main.c
int sum(int *a, int n);
int array[2] = {1, 2};

int main()
{
	int val = sum(array, 2);
	return val;
}

//sum.c
int sum(int *a, int n)
{
	int i, s = 0;
	for (i = 0; i < n; i++) {
		s += a[i];
	}
	return s;
}

typedef struct {
	int name;		//符号名称
	char type: 4,	//类型：函数或数据
		 binding: 4;//本地or全局
	char reserved;
	short section;	//属于那个节
	long value;		//距离节起始位置的偏移；对于可执行文件则是绝对运行时地址
	long size;		//目标大小
} Elf64_Symbol;

typedef struct {
	long offset;		//这个要修改的条目的位置，相对节起始位置的偏移
	long type: 32,		//
		 symbol: 32;	//引用的符号名称，对应符号表中的index
	long addend;		//常数
} Elf64_Rela

foreach section s { //遍历每个节
	foreach relocation entry r { //遍历每个节的每个重定位条目
		refptr = s + r.offset;  //计算引用所在的地址，即节的起始地址加上偏移
		//重定位一个相对地址的引用
		if (r.type == R_X86_64_PC32) {
			refaddr = ADDR(s) + r.offset;  //引用所在的运行时地址，即节的运行时地址加上偏移
			*refptr = ADDR(r.symbol) + r.addend - refaddr;  //引用的最终地址 - 引用所在地址，差值填到引用所在地址
		}
		//重定位一个绝对引用
		if (r.type == R_X86_64_32) {
			*refptr = ADDR(r.symbol) + r.addend
		}
	}
}

r.offset = 0x13
r.symbol = sum
r.type = R_X86_64_PC32
r.addend = -4

r.offset = e
r.symbol = array
r.type = R_X86_64_32
r.addend = 0

void sigsegv_handler_fun(int signum) {
    char cmd[64] = {0};
    sprintf(cmd, "echo catch sigsegv > /tmp/dddd");
    system(cmd);
    exit(0);
}

int main(int argc, char *argv[])
{
    ...
    signal(SIGSEGV, sigsegv_handler_fun);
    ...
}

//AVL节点对象
struct avl_node {
    struct list_head list; //libubox的AVL库将所有节点串成一个双向链表
    struct avl_node *parent; //指向父节点
    struct avl_node *left; //左子节点
    struct avl_node *right; //右子节点
    const void *key; //节点存储的值
    signed char balance; //平衡因子: 取值0 -1 +1
                         //-1表示左子树深度大1，+1表示右子树深度大1，0表示一样大
    bool leader;
};

//AVL树对象
typedef int (*avl_tree_comp) (const void *k1, const void *k2, void *ptr);
struct avl_tree {
    struct list_head list_head; //libubox的AVL库将所有节点串成一个有序双向链表，方便遍历
    struct avl_node *root; //树的根节点
    unsigned int count; //树中节点的个数
    bool allow_dups; //树中是否允许key相同的节点存在，libubox的AVL库支持插入相同的节点
    avl_tree_comp comp; //节点大小比较函数
    void *cmp_ptr; //比较函数的第三个入参
};

//在一棵AVL树中，查找值为key的节点，找到则返回找到的节点，找不到则返回空
struct avl_node *avl_find(const struct avl_tree *tree, const void *key)
    struct avl_node *node;
    int diff;
    if (tree->root == NULL)
        return NULL; //树为空则直接返回空
    //递归查找节点。入参是节点、key值、比较函数，出参是比较值
    node = avl_find_rec(tree->root, key, tree->comp, tree->cmp_ptr, &diff)
        int diff = (*comp) (key, node->key, cmp_ptr); //计算传入的key和节点的key的差值
        if (diff < 0) //如果差值小于0，即匹配的key值应该在节点的左子树，继续去左子树寻找
            if (node->left != NULL)
                return avl_find_rec(node->right, key, comp, cmp_ptr, cmp_result);
            else
                return node //如果没有左子树了，则返回这个叶节点
        if (diff > 0) //如果差值大于0，即匹配的key值应该在节点的右子树，继续去右子树寻找
            if (node->right != NULL)
                return avl_find_rec(node->right, key, comp, cmp_ptr, cmp_result)
            else
                return node //如果没有右子树了，则返回这个叶节点
        return node //如果差值等于0，则返回这个匹配的节点

    return diff == 0 ? node : NULL //如果差值不等于0，表示没找到匹配的节点，返回空

int avl_insert(struct avl_tree *tree, struct avl_node *new)
    //对要插入new节点除了key属性以外的其它属性赋初值
    new->parent = NULL;
    new->left = NULL;
    new->right = NULL;
    new->balance = 0;
    new->leader = true;

    if (tree->root == NULL)
        //如果这个树没有节点存在
        list_add(&new->list, &tree->list_head) //new节点直接插入树中
        tree->root = new //树的根节点指向这个new节点
        tree->count = 1 //此时树的节点个数变为1
        return 0

    //递归查找与new节点相同或相近的节点node
    node = avl_find_rec(tree->root, new->key, tree->comp, tree->cmp_ptr, &diff);
    last = node;

    //遍历last的next节点，即如果有key值相同的节点，取所有相同key值节点的最后一个
    while (!list_is_last(&last->list, &tree->list_head)) {
        next = avl_next(last);
        if (next->leader)
          break;
        last = next;

    diff = (*tree->comp) (new->key, node->key, tree->cmp_ptr); //再次计算差值，这一步是多余的？
    if (diff == 0)
        if (!tree->allow_dups)
          return -1; //不允许插入相同key值节点时，直接返回-1
        new->leader = 0; //leader置0表示已存在相同key值的节点
        avl_insert_after(tree, last, new); //将此节点插入双向链表
        return 0;

    if (node->balance == 1)
        //节点的平衡度为1，表示右子树深度大。此时直接将new节点插入node节点左侧即可
        avl_insert_before(tree, node, new);
        node->balance = 0; //插入new节点后node节点左右深度相同
        new->parent = node;
        node->left = new; //将new节点插入到node节点左侧
        return 0;

    if (node->balance == -1)
        //节点的平衡度为-1，表示左子树深度大。此时直接将new节点插入node节点右侧即可
        avl_insert_after(tree, last, new);
        node->balance = 0; //插入new节点后node节点左右深度相同
        new->parent = node;
        node->right = new; //将new节点插入到node节点的右侧
        return 0;

    if (diff < 0) { //new节点小于node节点，插入node节点左侧
        avl_insert_before(tree, node, new);
        node->balance = -1; //左子树深度变大
        new->parent = node;
        node->left = new; //将new节点插入到node节点左侧
        post_insert(tree, node); //插入后处理，后面专门看这个函数
        return 0;

    //new节点大于node节点，插入node节点右侧
    avl_insert_after(tree, last, new);
    node->balance = 1;
    new->parent = node;
    node->right = new;
    post_insert(tree, node); //插入后处理

/*
 总结下插入逻辑：
 在没有相同节点的情况下，插入分两大类。
 大类一：node节点平衡因子不为0，插入new后node节点的平衡因子变为0，不需要后处理
 大类二：node节点平衡因子为0，插入new后node节点平衡因子不为0，需要后处理
 */

//接下来看下后处理函数，这里是AVL树最具特点的地方，
//即插入一个新节点后，如果二叉树不再平衡，只需要特定的左旋右旋操作，二叉树就可再次平衡
static void post_insert(struct avl_tree *tree, struct avl_node *node)
    struct avl_node *parent = node->parent; //得到node节点的父节点
    if (parent == NULL)
        return //如果父节点为空，说明node节点就是整个AVL树的根节点，此时什么都不用做

    if (node == parent->left)
        //如果node节点在左侧，即插入的new节点在父节点的左子树
        parent->balance--; //父节点左侧深度增加
        if (parent->balance == 0)
            return; //如果父节点左右深度相同，则什么都不用做
        if (parent->balance == -1)
            //如果左子树深度大1，递归调用post_insert，也就是修改所有父节点的balance为-1
            post_insert(tree, parent);
            return;

        //函数走到此，说明parent->balance=-2，父节点不再平衡，需要旋转操作
        if (node->balance == -1)
            //如果node节点左子树深度大1，则进行右旋操作，后面单独看旋转操作函数
            avl_rotate_right(tree, parent);
            return;
        //如果node节点右子树大1，则先左旋，再右旋
        avl_rotate_left(tree, node);
        avl_rotate_right(tree, node->parent->parent);
        return;

    //以下是node节点在parent节点右侧的情况，与上面的代码对称
    parent->balance++; //父节点的右侧深度增加
    if (parent->balance == 0)
        return; //如果父节点左右深度相同，则什么都不用做
    if (parent->balance == 1)
        //如果左子树深度大1，递归调用post_insert，也就是修改所有父节点的balance为1
        post_insert(tree, parent);
        return;

    //函数走到此，说明parent->balance=2，父节点不再平衡，需要旋转操作
    if (node->balance == 1)
        //如果node节点右子树深度大1，则进行左旋操作
        avl_rotate_left(tree, parent);
        return;
    //如果node节点左子树大1，则先右旋，再左旋
    avl_rotate_right(tree, node);
    avl_rotate_left(tree, node->parent->parent);

//接下来看一下旋转操作的代码
//左旋
static void avl_rotate_left(struct avl_tree *tree, struct avl_node *node)
    struct avl_node *right, *parent;
    right = node->right; //记录node节点的右节点和父节点
    parent = node->parent;

    right->parent = parent; //右节点和node节点互换位置
    node->parent = right;
    if (parent == NULL) //如果父节点是空，则修改树的根节点
        tree->root = right;
    else
        //将父节点的子节点node修改为right
        if (parent->left == node)
            parent->left = right;
        else
            parent->right = right;
    node->right = right->left; //right的左节点赋给node右节点
    right->left = node; //node置为right节点的左节点
    if (node->right != NULL)
        node->right->parent = node; //如果node的右节点不为空，改一下其父节点
    node->balance -= 1 + avl_max(right->balance, 0); //修正平衡因子，至少减1，即右子树深度至少减1
    right->balance -= 1 - avl_min(node->balance, 0);

//右旋。与左旋相反
static void avl_rotate_right(struct avl_tree *tree, struct avl_node *node)
    struct avl_node *left, *parent;
    left = node->left;
    parent = node->parent;

    left->parent = parent; //左节点和node节点互换
    node->parent = left;
    if (parent == NULL)
        tree->root = left;
    else
        if (parent->left == node)
            parent->left = left;
        else
            parent->right = left;
    node->left = left->right;
    left->right = node;
    if (node->left != NULL)
        node->left->parent = node;
    node->balance += 1 - avl_min(left->balance, 0);
    left->balance += 1 + avl_max(node->balance, 0);

//删除节点逻辑也比较长，这里只介绍下梗概
void avl_delete(struct avl_tree *tree, struct avl_node *node)
    //如果node是一个重复节点，则直接删除，不涉及二叉树结构的变动

    //真正从二叉树删除一个节点
    avl_delete_worker(tree, node)
        //node是叶子节点时：
        if (node->left == NULL && node->right == NULL)
            //根据node父节点的平衡因子，进行后处理和旋转操作
        //node的左节点为空时：
        if (node->left == NULL)
            //将node的右节点代替node节点，并调用avl_post_delete
        //node的右节点为空时：
        if (node->right == NULL)
            //将node的左节点代替node节点，并调用avl_post_delete
        //node的左右节点都不为空时
        min = avl_local_min(node->right); //取node右子树的最小值节点min，此节点要么是叶子节点要么左节点为空
        avl_delete_worker(tree, min); //嵌套调用avl_delete_worker，删除这个min节点
        //用min节点代替node节点，从而删除了node节点

//这里入参first和last是指向包含avl_node结构的大结构体的指针，即遍历的起始大结构体和结束大结构体
//element是指向每一个遍历到的大结构体，node_member是avl_node结构体在大结构体中的成员名
#define avl_for_element_range(first, last, element, node_member) \
  for (element = (first); \
       element->node_member.list.prev != &(last)->node_member.list; \
       element = avl_next_element(element, node_member))

//avl_next_element，即取avl_node.list->next，然后利用container_of取到大结构体
#define avl_next_element(element, node_member) \
  container_of((&(element)->node_member.list)->next, typeof(*(element)), node_member.list)

//全节点遍历即调用avl_for_element_range，然后入参是AVL树的第一个节点和最后一个节点
//即从树的第一个节点 遍历到树的最后一个节点
#define avl_for_each_element(tree, element, node_member) \
  avl_for_element_range(avl_first_element(tree, element, node_member), \
                        avl_last_element(tree, element,  node_member), \
                        element, node_member)

//ubus/ubusd_proto.c
static int ubusd_handle_lookup()
	char *objpath;
	objpath = blob_data(attr[UBUS_ATTR_OBJPATH]);
	...
	obj = avl_find_element(&path, objpath, obj, path) //AVL树查找
	...

static inline int
NF_HOOK(uint8_t pf, unsigned int hook, struct net *net, struct sock *sk, struct sk_buff *skb,
    struct net_device *in, struct net_device *out,
    int (*okfn)(struct net *, struct sock *, struct sk_buff *))
{
    int ret = nf_hook(pf, hook, net, sk, skb, in, out, okfn);
    if (ret == 1)
        ret = okfn(net, sk, skb);
    return ret;
}

//nf_hook并不调用okfn回调函数，NF_HOOK宏判断nf_hook返回值=1(表示允许包通过)调用okfn
static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net,
              struct sock *sk, struct sk_buff *skb,
              struct net_device *indev, struct net_device *outdev,
              int (*okfn)(struct net *, struct sock *, struct sk_buff *))
    switch (pf)
    case NFPROTO_IPV4:
        hook_head = rcu_dereference(net->nf.hooks_ipv4[hook]);

    struct nf_hook_state state;
    nf_hook_state_init(&state, hook, pf, indev, outdev, sk, net, okfn);
    ret = nf_hook_slow(skb, &state, hook_head, 0);
    return ret

int nf_hook_slow(struct sk_buff *skb, struct nf_hook_state *state,
         const struct nf_hook_entries *e, unsigned int s)
    for (; s < e->num_hook_entries; s++)
        //依次执行注册的hook函数，如果返回值是NF_ACCEPT，则表示调用者可进一步执行okfn
        verdict = nf_hook_entry_hookfn(&e->hooks[s], skb, state);
            return entry->hook(entry->priv, skb, state);

int nf_register_net_hook(struct net *net, const struct nf_hook_ops *reg)
    __nf_register_net_hook(net, reg->pf, reg)
        struct nf_hook_entries *p, *new_hooks;
        struct nf_hook_entries __rcu **pp;
        pp = nf_hook_entry_head(net, pf, reg->hooknum, reg->dev)
            return net->nf.hooks_ipv4 + hooknum; //以pf==NFPROTO_IPV4为例。钩子挂载点保存在struct net对象中
        p = nf_entry_dereference(*pp);
        new_hooks = nf_hook_entries_grow(p, reg); //将新的nf_hook_ops按照优先级插入到hook entries中

typedef unsigned int nf_hookfn(void *priv,
                   struct sk_buff *skb,
                   const struct nf_hook_state *state);

struct nf_hook_ops {
    /* User fills in from here down. */
    nf_hookfn       *hook; //要注册的钩子回调函数
    struct net_device   *dev;
    void            *priv;
    u_int8_t        pf; //协议簇，对于IPv4来说，它为NFPROTO_IPV4; IPV6, NFPROTO_IPV6
    bool            nat_hook;
    unsigned int        hooknum; //netfilter的5个挂载点之一
    /* Hooks are ordered in ascending priority. */
    int         priority; //按优先级升序排列回调函数，priority值越小回调函数越先被调用
};

static const struct nf_hook_ops ipv4_conntrack_ops[] = {
    {
        .hook       = ipv4_conntrack_in,
        .pf     = NFPROTO_IPV4,
        .hooknum    = NF_INET_PRE_ROUTING,
        .priority   = NF_IP_PRI_CONNTRACK,
    },
    {
        .hook       = ipv4_conntrack_local,
        .pf     = NFPROTO_IPV4,
        .hooknum    = NF_INET_LOCAL_OUT,
        .priority   = NF_IP_PRI_CONNTRACK,
    },
    {
        .hook       = ipv4_helper,
        .pf     = NFPROTO_IPV4,
        .hooknum    = NF_INET_POST_ROUTING,
        .priority   = NF_IP_PRI_CONNTRACK_HELPER,
    },
    {
        .hook       = ipv4_confirm,
        .pf     = NFPROTO_IPV4,
        .hooknum    = NF_INET_POST_ROUTING,
        .priority   = NF_IP_PRI_CONNTRACK_CONFIRM,
    },
    {
        .hook       = ipv4_helper,
        .pf     = NFPROTO_IPV4,
        .hooknum    = NF_INET_LOCAL_IN,
        .priority   = NF_IP_PRI_CONNTRACK_HELPER,
    },
    {
        .hook       = ipv4_confirm,
        .pf     = NFPROTO_IPV4,
        .hooknum    = NF_INET_LOCAL_IN,
        .priority   = NF_IP_PRI_CONNTRACK_CONFIRM,
    },
};

//nf_conntrack_l3proto_ipv4.c
//nf_conntrack_l3proto_ipv4_init为nf_conntrack_ipv4.ko的初始化函数
module_init(nf_conntrack_l3proto_ipv4_init);


static int __init nf_conntrack_l3proto_ipv4_init(void)
    ...
    ret = nf_ct_l3proto_register(&nf_conntrack_l3proto_ipv4);
        rcu_assign_pointer(nf_ct_l3protos[proto->l3proto], proto); //注册到全局变量nf_ct_l3protos中


struct nf_conntrack_l3proto nf_conntrack_l3proto_ipv4 __read_mostly = {
    .l3proto     = PF_INET,
    .pkt_to_tuple    = ipv4_pkt_to_tuple,
    .invert_tuple    = ipv4_invert_tuple,
    .get_l4proto     = ipv4_get_l4proto,
#if IS_ENABLED(CONFIG_NF_CT_NETLINK)
    .tuple_to_nlattr = ipv4_tuple_to_nlattr,
    .nlattr_to_tuple = ipv4_nlattr_to_tuple,
    .nla_policy  = ipv4_nla_policy,
    .nla_size    = NLA_ALIGN(NLA_HDRLEN + sizeof(u32)) + /* CTA_IP_V4_SRC */
               NLA_ALIGN(NLA_HDRLEN + sizeof(u32)),  /* CTA_IP_V4_DST */
#endif
    .net_ns_get  = ipv4_hooks_register, //这里注册的函数用于注册连接跟踪的netfliter钩子
    .net_ns_put  = ipv4_hooks_unregister,
    .me      = THIS_MODULE,
};

//先看下ipv4_hooks_register
static int ipv4_hooks_register(struct net *net)
    struct conntrack4_net *cnet = net_generic(net, conntrack4_net_id);
    cnet->users++;
    if (cnet->users > 1)
        goto out_unlock; //只在第一次调用的时候往下走，之后的调用只是users技术+1
    //注册连接跟踪的netfilter钩子
    nf_register_net_hooks(net, ipv4_conntrack_ops, ARRAY_SIZE(ipv4_conntrack_ops));

//再看下调用nf_conntrack_l3proto_ipv4.net_ns_get的地方
int nf_ct_netns_get(struct net *net, u8 nfproto)
    if (nfproto == NFPROTO_INET)
        nf_ct_netns_do_get(net, NFPROTO_IPV4)
        nf_ct_netns_do_get(net, NFPROTO_IPV6)

static int nf_ct_netns_do_get(struct net *net, u8 nfproto)
    const struct nf_conntrack_l3proto *l3proto;
    l3proto = __nf_ct_l3proto_find(nfproto); //对于NFPROTO_IPV4，这里返回的是nf_conntrack_l3proto_ipv4
    l3proto->net_ns_get(net); //调用net_ns_get

//调用nf_ct_netns_get地方有很多，主要应该是通过nft_ct_get_init和nft_nat_init

struct nf_conntrack_tuple {
    struct nf_conntrack_man src;  //tuple的可操作部分

    /* 以下是tuple的固定部分 */
    struct {
        union nf_inet_addr u3;
        union {
            /* Add other protocols here. */
            __be16 all;
            struct {
                __be16 port;
            } tcp;
            struct {
                __be16 port;
            } udp;
            struct {
                u_int8_t type, code;
            } icmp;
            struct {
                __be16 port;
            } dccp;
            struct {
                __be16 port;
            } sctp;
            struct {
                __be16 key;
            } gre;
        } u;
        u_int8_t protonum; //protocol
        u_int8_t dir;
    } dst;
};

struct nf_conn {
    struct nf_conntrack ct_general;
    spinlock_t  lock;
    u16     cpu;
    struct nf_conntrack_zone zone;
    struct nf_conntrack_tuple_hash tuplehash[IP_CT_DIR_MAX]; //hashlist节点
    unsigned long status;
    u32 timeout;

    possible_net_t ct_net;
    struct hlist_node   nat_bysource;
    /* all members below initialized via memset */
    struct { } __nfct_init_offset;
    struct nf_conn *master;
    u_int32_t mark;
    u_int32_t secmark;
    struct nf_ct_ext *ext;
    union nf_conntrack_proto proto;
};

unsigned int nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum, struct sk_buff *skb)
    l3proto = __nf_ct_l3proto_find(pf);  //对于pf=PF_INET,PF_INET,返回的是全局变量nf_conntrack_l3proto_ipv4
    l3proto->get_l4proto(skb, skb_network_offset(skb), &dataoff, &protonum); //.get_l4proto=ipv4_get_l4proto
        //对于IPv4 ->get_l4proto=ipv4_get_l4proto
        *dataoff = nhoff + (iph->ihl << 2);
        *protonum = iph->protocol; //protonum即四层协议
    l4proto = __nf_ct_l4proto_find(pf, protonum); //以IPPROTO_TCP为例，返回的是全局变量nf_conntrack_l4proto_tcp4

    resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum, l3proto, l4proto);
        struct nf_conntrack_tuple tuple;
        struct nf_conntrack_tuple_hash *h;
        nf_ct_get_tuple() //填充tuple
        hash = hash_conntrack_raw(&tuple, net); //对tuple进行hash散列运算，调用的内核提供的jhash2()
        h = __nf_conntrack_find_get(net, zone, &tuple, hash); //在全局变量nf_conntrack_hash hash表下查找连接是否存在
        if (!h) //如果连接不存在，则新建一个连接，保存到unconfirmed list
            h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,skb, dataoff, hash);
        ct = nf_ct_tuplehash_to_ctrack(h); //利用container_of得到真正的连接对象
        ...//一系列ctinfo赋值逻辑，对于新建的连接ctinfo = IP_CT_NEW
        nf_ct_set(skb, ct, ctinfo); //将连接对象和连接状态值，保存到skb中
            skb->_nfct = (unsigned long)ct | info; //借助指针低4位一定为0的逻辑，低4位存整数值
    timeouts = nf_ct_timeout_lookup(net, ct, l4proto);
    l4proto->packet(ct, skb, dataoff, ctinfo, pf, timeouts); //以TCP为例，->packet==tcp_packet()

ipv4_confirm
    nf_conntrack_confirm

static inline int nf_conntrack_confirm(struct sk_buff *skb)
    ct = nf_ct_get(skb, &ctinfo);
    ...
    nf_ct_del_from_dying_or_unconfirmed_list(ct); //从unconfirmed或dying表中删除连接
    ...
    __nf_conntrack_hash_insert(ct, hash, reply_hash); //插入到nf_conntrack_hash
    ...

struct xt_table {
    struct list_head list;
    /* What hooks you will enter on */
    unsigned int valid_hooks;
    /* Man behind the curtain... */
    struct xt_table_info *private; //
    struct module *me;
    u_int8_t af;        /* address/protocol family */
    int priority;       /* hook order */
    /* called when table is needed in the given netns */
    int (*table_init)(struct net *net);
    const char name[XT_TABLE_MAXNAMELEN];
};

int ipt_register_table(struct net *net, const struct xt_table *table,
               const struct ipt_replace *repl,
               const struct nf_hook_ops *ops, struct xt_table **res)
    xt_register_table(net, table, &bootstrap, newinfo);
        list_add(&table->list, &net->xt.tables[table->af]); //注册到net->xt.tables上
    nf_register_net_hooks(net, ops, hweight32(table->valid_hooks)) //注册netfilter钩子

//filter表的定义
#define FILTER_VALID_HOOKS ((1 << NF_INET_LOCAL_IN) | \
                (1 << NF_INET_FORWARD) | \
                (1 << NF_INET_LOCAL_OUT))
static const struct xt_table packet_filter = {
    .name       = "filter",
    .valid_hooks    = FILTER_VALID_HOOKS, //按照FILTER_VALID_HOOKS定义，在netfilter的3个挂载点挂载钩子
    .me     = THIS_MODULE,
    .af     = NFPROTO_IPV4,
    .priority   = NF_IP_PRI_FILTER,
    .table_init = iptable_filter_table_init,
};

//初始化
static int __init iptable_filter_init(void)
    //这一步主要是初始化netfilter钩子挂载对象，3个挂载点的回调函数都是iptable_filter_hook
    filter_ops = xt_hook_ops_alloc(&packet_filter, iptable_filter_hook);
    register_pernet_subsys(&iptable_filter_net_ops)
        iptable_filter_net_init
            iptable_filter_table_init(net)
                //注册filter表
                ipt_register_table(net, &packet_filter, repl, filter_ops,
                 &net->ipv4.iptable_filter);

iptables -A INPUT -p udp --dport=5001 -j LOG --log-level 1

iptables -A INPUT -p tcp -m conntrack --ctstate ESTABLISHED -j LOG --log-level 1

static const struct xt_table nf_nat_ipv4_table = {
    .name       = "nat",
    .valid_hooks    = (1 << NF_INET_PRE_ROUTING) |
              (1 << NF_INET_POST_ROUTING) |
              (1 << NF_INET_LOCAL_OUT) |
              (1 << NF_INET_LOCAL_IN),
    .me     = THIS_MODULE,
    .af     = NFPROTO_IPV4,
    .table_init = iptable_nat_table_init,
};

static const struct nf_hook_ops nf_nat_ipv4_ops[] = {
    /* Before packet filtering, change destination */
    {
        .hook       = iptable_nat_ipv4_in,
        .pf     = NFPROTO_IPV4,
        .nat_hook   = true,
        .hooknum    = NF_INET_PRE_ROUTING,
        .priority   = NF_IP_PRI_NAT_DST,
    },
    /* After packet filtering, change source */
    {
        .hook       = iptable_nat_ipv4_out,
        .pf     = NFPROTO_IPV4,
        .nat_hook   = true,
        .hooknum    = NF_INET_POST_ROUTING,
        .priority   = NF_IP_PRI_NAT_SRC,
    },
    /* Before packet filtering, change destination */
    {
        .hook       = iptable_nat_ipv4_local_fn,
        .pf     = NFPROTO_IPV4,
        .nat_hook   = true,
        .hooknum    = NF_INET_LOCAL_OUT,
        .priority   = NF_IP_PRI_NAT_DST,
    },
    /* After packet filtering, change source */
    {
        .hook       = iptable_nat_ipv4_fn,
        .pf     = NFPROTO_IPV4,
        .nat_hook   = true,
        .hooknum    = NF_INET_LOCAL_IN,
        .priority   = NF_IP_PRI_NAT_SRC,
    },
};

static int __init iptable_nat_init(void)
    iptable_nat_table_init(&init_net)
        struct ipt_replace *repl;
        repl = ipt_alloc_initial_table(&nf_nat_ipv4_table);
        //调用ipt_register_table注册nat表
        ret = ipt_register_table(net, &nf_nat_ipv4_table, repl,
                 nf_nat_ipv4_ops, &net->ipv4.nat_table);

static const struct nf_nat_l3proto nf_nat_l3proto_ipv4 = {
    .l3proto        = NFPROTO_IPV4,
    .in_range       = nf_nat_ipv4_in_range,
    .secure_port        = nf_nat_ipv4_secure_port,
    .manip_pkt      = nf_nat_ipv4_manip_pkt, //修改ip包
    .csum_update        = nf_nat_ipv4_csum_update,
    .csum_recalc        = nf_nat_ipv4_csum_recalc,
#if IS_ENABLED(CONFIG_NF_CT_NETLINK)
    .nlattr_to_range    = nf_nat_ipv4_nlattr_to_range,
#endif
#ifdef CONFIG_XFRM
    .decode_session     = nf_nat_ipv4_decode_session,
#endif
};

//专门看下这个修改ip包的函数nf_nat_ipv4_manip_pkt
static bool nf_nat_ipv4_manip_pkt(struct sk_buff *skb,
                  unsigned int iphdroff,
                  const struct nf_nat_l4proto *l4proto,
                  const struct nf_conntrack_tuple *target,
                  enum nf_nat_manip_type maniptype)
    ...
    if (maniptype == NF_NAT_MANIP_SRC) {
        csum_replace4(&iph->check, iph->saddr, target->src.u3.ip);
        iph->saddr = target->src.u3.ip; //修改源IP
    } else {
        csum_replace4(&iph->check, iph->daddr, target->dst.u3.ip);
        iph->daddr = target->dst.u3.ip; //修改目标IP
    }

//TCP
const struct nf_nat_l4proto nf_nat_l4proto_tcp = {
    .l4proto        = IPPROTO_TCP,
    .manip_pkt      = tcp_manip_pkt, //修改IP包
    .in_range       = nf_nat_l4proto_in_range,
    .unique_tuple       = tcp_unique_tuple,
#if IS_ENABLED(CONFIG_NF_CT_NETLINK)
    .nlattr_to_range    = nf_nat_l4proto_nlattr_to_range,
#endif
};

//看下tcp_manip_pkt, udp的类似
static bool tcp_manip_pkt(struct sk_buff *skb,
          const struct nf_nat_l3proto *l3proto,
          unsigned int iphdroff, unsigned int hdroff,
          const struct nf_conntrack_tuple *tuple,
          enum nf_nat_manip_type maniptype)
    ...
    if (maniptype == NF_NAT_MANIP_SRC) {
        /* Get rid of src port */
        newport = tuple->src.u.tcp.port;
        portptr = &hdr->source;
    } else {
        /* Get rid of dst port */
        newport = tuple->dst.u.tcp.port;
        portptr = &hdr->dest;
    }
    oldport = *portptr;
    *portptr = newport; //修改端口号

unsigned int nf_nat_ipv4_fn(void *priv, struct sk_buff *skb, const struct nf_hook_state *state,
           unsigned int (*do_chain)(void *priv,
                    struct sk_buff *skb,
                    const struct nf_hook_state *state,
                    struct nf_conn *ct))
    struct nf_conn *ct;
    enum ip_conntrack_info ctinfo;
    ct = nf_ct_get(skb, &ctinfo);
    if (!ct)
        return NF_ACCEPT; //没有连接跟踪就直接返回

    switch (ctinfo)
    case IP_CT_NEW:
        if (!nf_nat_initialized(ct, maniptype))
            //do_chain最终调用ipt_do_table，在nat标准查找指定条目，找到则调用target的回调函数
            do_chain(priv, skb, state, ct);

    //执行报文修改操作
    nf_nat_packet(ct, ctinfo, state->hook, skb);
        //这里的l3proto对应前面讲的nf_nat_l3proto_ipv4
        l3proto = __nf_nat_l3proto_find(target.src.l3num);
        //如果是TCP的话，l4proto是nf_nat_l4proto_tcp
        l4proto = __nf_nat_l4proto_find(target.src.l3num,target.dst.protonum)
        l3proto->manip_pkt(skb, 0, l4proto, &target, mtype) //调用manip_pkt函数

struct iphdr {
    __u8    ihl:4,  //header length, 以4字节为单位，最小为4，最大为15
        version:4;  //总是4
    __u8    tos;
    __be16  tot_len;  //包括报头在内的数据包总长度
    __be16  id;  //对于分段来说，所有分段的id值都必须相同
    __be16  frag_off;  //后13bit为分段的偏移量，以8Byte为单位
    __u8    ttl;  //存活时间，每个转发节点都会将ttl减1
    __u8    protocol;  //包所属的第四层协议
    __sum16 check;  //报头校验和
    __be32  saddr;  //源IP地址
    __be32  daddr;  //目的IP地址
    /*The options start here. */ //IP选项，可选
};

//net/ipv4/af_inet.c

static int inet_init(void)
    proto_register(&tcp_prot, 1)  //所有注册的协议可以通过cat /proc/net/protocols查看
    proto_register(&udp_prot, 1)
    proto_register(&raw_prot, 1)
    proto_register(&ping_prot, 1)
    sock_register(&inet_family_ops)

    //注册各协议的接收处理函数，最终赋值到全局变量inet_protos[protocol]
    inet_add_protocol(&icmp_protocol, IPPROTO_ICMP)  
    inet_add_protocol(&udp_protocol, IPPROTO_UDP)
    inet_add_protocol(&tcp_protocol, IPPROTO_TCP)
    inet_add_protocol(&igmp_protocol, IPPROTO_IGMP)
    
    //注册各协议的socket interface接口
    for (q = inetsw_array; q < &inetsw_array[INETSW_ARRAY_LEN]; ++q)
        inet_register_protosw(q);

    arp_init()  //arp模块初始化
        dev_add_pack(&arp_packet_type)  //注册ETH_P_ARP=0x0806类型的处理函数
        arp_proc_init()  //cat /proc/net/arp, 查看arp表项
        register_netdevice_notifier(&arp_netdev_notifier)  //注册netdevice事件监听
    ip_init()
        ip_rt_init //ip路由相关的初始化
            devinet_init
                //注册netdevice时间回调
                register_netdevice_notifier(&ip_netdev_notifier);
                    //这里ip_netdev_notifier的回调函数是inetdev_event，看下这个回调
                    int inetdev_event(struct notifier_block *this, unsigned long event,void *ptr)
                        if (event == NETDEV_REGISTER)
                            //给net_device创建in_device，in_device即ip相关的配置，比如ip地址
                            //可以通过ip或ifconfig来修改
                            in_dev = inetdev_init(dev);
    tcp_init()
    udp_init()
    ping_init()
    icmp_init()
        icmp_sk_init(struct net *net)
            //每CPU注册一个ICMP RAW socket，用于处理接收的ICMP报文
            inet_ctl_sock_create
    ip_mr_init  //组播路由初始化

    //注册IP协议ETH_P_IP=0x0800接受处理函数ip_rcv，即注册IP报文接收入口函数
    dev_add_pack(&ip_packet_type)

void dev_add_pack(struct packet_type *pt)
    struct list_head *head = ptype_head(pt)
         return &ptype_base[ntohs(pt->type)]  //ptype_base是一个全局变量数组，记录每个协议的处理函数
    list_add_rcu(&pt->list, head)  //将处理函数pt赋值到全局变量ptype_base中

gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
    napi_skb_finish(dev_gro_receive(napi, skb), skb);
        netif_receive_skb_internal
            __netif_receive_skb
                __netif_receive_skb_core

static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
    //ptype_all是所有包类型的接收处理，对应tcpdump、raw socket等处理
    list_for_each_entry_rcu(ptype, &ptype_all, list)
        if (pt_prev)
            ret = deliver_skb(skb, pt_prev, orig_dev);
        pt_prev = ptype;

    //如果这个设备有注册rx_handler，通过接口netdev_rx_handler_register注册
    //则将包交给注册的rx_handler处理。例如加入网桥的接口会被注册rx_handler
    rx_handler = rcu_dereference(skb->dev->rx_handler);
    if (rx_handler)
        rx_handler(&skb)

    //交给ptype_base里注册的对应协议类型的处理函数
    deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
                &ptype_base[ntohs(type) & PTYPE_HASH_MASK]);

static struct packet_type ip_packet_type __read_mostly = {
    .type = cpu_to_be16(ETH_P_IP),
    .func = ip_rcv, //IP协议报文的入口
};

int ip_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev)
    iph = ip_hdr(skb);
    if (iph->ihl < 5 || iph->version != 4) //如果包长度小于20或版本不是4，则报头不合法
        goto inhdr_error;
    if (unlikely(ip_fast_csum((u8 *)iph, iph->ihl))) //报头校验和校验
        goto csum_error;

    //netfilter的第一个钩子挂载点，NF_INET_PRE_ROUTING
    //如果没被netfilter过滤，最终调用ip_rcv_finish
    return NF_HOOK(NFPROTO_IPV4, NF_INET_PRE_ROUTING,
               net, NULL, skb, dev, NULL,
               ip_rcv_finish);

static int ip_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)

    if (!skb_valid_dst(skb))
        //在路由选择子系统进行查找
        err = ip_route_input_noref(skb, iph->daddr, iph->saddr,iph->tos, dev);
            //ip_route_input_noref中如果判断此IP包是发给本地，
            //则skb->_skb_refdst的input函数赋值为ip_local_deliver
            //如果需要转发，则iput函数赋值为ip_forward

    if (iph->ihl > 5 && ip_rcv_options(skb, dev)) //ip报头的选项处理
        goto drop;

    //调用路由选择子系统查找到的input函数
    return dst_input(skb);
        skb_dst(skb)->input(skb) //即调用skb->_skb_refdst对象的input函数

//这里看一下struct dst_entry，即路由查找的结果
struct dst_entry {
    ...
    int (*input)(struct sk_buff *); //路由查找后的接收处理函数，发给本机的包对应函数为ip_local_deliver
    int (*output)(struct net *net, struct sock *sk, struct sk_buff *skb);

    ...
}

int ip_local_deliver(struct sk_buff *skb)
    if (ip_is_fragment(ip_hdr(skb)))
        ip_defrag //如果是分片报文，则交给解分片函数处理

    //netfilter的第二个钩子挂载点NF_INET_LOCAL_IN
    return NF_HOOK(NFPROTO_IPV4, NF_INET_LOCAL_IN,
               net, NULL, skb, skb->dev, NULL,
               ip_local_deliver_finish);

static int ip_local_deliver_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
    //先查并发给raw socket
    raw = raw_local_deliver(skb, protocol);

    ipprot = rcu_dereference(inet_protos[protocol]);  //inet_protos: 各协议注册的处理函数
    if (ipprot)
        ret = ipprot->handler(skb);  //调用注册的协议处理函数，通过接口inet_add_protocol注册
                                     //比如IPPROTO_ICMP的处理函数icmp_rcv，IPPROTO_TCP的是tcp_v4_rcv

int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)
    struct rtable *rt;
    //在路由子系统中查找路由
    rt = ip_route_output_ports
        ip_route_output_flow
            __ip_route_output_key
                ip_route_output_key_hash
                    ip_route_output_key_hash_rcu(net, fl4, &res, skb);
                        fib_lookup(net, fl4, res, 0);
    skb_dst_set_noref(skb, &rt->dst);
    ... //此处是一些填写ip header的逻辑
    res = ip_local_out(net, sk, skb);
        __ip_local_out
            //netfilter钩子挂载点NF_INET_LOCAL_OUT
            nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT,
               net, sk, skb, NULL, skb_dst(skb)->dev,
               dst_output);
        dst_output(net, sk, skb)
            skb_dst(skb)->output(net, sk, skb)  //一般地，这里的output是ip_output

//继续看下ip_output
int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb)
    //netfilter的钩子挂载点NF_INET_POST_ROUTING
    NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,
                net, sk, skb, NULL, dev,
                ip_finish_output,
                !(IPCB(skb)->flags & IPSKB_REROUTED));

//ip_finish_output
static int ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
    ip_finish_output2
        //从字面上理解，找到目的邻居，然后发给邻居
        neigh = __ipv4_neigh_lookup_noref(dev, nexthop);
        res = neigh_output(neigh, skb);
            dev_queue_xmit //最终交给网卡驱动

int ip_append_data(struct sock *sk, struct flowi4 *fl4,
           int getfrag(void *from, char *to, int offset, int len,
                   int odd, struct sk_buff *skb),
           void *from, int length, int transhdrlen,
           struct ipcm_cookie *ipc, struct rtable **rtp,
           unsigned int flags)
    __ip_append_data
        //此函数很长，此处略过。

int ip_forward(struct sk_buff *skb)
    //netfilter挂载点 NF_NET_FORWARD
    return NF_HOOK(NFPROTO_IPV4, NF_INET_FORWARD,
               net, NULL, skb, skb->dev, rt->dst.dev,
               ip_forward_finish);

static int ip_forward_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
    dst_output(net, sk, skb)
    //dst_output在上节讲到过，最终会调用到dev_queue_xmit交给网卡驱动

struct in6_addr {
    union {
        __u8        u6_addr8[16]; //用union定义了3种形式的128bit长度
        __be16      u6_addr16[8];
        __be32      u6_addr32[4];
    } in6_u;
#define s6_addr         in6_u.u6_addr8
#define s6_addr16       in6_u.u6_addr16
#define s6_addr32       in6_u.u6_addr32
};

struct ipv6hdr {
    __u8            priority:4, //流量优先级
                    version:4; //版本号，总是6
    __u8            flow_lbl[3]; //流标签
    __be16          payload_len; //数据包的长度，不包含包头
    __u8            nexthdr; //扩展报头或者上层协议编号
    __u8            hop_limit; //相当于ttl

    struct  in6_addr    saddr; //128bit源地址
    struct  in6_addr    daddr; //128bit目的地址
};

static int __init inet6_init(void)
    //和IPv4初始化类似，一堆协议注册。没跟到这里注册的协议后面怎么用。
    proto_register(&tcpv6_prot, 1)
    proto_register(&udpv6_prot, 1)
    proto_register(&udplitev6_prot, 1)
    proto_register(&rawv6_prot, 1)
    proto_register(&pingv6_prot, 1)

    rawv6_init()
    sock_register(&inet6_family_ops)
    inet6_net_init
    ip6_mr_init
    icmpv6_init
        icmpv6_sk_init
        //注册ICMPv6的接收处理函数icmpv6_rcv
        inet6_add_protocol(&icmpv6_protocol, IPPROTO_ICMPV6)
    ndisc_init()
    igmp6_init
    ipv6_netfilter_init
    ip6_route_init
    ipv6_frag_init
        inet6_add_protocol(&frag_protocol, IPPROTO_FRAGMENT)
    udpv6_init
        inet6_add_protocol(&udpv6_protocol, IPPROTO_UDP)
    tcpv6_init
        inet6_add_protocol(&tcpv6_protocol, IPPROTO_TCP)
    ipv6_packet_init()
        //注册IPv6协议的接收处理函数 ipv6_rcv
        //ETH_P_IPV6 = 0x86DD
        dev_add_pack(&ipv6_packet_type);

int ipv6_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev)
    ... //此处省略一系列校验和检查
    //netfilter挂载点NF_INET_PRE_ROUTING，最终调用ip6_rcv_finish
    return NF_HOOK(NFPROTO_IPV6, NF_INET_PRE_ROUTING,
               net, NULL, skb, dev, NULL,
               ip6_rcv_finish);

int ip6_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
    //路由查找
    ip6_route_input(skb);
        ip6_route_input_lookup
            fib6_rule_lookup

    return dst_input(skb);
        skb_dst(skb)->input(skb); //调用路由查找后的input函数
        //如果是给当前主机的包input为ip6_input
        //如果需要转发input为ip6_forward
        //如果数据包目的地址为组播input为ip6_mc_input

int ip6_input(struct sk_buff *skb)
    //netfilter挂载点NF_INET_LOCAL_IN
    return NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_IN,
               dev_net(skb->dev), NULL, skb, skb->dev, NULL,
               ip6_input_finish);

static int ip6_input_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
    raw6_local_deliver //先投递给原始套接字
    ipprot = rcu_dereference(inet6_protos[nexthdr])
    if (ipprot) 
        ipprot->handler(skb) //调用在IPv6初始化时注册的协议处理函数

int fd = socket(AF_NETLINK, SOCK_RAW, MY_NETLINK)

//socket接口的原型为：
int socket(int domain, int type, int protocol);
//这里用到了第三个入参protocol，即创建Netlink socket时要指定协议号

struct sock *
netlink_kernel_create(struct net *net, int unit, struct netlink_kernel_cfg *cfg)
//这里的第二个参数unit为协议号，与用户空间的protocol相同

struct nlmsghdr {
    __u32       nlmsg_len;  /* Length of message including header */
    __u16       nlmsg_type; /* Message content */
    __u16       nlmsg_flags;    /* Additional flags */
    __u32       nlmsg_seq;  /* Sequence number */
    __u32       nlmsg_pid;  /* Sending process port ID */
};

//定义自己的netlink协议号
#define MY_NETLINK 31

//接收回调，即内核侧收到用户发来的netlink消息回调
static void netlink_recv_msg(struct sk_buff *skb)
{
    ...
    nlh = (struct nlmsghdr *)skb->data;  //取netlink消息头
    pid = nlh->nlmsg_pid; /* pid of sending process */
    msg = (char *)nlmsg_data(nlh);  //取netlink消息data部分
    msg_size = strlen(msg);
    
    printk(KERN_INFO "netlink_kernel: Received from pid %d: %s\n", pid, msg);
    ...
}

//定义netlink_kernel_cfg，即声明接收回调
struct netlink_kernel_cfg cfg = {
    .input = netlink_recv_msg,
};

//创建内核测netlink socket
g_nl_sock = netlink_kernel_create(&init_net, MY_NETLINK, &cfg);
if (!g_nl_sock) {
    printk(KERN_ALERT "netlink_kernel: Error creating socket.\n");
    return -10;
}

int main(int argc, char* argv[])
{
    //创建socket
    sock_fd = socket(PF_NETLINK, SOCK_RAW, MY_NETLINK);

    memset(&src_addr, 0, sizeof(src_addr));
    src_addr.nl_family = AF_NETLINK;
    src_addr.nl_pid = getpid(); /* self pid */
    //绑定端口
    bind(sock_fd, (struct sockaddr*)&src_addr, sizeof(src_addr));
    
    //设置目标地址为内核netlink
    memset(&dest_addr, 0, sizeof(dest_addr));
    dest_addr.nl_family = AF_NETLINK;
    dest_addr.nl_pid = 0; /* For Linux Kernel */
    dest_addr.nl_groups = 0; /* unicast */
    
    //可以通过sendmsg或sendto两种接口向内核发送消息
    //相应的有recvmsg和recvfrom两种接口接收来自内核的消息
    //详见github，此处略
}

//创建netlink socket, libnl中用struct nl_sock表示一个socket
#include <netlink/socket.h>
struct nl_sock *nl_socket_alloc(void)
void nl_socket_free(struct nl_sock *sk)

//回调配置
struct nl_cb *nl_socket_get_cb(const struct nl_sock *sk);
void nl_socket_set_cb(struct nl_sock *sk, struct nl_cb *cb);
int nl_socket_modify_cb(struct nl_sock *, enum nl_cb_type, enum nl_cb_kind, nl_recvmsg_msg_cb_t, void *);

//发送
int nl_send_auto(struct nl_sock *sk, struct nl_msg *msg)
int nl_send(struct nl_sock *sk, struct nl_msg *msg)
int nl_send_iovec(struct nl_sock *sk, struct nl_msg *msg, struct iovec *iov, unsigned iovlen)
int nl_sendmsg(struct nl_sock *sk, struct nl_msg *msg, struct msghdr *hdr) //nl_sendmsg里调用Linux原生sendmsg接口
int nl_sendto(struct nl_sock *sk, void *buf, size_t size) //nl_sendto调用Linux原生sendto接口
int nl_send_simple(struct nl_sock *sk, int type, int flags, void *buf, size_t size)

//接收
int nl_recvmsgs_default(struct nl_sock *sk)

int nl_recvmsgs(struct nl_sock *sk, struct nl_cb *cb)
//如果socket是阻塞的，就阻塞式接收。recv到数据之后，通过cb进行处理

#include <netlink/netlink.h>
#include <netlink/msg.h>

#define MY_NETLINK 31
#define MY_NETLINK_TYPE_SET 0

//接收回调
static int my_input(struct nl_msg *msg, void *arg)
{
    struct nlmsghdr *nlh = nlmsg_hdr(msg);
    char *data = nlmsg_data(nlh);
    int datalen = nlmsg_datalen(nlh);

    printf("input cb: datalen:%d, data:%d\n", datalen, data);

    return 0;
}

int main(int argc, char* argv[])
{
    struct nl_sock *sk;
    int ret;
    //创建并绑定socket
    sk = nl_socket_alloc();
    ret = nl_connect(sk, MY_NETLINK);

    //修改接收回调函数，收到任何消息都会回调my_input
    nl_socket_modify_cb(sk, NL_CB_MSG_IN, NL_CB_CUSTOM, my_input, NULL);

    char msg[] = "Hello libnl!\n"
    ret = nl_send_simple(sk, MY_NETLINK_TYPE_SET, 0, msg, sizeof(msg));

    //阻塞式等待接收。接收到内核发来的消息后，会进入接收回调my_input
    nl_recvmsgs_default(sk);

    nl_socket_free(sk);
}

  0                   1                   2                   3
  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                Netlink message header (nlmsghdr)              |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |           Generic Netlink message header (genlmsghdr)         |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |             Optional user specific message header             |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |           Optional Generic Netlink message payload            |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 struct genlmsghdr {
    __u8    cmd;
    __u8    version;
    __u16   reserved;
};

//genl的message payload基于netlink的属性机制，即payload是由一个个nlattr组成
/*
 *  <------- NLA_HDRLEN ------> <-- NLA_ALIGN(payload)-->
 * +---------------------+- - -+- - - - - - - - - -+- - -+
 * |        Header       | Pad |     Payload       | Pad |
 * |   (struct nlattr)   | ing |                   | ing |
 * +---------------------+- - -+- - - - - - - - - -+- - -+
 *  <-------------- nlattr->nla_len -------------->
 */
struct nlattr {
    __u16           nla_len;
    __u16           nla_type;
};

/* Step1: 定义操作 */
/* attributes */
enum {
    EXMPL_A_UNSPEC,
    EXMPL_A_MSG,
    _EXMPL_A_MAX,
};
#define EXMPL_A_MAX (_EXMPL_A_MAX - 1)
/* attribute policy */
static struct nla_policy exmpl_genl_policy[EXMPL_A_MAX + 1] = {
    [EXMPL_A_MSG] = {.type = NLA_NUL_STRING},
};
// handler
static int exmpl_echo(struct sk_buff *skb, struct genl_info *info);
// commands
enum {
    EXMPL_C_UNSPEC,
    EXMPL_C_ECHO,
    _EXMPL_C_MAX,
};
#define EXMPL_C_MAX (_EXMPL_C_MAX - 1)
// operation definition
struct genl_ops exmpl_genl_ops[EXMPL_C_MAX] = {
    {
        .cmd = EXMPL_C_ECHO,
        .doit = exmpl_echo,
        .policy = exmpl_genl_policy,
    }
};

#define FAMILY_NAME "my_genl"
/* Step2: 定义family */
// family definition
static struct genl_family my_genl_family = {
    .id = 0,
    .hdrsize = 0,  //表示没有用户自定义的额外header
    .name = FAMILY_NAME,
    .version = 1,
    .ops = exmpl_genl_ops,
    .n_ops = ARRAY_SIZE(exmpl_genl_ops),
    .maxattr = EXMPL_A_MAX + 1,
};

// handler的具体定义
static int exmpl_echo(struct sk_buff *skb, struct genl_info *info)
{
    struct nlattr *na;
    struct sk_buff *reply_skb;
    void *msg_head;
    int ret;

    printk("%s in.\n", __func__);

    //内核已经解析好了每个attr
    na = info->attrs[EXMPL_A_MSG];
    if (!na) {
        printk("Error: attr EXMPL_A_MSG is null\n");
        return -EINVAL;
    }
    printk("Recv message: %s\n", nla_data(na));

    //将收到的消息发回去
    reply_skb = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
    //填写genl消息头
    msg_head = genlmsg_put(reply_skb, info->snd_portid, info->snd_seq, &my_genl_family, 0, EXMPL_C_ECHO);
    //向skb尾部填写attr
    nla_put_string(reply_skb, EXMPL_A_MSG, nla_data(na));
    //Finalize the message: 更新nlmsghdr中的nlmsg_len字段
    genlmsg_end(reply_skb, msg_head);
    //Send the message back
    ret = genlmsg_reply(reply_skb, info);
    if (ret != 0) {
        printk("genlmsg_reply return fail: %d\n", ret);
        return -ret;
    }

    return 0;
}

/* Step3: 注册famliy */
int ret;
ret = genl_register_family(&my_genl_family);
if (err != 0) {
    printk("genl_register_family fail, ret:%d\n", ret);
    return ret;
}

#define MY_FAMILY_NAME "my_genl"

//用户侧需要定义和内核侧相同的属性以及命令，所以通常把这一部分摘成一个独立的.h，内核和app共用
//这里没有摘成一个独立的.h，用户侧也重复定义一份
/* attributes */
enum {
    EXMPL_A_UNSPEC,
    EXMPL_A_MSG,
    _EXMPL_A_MAX,
};
#define EXMPL_A_MAX (_EXMPL_A_MAX - 1)
// define attribute policy
static struct nla_policy exmpl_genl_policy[EXMPL_A_MAX + 1] = {
    [EXMPL_A_MSG] = {.type = NLA_STRING},
};
// commands
enum {
    EXMPL_C_UNSPEC,
    EXMPL_C_ECHO,
    _EXMPL_C_MAX,
};
#define EXMPL_C_MAX (_EXMPL_C_MAX - 1)

//接收回调定义
int recv_callback(struct nl_msg* recv_msg, void* arg)
{
    struct nlmsghdr *nlh = nlmsg_hdr(recv_msg);
    struct nlattr *tb_msg[EXMPL_A_MAX + 1];

    if (nlh->nlmsg_type == NLMSG_ERROR) {
        printf("Received NLMSG_ERROR message!\n");
        return NL_STOP;
    }

    struct genlmsghdr *gnlh = (struct genlmsghdr*)nlmsg_data(nlh);
    //按照每attr解析内核发来的genl消息
    nla_parse(tb_msg, EXMPL_A_MAX,
              genlmsg_attrdata(gnlh, 0),
              genlmsg_attrlen(gnlh, 0),
              exmpl_genl_policy);

    //判断是否包含属性EXMPL_A_MSG
    if (tb_msg[EXMPL_A_MSG]) {
        // parse it as string
        char * payload_msg = nla_get_string(tb_msg[EXMPL_A_MSG]);
        printf("Kernel replied: %s\n", payload_msg);
    } else {
        printf("Attribute EXMPL_A_MSG is missing\n");
    }

    return NL_OK;
}

int main(int argc, char* argv[])
{
    //创建并连接genl socket
    struct nl_sock *sk = nl_socket_alloc();
    genl_connect(sk);
    //根据FAMILY_NAME获得对应的famlily_id
    int family_id;
    family_id = genl_ctrl_resolve(sk, FAMILY_NAME);
    if (family_id < 0) {
        printf("generic netlink family '" FAMILY_NAME "' NOT REGISTERED\n");
        nl_socket_free(sk);
        exit(-1);
    } else {
        printf("Family-ID of generic netlink family '" FAMILY_NAME "' is: %d\n", family_id);
    }

    //设置接收回调 
    nl_socket_modify_cb(sk, NL_CB_MSG_IN, NL_CB_CUSTOM, recv_callback, NULL);

    //发送消息
    struct nl_msg *msg = nlmsg_alloc();
    genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, family_id,
                0, NLM_F_REQUEST, EXMPL_C_ECHO, 1);
    NLA_PUT_STRING(msg, EXMPL_A_MSG, "genl message from user to kernel");
    int res = nl_send_auto(sk, msg);
    nlmsg_free(msg);
    if (res < 0) {
        printf("nl_send_auto fail, ret:%d\n", res);
    } else {
        printf("nl_send_auto OK, ret: %d\n", res);
    }

    //接收消息。接收到内核发来的消息后，触发回调recv_callback
    nl_recvmsgs_default(sk);

nla_put_failure: //referenced by NLA_PUT_STRING 
    nl_socket_free(sk);

    return 0;
}