/* We are about to mangle packet. Copy it! */
if (skb_cow(skb, LL_RESERVED_SPACE(rt->dst.dev)+rt->dst.header_len))
	goto drop;
iph = ip_hdr(skb);

/* Decrease ttl after skb cow done */
ip_decrease_ttl(iph);

$ readlink /sys/bus/pci/devices/0000:06:0d.0/iommu_group
../../../../kernel/iommu_groups/26

modprobe vfio-pci

$ lspci -n -s 0000:06:0d.0
06:0d.0 0401: 1102:0002 (rev 08)
# echo 0000:06:0d.0 > /sys/bus/pci/devices/0000:06:0d.0/driver/unbind
# echo 1102 0002 > /sys/bus/pci/drivers/vfio-pci/new_id

$ ls -l /sys/bus/pci/devices/0000:06:0d.0/iommu_group/devices
total 0
lrwxrwxrwx. 1 root root 0 Apr 23 16:13 0000:00:1e.0 ->
	../../../../devices/pci0000:00/0000:00:1e.0
lrwxrwxrwx. 1 root root 0 Apr 23 16:13 0000:06:0d.0 ->
	../../../../devices/pci0000:00/0000:00:1e.0/0000:06:0d.0
lrwxrwxrwx. 1 root root 0 Apr 23 16:13 0000:06:0d.1 ->
	../../../../devices/pci0000:00/0000:00:1e.0/0000:06:0d.1

# chown user:user /dev/vfio/26

int container, group, device, i;
struct vfio_group_status group_status =
				{ .argsz = sizeof(group_status) };
struct vfio_iommu_type1_info iommu_info = { .argsz = sizeof(iommu_info) };
struct vfio_iommu_type1_dma_map dma_map = { .argsz = sizeof(dma_map) };
struct vfio_device_info device_info = { .argsz = sizeof(device_info) };

/* Create a new container */
container = open("/dev/vfio/vfio", O_RDWR);

if (ioctl(container, VFIO_GET_API_VERSION) != VFIO_API_VERSION)
	/* Unknown API version */

if (!ioctl(container, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU))
	/* Doesn't support the IOMMU driver we want. */

/* Open the group */
group = open("/dev/vfio/26", O_RDWR);

/* Test the group is viable and available */
ioctl(group, VFIO_GROUP_GET_STATUS, &group_status);

if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE))
	/* Group is not viable (ie, not all devices bound for vfio) */

/* Add the group to the container */
ioctl(group, VFIO_GROUP_SET_CONTAINER, &container);

/* Enable the IOMMU model we want */
ioctl(container, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);

/* Get addition IOMMU info */
ioctl(container, VFIO_IOMMU_GET_INFO, &iommu_info);

/* Allocate some space and setup a DMA mapping */
dma_map.vaddr = mmap(0, 1024 * 1024, PROT_READ | PROT_WRITE,
		     MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
dma_map.size = 1024 * 1024;
dma_map.iova = 0; /* 1MB starting at 0x0 from device view */
dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;

ioctl(container, VFIO_IOMMU_MAP_DMA, &dma_map);

/* Get a file descriptor for the device */
device = ioctl(group, VFIO_GROUP_GET_DEVICE_FD, "0000:06:0d.0");

/* Test and setup the device */
ioctl(device, VFIO_DEVICE_GET_INFO, &device_info);

for (i = 0; i < device_info.num_regions; i++) {
	struct vfio_region_info reg = { .argsz = sizeof(reg) };

	reg.index = i;

	ioctl(device, VFIO_DEVICE_GET_REGION_INFO, &reg);

	/* Setup mappings... read/write offsets, mmaps
	 * For PCI devices, config space is a region */
}

for (i = 0; i < device_info.num_irqs; i++) {
	struct vfio_irq_info irq = { .argsz = sizeof(irq) };

	irq.index = i;

	ioctl(device, VFIO_DEVICE_GET_IRQ_INFO, &irq);

	/* Setup IRQs... eventfds, VFIO_DEVICE_SET_IRQS */
}

/* Gratuitous device reset and go... */
ioctl(device, VFIO_DEVICE_RESET);

static const struct vfio_device_ops vfio_pci_ops = {
	.name		= "vfio-pci",
	.open		= vfio_pci_open,
	.release	= vfio_pci_release,
	.ioctl		= vfio_pci_ioctl,
	.read		= vfio_pci_read,
	.write		= vfio_pci_write,
	.mmap		= vfio_pci_mmap,
	.request	= vfio_pci_request,
};

struct prio_sched_data {
	int bands;
	struct tcf_proto __rcu *filter_list;
	u8  prio2band[TC_PRIO_MAX+1];
	struct Qdisc *queues[TCQ_PRIO_BANDS];
};

Rc = min(CRc, ARc + Bc)        [eq1]

       Qc Rp
Bc = -----------------------------  iff min[Pi over D(p)]>=Pc  [eq2]
     sum[Qi over D(p) where Pi=Pc]

Bc = 0   otherwise      [eq3]

/* used internaly to keep status of single class */
enum htb_cmode {
	HTB_CANT_SEND,		/* class can't send and can't borrow */
	HTB_MAY_BORROW,		/* class can't send but may borrow */
	HTB_CAN_SEND		/* class can send */
};

struct htb_sched {
	struct Qdisc_class_hash clhash;
	int			defcls;		/* class where unclassified flows go to */
	int			rate2quantum;	/* quant = rate / rate2quantum */

	/* filters for qdisc itself */
	struct tcf_proto __rcu	*filter_list;

#define HTB_WARN_TOOMANYEVENTS	0x1
	unsigned int		warned;	/* only one warning */
	int			direct_qlen;
	struct work_struct	work;

	/* non shaped skbs; let them go directly thru */
	struct qdisc_skb_head	direct_queue;
	long			direct_pkts;

	struct qdisc_watchdog	watchdog;

	s64			now;	/* cached dequeue time */
	struct list_head	drops[TC_HTB_NUMPRIO];/* active leaves (for drops) */

	/* time of nearest event per level (row) */
	s64			near_ev_cache[TC_HTB_MAXDEPTH];

	int			row_mask[TC_HTB_MAXDEPTH];

	struct htb_level	hlevel[TC_HTB_MAXDEPTH];
};

// handle 是一组用户指定的标示符，格式为 major:minor。
// 如果是一条 queueing discipline，minor 需要一直为 0。
# tc qdisc add dev eth0 root handle 1: htb

// parent 指明该新增的 class 添加到那一个父 handle 上去
// classid 指明该 class handle 的唯一 ID，minor 需要是非零值
// ceil 定义 rate 的上界
# tc class add dev eth0 parent 1:1 classid 1:6 htb rate 256kbit ceil 512kbit

// 新建一个带宽为 100kbps 的 root class, 其 classid 为 1:1
# tc class add dev eth0 parent 1: classid 1:1 htb rate 100kbps ceil 100kbps
// 接着建立两个子 class，指定其 parent 为 1:1，ceil 用来限制子类最大的带宽
# tc class add dev eth0 parent 1:1 classid 1:10 htb rate 40kbps ceil 100kbps
# tc class add dev eth0 parent 1:1 classid 1:11 htb rate 60kbps ceil 100kbps
// 随后建立 filter 指定哪些类型的 packet 进入那个 class
# tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip src 1.2.3.4 match ip dport 80 0xffff flowid 1:10
# tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip src 1.2.3.4 flow 1:11
// 最后为这些 class 添加 queuing disciplines
# tc qdisc add dev eth0 parent 1:10 handle 20: pfifo limit 5
# tc qdisc add dev eth0 parent 1:11 handle 30: sfq perturb 10

dev_queue_xmit ->
	__dev_queue_xmit ->
    	__dev_xmit_skb ->
    		qdisc->enqueue
    		__qdisc_run ->
    		  qdisc_restart ->
    		    qdisc->dequeue ->
    		      sch_direct_xmit

mkdir /sys/fs/cgroup/net_cls
mount -t cgroup -onet_cls net_cls /sys/fs/cgroup/net_cls
mkdir /sys/fs/cgroup/net_cls/0
echo 0x100001 >  /sys/fs/cgroup/net_cls/0/net_cls.classid

cat /sys/fs/cgroup/net_cls/0/net_cls.classid
1048577

tc qdisc add dev eth0 root handle 10: htb
tc class add dev eth0 parent 10: classid 10:1 htb rate 40mbit

tc filter add dev eth0 parent 10: protocol ip prio 10 handle 1: cgroup

iptables -A OUTPUT -m cgroup ! --cgroup 0x100001 -j DROP

~]# cat /cgroup/net_prio/iscsi/net_prio.ifpriomap
eth0 5
eth1 4
eth2 6

~]# echo "eth0 5" > /cgroup/net_prio/iscsi/net_prio.ifpriomap

static int cgroup_mkdir(struct inode *dir, struct dentry *dentry, int mode)
{ 
        struct cgroup *c_parent = dentry->d_parent->d_fsdata;
  
        /* the vfs holds inode->i_mutex already */
        return cgroup_create(c_parent, dentry, mode | S_IFDIR);
} 

static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
                             mode_t mode)
{
	/* 获取父cgroup对应的层级hierarchy */
	/* 这里叫做 cgroupfs 其实是匹配的，因为创建 hierachy 就是 mount 了一个文件系统的动作 */
	struct cgroupfs_root *root = parent->root;
	/* 初始化一个cgroup结构 cgrp */
        init_cgroup_housekeeping(cgrp);
        cgrp->parent = parent; /* 设置父 cgroup */
        cgrp->root = parent->root; /* 继承 parent 的 hierachy */
        cgrp->top_cgroup = parent->top_cgroup; /* 继承父 parent 对应 hierachy 的root cgroup */
	/* 继承父 parent 的 notify_on_release 设置 */
        if (notify_on_release(parent))
                set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
	/* 对所属的hierachy的子系统进行初始化 */
        for_each_subsys(root, ss) {
                struct cgroup_subsys_state *css = ss->create(ss, cgrp);
                if (IS_ERR(css)) {
                        err = PTR_ERR(css);
                        goto err_destroy;
                }
                init_cgroup_css(css, ss, cgrp);
                if (ss->use_id)
                        if (alloc_css_id(ss, parent, cgrp))
                                goto err_destroy;
                /* At error, ->destroy() callback has to free assigned ID. */
        }
	/* 加入到父 cgroup 的子列表里 */
        cgroup_lock_hierarchy(root);
        list_add(&cgrp->sibling, &cgrp->parent->children);
        cgroup_unlock_hierarchy(root);
	/* 创建 cgroup 目录 */
	cgroup_create_dir(cgrp, dentry, mode);
	/* 创建目录下对应的文件,比如common的部分(tasks),或者子系统的部分(cpu.shares,freezer.state)*/
        cgroup_populate_dir(cgrp);

static struct cgroup_subsys_state *cpuset_create(
        struct cgroup_subsys *ss,
        struct cgroup *cont)
{
        struct cpuset *cs;
        struct cpuset *parent;

        if (!cont->parent) {
                return &top_cpuset.css;
        }
        parent = cgroup_cs(cont->parent);
        cs = kmalloc(sizeof(*cs), GFP_KERNEL);
        if (!cs)
                return ERR_PTR(-ENOMEM);
        if (!alloc_cpumask_var(&cs->cpus_allowed, GFP_KERNEL)) {
                kfree(cs);
                return ERR_PTR(-ENOMEM);
        }

        cs->flags = 0;
        if (is_spread_page(parent))
                set_bit(CS_SPREAD_PAGE, &cs->flags);
        if (is_spread_slab(parent))
                set_bit(CS_SPREAD_SLAB, &cs->flags);
        set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
        cpumask_clear(cs->cpus_allowed);
        nodes_clear(cs->mems_allowed);
        fmeter_init(&cs->fmeter);
        cs->relax_domain_level = -1;

        cs->parent = parent;
        number_of_cpusets++;
        return &cs->css ;
}

/* wait 是我们想要让 task 睡眠的 queue entry, q 是等待队列 */
DEFINE_WAIT(wait);
/* 添加到等待队列中 */
add_wait_queue(q, &wait);

/*
 * 这里要检查condition是因为可能在唤醒之后这个condition的条件又不成立了,
 * 这个和条件变量一样,即使条件满足被wake up了,也可能被其他进程修改了该条件.
 */
while (!condition) {
	prepare_to_wait(&q, &wait, TASK_INTERRUPTIBLE);
	if (signal_pending(current))
		/* 处理信号 */
	/* 进行调度 */
	schedule();
}

finish_wait(&g, &wait);

/* Refrigerator is place where frozen processes are stored :-). */
void refrigerator(void)        
{
        /* Hmm, should we be allowed to suspend when there are realtime
           processes around? */
        long save;             

        task_lock(current);    
        if (freezing(current)) {        
                frozen_process();               
                task_unlock(current);           
        } else {               
                task_unlock(current);           
                return;        
        }
        save = current->state; 
        pr_debug("%s entered refrigerator\n", current->comm);
  
        spin_lock_irq(&current->sighand->siglock);
        recalc_sigpending(); /* We sent fake signal, clean it up */
        spin_unlock_irq(&current->sighand->siglock);
  
        /* prevent accounting of that task to load */
        current->flags |= PF_FREEZING;  
  
        for (;;) {             
                set_current_state(TASK_UNINTERRUPTIBLE);
                if (!frozen(current))           
                        break; 
                schedule();    
        }

        /* Remove the accounting blocker */
        current->flags &= ~PF_FREEZING;

        pr_debug("%s left refrigerator\n", current->comm);
        __set_current_state(save);
}

enum freezer_state {           
        CGROUP_THAWED = 0,     
        CGROUP_FREEZING,       
        CGROUP_FROZEN,         
};
  
struct freezer {               
        struct cgroup_subsys_state css; 
        enum freezer_state state;       
        spinlock_t lock; /* protects _writes_ to state */
};

/* 文件的读写函数 */
static struct cftype files[] = {
        {
                .name = "state", 
                .read_seq_string = freezer_read,
                .write_string = freezer_write,  
        },                     
};
  
/* 添加子系统文件到cgroup目录中 */
static int freezer_populate(struct cgroup_subsys *ss, struct cgroup *cgroup)
{ 
        if (!cgroup->parent)   
                return 0;      
        return cgroup_add_files(cgroup, ss, files, ARRAY_SIZE(files));
} 

static int freezer_read(struct cgroup *cgroup, struct cftype *cft,
                        struct seq_file *m)
{
        struct freezer *freezer;
        enum freezer_state state;

        if (!cgroup_lock_live_group(cgroup))
                return -ENODEV;

        freezer = cgroup_freezer(cgroup);
        spin_lock_irq(&freezer->lock);
        state = freezer->state;
        if (state == CGROUP_FREEZING) {
                /* We change from FREEZING to FROZEN lazily if the cgroup was
                 * only partially frozen when we exitted write. */
                update_freezer_state(cgroup, freezer);
                state = freezer->state;
        }
        spin_unlock_irq(&freezer->lock);
        cgroup_unlock();

        seq_puts(m, freezer_state_strs[state]);
        seq_putc(m, '\n');
        return 0;
}

static int freezer_write(struct cgroup *cgroup,
                         struct cftype *cft,             
                         const char *buffer)             
{ 
        int retval;            
        enum freezer_state goal_state;  
  
        if (strcmp(buffer, freezer_state_strs[CGROUP_THAWED]) == 0)
                goal_state = CGROUP_THAWED;     
        else if (strcmp(buffer, freezer_state_strs[CGROUP_FROZEN]) == 0)
                goal_state = CGROUP_FROZEN;     
        else                   
                return -EINVAL;
  
        if (!cgroup_lock_live_group(cgroup))
                return -ENODEV;
        retval = freezer_change_state(cgroup, goal_state);
        cgroup_unlock();       
        return retval;         
} 

static int try_to_freeze_cgroup(struct cgroup *cgroup, struct freezer *freezer)
{
        struct cgroup_iter it;
        struct task_struct *task;
        unsigned int num_cant_freeze_now = 0;

        freezer->state = CGROUP_FREEZING;
        cgroup_iter_start(cgroup, &it);
        while ((task = cgroup_iter_next(cgroup, &it))) {
		/* 尝试freeze task */
                if (!freeze_task(task, true))
                        continue;
                if (is_task_frozen_enough(task))
                        continue;
                if (!freezing(task) && !freezer_should_skip(task))
                        num_cant_freeze_now++;
        }
        cgroup_iter_end(cgroup, &it);

        return num_cant_freeze_now ? -EBUSY : 0;
}

bool freeze_task(struct task_struct *p, bool sig_only)
{
        /*
         * We first check if the task is freezing and next if it has already
         * been frozen to avoid the race with frozen_process() which first marks
         * the task as frozen and next clears its TIF_FREEZE.
         */
        if (!freezing(p)) {
                rmb();
		/* 如果frozen标记了
		 * 说明已经冻结,就返回失败
		 */
                if (frozen(p))
                        return false;

                if (!sig_only || should_send_signal(p))
                        set_freeze_flag(p);
                else
                        return false;
        }

        if (should_send_signal(p)) {
                if (!signal_pending(p))
                        fake_signal_wake_up(p);
        } else if (sig_only) {
                return false;
        } else {
                wake_up_state(p, TASK_INTERRUPTIBLE);
        }

        return true;
}

struct sched_entity {
        struct load_weight      load;           /* for load-balancing */
        struct rb_node          run_node;
        struct list_head        group_node;
        unsigned int            on_rq;

        u64                     exec_start;
        u64                     sum_exec_runtime;
        u64                     vruntime;
        u64                     prev_sum_exec_runtime;

        u64                     nr_migrations;

#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)
        /* Per-entity load-tracking */
        struct sched_avg        avg;
#endif
#ifdef CONFIG_SCHEDSTATS       
        struct sched_statistics statistics;
#endif                         

#ifdef CONFIG_FAIR_GROUP_SCHED 
        struct sched_entity     *parent;
        /* rq on which this entity is (to be) queued: */
        struct cfs_rq           *cfs_rq;
        /* rq "owned" by this entity/group: */
        struct cfs_rq           *my_q;  
#endif
};

   struct cfs_rq {
struct load_weight load;
unsigned long runnable_load_avg;
unsigned long blocked_load_avg;
unsigned long tg_load_contrib;
/* ... */
   };

   struct task_group {
struct sched_entity **se;
struct cfs_rq **cfs_rq;
unsigned long shares;
atomic_long_t load_avg;
/* ... */
   };

static const int prio_to_weight[40] = {
 /* -20 */     88761,     71755,     56483,     46273,     36291,
 /* -15 */     29154,     23254,     18705,     14949,     11916,
 /* -10 */      9548,      7620,      6100,      4904,      3906,
 /*  -5 */      3121,      2501,      1991,      1586,      1277,
 /*   0 */      1024,       820,       655,       526,       423,
 /*   5 */       335,       272,       215,       172,       137,
 /*  10 */       110,        87,        70,        56,        45,
 /*  15 */        36,        29,        23,        18,        15,
};

   struct load_weight {
unsigned long weight;
   };

struct sched_avg {
u32 runnable_sum, runnable_avg_period;
unsigned long load_avg_contrib;
};

tg = cfs_rq->tg;

cfs_rq->tg_load_contrib = cfs_rq->runnable_load_avg + cfs_rq->blocked_load_avg;
    tg->load_avg += cfs_rq->tg_load_contrib;

se->avg.load_avg_contrib =
 (cfs_rq->tg_load_contrib * tg->shares / tg->load_avg);

static struct cftype cpu_files[] = {
#ifdef CONFIG_FAIR_GROUP_SCHED
        {
                .name = "shares",
                .read_u64 = cpu_shares_read_u64,
                .write_u64 = cpu_shares_write_u64,
        },
#endif
#ifdef CONFIG_CFS_BANDWIDTH
        {
                .name = "cfs_quota_us",
                .read_s64 = cpu_cfs_quota_read_s64,
                .write_s64 = cpu_cfs_quota_write_s64,
        },
        {
                .name = "cfs_period_us",
                .read_u64 = cpu_cfs_period_read_u64,
                .write_u64 = cpu_cfs_period_write_u64,
        },
        {
                .name = "stat",
                .read_map = cpu_stats_show,
        },
#endif

cpu_shares_write_u64 实际调用的是
	-> sched_group_set_shares(cgroup_tg(cgrp), scale_load(shareval))
		-> update_cfs_shares 获取 cgroup 的task group结构，调整权重

static void update_cfs_shares(struct cfs_rq *cfs_rq)
{
        struct task_group *tg;
        struct sched_entity *se;
        long shares;

        tg = cfs_rq->tg;
        se = tg->se[cpu_of(rq_of(cfs_rq))];
        if (!se || throttled_hierarchy(cfs_rq))
                return;
#ifndef CONFIG_SMP
        if (likely(se->load.weight == tg->shares))
                return;
#endif
        /* 根据tg->shares 和 rq 的负载计算出新的权重 */
        shares = calc_cfs_shares(cfs_rq, tg);
	
        reweight_entity(cfs_rq_of(se), se, shares);
}

struct cfs_bandwidth {
#ifdef CONFIG_CFS_BANDWIDTH
        raw_spinlock_t lock;
        ktime_t period;
        /* quota 是被赋予的时间，runtime 是实际运行的时间 */
        u64 quota, runtime;
        s64 hierarchal_quota;
        /* 到期时间 */ 
        u64 runtime_expires;
  
        int idle, timer_active;
        struct hrtimer period_timer, slack_timer;
        struct list_head throttled_cfs_rq;

        /* statistics */
        int nr_periods, nr_throttled;   
        u64 throttled_time;
#endif
};

/* track cpu usage of a group of tasks and its child groups */
struct cpuacct {
        struct cgroup_subsys_state css;
        /* cpuusage holds pointer to a u64-type object on every cpu */
        u64 __percpu *cpuusage;
        struct kernel_cpustat __percpu *cpustat;
};

static struct cftype files[] = {
        {
                .name = "usage",
                .read_u64 = cpuusage_read,
                .write_u64 = cpuusage_write,
        },
        {
                .name = "usage_percpu",
                .read_seq_string = cpuacct_percpu_seq_read,
        },
        {
                .name = "stat",
                .read_map = cpuacct_stats_show,
        },
        { }     /* terminate */
};

/*
 * charge this task's execution time to its accounting group.
 *
 * called with rq->lock held.
 */
void cpuacct_charge(struct task_struct *tsk, u64 cputime)
{
        struct cpuacct *ca;
        int cpu;
        /* 获取当前task属于的cpu */
        cpu = task_cpu(tsk);

        rcu_read_lock();
        /* task的cpuacct结构 */
        ca = task_ca(tsk);
        /* 所有父节点的值都应该相应变化
         * 上溯父节点更新统计值
         */
        while (true) {
                u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
                *cpuusage += cputime;

                ca = parent_ca(ca);
                if (!ca)
                        break;
        }

        rcu_read_unlock();
}

/*
 * Add user/system time to cpuacct.
 *
 * Note: it's the caller that updates the account of the root cgroup.
 */
void cpuacct_account_field(struct task_struct *p, int index, u64 val)
{
        struct kernel_cpustat *kcpustat;
        struct cpuacct *ca;

        rcu_read_lock();
        ca = task_ca(p);
        while (ca != &root_cpuacct) {
                kcpustat = this_cpu_ptr(ca->cpustat);
                kcpustat->cpustat[index] += val;
                ca = __parent_ca(ca);
        }
        rcu_read_unlock();
}   

struct cpuset {
        struct cgroup_subsys_state css;
        
        unsigned long flags;            /* "unsigned long" so bitops work */
        cpumask_var_t cpus_allowed;     /* CPUs allowed to tasks in cpuset */
        nodemask_t mems_allowed;        /* Memory Nodes allowed to tasks */
        
        struct fmeter fmeter;           /* memory_pressure filter */
        
        /*
         * Tasks are being attached to this cpuset.  Used to prevent
         * zeroing cpus/mems_allowed between ->can_attach() and ->attach().
         */
        int attach_in_progress;
        
        /* partition number for rebuild_sched_domains() */
        int pn;
        
        /* for custom sched domain */
        int relax_domain_level;
        
        struct work_struct hotplug_work;
};  

static inline int zref_in_nodemask(struct zoneref *zref, nodemask_t *nodes)
{ 
#ifdef CONFIG_NUMA
        return node_isset(zonelist_node_idx(zref), *nodes);
#else
        return 1;
#endif /* CONFIG_NUMA */
} 

static int cpuset_spread_node(int *rotor)
{ 
        int node;              

        node = next_node(*rotor, current->mems_allowed);
        if (node == MAX_NUMNODES)       
                node = first_node(current->mems_allowed);
        *rotor = node;         
        return node;           
} 
  
int cpuset_mem_spread_node(void)
{ 
        if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE)
                current->cpuset_mem_spread_rotor =
                        node_random(&current->mems_allowed);
  
        return cpuset_spread_node(&current->cpuset_mem_spread_rotor);
} 

/*
 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
 */
static inline
int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
{ 
        int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags);
  
        /*
         * In order not to call set_task_cpu() on a blocking task we need
         * to rely on ttwu() to place the task on a valid ->cpus_allowed
         * cpu.
         *
         * Since this is common to all placement strategies, this lives here.
         *
         * [ this allows ->select_task() to simply return task_cpu(p) and
         *   not worry about this generic constraint ]
         */
        if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) ||
                     !cpu_online(cpu)))              
                cpu = select_fallback_rq(task_cpu(p), p);

        return cpu;
}

// 向上遍历更高层次的domain,如果发现同属一个domain
// 就是affine目标
      for_each_domain(cpu, tmp) {
               /*
                * If both cpu and prev_cpu are part of this domain,
                * cpu is a valid SD_WAKE_AFFINE target.
                */
               if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
                   cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
                       affine_sd = tmp;
                       break;
               }

               if (tmp->flags & sd_flag)
                       sd = tmp;
       }
// 如果上面的条件满足,从prev_cpu中选出一个idle的new_cpu来运行.
       if (affine_sd) {
               if (cpu != prev_cpu && wake_affine(affine_sd, p, sync))
                       prev_cpu = cpu;                 
	// 在同一个级别的sched domain向下找到一个idle的CPU.
               new_cpu = select_idle_sibling(p, prev_cpu);
	// 快速路径,有idle的CPU就不用负载均衡了.
               goto unlock;
       }
// 遍历层级
       while (sd) {
	//  找到负载最小的CPU
               group = find_idlest_group(sd, p, cpu, load_idx);
               if (!group) {
                       sd = sd->child;
                       continue;
               }

               new_cpu = find_idlest_cpu(group, p, cpu);
	/* 如果最闲置的CPU没有变的话,或者没有找到的话,就向下遍历.
               if (new_cpu == -1 || new_cpu == cpu) {
                       /* Now try balancing at a lower domain level of cpu */
                       sd = sd->child;
                       continue;
               }

               /* Now try balancing at a lower domain level of new_cpu */
               cpu = new_cpu;
               weight = sd->span_weight;
               sd = NULL;
	// 如果选出的节点weight比其他节点都大的话.
	// 再向下一个层级遍历.
               for_each_domain(cpu, tmp) {
                       if (weight <= tmp->span_weight)
                               break;
                       if (tmp->flags & sd_flag)
                               sd = tmp;
               }
               /* while loop will break here if sd == NULL */
       }

/*
 * The core object. the cgroup that wishes to account for some
 * resource may include this counter into its structures and use
 * the helpers described beyond
 */

struct res_counter {
        /*
         * the current resource consumption level
         */
        unsigned long long usage;
        /*
         * the maximal value of the usage from the counter creation
         */
        unsigned long long max_usage;
        /*
         * the limit that usage cannot exceed
         */
        unsigned long long limit;
        /*
         * the limit that usage can be exceed
         */
        unsigned long long soft_limit;
        /*
         * the number of unsuccessful attempts to consume the resource
         */
        unsigned long long failcnt;
        /*
         * the lock to protect all of the above.
         * the routines below consider this to be IRQ-safe
         */
        spinlock_t lock;
        /*
         * Parent counter, used for hierarchial resource accounting
         */
        struct res_counter *parent;
};

static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
{ 
        u64 val;
  
        if (!mem_cgroup_is_root(memcg)) {
                if (!swap)
                        return res_counter_read_u64(&memcg->res, RES_USAGE);
                else
                        return res_counter_read_u64(&memcg->memsw, RES_USAGE);
        }
  
        /*
         * Transparent hugepages are still accounted for in MEM_CGROUP_STAT_RSS
         * as well as in MEM_CGROUP_STAT_RSS_HUGE.
         */
	// 如果是root就把所有的内存使用量都算进来.
        val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
        val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
  
        if (swap)
                val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
  
        return val << PAGE_SHIFT;       
} 

struct mem_cgroup {
        struct cgroup_subsys_state css; // 通过css关联cgroup.
        struct res_counter res; // mem统计变量
	res_counter memsw; // mem+sw的和
        struct res_counter kmem; // 内核内存统计量
	...
}

static ssize_t mem_cgroup_read(struct cgroup *cont, struct cftype *cft,
                               struct file *file, char __user *buf,
                               size_t nbytes, loff_t *ppos)
{
	// 获取cgroup对应的mem_cgroup.
        struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
        char str[64];
        u64 val;
        int name, len;
        enum res_type type;
  
	// 获取读取的类型,memory.usage_in_bytes就是_MEM
        type = MEMFILE_TYPE(cft->private);
	// 名称
        name = MEMFILE_ATTR(cft->private);
  
        switch (type) {
        case _MEM:
                if (name == RES_USAGE)          
                        val = mem_cgroup_usage(memcg, false); 
                else
                        val = res_counter_read_u64(&memcg->res, name);
                break;
        case _MEMSWAP:
                if (name == RES_USAGE)          
                        val = mem_cgroup_usage(memcg, true);
                else
                        val = res_counter_read_u64(&memcg->memsw, name);
                break;
        case _KMEM:
                val = res_counter_read_u64(&memcg->kmem, name);
                break;
        default:
                BUG();
        }

        len = scnprintf(str, sizeof(str), "%llu\n", (unsigned long long)val);
        return simple_read_from_buffer(buf, nbytes, ppos, str, len);
}