1  struct percpu_counter {
       2  	signed long long count;
       3  };
       4  struct blkg_rwstat {
       5  	struct percpu_counter cpu_cnt[4];
       6  };
       7  struct cfq_group {
       8  	struct blkg_rwstat service_time;
       9  };
      10  struct cfq_queue {
      11  	struct cfq_group *group;
      12  };
      13  struct request {
      14  	struct cfq_queue *active_queue;
      15  	unsigned long long cmd_flags;
      16  	void *priv;
      17  };
      18  static void blkg_rwstat_add(struct blkg_rwstat *rwstat, int rw, unsigned long long val)
      19  {
      20  	struct percpu_counter *cnt;
      21  	if (rw & 1)
      22  		cnt = &rwstat->cpu_cnt[1];
      23  	else
      24  		cnt = &rwstat->cpu_cnt[0];
      25  	cnt->count += val;
      26  	if (rw & 2)
      27  		cnt = &rwstat->cpu_cnt[2];
      28  	else
      29  		cnt = &rwstat->cpu_cnt[3];
      30  	cnt->count += val;
      31  }
      32  extern unsigned long long rq_start_time_ns(void);
      33  extern unsigned long long rq_io_start_time_ns(void);
      34  extern int rq_is_sync(void);
      35  extern void cfq_arm_slice_timer(void);
      36  void cfq_completed_request(struct request *rq)
      37  {
      38  	struct cfq_queue *queue = rq->priv;
      39  	int sync = rq_is_sync();
      40  	struct cfq_group *group = queue->group;
      41  	long long start_time = rq_start_time_ns();
      42  	long long io_start_time = rq_io_start_time_ns();
      43  	int rw = rq->cmd_flags;
      44  
      45  	if (io_start_time < 1)
      46  		blkg_rwstat_add(&group->service_time, rw, 1 - io_start_time);
      47  	blkg_rwstat_add(0, rw, io_start_time - start_time);
      48  
      49  	if (rq->active_queue == queue && sync)
      50  		cfq_arm_slice_timer();
      51  }
      52