nohz: Avoid duplication of code related to got_idle_tick

[uclinux-h8/linux.git] / kernel / time / tick-sched.c

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index f56d2c6..956831c 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -113,8 +113,7 @@ static ktime_t tick_init_jiffy_update(void)
        return period;
 }
 
-
-static void tick_sched_do_timer(ktime_t now)
+static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
 {
        int cpu = smp_processor_id();
 
@@ -134,6 +133,9 @@ static void tick_sched_do_timer(ktime_t now)
        /* Check, if the jiffies need an update */
        if (tick_do_timer_cpu == cpu)
                tick_do_update_jiffies64(now);
+
+       if (ts->inidle)
+               ts->got_idle_tick = 1;
 }
 
 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
@@ -465,7 +467,9 @@ __setup("nohz=", setup_tick_nohz);
 
 bool tick_nohz_tick_stopped(void)
 {
-       return __this_cpu_read(tick_cpu_sched.tick_stopped);
+       struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
+
+       return ts->tick_stopped;
 }
 
 bool tick_nohz_tick_stopped_cpu(int cpu)
@@ -913,16 +917,19 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
 
 static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
 {
-       struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
        ktime_t expires;
        int cpu = smp_processor_id();
 
-       WARN_ON_ONCE(ts->timer_expires_base);
-
-       if (!can_stop_idle_tick(cpu, ts))
-               goto out;
-
-       expires = tick_nohz_next_event(ts, cpu);
+       /*
+        * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
+        * tick timer expiration time is known already.
+        */
+       if (ts->timer_expires_base)
+               expires = ts->timer_expires;
+       else if (can_stop_idle_tick(cpu, ts))
+               expires = tick_nohz_next_event(ts, cpu);
+       else
+               return;
 
        ts->idle_calls++;
 
@@ -941,9 +948,6 @@ static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
        } else {
                tick_nohz_retain_tick(ts);
        }
-
-out:
-       ts->sleep_length = ktime_sub(dev->next_event, ts->idle_entrytime);
 }
 
 /**
@@ -956,6 +960,16 @@ void tick_nohz_idle_stop_tick(void)
        __tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched));
 }
 
+void tick_nohz_idle_retain_tick(void)
+{
+       tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched));
+       /*
+        * Undo the effect of get_next_timer_interrupt() called from
+        * tick_nohz_next_event().
+        */
+       timer_clear_idle();
+}
+
 /**
  * tick_nohz_idle_enter - prepare for entering idle on the current CPU
  *
@@ -1004,23 +1018,50 @@ bool tick_nohz_idle_got_tick(void)
 {
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 
-       if (ts->inidle > 1) {
-               ts->inidle = 1;
+       if (ts->got_idle_tick) {
+               ts->got_idle_tick = 0;
                return true;
        }
        return false;
 }
 
 /**
- * tick_nohz_get_sleep_length - return the length of the current sleep
+ * tick_nohz_get_sleep_length - return the expected length of the current sleep
+ * @delta_next: duration until the next event if the tick cannot be stopped
  *
  * Called from power state control code with interrupts disabled
  */
-ktime_t tick_nohz_get_sleep_length(void)
+ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
 {
+       struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
+       int cpu = smp_processor_id();
+       /*
+        * The idle entry time is expected to be a sufficient approximation of
+        * the current time at this point.
+        */
+       ktime_t now = ts->idle_entrytime;
+       ktime_t next_event;
+
+       WARN_ON_ONCE(!ts->inidle);
+
+       *delta_next = ktime_sub(dev->next_event, now);
 
-       return ts->sleep_length;
+       if (!can_stop_idle_tick(cpu, ts))
+               return *delta_next;
+
+       next_event = tick_nohz_next_event(ts, cpu);
+       if (!next_event)
+               return *delta_next;
+
+       /*
+        * If the next highres timer to expire is earlier than next_event, the
+        * idle governor needs to know that.
+        */
+       next_event = min_t(u64, next_event,
+                          hrtimer_next_event_without(&ts->sched_timer));
+
+       return ktime_sub(next_event, now);
 }
 
 /**
@@ -1123,12 +1164,9 @@ static void tick_nohz_handler(struct clock_event_device *dev)
        struct pt_regs *regs = get_irq_regs();
        ktime_t now = ktime_get();
 
-       if (ts->inidle)
-               ts->inidle = 2;
-
        dev->next_event = KTIME_MAX;
 
-       tick_sched_do_timer(now);
+       tick_sched_do_timer(ts, now);
        tick_sched_handle(ts, regs);
 
        /* No need to reprogram if we are running tickless  */
@@ -1223,10 +1261,7 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
        struct pt_regs *regs = get_irq_regs();
        ktime_t now = ktime_get();
 
-       if (ts->inidle)
-               ts->inidle = 2;
-
-       tick_sched_do_timer(now);
+       tick_sched_do_timer(ts, now);
 
        /*
         * Do not call, when we are not in irq context and have