#include <linux/jiffies.h>
#include <linux/time.h>
#include <linux/tick.h>
+#include <linux/stop_machine.h>
/* Structure holding internal timekeeping values. */
struct timekeeper {
/* Current clocksource used for timekeeping. */
struct clocksource *clock;
+ /* The shift value of the current clocksource. */
+ int shift;
/* Number of clock cycles in one NTP interval. */
cycle_t cycle_interval;
/* Difference between accumulated time and NTP time in ntp
* shifted nano seconds. */
s64 ntp_error;
+ /* Shift conversion between clock shifted nano seconds and
+ * ntp shifted nano seconds. */
+ int ntp_error_shift;
+ /* NTP adjusted clock multiplier */
+ u32 mult;
};
struct timekeeper timekeeper;
/* Do the ns -> cycle conversion first, using original mult */
tmp = NTP_INTERVAL_LENGTH;
tmp <<= clock->shift;
- tmp += clock->mult_orig/2;
- do_div(tmp, clock->mult_orig);
+ tmp += clock->mult/2;
+ do_div(tmp, clock->mult);
if (tmp == 0)
tmp = 1;
/* Go back from cycles -> shifted ns */
timekeeper.xtime_interval = (u64) interval * clock->mult;
timekeeper.raw_interval =
- ((u64) interval * clock->mult_orig) >> clock->shift;
+ ((u64) interval * clock->mult) >> clock->shift;
timekeeper.xtime_nsec = 0;
+ timekeeper.shift = clock->shift;
timekeeper.ntp_error = 0;
+ timekeeper.ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
+
+ /*
+ * The timekeeper keeps its own mult values for the currently
+ * active clocksource. These value will be adjusted via NTP
+ * to counteract clock drifting.
+ */
+ timekeeper.mult = clock->mult;
+}
+
+/* Timekeeper helper functions. */
+static inline s64 timekeeping_get_ns(void)
+{
+ cycle_t cycle_now, cycle_delta;
+ struct clocksource *clock;
+
+ /* read clocksource: */
+ clock = timekeeper.clock;
+ cycle_now = clock->read(clock);
+
+ /* calculate the delta since the last update_wall_time: */
+ cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
+
+ /* return delta convert to nanoseconds using ntp adjusted mult. */
+ return clocksource_cyc2ns(cycle_delta, timekeeper.mult,
+ timekeeper.shift);
+}
+
+static inline s64 timekeeping_get_ns_raw(void)
+{
+ cycle_t cycle_now, cycle_delta;
+ struct clocksource *clock;
+
+ /* read clocksource: */
+ clock = timekeeper.clock;
+ cycle_now = clock->read(clock);
+
+ /* calculate the delta since the last update_wall_time: */
+ cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
+
+ /* return delta convert to nanoseconds using ntp adjusted mult. */
+ return clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
}
/*
*/
struct timespec xtime __attribute__ ((aligned (16)));
struct timespec wall_to_monotonic __attribute__ ((aligned (16)));
-static unsigned long total_sleep_time; /* seconds */
+static struct timespec total_sleep_time;
/*
* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock.
}
#ifdef CONFIG_GENERIC_TIME
+
/**
* timekeeping_forward_now - update clock to the current time
*
cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
clock->cycle_last = cycle_now;
- nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
+ nsec = clocksource_cyc2ns(cycle_delta, timekeeper.mult,
+ timekeeper.shift);
/* If arch requires, add in gettimeoffset() */
nsec += arch_gettimeoffset();
timespec_add_ns(&xtime, nsec);
- nsec = clocksource_cyc2ns(cycle_delta, clock->mult_orig, clock->shift);
+ nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
timespec_add_ns(&raw_time, nsec);
}
*/
void getnstimeofday(struct timespec *ts)
{
- cycle_t cycle_now, cycle_delta;
- struct clocksource *clock;
unsigned long seq;
s64 nsecs;
seq = read_seqbegin(&xtime_lock);
*ts = xtime;
-
- /* read clocksource: */
- clock = timekeeper.clock;
- cycle_now = clock->read(clock);
-
- /* calculate the delta since the last update_wall_time: */
- cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
-
- /* convert to nanoseconds: */
- nsecs = clocksource_cyc2ns(cycle_delta, clock->mult,
- clock->shift);
+ nsecs = timekeeping_get_ns();
/* If arch requires, add in gettimeoffset() */
nsecs += arch_gettimeoffset();
ktime_t ktime_get(void)
{
- cycle_t cycle_now, cycle_delta;
- struct clocksource *clock;
unsigned int seq;
s64 secs, nsecs;
seq = read_seqbegin(&xtime_lock);
secs = xtime.tv_sec + wall_to_monotonic.tv_sec;
nsecs = xtime.tv_nsec + wall_to_monotonic.tv_nsec;
-
- /* read clocksource: */
- clock = timekeeper.clock;
- cycle_now = clock->read(clock);
-
- /* calculate the delta since the last update_wall_time: */
- cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
-
- /* convert to nanoseconds: */
- nsecs += clocksource_cyc2ns(cycle_delta, clock->mult,
- clock->shift);
+ nsecs += timekeeping_get_ns();
} while (read_seqretry(&xtime_lock, seq));
/*
*/
void ktime_get_ts(struct timespec *ts)
{
- cycle_t cycle_now, cycle_delta;
- struct clocksource *clock;
struct timespec tomono;
unsigned int seq;
s64 nsecs;
seq = read_seqbegin(&xtime_lock);
*ts = xtime;
tomono = wall_to_monotonic;
-
- /* read clocksource: */
- clock = timekeeper.clock;
- cycle_now = clock->read(clock);
-
- /* calculate the delta since the last update_wall_time: */
- cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
-
- /* convert to nanoseconds: */
- nsecs = clocksource_cyc2ns(cycle_delta, clock->mult,
- clock->shift);
+ nsecs = timekeeping_get_ns();
} while (read_seqretry(&xtime_lock, seq));
*
* Accumulates current time interval and initializes new clocksource
*/
-static void change_clocksource(void)
+static int change_clocksource(void *data)
{
struct clocksource *new, *old;
- new = clocksource_get_next();
-
- if (!new || timekeeper.clock == new)
- return;
+ new = (struct clocksource *) data;
timekeeping_forward_now();
+ if (!new->enable || new->enable(new) == 0) {
+ old = timekeeper.clock;
+ timekeeper_setup_internals(new);
+ if (old->disable)
+ old->disable(old);
+ }
+ return 0;
+}
- if (new->enable && !new->enable(new))
+/**
+ * timekeeping_notify - Install a new clock source
+ * @clock: pointer to the clock source
+ *
+ * This function is called from clocksource.c after a new, better clock
+ * source has been registered. The caller holds the clocksource_mutex.
+ */
+void timekeeping_notify(struct clocksource *clock)
+{
+ if (timekeeper.clock == clock)
return;
- /*
- * The frequency may have changed while the clocksource
- * was disabled. If so the code in ->enable() must update
- * the mult value to reflect the new frequency. Make sure
- * mult_orig follows this change.
- */
- new->mult_orig = new->mult;
-
- old = timekeeper.clock;
- timekeeper_setup_internals(new);
-
- /*
- * Save mult_orig in mult so that the value can be restored
- * regardless if ->enable() updates the value of mult or not.
- */
- old->mult = old->mult_orig;
- if (old->disable)
- old->disable(old);
-
+ stop_machine(change_clocksource, clock, NULL);
tick_clock_notify();
}
+
#else /* GENERIC_TIME */
+
static inline void timekeeping_forward_now(void) { }
-static inline void change_clocksource(void) { }
/**
* ktime_get - get the monotonic time in ktime_t format
ts->tv_nsec + tomono.tv_nsec);
}
EXPORT_SYMBOL_GPL(ktime_get_ts);
+
#endif /* !GENERIC_TIME */
/**
{
unsigned long seq;
s64 nsecs;
- cycle_t cycle_now, cycle_delta;
- struct clocksource *clock;
do {
seq = read_seqbegin(&xtime_lock);
-
- /* read clocksource: */
- clock = timekeeper.clock;
- cycle_now = clock->read(clock);
-
- /* calculate the delta since the last update_wall_time: */
- cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
-
- /* convert to nanoseconds: */
- nsecs = clocksource_cyc2ns(cycle_delta, clock->mult_orig,
- clock->shift);
-
+ nsecs = timekeeping_get_ns_raw();
*ts = raw_time;
} while (read_seqretry(&xtime_lock, seq));
}
/**
- * read_persistent_clock - Return time in seconds from the persistent clock.
+ * read_persistent_clock - Return time from the persistent clock.
*
* Weak dummy function for arches that do not yet support it.
- * Returns seconds from epoch using the battery backed persistent clock.
- * Returns zero if unsupported.
+ * Reads the time from the battery backed persistent clock.
+ * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
*
* XXX - Do be sure to remove it once all arches implement it.
*/
-unsigned long __attribute__((weak)) read_persistent_clock(void)
+void __attribute__((weak)) read_persistent_clock(struct timespec *ts)
{
- return 0;
+ ts->tv_sec = 0;
+ ts->tv_nsec = 0;
+}
+
+/**
+ * read_boot_clock - Return time of the system start.
+ *
+ * Weak dummy function for arches that do not yet support it.
+ * Function to read the exact time the system has been started.
+ * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
+ *
+ * XXX - Do be sure to remove it once all arches implement it.
+ */
+void __attribute__((weak)) read_boot_clock(struct timespec *ts)
+{
+ ts->tv_sec = 0;
+ ts->tv_nsec = 0;
}
/*
{
struct clocksource *clock;
unsigned long flags;
- unsigned long sec = read_persistent_clock();
+ struct timespec now, boot;
+
+ read_persistent_clock(&now);
+ read_boot_clock(&boot);
write_seqlock_irqsave(&xtime_lock, flags);
clock = clocksource_default_clock();
if (clock->enable)
clock->enable(clock);
- /* set mult_orig on enable */
- clock->mult_orig = clock->mult;
-
timekeeper_setup_internals(clock);
- xtime.tv_sec = sec;
- xtime.tv_nsec = 0;
+ xtime.tv_sec = now.tv_sec;
+ xtime.tv_nsec = now.tv_nsec;
raw_time.tv_sec = 0;
raw_time.tv_nsec = 0;
+ if (boot.tv_sec == 0 && boot.tv_nsec == 0) {
+ boot.tv_sec = xtime.tv_sec;
+ boot.tv_nsec = xtime.tv_nsec;
+ }
set_normalized_timespec(&wall_to_monotonic,
- -xtime.tv_sec, -xtime.tv_nsec);
+ -boot.tv_sec, -boot.tv_nsec);
update_xtime_cache(0);
- total_sleep_time = 0;
+ total_sleep_time.tv_sec = 0;
+ total_sleep_time.tv_nsec = 0;
write_sequnlock_irqrestore(&xtime_lock, flags);
}
/* time in seconds when suspend began */
-static unsigned long timekeeping_suspend_time;
+static struct timespec timekeeping_suspend_time;
/**
* timekeeping_resume - Resumes the generic timekeeping subsystem.
static int timekeeping_resume(struct sys_device *dev)
{
unsigned long flags;
- unsigned long now = read_persistent_clock();
+ struct timespec ts;
+
+ read_persistent_clock(&ts);
clocksource_resume();
write_seqlock_irqsave(&xtime_lock, flags);
- if (now && (now > timekeeping_suspend_time)) {
- unsigned long sleep_length = now - timekeeping_suspend_time;
-
- xtime.tv_sec += sleep_length;
- wall_to_monotonic.tv_sec -= sleep_length;
- total_sleep_time += sleep_length;
+ if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) {
+ ts = timespec_sub(ts, timekeeping_suspend_time);
+ xtime = timespec_add_safe(xtime, ts);
+ wall_to_monotonic = timespec_sub(wall_to_monotonic, ts);
+ total_sleep_time = timespec_add_safe(total_sleep_time, ts);
}
update_xtime_cache(0);
/* re-base the last cycle value */
{
unsigned long flags;
- timekeeping_suspend_time = read_persistent_clock();
+ read_persistent_clock(&timekeeping_suspend_time);
write_seqlock_irqsave(&xtime_lock, flags);
timekeeping_forward_now();
* Now calculate the error in (1 << look_ahead) ticks, but first
* remove the single look ahead already included in the error.
*/
- tick_error = tick_length >>
- (NTP_SCALE_SHIFT - timekeeper.clock->shift + 1);
+ tick_error = tick_length >> (timekeeper.ntp_error_shift + 1);
tick_error -= timekeeper.xtime_interval >> 1;
error = ((error - tick_error) >> look_ahead) + tick_error;
s64 error, interval = timekeeper.cycle_interval;
int adj;
- error = timekeeper.ntp_error >>
- (NTP_SCALE_SHIFT - timekeeper.clock->shift - 1);
+ error = timekeeper.ntp_error >> (timekeeper.ntp_error_shift - 1);
if (error > interval) {
error >>= 2;
if (likely(error <= interval))
} else
return;
- timekeeper.clock->mult += adj;
+ timekeeper.mult += adj;
timekeeper.xtime_interval += interval;
timekeeper.xtime_nsec -= offset;
timekeeper.ntp_error -= (interval - offset) <<
- (NTP_SCALE_SHIFT - timekeeper.clock->shift);
+ timekeeper.ntp_error_shift;
}
/**
{
struct clocksource *clock;
cycle_t offset;
- s64 nsecs;
+ u64 nsecs;
/* Make sure we're fully resumed: */
if (unlikely(timekeeping_suspended))
#else
offset = timekeeper.cycle_interval;
#endif
- timekeeper.xtime_nsec = (s64)xtime.tv_nsec << clock->shift;
+ timekeeper.xtime_nsec = (s64)xtime.tv_nsec << timekeeper.shift;
/* normally this loop will run just once, however in the
* case of lost or late ticks, it will accumulate correctly.
*/
while (offset >= timekeeper.cycle_interval) {
- u64 nsecps = (u64)NSEC_PER_SEC << clock->shift;
+ u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift;
/* accumulate one interval */
offset -= timekeeper.cycle_interval;
/* accumulate error between NTP and clock interval */
timekeeper.ntp_error += tick_length;
timekeeper.ntp_error -= timekeeper.xtime_interval <<
- (NTP_SCALE_SHIFT - clock->shift);
+ timekeeper.ntp_error_shift;
}
/* correct the clock when NTP error is too big */
if (unlikely((s64)timekeeper.xtime_nsec < 0)) {
s64 neg = -(s64)timekeeper.xtime_nsec;
timekeeper.xtime_nsec = 0;
- timekeeper.ntp_error += neg << (NTP_SCALE_SHIFT - clock->shift);
+ timekeeper.ntp_error += neg << timekeeper.ntp_error_shift;
}
/* store full nanoseconds into xtime after rounding it up and
* add the remainder to the error difference.
*/
- xtime.tv_nsec = ((s64)timekeeper.xtime_nsec >> clock->shift) + 1;
- timekeeper.xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
- timekeeper.ntp_error += timekeeper.xtime_nsec <<
- (NTP_SCALE_SHIFT - clock->shift);
+ xtime.tv_nsec = ((s64) timekeeper.xtime_nsec >> timekeeper.shift) + 1;
+ timekeeper.xtime_nsec -= (s64) xtime.tv_nsec << timekeeper.shift;
+ timekeeper.ntp_error += timekeeper.xtime_nsec <<
+ timekeeper.ntp_error_shift;
- nsecs = clocksource_cyc2ns(offset, clock->mult, clock->shift);
+ nsecs = clocksource_cyc2ns(offset, timekeeper.mult, timekeeper.shift);
update_xtime_cache(nsecs);
/* check to see if there is a new clocksource to use */
- change_clocksource();
update_vsyscall(&xtime, timekeeper.clock);
}
*/
void getboottime(struct timespec *ts)
{
- set_normalized_timespec(ts,
- - (wall_to_monotonic.tv_sec + total_sleep_time),
- - wall_to_monotonic.tv_nsec);
+ struct timespec boottime = {
+ .tv_sec = wall_to_monotonic.tv_sec + total_sleep_time.tv_sec,
+ .tv_nsec = wall_to_monotonic.tv_nsec + total_sleep_time.tv_nsec
+ };
+
+ set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec);
}
/**
*/
void monotonic_to_bootbased(struct timespec *ts)
{
- ts->tv_sec += total_sleep_time;
+ *ts = timespec_add_safe(*ts, total_sleep_time);
}
unsigned long get_seconds(void)
}
EXPORT_SYMBOL(get_seconds);
+struct timespec __current_kernel_time(void)
+{
+ return xtime_cache;
+}
struct timespec current_kernel_time(void)
{
return now;
}
EXPORT_SYMBOL(current_kernel_time);
+
+struct timespec get_monotonic_coarse(void)
+{
+ struct timespec now, mono;
+ unsigned long seq;
+
+ do {
+ seq = read_seqbegin(&xtime_lock);
+
+ now = xtime_cache;
+ mono = wall_to_monotonic;
+ } while (read_seqretry(&xtime_lock, seq));
+
+ set_normalized_timespec(&now, now.tv_sec + mono.tv_sec,
+ now.tv_nsec + mono.tv_nsec);
+ return now;
+}
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff