1 // SPDX-License-Identifier: GPL-2.0-only
28 #include <asm/intel-family.h>
56 	struct cyc2ns_data data[2];	/*  0 + 2*16 = 32 */  member
69 __always_inline void cyc2ns_read_begin(struct cyc2ns_data *data)  in cyc2ns_read_begin()  argument
79 		data->cyc2ns_offset = this_cpu_read(cyc2ns.data[idx].cyc2ns_offset);  in cyc2ns_read_begin()
80 		data->cyc2ns_mul    = this_cpu_read(cyc2ns.data[idx].cyc2ns_mul);  in cyc2ns_read_begin()
81 		data->cyc2ns_shift  = this_cpu_read(cyc2ns.data[idx].cyc2ns_shift);  in cyc2ns_read_begin()
106  *  cyc2ns_scale needs to be a 32-bit value so that 32-bit multiplication
107  *  (64-bit result) can be used.
112  *                      -johnstul@us.ibm.com "math is hard, lets go shopping!"
117 	struct cyc2ns_data data;  in cycles_2_ns()  local
120 	cyc2ns_read_begin(&data);  in cycles_2_ns()
122 	ns = data.cyc2ns_offset;  in cycles_2_ns()
123 	ns += mul_u64_u32_shr(cyc, data.cyc2ns_mul, data.cyc2ns_shift);  in cycles_2_ns()
133 	struct cyc2ns_data data;  in __set_cyc2ns_scale()  local
143 	clocks_calc_mult_shift(&data.cyc2ns_mul, &data.cyc2ns_shift, khz,  in __set_cyc2ns_scale()
149 	 * conversion algorithm shifting a 32-bit value (now specifies a 64-bit  in __set_cyc2ns_scale()
150 	 * value) - refer perf_event_mmap_page documentation in perf_event.h.  in __set_cyc2ns_scale()
152 	if (data.cyc2ns_shift == 32) {  in __set_cyc2ns_scale()
153 		data.cyc2ns_shift = 31;  in __set_cyc2ns_scale()
154 		data.cyc2ns_mul >>= 1;  in __set_cyc2ns_scale()
157 	data.cyc2ns_offset = ns_now -  in __set_cyc2ns_scale()
158 		mul_u64_u32_shr(tsc_now, data.cyc2ns_mul, data.cyc2ns_shift);  in __set_cyc2ns_scale()
162 	raw_write_seqcount_latch(&c2n->seq);  in __set_cyc2ns_scale()
163 	c2n->data[0] = data;  in __set_cyc2ns_scale()
164 	raw_write_seqcount_latch(&c2n->seq);  in __set_cyc2ns_scale()
165 	c2n->data[1] = data;  in __set_cyc2ns_scale()
189 	seqcount_latch_init(&c2n->seq);  in cyc2ns_init_boot_cpu()
202 	struct cyc2ns_data *data = c2n->data;  in cyc2ns_init_secondary_cpus()  local
206 			seqcount_latch_init(&c2n->seq);  in cyc2ns_init_secondary_cpus()
208 			c2n->data[0] = data[0];  in cyc2ns_init_secondary_cpus()
209 			c2n->data[1] = data[1];  in cyc2ns_init_secondary_cpus()
215  * Scheduler clock - returns current time in nanosec units.
236 	return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);  in native_sched_clock()
329 		if ((t2 - t1) < thresh)  in tsc_read_refs()
344 	hpet2 -= hpet1;  in calc_hpet_ref()
364 	pm2 -= pm1;  in calc_pmtimer_ref()
427 		delta = t2 - tsc;  in pit_calibrate_tsc()
449 	delta = t2 - t1;  in pit_calibrate_tsc()
457  * non-virtualized hardware.
461  *  - the PIT is running at roughly 1.19MHz
463  *  - each IO is going to take about 1us on real hardware,
466  *    update - anything else implies a unacceptably slow CPU
467  *    or PIT for the fast calibration to work.
469  *  - with 256 PIT ticks to read the value, we have 214us to
473  *  - We're doing 2 reads per loop (LSB, MSB), and we expect
478  *  - if the PIT is stuck, and we see *many* more reads, we
480  *    then consider it a failure when they don't see the
485  * high accuracy, and we didn't miss any events. We can thus
507 	*deltap = get_cycles() - prev_tsc;  in pit_expect_msb()
539 	 * Counter 2, mode 0 (one-shot), binary count  in quick_pit_calibrate()
543 	 * final output frequency as a decrement-by-one),  in quick_pit_calibrate()
556 	 * to do that is to just read back the 16-bit counter  in quick_pit_calibrate()
563 			if (!pit_expect_msb(0xff-i, &delta, &d2))  in quick_pit_calibrate()
566 			delta -= tsc;  in quick_pit_calibrate()
589 			if (!pit_verify_msb(0xfe - i))  in quick_pit_calibrate()
594 	pr_info("Fast TSC calibration failed\n");  in quick_pit_calibrate()
607 	 * kHz = ticks / time-in-seconds / 1000;  in quick_pit_calibrate()
608 	 * kHz = (t2 - t1) / (I * 256 / PIT_TICK_RATE) / 1000  in quick_pit_calibrate()
609 	 * kHz = ((t2 - t1) * PIT_TICK_RATE) / (I * 256 * 1000)  in quick_pit_calibrate()
613 	pr_info("Fast TSC calibration using PIT\n");  in quick_pit_calibrate()
643 	 * Denverton SoCs don't report crystal clock, and also don't support  in native_calibrate_tsc()
660 	 * Some Intel SoCs like Skylake and Kabylake don't report the crystal  in native_calibrate_tsc()
724 	 * Run 5 calibration loops to get the lowest frequency value  in pit_hpet_ptimer_calibrate_cpu()
725 	 * (the best estimate). We use two different calibration modes  in pit_hpet_ptimer_calibrate_cpu()
744 	 * calibration delay loop as we have to wait for a certain  in pit_hpet_ptimer_calibrate_cpu()
759 		 * calibration, which will take at least 50ms, and  in pit_hpet_ptimer_calibrate_cpu()
768 		/* Pick the lowest PIT TSC calibration so far */  in pit_hpet_ptimer_calibrate_cpu()
779 		tsc2 = (tsc2 - tsc1) * 1000000LL;  in pit_hpet_ptimer_calibrate_cpu()
792 		 * If both calibration results are inside a 10% window  in pit_hpet_ptimer_calibrate_cpu()
793 		 * then we can be sure, that the calibration  in pit_hpet_ptimer_calibrate_cpu()
798 			pr_info("PIT calibration matches %s. %d loops\n",  in pit_hpet_ptimer_calibrate_cpu()
823 		/* We don't have an alternative source, disable TSC */  in pit_hpet_ptimer_calibrate_cpu()
831 			pr_warn("HPET/PMTIMER calibration failed\n");  in pit_hpet_ptimer_calibrate_cpu()
836 		pr_info("using %s reference calibration\n",  in pit_hpet_ptimer_calibrate_cpu()
842 	/* We don't have an alternative source, use the PIT calibration value */  in pit_hpet_ptimer_calibrate_cpu()
844 		pr_info("Using PIT calibration value\n");  in pit_hpet_ptimer_calibrate_cpu()
848 	/* The alternative source failed, use the PIT calibration value */  in pit_hpet_ptimer_calibrate_cpu()
850 		pr_warn("HPET/PMTIMER calibration failed. Using PIT calibration.\n");  in pit_hpet_ptimer_calibrate_cpu()
855 	 * The calibration values differ too much. In doubt, we use  in pit_hpet_ptimer_calibrate_cpu()
859 	pr_warn("PIT calibration deviates from %s: %lu %lu\n",  in pit_hpet_ptimer_calibrate_cpu()
861 	pr_info("Using PIT calibration value\n");  in pit_hpet_ptimer_calibrate_cpu()
866  * native_calibrate_cpu_early - can calibrate the cpu early in boot
884  * native_calibrate_cpu - calibrate the cpu
908 	else if (abs(cpu_khz - tsc_khz) * 10 > tsc_khz)  in recalibrate_cpu_khz()
948 	 * We're coming out of suspend, there's no concurrency yet; don't  in tsc_restore_sched_clock_state()
950 	 * data fields.  in tsc_restore_sched_clock_state()
953 	this_cpu_write(cyc2ns.data[0].cyc2ns_offset, 0);  in tsc_restore_sched_clock_state()
954 	this_cpu_write(cyc2ns.data[1].cyc2ns_offset, 0);  in tsc_restore_sched_clock_state()
956 	offset = cyc2ns_suspend - sched_clock();  in tsc_restore_sched_clock_state()
959 		per_cpu(cyc2ns.data[0].cyc2ns_offset, cpu) = offset;  in tsc_restore_sched_clock_state()
960 		per_cpu(cyc2ns.data[1].cyc2ns_offset, cpu) = offset;  in tsc_restore_sched_clock_state()
983 				void *data)  in time_cpufreq_notifier()  argument
985 	struct cpufreq_freqs *freq = data;  in time_cpufreq_notifier()
993 		ref_freq = freq->old;  in time_cpufreq_notifier()
998 	if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||  in time_cpufreq_notifier()
999 	    (val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) {  in time_cpufreq_notifier()
1001 			cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);  in time_cpufreq_notifier()
1003 		tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new);  in time_cpufreq_notifier()
1004 		if (!(freq->flags & CPUFREQ_CONST_LOOPS))  in time_cpufreq_notifier()
1007 		set_cyc2ns_scale(tsc_khz, freq->policy->cpu, rdtsc());  in time_cpufreq_notifier()
1047 	 * Don't enable ART in a VM, non-stop TSC and TSC_ADJUST required,  in detect_art()
1078  * structure to avoid a nasty time-warp. This can be observed in a
1089  * checking the result of read_tsc() - cycle_last for being negative.
1128 	.name			= "tsc-early",
1197 		/* Geode_LX - the OLPC CPU has a very reliable TSC */  in check_system_tsc_reliable()
1207 	 *  - TSC running at constant frequency  in check_system_tsc_reliable()
1208 	 *  - TSC which does not stop in C-States  in check_system_tsc_reliable()
1209 	 *  - the TSC_ADJUST register which allows to detect even minimal  in check_system_tsc_reliable()
1211 	 *  - not more than two sockets. As the number of sockets cannot be  in check_system_tsc_reliable()
1276  * convert_art_ns_to_tsc() - Convert ART in nanoseconds to TSC.
1280  * software requests a cross-timestamp, this function converts system timestamp
1288  * struct system_counterval_t - system counter value with the pointer to the
1317  * tsc_refine_calibration_work - Further refine tsc freq calibration
1318  * @work - ignored.
1322  * timer based, instead of loop based, we don't block the boot
1323  * process while this longer calibration is done.
1325  * If there are any calibration anomalies (too many SMIs, etc),
1326  * or the refined calibration is off by 1% of the fast early
1327  * calibration, we throw out the new calibration and use the
1328  * early calibration.
1338 	/* Don't bother refining TSC on unstable systems */  in tsc_refine_calibration_work()
1369 	delta = tsc_stop - tsc_start;  in tsc_refine_calibration_work()
1377 	if (abs(tsc_khz - freq) > tsc_khz/100)  in tsc_refine_calibration_work()
1381 	pr_info("Refined TSC clocksource calibration: %lu.%03lu MHz\n",  in tsc_refine_calibration_work()
1417 	 * the refined calibration and directly register it as a clocksource.  in init_tsc_clocksource()
1449 		/* We should not be here with non-native cpu calibration */  in determine_cpu_tsc_frequencies()
1455 	 * Trust non-zero tsc_khz as authoritative,  in determine_cpu_tsc_frequencies()
1461 	else if (abs(cpu_khz - tsc_khz) * 10 > tsc_khz)  in determine_cpu_tsc_frequencies()
1499 	/* Don't change UV TSC multi-chassis synchronization */  in tsc_early_init()
1558  * we can skip clock calibration if another cpu in the same socket has already
1560  * cpus in the socket - this should be a safe assumption.