Attachment #888012 for bug #916954




	u64				sum_exec_runtime;
	u64				prev_sum_exec_runtime;
	u64				vruntime;
#ifdef CONFIG_SCHED_BORE
	u64				burst_time;
	u8				prev_burst_penalty;
	u8				curr_burst_penalty;
	u8				burst_penalty;
	u8				burst_score;
	u32				burst_load;
	bool			on_cfs_rq;
	u8				child_burst;
	u32				child_burst_cnt;
	u64				child_burst_last_cached;
#endif // CONFIG_SCHED_BORE
	s64				vlag;
	u64				slice;






	  If unsure, say N here.

config SCHED_BORE
	bool "Burst-Oriented Response Enhancer"
	default y
	help
	  In Desktop and Mobile computing, one might prefer interactive
	  tasks to keep responsive no matter what they run in the background.

	  Enabling this kernel feature modifies the scheduler to discriminate
	  tasks by their burst time (runtime since it last went sleeping or
	  yielding state) and prioritize those that run less bursty.
	  Such tasks usually include window compositor, widgets backend,
	  terminal emulator, video playback, games and so on.
	  With a little impact to scheduling fairness, it may improve
	  responsiveness especially under heavy background workload.

	  You can turn it off by setting the sysctl kernel.sched_bore = 0.

	  If unsure, say Y here.

config SCHED_AUTOGROUP
	bool "Automatic process group scheduling"
	select CGROUPS




	return try_to_wake_up(p, state, 0);
}

#ifdef CONFIG_SCHED_BORE
extern bool sched_bore;
extern u8   sched_burst_fork_atavistic;
extern uint sched_burst_cache_lifetime;

static void __init sched_init_bore(void) {
	init_task.se.burst_time = 0;
	init_task.se.prev_burst_penalty = 0;
	init_task.se.curr_burst_penalty = 0;
	init_task.se.burst_penalty = 0;
	init_task.se.burst_score = 0;
	init_task.se.on_cfs_rq = false;
	init_task.se.child_burst_last_cached = 0;
	init_task.se.burst_load = 0;
}

void inline sched_fork_bore(struct task_struct *p) {
	p->se.burst_time = 0;
	p->se.curr_burst_penalty = 0;
	p->se.burst_score = 0;
	p->se.on_cfs_rq = false;
	p->se.child_burst_last_cached = 0;
	p->se.burst_load = 0;
}

static u32 count_child_tasks(struct task_struct *p) {
	struct task_struct *child;
	u32 cnt = 0;
	list_for_each_entry(child, &p->children, sibling) {cnt++;}
	return cnt;
}

static inline bool task_is_inheritable(struct task_struct *p) {
	return (p->sched_class == &fair_sched_class);
}

static inline bool child_burst_cache_expired(struct task_struct *p, u64 now) {
	u64 expiration_time =
		p->se.child_burst_last_cached + sched_burst_cache_lifetime;
	return ((s64)(expiration_time - now) < 0);
}

static void __update_child_burst_cache(
	struct task_struct *p, u32 cnt, u32 sum, u64 now) {
	u8 avg = 0;
	if (cnt) avg = sum / cnt;
	p->se.child_burst = max(avg, p->se.burst_penalty);
	p->se.child_burst_cnt = cnt;
	p->se.child_burst_last_cached = now;
}

static inline void update_child_burst_direct(struct task_struct *p, u64 now) {
	struct task_struct *child;
	u32 cnt = 0;
	u32 sum = 0;

	list_for_each_entry(child, &p->children, sibling) {
		if (!task_is_inheritable(child)) continue;
		cnt++;
		sum += child->se.burst_penalty;
	}

	__update_child_burst_cache(p, cnt, sum, now);
}

static inline u8 __inherit_burst_direct(struct task_struct *p, u64 now) {
	struct task_struct *parent = p->real_parent;
	if (child_burst_cache_expired(parent, now))
		update_child_burst_direct(parent, now);

	return parent->se.child_burst;
}

static void update_child_burst_topological(
	struct task_struct *p, u64 now, u32 depth, u32 *acnt, u32 *asum) {
	struct task_struct *child, *dec;
	u32 cnt = 0, dcnt = 0;
	u32 sum = 0;

	list_for_each_entry(child, &p->children, sibling) {
		dec = child;
		while ((dcnt = count_child_tasks(dec)) == 1)
			dec = list_first_entry(&dec->children, struct task_struct, sibling);
		
		if (!dcnt || !depth) {
			if (!task_is_inheritable(dec)) continue;
			cnt++;
			sum += dec->se.burst_penalty;
			continue;
		}
		if (!child_burst_cache_expired(dec, now)) {
			cnt += dec->se.child_burst_cnt;
			sum += (u32)dec->se.child_burst * dec->se.child_burst_cnt;
			continue;
		}
		update_child_burst_topological(dec, now, depth - 1, &cnt, &sum);
	}

	__update_child_burst_cache(p, cnt, sum, now);
	*acnt += cnt;
	*asum += sum;
}

static inline u8 __inherit_burst_topological(struct task_struct *p, u64 now) {
	struct task_struct *anc = p->real_parent;
	u32 cnt = 0, sum = 0;

	while (anc->real_parent != anc && count_child_tasks(anc) == 1)
		anc = anc->real_parent;

	if (child_burst_cache_expired(anc, now))
		update_child_burst_topological(
			anc, now, sched_burst_fork_atavistic - 1, &cnt, &sum);

	return anc->se.child_burst;
}

static inline void inherit_burst(struct task_struct *p) {
	u8 burst_cache;
	u64 now = ktime_get_ns();

	read_lock(&tasklist_lock);
	burst_cache = likely(sched_burst_fork_atavistic)?
		__inherit_burst_topological(p, now):
		__inherit_burst_direct(p, now);
	read_unlock(&tasklist_lock);

	p->se.prev_burst_penalty = max(p->se.prev_burst_penalty, burst_cache);
}

static void sched_post_fork_bore(struct task_struct *p) {
	if (p->sched_class == &fair_sched_class && likely(sched_bore))
		inherit_burst(p);
	p->se.burst_penalty = p->se.prev_burst_penalty;
}
#endif // CONFIG_SCHED_BORE

/*
 * Perform scheduler related setup for a newly forked process p.
 * p is forked by current.

	p->se.prev_sum_exec_runtime	= 0;
	p->se.nr_migrations		= 0;
	p->se.vruntime			= 0;
#ifdef CONFIG_SCHED_BORE
	sched_fork_bore(p);
#endif // CONFIG_SCHED_BORE
	p->se.vlag			= 0;
	p->se.slice			= sysctl_sched_base_slice;
	INIT_LIST_HEAD(&p->se.group_node);


void sched_post_fork(struct task_struct *p)
{
#ifdef CONFIG_SCHED_BORE
	sched_post_fork_bore(p);
#endif // CONFIG_SCHED_BORE
	uclamp_post_fork(p);
}


	BUG_ON(&dl_sched_class != &stop_sched_class + 1);
#endif

#ifdef CONFIG_SCHED_BORE
	sched_init_bore();
	printk(KERN_INFO "BORE (Burst-Oriented Response Enhancer) CPU Scheduler modification 4.5.0 by Masahito Suzuki");
#endif // CONFIG_SCHED_BORE

	wait_bit_init();

#ifdef CONFIG_FAIR_GROUP_SCHED




};

#ifdef CONFIG_SMP
#ifdef CONFIG_SCHED_BORE
static ssize_t sched_min_base_slice_write(struct file *filp, const char __user *ubuf,
				   size_t cnt, loff_t *ppos)
{
	char buf[16];
	unsigned int value;

	if (cnt > 15)
		cnt = 15;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;
	buf[cnt] = '\0';

	if (kstrtouint(buf, 10, &value))
		return -EINVAL;

	if (!value)
		return -EINVAL;

	sysctl_sched_min_base_slice = value;
	sched_update_min_base_slice();

	*ppos += cnt;
	return cnt;
}

static int sched_min_base_slice_show(struct seq_file *m, void *v)
{
	seq_printf(m, "%d\n", sysctl_sched_min_base_slice);
	return 0;
}

static int sched_min_base_slice_open(struct inode *inode, struct file *filp)
{
	return single_open(filp, sched_min_base_slice_show, NULL);
}

static const struct file_operations sched_min_base_slice_fops = {
	.open		= sched_min_base_slice_open,
	.write		= sched_min_base_slice_write,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};
#else // !CONFIG_SCHED_BORE
static ssize_t sched_scaling_write(struct file *filp, const char __user *ubuf,
				   size_t cnt, loff_t *ppos)
{

	.llseek		= seq_lseek,
	.release	= single_release,
};
#endif // CONFIG_SCHED_BORE
#endif /* SMP */

#ifdef CONFIG_PREEMPT_DYNAMIC

	debugfs_create_file("preempt", 0644, debugfs_sched, NULL, &sched_dynamic_fops);
#endif

#ifdef CONFIG_SCHED_BORE
	debugfs_create_file("min_base_slice_ns", 0644, debugfs_sched, NULL, &sched_min_base_slice_fops);
	debugfs_create_u32("base_slice_ns", 0400, debugfs_sched, &sysctl_sched_base_slice);
#else // !CONFIG_SCHED_BORE
 	debugfs_create_u32("base_slice_ns", 0644, debugfs_sched, &sysctl_sched_base_slice);
#endif // CONFIG_SCHED_BORE

#ifndef CONFIG_SCHED_ALT
	debugfs_create_u32("latency_warn_ms", 0644, debugfs_sched, &sysctl_resched_latency_warn_ms);
	debugfs_create_u33("latency_warn_once", 0644, debugfs_sched, &sysctl_resched_latency_warn_once);

#ifdef CONFIG_SMP
#if !defined(CONFIG_SCHED_BORE)
 	debugfs_create_file("tunable_scaling", 0644, debugfs_sched, NULL, &sched_scaling_fops);
#endif // CONFIG_SCHED_BORE
	debugfs_create_u32("migration_cost_ns", 0644, debugfs_sched, &sysctl_sched_migration_cost);
	debugfs_create_u32("nr_migrate", 0644, debugfs_sched, &sysctl_sched_nr_migrate);


		SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)),
		SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime)));

#ifdef CONFIG_SCHED_BORE
	SEQ_printf(m, " %2d", p->se.burst_score);
#endif // CONFIG_SCHED_BORE
#ifdef CONFIG_NUMA_BALANCING
	SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
#endif


	P(se.load.weight);
#ifdef CONFIG_SMP
#ifdef CONFIG_SCHED_BORE
	P(se.burst_load);
	P(se.burst_score);
#endif // CONFIG_SCHED_BORE
	P(se.avg.load_sum);
	P(se.avg.runnable_sum);
	P(se.avg.util_sum);

Lines 19-24 Link Here

(-)a/kernel/sched/fair.c (-14 / +288 lines)
19	*	19	*
20	* Adaptive scheduling granularity, math enhancements by Peter Zijlstra	20	* Adaptive scheduling granularity, math enhancements by Peter Zijlstra
21	* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra	21	* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
		22	*
		23	* Burst-Oriented Response Enhancer (BORE) CPU Scheduler
		24	* Copyright (C) 2021-2024 Masahito Suzuki <firelzrd@gmail.com>
22	*/	25	*/
23	#include <linux/energy_model.h>	26	#include <linux/energy_model.h>
24	#include <linux/mmap_lock.h>	27	#include <linux/mmap_lock.h>
Lines 64-83 Link Here
64	* SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)	67	* SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)
65	* SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus	68	* SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
66	*	69	*
67	* (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))	70	* (BORE default SCHED_TUNABLESCALING_NONE = *1 constant)
		71	* (EEVDF default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
68	*/	72	*/
		73	#ifdef CONFIG_SCHED_BORE
		74	unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_NONE;
		75	#else // !CONFIG_SCHED_BORE
69	unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;	76	unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;
		77	#endif // CONFIG_SCHED_BORE
70		78
71	/*	79	/*
72	* Minimal preemption granularity for CPU-bound tasks:	80	* Minimal preemption granularity for CPU-bound tasks:
73	*	81	*
74	* (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)	82	* (BORE default: max(1 sec / HZ, min_base_slice) constant, units: nanoseconds)
		83	* (EEVDF default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)
75	*/	84	*/
		85	#ifdef CONFIG_SCHED_BORE
		86	unsigned int sysctl_sched_base_slice = 1000000000ULL / HZ;
		87	static unsigned int configured_sched_base_slice = 1000000000ULL / HZ;
		88	unsigned int sysctl_sched_min_base_slice = 2000000ULL;
		89	#else // !CONFIG_SCHED_BORE
76	unsigned int sysctl_sched_base_slice = 750000ULL;	90	unsigned int sysctl_sched_base_slice = 750000ULL;
77	static unsigned int normalized_sysctl_sched_base_slice = 750000ULL;	91	static unsigned int normalized_sysctl_sched_base_slice = 750000ULL;
		92	#endif // CONFIG_SCHED_BORE
78		93
79	const_debug unsigned int sysctl_sched_migration_cost = 500000UL;	94	const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
80		95
		96	#ifdef CONFIG_SCHED_BORE
		97	u8 __read_mostly sched_bore = 1;
		98	u8 __read_mostly sched_burst_score_rounding = 0;
		99	u8 __read_mostly sched_burst_smoothness_long = 1;
		100	u8 __read_mostly sched_burst_smoothness_short = 0;
		101	u8 __read_mostly sched_burst_fork_atavistic = 2;
		102	u8 __read_mostly sched_burst_penalty_offset = 22;
		103	uint __read_mostly sched_burst_penalty_scale = 1280;
		104	uint __read_mostly sched_burst_cache_lifetime = 60000000;
		105	u8 __read_mostly sched_vlag_deviation_limit = 8;
		106	static int __maybe_unused thirty_two = 32;
		107	static int __maybe_unused sixty_four = 64;
		108	static int __maybe_unused maxval_12_bits = 4095;
		109
		110	#define MAX_BURST_PENALTY (39U <<2)
		111
		112	static inline u32 log2plus1_u64_u32f8(u64 v) {
		113	u32 msb = fls64(v);
		114	s32 excess_bits = msb - 9;
		115	u8 fractional = (0 <= excess_bits)? v >> excess_bits: v << -excess_bits;
		116	return msb << 8 \| fractional;
		117	}
		118
		119	static inline u32 calc_burst_penalty(u64 burst_time) {
		120	u32 greed, tolerance, penalty, scaled_penalty;
		121
		122	greed = log2plus1_u64_u32f8(burst_time);
		123	tolerance = sched_burst_penalty_offset << 8;
		124	penalty = max(0, (s32)greed - (s32)tolerance);
		125	scaled_penalty = penalty * sched_burst_penalty_scale >> 16;
		126
		127	return min(MAX_BURST_PENALTY, scaled_penalty);
		128	}
		129
		130	static inline u64 scale_slice(u64 delta, struct sched_entity *se) {
		131	return mul_u64_u32_shr(delta, sched_prio_to_wmult[se->burst_score], 22);
		132	}
		133
		134	static inline u64 __unscale_slice(u64 delta, u8 score) {
		135	return mul_u64_u32_shr(delta, sched_prio_to_weight[score], 10);
		136	}
		137
		138	static inline u64 unscale_slice(u64 delta, struct sched_entity *se) {
		139	return __unscale_slice(delta, se->burst_score);
		140	}
		141
		142	static void avg_vruntime_add(struct cfs_rq cfs_rq, struct sched_entity se);
		143	static void avg_vruntime_sub(struct cfs_rq cfs_rq, struct sched_entity se);
		144
		145	static void update_burst_score(struct sched_entity *se) {
		146	struct cfs_rq *cfs_rq = cfs_rq_of(se);
		147	u8 prev_score = se->burst_score;
		148	u32 penalty = se->burst_penalty;
		149	if (sched_burst_score_rounding) penalty += 0x2U;
		150	se->burst_score = penalty >> 2;
		151
		152	if ((se->burst_score != prev_score) && se->on_cfs_rq) {
		153	avg_vruntime_sub(cfs_rq, se);
		154	avg_vruntime_add(cfs_rq, se);
		155	}
		156	}
		157
		158	static void update_burst_penalty(struct sched_entity *se) {
		159	se->curr_burst_penalty = calc_burst_penalty(se->burst_time);
160	se->burst_penalty = max(se->prev_burst_penalty, se->curr_burst_penalty);
161	update_burst_score(se);
162	}
163
164	static inline u32 binary_smooth(u32 new, u32 old) {
165	int increment = new - old;
166	return (0 <= increment)?
167	old + ( increment >> (int)sched_burst_smoothness_long):
168	old - (-increment >> (int)sched_burst_smoothness_short);
169	}
170
171	static void restart_burst(struct sched_entity *se) {
172	se->burst_penalty = se->prev_burst_penalty =
173	binary_smooth(se->curr_burst_penalty, se->prev_burst_penalty);
174	se->curr_burst_penalty = 0;
175	se->burst_time = 0;
176	update_burst_score(se);
177	}
178
179	static void restart_burst_rescale_deadline(struct sched_entity *se) {
180	s64 vscaled, wremain, vremain = se->deadline - se->vruntime;
181	u8 prev_score = se->burst_score;
182	restart_burst(se);
183	if (prev_score > se->burst_score) {
184	wremain = __unscale_slice(abs(vremain), prev_score);
185	vscaled = scale_slice(wremain, se);
186	if (unlikely(vremain < 0))
187	vscaled = -vscaled;
188	se->deadline = se->vruntime + vscaled;
189	}
190	}
191	#endif // CONFIG_SCHED_BORE
192
81	int sched_thermal_decay_shift;	193	int sched_thermal_decay_shift;
82	static int __init setup_sched_thermal_decay_shift(char *str)	194	static int __init setup_sched_thermal_decay_shift(char *str)
83	{	195	{
Lines 137-142 static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; Link Here
137		249
138	#ifdef CONFIG_SYSCTL	250	#ifdef CONFIG_SYSCTL
139	static struct ctl_table sched_fair_sysctls[] = {	251	static struct ctl_table sched_fair_sysctls[] = {
		252	#ifdef CONFIG_SCHED_BORE
		253	{
		254	.procname = "sched_bore",
		255	.data = &sched_bore,
		256	.maxlen = sizeof(u8),
		257	.mode = 0644,
		258	.proc_handler = proc_dou8vec_minmax,
		259	.extra1 = SYSCTL_ZERO,
		260	.extra2 = SYSCTL_ONE,
		261	},
		262	{
		263	.procname = "sched_burst_score_rounding",
		264	.data = &sched_burst_score_rounding,
		265	.maxlen = sizeof(u8),
		266	.mode = 0644,
		267	.proc_handler = proc_dou8vec_minmax,
		268	.extra1 = SYSCTL_ZERO,
		269	.extra2 = SYSCTL_ONE,
		270	},
		271	{
		272	.procname = "sched_burst_smoothness_long",
		273	.data = &sched_burst_smoothness_long,
		274	.maxlen = sizeof(u8),
		275	.mode = 0644,
		276	.proc_handler = proc_dou8vec_minmax,
		277	.extra1 = SYSCTL_ZERO,
		278	.extra2 = SYSCTL_ONE,
		279	},
		280	{
		281	.procname = "sched_burst_smoothness_short",
		282	.data = &sched_burst_smoothness_short,
		283	.maxlen = sizeof(u8),
		284	.mode = 0644,
		285	.proc_handler = proc_dou8vec_minmax,
		286	.extra1 = SYSCTL_ZERO,
		287	.extra2 = SYSCTL_ONE,
		288	},
		289	{
		290	.procname = "sched_burst_fork_atavistic",
		291	.data = &sched_burst_fork_atavistic,
		292	.maxlen = sizeof(u8),
		293	.mode = 0644,
		294	.proc_handler = proc_dou8vec_minmax,
		295	.extra1 = SYSCTL_ZERO,
		296	.extra2 = SYSCTL_THREE,
		297	},
		298	{
		299	.procname = "sched_burst_penalty_offset",
		300	.data = &sched_burst_penalty_offset,
		301	.maxlen = sizeof(u8),
		302	.mode = 0644,
		303	.proc_handler = proc_dou8vec_minmax,
		304	.extra1 = SYSCTL_ZERO,
		305	.extra2 = &sixty_four,
		306	},
		307	{
		308	.procname = "sched_burst_penalty_scale",
		309	.data = &sched_burst_penalty_scale,
		310	.maxlen = sizeof(uint),
		311	.mode = 0644,
		312	.proc_handler = proc_douintvec_minmax,
		313	.extra1 = SYSCTL_ZERO,
		314	.extra2 = &maxval_12_bits,
		315	},
316	{
317	.procname = "sched_burst_cache_lifetime",
318	.data = &sched_burst_cache_lifetime,
319	.maxlen = sizeof(uint),
320	.mode = 0644,
321	.proc_handler = proc_douintvec,
322	},
323	{
324	.procname = "sched_vlag_deviation_limit",
325	.data = &sched_vlag_deviation_limit,
326	.maxlen = sizeof(u8),
327	.mode = 0644,
328	.proc_handler = proc_dou8vec_minmax,
329	.extra1 = SYSCTL_ZERO,
330	.extra2 = &thirty_two,
331	},
332	#endif // CONFIG_SCHED_BORE
140	#ifdef CONFIG_CFS_BANDWIDTH	333	#ifdef CONFIG_CFS_BANDWIDTH
141	{	334	{
142	.procname = "sched_cfs_bandwidth_slice_us",	335	.procname = "sched_cfs_bandwidth_slice_us",
Lines 195-200 static inline void update_load_set(struct load_weight *lw, unsigned long w) Link Here
195	*	388	*
196	* This idea comes from the SD scheduler of Con Kolivas:	389	* This idea comes from the SD scheduler of Con Kolivas:
197	*/	390	*/
		391	#ifdef CONFIG_SCHED_BORE
		392	static void update_sysctl(void) {
		393	sysctl_sched_base_slice =
		394	max(sysctl_sched_min_base_slice, configured_sched_base_slice);
		395	}
		396	void sched_update_min_base_slice(void) { update_sysctl(); }
		397	#else // !CONFIG_SCHED_BORE
198	static unsigned int get_update_sysctl_factor(void)	398	static unsigned int get_update_sysctl_factor(void)
199	{	399	{
200	unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8);	400	unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8);
Lines 225-230 static void update_sysctl(void) Link Here
225	SET_SYSCTL(sched_base_slice);	425	SET_SYSCTL(sched_base_slice);
226	#undef SET_SYSCTL	426	#undef SET_SYSCTL
227	}	427	}
		428	#endif // CONFIG_SCHED_BORE
228		429
229	void __init sched_init_granularity(void)	430	void __init sched_init_granularity(void)
230	{	431	{
Lines 298-303 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) Link Here
298	if (unlikely(se->load.weight != NICE_0_LOAD))	499	if (unlikely(se->load.weight != NICE_0_LOAD))
299	delta = __calc_delta(delta, NICE_0_LOAD, &se->load);	500	delta = __calc_delta(delta, NICE_0_LOAD, &se->load);
300		501
		502	#ifdef CONFIG_SCHED_BORE
		503	if (likely(sched_bore)) delta = scale_slice(delta, se);
		504	#endif // CONFIG_SCHED_BORE
301	return delta;	505	return delta;
302	}	506	}
303		507
Lines 624-633 static inline s64 entity_key(struct cfs_rq cfs_rq, struct sched_entity se) Link Here
624	*	828	*
625	* As measured, the max (key * weight) value was ~44 bits for a kernel build.	829	* As measured, the max (key * weight) value was ~44 bits for a kernel build.
626	*/	830	*/
		831	#if !defined(CONFIG_SCHED_BORE)
		832	#define entity_weight(se) scale_load_down(se->load.weight)
		833	#else // CONFIG_SCHED_BORE
		834	static unsigned long entity_weight(struct sched_entity *se) {
		835	unsigned long weight = se->load.weight;
		836	if (likely(sched_bore)) weight = unscale_slice(weight, se);
		837	#ifdef CONFIG_64BIT
		838	weight >>= SCHED_FIXEDPOINT_SHIFT - 3;
		839	#endif // CONFIG_64BIT
		840	return weight;
		841	}
		842	#endif // CONFIG_SCHED_BORE
		843
627	static void	844	static void
628	avg_vruntime_add(struct cfs_rq cfs_rq, struct sched_entity se)	845	avg_vruntime_add(struct cfs_rq cfs_rq, struct sched_entity se)
629	{	846	{
630	unsigned long weight = scale_load_down(se->load.weight);	847	unsigned long weight = entity_weight(se);
		848	#ifdef CONFIG_SCHED_BORE
		849	se->burst_load = weight;
		850	#endif // CONFIG_SCHED_BORE
631	s64 key = entity_key(cfs_rq, se);	851	s64 key = entity_key(cfs_rq, se);
632		852
633	cfs_rq->avg_vruntime += key * weight;	853	cfs_rq->avg_vruntime += key * weight;
Lines 637-643 avg_vruntime_add(struct cfs_rq cfs_rq, struct sched_entity se) Link Here
637	static void	857	static void
638	avg_vruntime_sub(struct cfs_rq cfs_rq, struct sched_entity se)	858	avg_vruntime_sub(struct cfs_rq cfs_rq, struct sched_entity se)
639	{	859	{
640	unsigned long weight = scale_load_down(se->load.weight);	860	#if !defined(CONFIG_SCHED_BORE)
		861	unsigned long weight = entity_weight(se);
		862	#else // CONFIG_SCHED_BORE
		863	unsigned long weight = se->burst_load;
		864	se->burst_load = 0;
		865	#endif // CONFIG_SCHED_BORE
641	s64 key = entity_key(cfs_rq, se);	866	s64 key = entity_key(cfs_rq, se);
642		867
643	cfs_rq->avg_vruntime -= key * weight;	868	cfs_rq->avg_vruntime -= key * weight;
Lines 657-670 void avg_vruntime_update(struct cfs_rq *cfs_rq, s64 delta) Link Here
657	* Specifically: avg_runtime() + 0 must result in entity_eligible() := true	882	* Specifically: avg_runtime() + 0 must result in entity_eligible() := true
658	* For this to be so, the result of this function must have a left bias.	883	* For this to be so, the result of this function must have a left bias.
659	*/	884	*/
660	u64 avg_vruntime(struct cfs_rq *cfs_rq)	885	static u64 avg_key(struct cfs_rq *cfs_rq)
661	{	886	{
662	struct sched_entity *curr = cfs_rq->curr;	887	struct sched_entity *curr = cfs_rq->curr;
663	s64 avg = cfs_rq->avg_vruntime;	888	s64 avg = cfs_rq->avg_vruntime;
664	long load = cfs_rq->avg_load;	889	long load = cfs_rq->avg_load;
665		890
666	if (curr && curr->on_rq) {	891	if (curr && curr->on_rq) {
667	unsigned long weight = scale_load_down(curr->load.weight);	892	unsigned long weight = entity_weight(curr);
668		893
669	avg += entity_key(cfs_rq, curr) * weight;	894	avg += entity_key(cfs_rq, curr) * weight;
670	load += weight;	895	load += weight;
Lines 674-685 u64 avg_vruntime(struct cfs_rq *cfs_rq) Link Here
674	/* sign flips effective floor / ceil */	899	/* sign flips effective floor / ceil */
675	if (avg < 0)	900	if (avg < 0)
676	avg -= (load - 1);	901	avg -= (load - 1);
677	avg = div_s64(avg, load);	902	avg = div64_s64(avg, load);
678	}	903	}
679		904
680	return cfs_rq->min_vruntime + avg;	905	return avg;
681	}	906	}
682		907
		908	u64 avg_vruntime(struct cfs_rq *cfs_rq) {
		909	return cfs_rq->min_vruntime + avg_key(cfs_rq);
		910	}
683	/*	911	/*
684	* lag_i = S - s_i = w_i * (V - v_i)	912	* lag_i = S - s_i = w_i * (V - v_i)
685	*	913	*
Lines 704-709 static void update_entity_lag(struct cfs_rq cfs_rq, struct sched_entity se) Link Here
704	lag = avg_vruntime(cfs_rq) - se->vruntime;	932	lag = avg_vruntime(cfs_rq) - se->vruntime;
705		933
706	limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se);	934	limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se);
		935	#ifdef CONFIG_SCHED_BORE
		936	if (likely(sched_bore)) limit >>= 1;
		937	#endif // CONFIG_SCHED_BORE
707	se->vlag = clamp(lag, -limit, limit);	938	se->vlag = clamp(lag, -limit, limit);
708	}	939	}
709		940
Lines 731-737 static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) Link Here
731	long load = cfs_rq->avg_load;	962	long load = cfs_rq->avg_load;
732		963
733	if (curr && curr->on_rq) {	964	if (curr && curr->on_rq) {
734	unsigned long weight = scale_load_down(curr->load.weight);	965	unsigned long weight = entity_weight(curr);
735		966
736	avg += entity_key(cfs_rq, curr) * weight;	967	avg += entity_key(cfs_rq, curr) * weight;
737	load += weight;	968	load += weight;
Lines 827-836 static void __enqueue_entity(struct cfs_rq cfs_rq, struct sched_entity se) Link Here
827	se->min_vruntime = se->vruntime;	1058	se->min_vruntime = se->vruntime;
828	rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,	1059	rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,
829	__entity_less, &min_vruntime_cb);	1060	__entity_less, &min_vruntime_cb);
		1061	#ifdef CONFIG_SCHED_BORE
		1062	se->on_cfs_rq = true;
		1063	#endif // CONFIG_SCHED_BORE
830	}	1064	}
831		1065
832	static void __dequeue_entity(struct cfs_rq cfs_rq, struct sched_entity se)	1066	static void __dequeue_entity(struct cfs_rq cfs_rq, struct sched_entity se)
833	{	1067	{
		1068	#ifdef CONFIG_SCHED_BORE
		1069	se->on_cfs_rq = false;
		1070	#endif // CONFIG_SCHED_BORE
834	rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,	1071	rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,
835	&min_vruntime_cb);	1072	&min_vruntime_cb);
836	avg_vruntime_sub(cfs_rq, se);	1073	avg_vruntime_sub(cfs_rq, se);
Lines 955-960 struct sched_entity __pick_last_entity(struct cfs_rq cfs_rq) Link Here
955	* Scheduling class statistics methods:	1192	* Scheduling class statistics methods:
956	*/	1193	*/
957	#ifdef CONFIG_SMP	1194	#ifdef CONFIG_SMP
		1195	#if !defined(CONFIG_SCHED_BORE)
958	int sched_update_scaling(void)	1196	int sched_update_scaling(void)
959	{	1197	{
960	unsigned int factor = get_update_sysctl_factor();	1198	unsigned int factor = get_update_sysctl_factor();
Lines 966-971 int sched_update_scaling(void) Link Here
966		1204
967	return 0;	1205	return 0;
968	}	1206	}
		1207	#endif // CONFIG_SCHED_BORE
969	#endif	1208	#endif
970	#endif	1209	#endif
971		1210
Lines 1165-1171 static void update_curr(struct cfs_rq *cfs_rq) Link Here
1165	if (unlikely(delta_exec <= 0))	1404	if (unlikely(delta_exec <= 0))
1166	return;	1405	return;
1167		1406
		1407	#ifdef CONFIG_SCHED_BORE
		1408	curr->burst_time += delta_exec;
		1409	update_burst_penalty(curr);
		1410	curr->vruntime += max(1ULL, calc_delta_fair(delta_exec, curr));
		1411	#else // !CONFIG_SCHED_BORE
1168	curr->vruntime += calc_delta_fair(delta_exec, curr);	1412	curr->vruntime += calc_delta_fair(delta_exec, curr);
		1413	#endif // CONFIG_SCHED_BORE
1169	update_deadline(cfs_rq, curr);	1414	update_deadline(cfs_rq, curr);
1170	update_min_vruntime(cfs_rq);	1415	update_min_vruntime(cfs_rq);
1171		1416
Lines 5157-5164 static inline void update_misfit_status(struct task_struct p, struct rq rq) {} Link Here
5157	static void	5402	static void
5158	place_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags)	5403	place_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags)
5159	{	5404	{
5160	u64 vslice, vruntime = avg_vruntime(cfs_rq);	5405	s64 lag = 0, key = avg_key(cfs_rq);
5161	s64 lag = 0;	5406	u64 vslice, vruntime = cfs_rq->min_vruntime + key;
5162		5407
5163	se->slice = sysctl_sched_base_slice;	5408	se->slice = sysctl_sched_base_slice;
5164	vslice = calc_delta_fair(se->slice, se);	5409	vslice = calc_delta_fair(se->slice, se);
Lines 5171-5176 place_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags) Link Here
5171	*	5416	*
5172	* EEVDF: placement strategy #1 / #2	5417	* EEVDF: placement strategy #1 / #2
5173	*/	5418	*/
		5419	#ifdef CONFIG_SCHED_BORE
		5420	if (unlikely(!sched_bore) \|\| se->vlag)
		5421	#endif // CONFIG_SCHED_BORE
5174	if (sched_feat(PLACE_LAG) && cfs_rq->nr_running) {	5422	if (sched_feat(PLACE_LAG) && cfs_rq->nr_running) {
5175	struct sched_entity *curr = cfs_rq->curr;	5423	struct sched_entity *curr = cfs_rq->curr;
5176	unsigned long load;	5424	unsigned long load;
Lines 5231-5242 place_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags) Link Here
5231	*/	5479	*/
5232	load = cfs_rq->avg_load;	5480	load = cfs_rq->avg_load;
5233	if (curr && curr->on_rq)	5481	if (curr && curr->on_rq)
5234	load += scale_load_down(curr->load.weight);	5482	load += entity_weight(curr);
5235		5483
5236	lag *= load + scale_load_down(se->load.weight);	5484	lag *= load + entity_weight(se);
5237	if (WARN_ON_ONCE(!load))	5485	if (WARN_ON_ONCE(!load))
5238	load = 1;	5486	load = 1;
5239	lag = div_s64(lag, load);	5487	lag = div64_s64(lag, load);
		5488	#ifdef CONFIG_SCHED_BORE
		5489	if (likely(sched_bore)) {
		5490	s64 limit = vslice << sched_vlag_deviation_limit;
		5491	lag = clamp(lag, -limit, limit);
		5492	}
		5493	#endif // CONFIG_SCHED_BORE
5240	}	5494	}
5241		5495
5242	se->vruntime = vruntime - lag;	5496	se->vruntime = vruntime - lag;
Lines 6803-6808 static void dequeue_task_fair(struct rq rq, struct task_struct p, int flags) Link Here
6803	bool was_sched_idle = sched_idle_rq(rq);	7057	bool was_sched_idle = sched_idle_rq(rq);
6804		7058
6805	util_est_dequeue(&rq->cfs, p);	7059	util_est_dequeue(&rq->cfs, p);
		7060	#ifdef CONFIG_SCHED_BORE
		7061	if (task_sleep) {
		7062	cfs_rq = cfs_rq_of(se);
		7063	if (cfs_rq->curr == se)
		7064	update_curr(cfs_rq);
		7065	restart_burst(se);
		7066	}
		7067	#endif // CONFIG_SCHED_BORE
6806		7068
6807	for_each_sched_entity(se) {	7069	for_each_sched_entity(se) {
6808	cfs_rq = cfs_rq_of(se);	7070	cfs_rq = cfs_rq_of(se);
Lines 8552-8567 static void yield_task_fair(struct rq *rq) Link Here
8552	/*	8814	/*
8553	* Are we the only task in the tree?	8815	* Are we the only task in the tree?
8554	*/	8816	*/
		8817	#if !defined(CONFIG_SCHED_BORE)
8555	if (unlikely(rq->nr_running == 1))	8818	if (unlikely(rq->nr_running == 1))
8556	return;	8819	return;
8557		8820
8558	clear_buddies(cfs_rq, se);	8821	clear_buddies(cfs_rq, se);
		8822	#endif // CONFIG_SCHED_BORE
8559		8823
8560	update_rq_clock(rq);	8824	update_rq_clock(rq);
8561	/*	8825	/*
8562	* Update run-time statistics of the 'current'.	8826	* Update run-time statistics of the 'current'.
8563	*/	8827	*/
8564	update_curr(cfs_rq);	8828	update_curr(cfs_rq);
		8829	#ifdef CONFIG_SCHED_BORE
		8830	restart_burst_rescale_deadline(se);
		8831	if (unlikely(rq->nr_running == 1))
		8832	return;
		8833
		8834	clear_buddies(cfs_rq, se);
		8835	#endif // CONFIG_SCHED_BORE
8565	/*	8836	/*
8566	* Tell update_rq_clock() that we've just updated,	8837	* Tell update_rq_clock() that we've just updated,
8567	* so we don't do microscopic update in schedule()	8838	* so we don't do microscopic update in schedule()
Lines 12651-12656 static void task_fork_fair(struct task_struct *p) Link Here
12651	curr = cfs_rq->curr;	12922	curr = cfs_rq->curr;
12652	if (curr)	12923	if (curr)
12653	update_curr(cfs_rq);	12924	update_curr(cfs_rq);
		12925	#ifdef CONFIG_SCHED_BORE
		12926	update_burst_score(se);
		12927	#endif // CONFIG_SCHED_BORE
12654	place_entity(cfs_rq, se, ENQUEUE_INITIAL);	12928	place_entity(cfs_rq, se, ENQUEUE_INITIAL);
12655	rq_unlock(rq, &rf);	12929	rq_unlock(rq, &rf);
12656	}	12930	}

Lines 6-12 Link Here

(-)a/kernel/sched/features.h (+4 lines)
6	*/	6	*/
7	SCHED_FEAT(PLACE_LAG, true)	7	SCHED_FEAT(PLACE_LAG, true)
8	SCHED_FEAT(PLACE_DEADLINE_INITIAL, true)	8	SCHED_FEAT(PLACE_DEADLINE_INITIAL, true)
		9	#ifdef CONFIG_SCHED_BORE
		10	SCHED_FEAT(RUN_TO_PARITY, false)
		11	#else // !CONFIG_SCHED_BORE
9	SCHED_FEAT(RUN_TO_PARITY, true)	12	SCHED_FEAT(RUN_TO_PARITY, true)
		13	#endif // CONFIG_SCHED_BORE
10		14
11	/*	15	/*
12	* Prefer to schedule the task we woke last (assuming it failed	16	* Prefer to schedule the task we woke last (assuming it failed

Lines 1965-1971 static inline void dirty_sched_domain_sysctl(int cpu) Link Here

(-)a/kernel/sched/sched.h (-1 / +7 lines)
1965	}	1965	}
1966	#endif	1966	#endif
1967		1967
		1968	#ifdef CONFIG_SCHED_BORE
		1969	extern void sched_update_min_base_slice(void);
		1970	#else // !CONFIG_SCHED_BORE
1968	extern int sched_update_scaling(void);	1971	extern int sched_update_scaling(void);
		1972	#endif // CONFIG_SCHED_BORE
1969		1973
1970	static inline const struct cpumask task_user_cpus(struct task_struct p)	1974	static inline const struct cpumask task_user_cpus(struct task_struct p)
1971	{	1975	{
Lines 2552-2557 extern const_debug unsigned int sysctl_sched_nr_migrate; Link Here
2552	extern const_debug unsigned int sysctl_sched_migration_cost;	2556	extern const_debug unsigned int sysctl_sched_migration_cost;
2553		2557
2554	extern unsigned int sysctl_sched_base_slice;	2558	extern unsigned int sysctl_sched_base_slice;
		2559	#ifdef CONFIG_SCHED_BORE
		2560	extern unsigned int sysctl_sched_min_base_slice;
		2561	#endif // CONFIG_SCHED_BORE
2555		2562
2556	#ifdef CONFIG_SCHED_DEBUG	2563	#ifdef CONFIG_SCHED_DEBUG
2557	extern int sysctl_resched_latency_warn_ms;	2564	extern int sysctl_resched_latency_warn_ms;
2558	-




			Force threading of all interrupt handlers except those
			marked explicitly IRQF_NO_THREAD.

	threadprintk	[KNL]
			Force threaded printing of all legacy consoles. Be
			aware that with this option, the shutdown, reboot, and
			panic messages may not be printed on the legacy
			consoles. Also, earlycon/earlyprintk printing will be
			delayed until a regular console or the kthread is
			available.

			Users can view /proc/consoles to see if their console
			driver is legacy or not. Non-legacy (NBCON) console
			drivers are already threaded and are shown with 'N'.

	topology=	[S390]
			Format: {off | on}
			Specify if the kernel should make use of the cpu




	select ARCH_SUPPORTS_ATOMIC_RMW
	select ARCH_SUPPORTS_HUGETLBFS if ARM_LPAE
	select ARCH_SUPPORTS_PER_VMA_LOCK
	select ARCH_SUPPORTS_RT if HAVE_POSIX_CPU_TIMERS_TASK_WORK
	select ARCH_USE_BUILTIN_BSWAP
	select ARCH_USE_CMPXCHG_LOCKREF
	select ARCH_USE_MEMTEST

	select HAS_IOPORT
	select HAVE_ARCH_AUDITSYSCALL if AEABI && !OABI_COMPAT
	select HAVE_ARCH_BITREVERSE if (CPU_32v7M || CPU_32v7) && !CPU_32v6
	select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU && !PREEMPT_RT
	select HAVE_ARCH_KFENCE if MMU && !XIP_KERNEL
	select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU
	select HAVE_ARCH_KASAN if MMU && !XIP_KERNEL

	select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE
	select HAVE_EFFICIENT_UNALIGNED_ACCESS if (CPU_V6 || CPU_V6K || CPU_V7) && MMU
	select HAVE_EXIT_THREAD
	select HAVE_FAST_GUP if ARM_LPAE && !(PREEMPT_RT && HIGHPTE)
	select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL
	select HAVE_FUNCTION_ERROR_INJECTION
	select HAVE_FUNCTION_GRAPH_TRACER

	select HAVE_PERF_EVENTS
	select HAVE_PERF_REGS
	select HAVE_PERF_USER_STACK_DUMP
	select HAVE_POSIX_CPU_TIMERS_TASK_WORK if !KVM
	select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE
	select HAVE_REGS_AND_STACK_ACCESS_API
	select HAVE_RSEQ

Lines 436-441 do_translation_fault(unsigned long addr, unsigned int fsr, Link Here

(-)a/arch/arm/mm/fault.c (+6 lines)
436	if (addr < TASK_SIZE)	436	if (addr < TASK_SIZE)
437	return do_page_fault(addr, fsr, regs);	437	return do_page_fault(addr, fsr, regs);
438		438
		439	if (interrupts_enabled(regs))
		440	local_irq_enable();
		441
439	if (user_mode(regs))	442	if (user_mode(regs))
440	goto bad_area;	443	goto bad_area;
441		444
Lines 506-511 do_translation_fault(unsigned long addr, unsigned int fsr, Link Here
506	static int	509	static int
507	do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)	510	do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
508	{	511	{
		512	if (interrupts_enabled(regs))
		513	local_irq_enable();
		514
509	do_bad_area(addr, fsr, regs);	515	do_bad_area(addr, fsr, regs);
510	return 0;	516	return 0;
511	}	517	}




 */
union vfp_state *vfp_current_hw_state[NR_CPUS];

/*
 * Claim ownership of the VFP unit.
 *
 * The caller may change VFP registers until vfp_unlock() is called.
 *
 * local_bh_disable() is used to disable preemption and to disable VFP
 * processing in softirq context. On PREEMPT_RT kernels local_bh_disable() is
 * not sufficient because it only serializes soft interrupt related sections
 * via a local lock, but stays preemptible. Disabling preemption is the right
 * choice here as bottom half processing is always in thread context on RT
 * kernels so it implicitly prevents bottom half processing as well.
 */
static void vfp_lock(void)
{
	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		local_bh_disable();
	else
		preempt_disable();
}

static void vfp_unlock(void)
{
	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		local_bh_enable();
	else
		preempt_enable();
}

/*
 * Is 'thread's most up to date state stored in this CPUs hardware?
 * Must be called from non-preemptible context.

/*
 * Process bitmask of exception conditions.
 */
static int vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr)
{
	int si_code = 0;



	if (exceptions == VFP_EXCEPTION_ERROR) {
		vfp_panic("unhandled bounce", inst);
		return FPE_FLTINV;
		return;
	}

	/*

	RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF);
	RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV);

	return si_code;
		vfp_raise_sigfpe(si_code, regs);
}

/*

static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
{
	u32 fpscr, orig_fpscr, fpsid, exceptions;
	int si_code2 = 0;
	int si_code = 0;

	pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);


		 * unallocated VFP instruction but with FPSCR.IXE set and not
		 * on VFP subarch 1.
		 */
		si_code = vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr);
		goto exit;
	}

	/*

	 */
	exceptions = vfp_emulate_instruction(trigger, fpscr, regs);
	if (exceptions)
		si_code2 = vfp_raise_exceptions(exceptions, trigger, orig_fpscr);

	/*
	 * If there isn't a second FP instruction, exit now. Note that
	 * the FPEXC.FP2V bit is valid only if FPEXC.EX is 1.
	 */
	if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V))
		goto exit;

	/*
	 * The barrier() here prevents fpinst2 being read

 emulate:
	exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);
	if (exceptions)
		si_code = vfp_raise_exceptions(exceptions, trigger, orig_fpscr);
exit:
	vfp_unlock();
	if (si_code2)
		vfp_raise_sigfpe(si_code2, regs);
	if (si_code)
		vfp_raise_sigfpe(si_code, regs);
}

static void vfp_enable(void *unused)

 */
void vfp_sync_hwstate(struct thread_info *thread)
{
	vfp_lock();

	if (vfp_state_in_hw(raw_smp_processor_id(), thread)) {

	if (vfp_state_in_hw(cpu, thread)) {
		u32 fpexc = fmrx(FPEXC);

		/*

		fmxr(FPEXC, fpexc);
	}

	vfp_unlock();
	put_cpu();
}

/* Ensure that the thread reloads the hardware VFP state on the next use. */

	if (!user_mode(regs))
		return vfp_kmode_exception(regs, trigger);

	vfp_lock();
	fpexc = fmrx(FPEXC);

	/*

		 * replay the instruction that trapped.
		 */
		fmxr(FPEXC, fpexc);
		vfp_unlock();
	} else {
		/* Check for synchronous or asynchronous exceptions */
		if (!(fpexc & (FPEXC_EX | FPEXC_DEX))) {

			if (!(fpscr & FPSCR_IXE)) {
				if (!(fpscr & FPSCR_LENGTH_MASK)) {
					pr_debug("not VFP\n");
					vfp_unlock();
					return -ENOEXEC;
				}
				fpexc |= FPEXC_DEX;
			}
		}
bounce:		regs->ARM_pc += 4;
		/* VFP_bounce() will invoke vfp_unlock() */
		VFP_bounce(trigger, fpexc, regs);
	}

	local_bh_enable();
	return 0;
}


	unsigned int cpu;
	u32 fpexc;

	vfp_lock();

	/*
	 * Kernel mode NEON is only allowed outside of hardirq context with

{
	/* Disable the NEON/VFP unit. */
	fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
	vfp_unlock();
}
EXPORT_SYMBOL(kernel_neon_end);


Lines 98-103 config ARM64 Link Here

(-)a/arch/arm64/Kconfig (+1 lines)
98	select ARCH_SUPPORTS_NUMA_BALANCING	98	select ARCH_SUPPORTS_NUMA_BALANCING
99	select ARCH_SUPPORTS_PAGE_TABLE_CHECK	99	select ARCH_SUPPORTS_PAGE_TABLE_CHECK
100	select ARCH_SUPPORTS_PER_VMA_LOCK	100	select ARCH_SUPPORTS_PER_VMA_LOCK
		101	select ARCH_SUPPORTS_RT
101	select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH	102	select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
102	select ARCH_WANT_COMPAT_IPC_PARSE_VERSION if COMPAT	103	select ARCH_WANT_COMPAT_IPC_PARSE_VERSION if COMPAT
103	select ARCH_WANT_DEFAULT_BPF_JIT	104	select ARCH_WANT_DEFAULT_BPF_JIT

Lines 166-171 config PPC Link Here

(-)a/arch/powerpc/Kconfig (+2 lines)
166	select ARCH_STACKWALK	166	select ARCH_STACKWALK
167	select ARCH_SUPPORTS_ATOMIC_RMW	167	select ARCH_SUPPORTS_ATOMIC_RMW
168	select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S \|\| PPC_8xx \|\| 40x	168	select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S \|\| PPC_8xx \|\| 40x
		169	select ARCH_SUPPORTS_RT if HAVE_POSIX_CPU_TIMERS_TASK_WORK
169	select ARCH_USE_BUILTIN_BSWAP	170	select ARCH_USE_BUILTIN_BSWAP
170	select ARCH_USE_CMPXCHG_LOCKREF if PPC64	171	select ARCH_USE_CMPXCHG_LOCKREF if PPC64
171	select ARCH_USE_MEMTEST	172	select ARCH_USE_MEMTEST
Lines 270-275 config PPC Link Here
270	select HAVE_PERF_USER_STACK_DUMP	271	select HAVE_PERF_USER_STACK_DUMP
271	select HAVE_REGS_AND_STACK_ACCESS_API	272	select HAVE_REGS_AND_STACK_ACCESS_API
272	select HAVE_RELIABLE_STACKTRACE	273	select HAVE_RELIABLE_STACKTRACE
		274	select HAVE_POSIX_CPU_TIMERS_TASK_WORK if !KVM
273	select HAVE_RSEQ	275	select HAVE_RSEQ
274	select HAVE_SETUP_PER_CPU_AREA if PPC64	276	select HAVE_SETUP_PER_CPU_AREA if PPC64
275	select HAVE_SOFTIRQ_ON_OWN_STACK	277	select HAVE_SOFTIRQ_ON_OWN_STACK

Lines 19-26 Link Here

(-)a/arch/powerpc/include/asm/stackprotector.h (-1 / +6 lines)
19	*/	19	*/
20	static __always_inline void boot_init_stack_canary(void)	20	static __always_inline void boot_init_stack_canary(void)
21	{	21	{
22	unsigned long canary = get_random_canary();	22	unsigned long canary;
23		23
		24	#ifndef CONFIG_PREEMPT_RT
		25	canary = get_random_canary();
		26	#else
		27	canary = ((unsigned long)&canary) & CANARY_MASK;
		28	#endif
24	current->stack_canary = canary;	29	current->stack_canary = canary;
25	#ifdef CONFIG_PPC64	30	#ifdef CONFIG_PPC64
26	get_paca()->canary = canary;	31	get_paca()->canary = canary;

Lines 261-272 static char *get_mmu_str(void) Link Here

(-)a/arch/powerpc/kernel/traps.c (-1 / +6 lines)
261		261
262	static int __die(const char str, struct pt_regs regs, long err)	262	static int __die(const char str, struct pt_regs regs, long err)
263	{	263	{
		264	const char *pr = "";
		265
264	printk("Oops: %s, sig: %ld [#%d]\n", str, err, ++die_counter);	266	printk("Oops: %s, sig: %ld [#%d]\n", str, err, ++die_counter);
265		267
		268	if (IS_ENABLED(CONFIG_PREEMPTION))
		269	pr = IS_ENABLED(CONFIG_PREEMPT_RT) ? " PREEMPT_RT" : " PREEMPT";
		270
266	printk("%s PAGE_SIZE=%luK%s%s%s%s%s%s %s\n",	271	printk("%s PAGE_SIZE=%luK%s%s%s%s%s%s %s\n",
267	IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? "LE" : "BE",	272	IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? "LE" : "BE",
268	PAGE_SIZE / 1024, get_mmu_str(),	273	PAGE_SIZE / 1024, get_mmu_str(),
269	IS_ENABLED(CONFIG_PREEMPT) ? " PREEMPT" : "",	274	pr,
270	IS_ENABLED(CONFIG_SMP) ? " SMP" : "",	275	IS_ENABLED(CONFIG_SMP) ? " SMP" : "",
271	IS_ENABLED(CONFIG_SMP) ? (" NR_CPUS=" __stringify(NR_CPUS)) : "",	276	IS_ENABLED(CONFIG_SMP) ? (" NR_CPUS=" __stringify(NR_CPUS)) : "",
272	debug_pagealloc_enabled() ? " DEBUG_PAGEALLOC" : "",	277	debug_pagealloc_enabled() ? " DEBUG_PAGEALLOC" : "",

Lines 222-227 config KVM_E500MC Link Here

(-)a/arch/powerpc/kvm/Kconfig (+1 lines)
222	config KVM_MPIC	222	config KVM_MPIC
223	bool "KVM in-kernel MPIC emulation"	223	bool "KVM in-kernel MPIC emulation"
224	depends on KVM && PPC_E500	224	depends on KVM && PPC_E500
		225	depends on !PREEMPT_RT
225	select HAVE_KVM_IRQCHIP	226	select HAVE_KVM_IRQCHIP
226	select HAVE_KVM_IRQ_ROUTING	227	select HAVE_KVM_IRQ_ROUTING
227	select HAVE_KVM_MSI	228	select HAVE_KVM_MSI

Lines 2-7 Link Here

(-)a/arch/powerpc/platforms/pseries/Kconfig (+1 lines)
2	config PPC_PSERIES	2	config PPC_PSERIES
3	depends on PPC64 && PPC_BOOK3S	3	depends on PPC64 && PPC_BOOK3S
4	bool "IBM pSeries & new (POWER5-based) iSeries"	4	bool "IBM pSeries & new (POWER5-based) iSeries"
		5	select GENERIC_ALLOCATOR
5	select HAVE_PCSPKR_PLATFORM	6	select HAVE_PCSPKR_PLATFORM
6	select MPIC	7	select MPIC
7	select OF_DYNAMIC	8	select OF_DYNAMIC

Lines 25-30 Link Here

(-)a/arch/powerpc/platforms/pseries/iommu.c (-11 / +20 lines)
25	#include <linux/of_address.h>	25	#include <linux/of_address.h>
26	#include <linux/iommu.h>	26	#include <linux/iommu.h>
27	#include <linux/rculist.h>	27	#include <linux/rculist.h>
		28	#include <linux/local_lock.h>
28	#include <asm/io.h>	29	#include <asm/io.h>
29	#include <asm/prom.h>	30	#include <asm/prom.h>
30	#include <asm/rtas.h>	31	#include <asm/rtas.h>
Lines 206-212 static int tce_build_pSeriesLP(unsigned long liobn, long tcenum, long tceshift, Link Here
206	return ret;	207	return ret;
207	}	208	}
208		209
209	static DEFINE_PER_CPU(__be64 *, tce_page);	210	struct tce_page {
		211	__be64 * page;
		212	local_lock_t lock;
		213	};
		214	static DEFINE_PER_CPU(struct tce_page, tce_page) = {
		215	.lock = INIT_LOCAL_LOCK(lock),
		216	};
210		217
211	static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,	218	static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
212	long npages, unsigned long uaddr,	219	long npages, unsigned long uaddr,
Lines 229-237 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, Link Here
229	direction, attrs);	236	direction, attrs);
230	}	237	}
231		238
232	local_irq_save(flags); /* to protect tcep and the page behind it */	239	/* to protect tcep and the page behind it */
		240	local_lock_irqsave(&tce_page.lock, flags);
233		241
234	tcep = __this_cpu_read(tce_page);	242	tcep = __this_cpu_read(tce_page.page);
235		243
236	/* This is safe to do since interrupts are off when we're called	244	/* This is safe to do since interrupts are off when we're called
237	* from iommu_alloc{,_sg}()	245	* from iommu_alloc{,_sg}()
Lines 240-251 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, Link Here
240	tcep = (__be64 *)__get_free_page(GFP_ATOMIC);	248	tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
241	/* If allocation fails, fall back to the loop implementation */	249	/* If allocation fails, fall back to the loop implementation */
242	if (!tcep) {	250	if (!tcep) {
243	local_irq_restore(flags);	251	local_unlock_irqrestore(&tce_page.lock, flags);
244	return tce_build_pSeriesLP(tbl->it_index, tcenum,	252	return tce_build_pSeriesLP(tbl->it_index, tcenum,
245	tceshift,	253	tceshift,
246	npages, uaddr, direction, attrs);	254	npages, uaddr, direction, attrs);
247	}	255	}
248	__this_cpu_write(tce_page, tcep);	256	__this_cpu_write(tce_page.page, tcep);
249	}	257	}
250		258
251	rpn = __pa(uaddr) >> tceshift;	259	rpn = __pa(uaddr) >> tceshift;
Lines 275-281 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, Link Here
275	tcenum += limit;	283	tcenum += limit;
276	} while (npages > 0 && !rc);	284	} while (npages > 0 && !rc);
277		285
278	local_irq_restore(flags);	286	local_unlock_irqrestore(&tce_page.lock, flags);
279		287
280	if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {	288	if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
281	ret = (int)rc;	289	ret = (int)rc;
Lines 459-474 static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn, Link Here
459	DMA_BIDIRECTIONAL, 0);	467	DMA_BIDIRECTIONAL, 0);
460	}	468	}
461		469
462	local_irq_disable(); /* to protect tcep and the page behind it */	470	/* to protect tcep and the page behind it */
463	tcep = __this_cpu_read(tce_page);	471	local_lock_irq(&tce_page.lock);
		472	tcep = __this_cpu_read(tce_page.page);
464		473
465	if (!tcep) {	474	if (!tcep) {
466	tcep = (__be64 *)__get_free_page(GFP_ATOMIC);	475	tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
467	if (!tcep) {	476	if (!tcep) {
468	local_irq_enable();	477	local_unlock_irq(&tce_page.lock);
469	return -ENOMEM;	478	return -ENOMEM;
470	}	479	}
471	__this_cpu_write(tce_page, tcep);	480	__this_cpu_write(tce_page.page, tcep);
472	}	481	}
473		482
474	proto_tce = TCE_PCI_READ \| TCE_PCI_WRITE;	483	proto_tce = TCE_PCI_READ \| TCE_PCI_WRITE;
Lines 511-517 static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn, Link Here
511		520
512	/* error cleanup: caller will clear whole range */	521	/* error cleanup: caller will clear whole range */
513		522
514	local_irq_enable();	523	local_unlock_irq(&tce_page.lock);
515	return rc;	524	return rc;
516	}	525	}
517		526

Lines 94-99 int arch_dup_task_struct(struct task_struct dst, struct task_struct src); Link Here

(-)a/arch/riscv/include/asm/thread_info.h (+2 lines)
94	* - pending work-to-be-done flags are in lowest half-word	94	* - pending work-to-be-done flags are in lowest half-word
95	* - other flags in upper half-word(s)	95	* - other flags in upper half-word(s)
96	*/	96	*/
		97	#define TIF_ARCH_RESCHED_LAZY 0 /* Lazy rescheduling */
97	#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */	98	#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */
98	#define TIF_SIGPENDING 2 /* signal pending */	99	#define TIF_SIGPENDING 2 /* signal pending */
99	#define TIF_NEED_RESCHED 3 /* rescheduling necessary */	100	#define TIF_NEED_RESCHED 3 /* rescheduling necessary */
Lines 104-109 int arch_dup_task_struct(struct task_struct dst, struct task_struct src); Link Here
104	#define TIF_32BIT 11 /* compat-mode 32bit process */	105	#define TIF_32BIT 11 /* compat-mode 32bit process */
105	#define TIF_RISCV_V_DEFER_RESTORE 12 /* restore Vector before returing to user */	106	#define TIF_RISCV_V_DEFER_RESTORE 12 /* restore Vector before returing to user */
106		107
		108	#define _TIF_ARCH_RESCHED_LAZY (1 << TIF_ARCH_RESCHED_LAZY)
107	#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)	109	#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
108	#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)	110	#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
109	#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)	111	#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)




	select ARCH_HAS_GIGANTIC_PAGE
	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
	select ARCH_SUPPORTS_PER_VMA_LOCK
	select ARCH_SUPPORTS_RT
	select HAVE_ARCH_SOFT_DIRTY
	select MODULES_USE_ELF_RELA
	select NEED_DMA_MAP_STATE

	select ARCH_USES_CFI_TRAPS		if X86_64 && CFI_CLANG
	select ARCH_SUPPORTS_LTO_CLANG
	select ARCH_SUPPORTS_LTO_CLANG_THIN
	select ARCH_SUPPORTS_RT
	select ARCH_USE_BUILTIN_BSWAP
	select ARCH_USE_CMPXCHG_LOCKREF		if X86_CMPXCHG64
	select ARCH_USE_MEMTEST

	select HAVE_STATIC_CALL
	select HAVE_STATIC_CALL_INLINE		if HAVE_OBJTOOL
	select HAVE_PREEMPT_DYNAMIC_CALL
	select HAVE_PREEMPT_AUTO
	select HAVE_RSEQ
	select HAVE_RUST			if X86_64
	select HAVE_SYSCALL_TRACEPOINTS




#define TIF_NOTIFY_RESUME	1	/* callback before returning to user */
#define TIF_SIGPENDING		2	/* signal pending */
#define TIF_NEED_RESCHED	3	/* rescheduling necessary */
#define TIF_ARCH_RESCHED_LAZY	4	/* Lazy rescheduling */
#define TIF_SINGLESTEP		5	/* reenable singlestep on user return*/
#define TIF_SSBD		6	/* Speculative store bypass disable */
#define TIF_SPEC_IB		9	/* Indirect branch speculation mitigation */
#define TIF_SPEC_L1D_FLUSH	10	/* Flush L1D on mm switches (processes) */
#define TIF_USER_RETURN_NOTIFY	11	/* notify kernel of userspace return */

#define _TIF_NOTIFY_RESUME	(1 << TIF_NOTIFY_RESUME)
#define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)
#define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)
#define _TIF_ARCH_RESCHED_LAZY	(1 << TIF_ARCH_RESCHED_LAZY)
#define _TIF_SINGLESTEP		(1 << TIF_SINGLESTEP)
#define _TIF_SSBD		(1 << TIF_SSBD)
#define _TIF_SPEC_IB		(1 << TIF_SPEC_IB)

Lines 108-114 static const struct dmi_system_id processor_power_dmi_table[] = { Link Here

(-)a/drivers/acpi/processor_idle.c (-1 / +1 lines)
108	*/	108	*/
109	static void __cpuidle acpi_safe_halt(void)	109	static void __cpuidle acpi_safe_halt(void)
110	{	110	{
111	if (!tif_need_resched()) {	111	if (!need_resched()) {
112	raw_safe_halt();	112	raw_safe_halt();
113	raw_local_irq_disable();	113	raw_local_irq_disable();
114	}	114	}




static int zram_read_page(struct zram *zram, struct page *page, u32 index,
			  struct bio *parent);

#ifdef CONFIG_PREEMPT_RT
static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages)
{
	size_t index;

	for (index = 0; index < num_pages; index++)
		spin_lock_init(&zram->table[index].lock);
}

static int zram_slot_trylock(struct zram *zram, u32 index)
{
	int ret;

	ret = spin_trylock(&zram->table[index].lock);
	if (ret)
		__set_bit(ZRAM_LOCK, &zram->table[index].flags);
	return ret;
}

static void zram_slot_lock(struct zram *zram, u32 index)
{
	spin_lock(&zram->table[index].lock);
	__set_bit(ZRAM_LOCK, &zram->table[index].flags);
}

static void zram_slot_unlock(struct zram *zram, u32 index)
{
	__clear_bit(ZRAM_LOCK, &zram->table[index].flags);
	spin_unlock(&zram->table[index].lock);
}

#else

static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages) { }

static int zram_slot_trylock(struct zram *zram, u32 index)
{
	return bit_spin_trylock(ZRAM_LOCK, &zram->table[index].flags);

{
	bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags);
}
#endif

static inline bool init_done(struct zram *zram)
{


	if (!huge_class_size)
		huge_class_size = zs_huge_class_size(zram->mem_pool);
	zram_meta_init_table_locks(zram, num_pages);
	return true;
}


Lines 69-74 struct zram_table_entry { Link Here

(-)a/drivers/block/zram/zram_drv.h (+3 lines)
69	unsigned long element;	69	unsigned long element;
70	};	70	};
71	unsigned long flags;	71	unsigned long flags;
		72	#ifdef CONFIG_PREEMPT_RT
		73	spinlock_t lock;
		74	#endif
72	#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME	75	#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
73	ktime_t ac_time;	76	ktime_t ac_time;
74	#endif	77	#endif

Lines 3-9 config DRM_I915 Link Here

(-)a/drivers/gpu/drm/i915/Kconfig (-1 lines)
3	tristate "Intel 8xx/9xx/G3x/G4x/HD Graphics"	3	tristate "Intel 8xx/9xx/G3x/G4x/HD Graphics"
4	depends on DRM	4	depends on DRM
5	depends on X86 && PCI	5	depends on X86 && PCI
6	depends on !PREEMPT_RT
7	select INTEL_GTT if X86	6	select INTEL_GTT if X86
8	select INTERVAL_TREE	7	select INTERVAL_TREE
9	# we need shmfs for the swappable backing store, and in particular	8	# we need shmfs for the swappable backing store, and in particular




	 */
	intel_psr_wait_for_idle_locked(new_crtc_state);

	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		local_irq_disable();

	crtc->debug.min_vbl = min;
	crtc->debug.max_vbl = max;

			break;
		}

		if (!IS_ENABLED(CONFIG_PREEMPT_RT))
			local_irq_enable();

		timeout = schedule_timeout(timeout);

		if (!IS_ENABLED(CONFIG_PREEMPT_RT))
			local_irq_disable();
	}

	finish_wait(wq, &wait);

	return;

irq_disable:
	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		local_irq_disable();
}

#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)

	 */
	intel_vrr_send_push(new_crtc_state);

	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		local_irq_enable();

	if (intel_vgpu_active(dev_priv))
		goto out;




 * all register accesses to the same cacheline to be serialized,
 * otherwise they may hang.
 */
static void intel_vblank_section_enter_irqsave(struct drm_i915_private *i915, unsigned long *flags)
	__acquires(i915->uncore.lock)
{
#ifdef I915
	spin_lock_irqsave(&i915->uncore.lock, *flags);
#else
	*flags = 0;
#endif
}

static void intel_vblank_section_exit_irqrestore(struct drm_i915_private *i915, unsigned long flags)
	__releases(i915->uncore.lock)
{
#ifdef I915
	spin_unlock_irqrestore(&i915->uncore.lock, flags);
#else
	if (flags)
		return;
#endif
}
static void intel_vblank_section_enter(struct drm_i915_private *i915)
	__acquires(i915->uncore.lock)
{

	 * timing critical raw register reads, potentially with
	 * preemption disabled, so the following code must not block.
	 */
	intel_vblank_section_enter_irqsave(dev_priv, &irqflags);
	intel_vblank_section_enter(dev_priv);

	if (IS_ENABLED(CONFIG_PREEMPT_RT))
		preempt_disable();

	/* Get optional system timestamp before query. */
	if (stime)

	if (etime)
		*etime = ktime_get();

	if (IS_ENABLED(CONFIG_PREEMPT_RT))
		preempt_enable();

	intel_vblank_section_exit_irqrestore(dev_priv, irqflags);
	local_irq_restore(irqflags);

	/*
	 * While in vblank, position will be negative

	unsigned long irqflags;
	int position;

	intel_vblank_section_enter_irqsave(dev_priv, &irqflags);
	intel_vblank_section_enter(dev_priv);

	position = __intel_get_crtc_scanline(crtc);

	intel_vblank_section_exit_irqrestore(dev_priv, irqflags);
	local_irq_restore(irqflags);

	return position;
}

Lines 317-326 void __intel_breadcrumbs_park(struct intel_breadcrumbs *b) Link Here

(-)a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c (-3 / +2 lines)
317	/* Kick the work once more to drain the signalers, and disarm the irq */	317	/* Kick the work once more to drain the signalers, and disarm the irq */
318	irq_work_sync(&b->irq_work);	318	irq_work_sync(&b->irq_work);
319	while (READ_ONCE(b->irq_armed) && !atomic_read(&b->active)) {	319	while (READ_ONCE(b->irq_armed) && !atomic_read(&b->active)) {
320	local_irq_disable();	320	irq_work_queue(&b->irq_work);
321	signal_irq_work(&b->irq_work);
322	local_irq_enable();
323	cond_resched();	321	cond_resched();
		322	irq_work_sync(&b->irq_work);
324	}	323	}
325	}	324	}
326		325




	 * and context switches) submission.
	 */

	spin_lock_irq(&sched_engine->lock);

	/*
	 * If the queue is higher priority than the last

				 * Even if ELSP[1] is occupied and not worthy
				 * of timeslices, our queue might be.
				 */
				spin_unlock_irq(&sched_engine->lock);
				return;
			}
		}


		if (last && !can_merge_rq(last, rq)) {
			spin_unlock(&ve->base.sched_engine->lock);
			spin_unlock_irq(&engine->sched_engine->lock);
			return; /* leave this for another sibling */
		}


	 */
	sched_engine->queue_priority_hint = queue_prio(sched_engine);
	i915_sched_engine_reset_on_empty(sched_engine);
	spin_unlock_irq(&sched_engine->lock);

	/*
	 * We can skip poking the HW if we ended up with exactly the same set

	}
}

static void execlists_dequeue_irq(struct intel_engine_cs *engine)
{
	local_irq_disable(); /* Suspend interrupts across request submission */
	execlists_dequeue(engine);
	local_irq_enable(); /* flush irq_work (e.g. breadcrumb enabling) */
}

static void clear_ports(struct i915_request **ports, int count)
{
	memset_p((void **)ports, NULL, count);

	}

	if (!engine->execlists.pending[0]) {
		execlists_dequeue(engine);
		start_timeslice(engine);
	}


Lines 362-368 static inline int intel_guc_send_busy_loop(struct intel_guc *guc, Link Here

(-)a/drivers/gpu/drm/i915/gt/uc/intel_guc.h (-1 / +1 lines)
362	{	362	{
363	int err;	363	int err;
364	unsigned int sleep_period_ms = 1;	364	unsigned int sleep_period_ms = 1;
365	bool not_atomic = !in_atomic() && !irqs_disabled();	365	bool not_atomic = !in_atomic() && !irqs_disabled() && !rcu_preempt_depth();
366		366
367	/*	367	/*
368	* FIXME: Have caller pass in if we are in an atomic context to avoid	368	* FIXME: Have caller pass in if we are in an atomic context to avoid

Lines 609-615 bool __i915_request_submit(struct i915_request *request) Link Here

(-)a/drivers/gpu/drm/i915/i915_request.c (-2 lines)
609		609
610	RQ_TRACE(request, "\n");	610	RQ_TRACE(request, "\n");
611		611
612	GEM_BUG_ON(!irqs_disabled());
613	lockdep_assert_held(&engine->sched_engine->lock);	612	lockdep_assert_held(&engine->sched_engine->lock);
614		613
615	/*	614	/*
Lines 718-724 void __i915_request_unsubmit(struct i915_request *request) Link Here
718	*/	717	*/
719	RQ_TRACE(request, "\n");	718	RQ_TRACE(request, "\n");
720		719
721	GEM_BUG_ON(!irqs_disabled());
722	lockdep_assert_held(&engine->sched_engine->lock);	720	lockdep_assert_held(&engine->sched_engine->lock);
723		721
724	/*	722	/*

Lines 6-11 Link Here

(-)a/drivers/gpu/drm/i915/i915_trace.h (-1 / +5 lines)
6	#if !defined(_I915_TRACE_H_) \|\| defined(TRACE_HEADER_MULTI_READ)	6	#if !defined(_I915_TRACE_H_) \|\| defined(TRACE_HEADER_MULTI_READ)
7	#define _I915_TRACE_H_	7	#define _I915_TRACE_H_
8		8
		9	#ifdef CONFIG_PREEMPT_RT
		10	#define NOTRACE
		11	#endif
		12
9	#include <linux/stringify.h>	13	#include <linux/stringify.h>
10	#include <linux/types.h>	14	#include <linux/types.h>
11	#include <linux/tracepoint.h>	15	#include <linux/tracepoint.h>
Lines 322-328 DEFINE_EVENT(i915_request, i915_request_add, Link Here
322	TP_ARGS(rq)	326	TP_ARGS(rq)
323	);	327	);
324		328
325	#if defined(CONFIG_DRM_I915_LOW_LEVEL_TRACEPOINTS)	329	#if defined(CONFIG_DRM_I915_LOW_LEVEL_TRACEPOINTS) && !defined(NOTRACE)
326	DEFINE_EVENT(i915_request, i915_request_guc_submit,	330	DEFINE_EVENT(i915_request, i915_request_guc_submit,
327	TP_PROTO(struct i915_request *rq),	331	TP_PROTO(struct i915_request *rq),
328	TP_ARGS(rq)	332	TP_ARGS(rq)

Lines 288-294 wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) Link Here

(-)a/drivers/gpu/drm/i915/i915_utils.h (-1 / +1 lines)
288	#define wait_for(COND, MS) _wait_for((COND), (MS) * 1000, 10, 1000)	288	#define wait_for(COND, MS) _wait_for((COND), (MS) * 1000, 10, 1000)
289		289
290	/* If CONFIG_PREEMPT_COUNT is disabled, in_atomic() always reports false. */	290	/* If CONFIG_PREEMPT_COUNT is disabled, in_atomic() always reports false. */
291	#if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT)	291	#if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT) && !defined(CONFIG_PREEMPT_RT)
292	# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) WARN_ON_ONCE((ATOMIC) && !in_atomic())	292	# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) WARN_ON_ONCE((ATOMIC) && !in_atomic())
293	#else	293	#else
294	# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) do { } while (0)	294	# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) do { } while (0)





#ifdef CONFIG_SERIAL_8250_CONSOLE

#ifdef CONFIG_SERIAL_8250_LEGACY_CONSOLE
static void univ8250_console_write(struct console *co, const char *s,
				   unsigned int count)
{


	serial8250_console_write(up, s, count);
}
#else
static bool univ8250_console_write_atomic(struct console *co,
					  struct nbcon_write_context *wctxt)
{
	struct uart_8250_port *up = &serial8250_ports[co->index];

	return serial8250_console_write_atomic(up, wctxt);
}

static bool univ8250_console_write_thread(struct console *co,
					  struct nbcon_write_context *wctxt)
{
	struct uart_8250_port *up = &serial8250_ports[co->index];

	return serial8250_console_write_thread(up, wctxt);
}

static void univ8250_console_driver_enter(struct console *con, unsigned long *flags)
{
	struct uart_port *up = &serial8250_ports[con->index].port;

	__uart_port_lock_irqsave(up, flags);
}

static void univ8250_console_driver_exit(struct console *con, unsigned long flags)
{
	struct uart_port *up = &serial8250_ports[con->index].port;

	__uart_port_unlock_irqrestore(up, flags);
}
#endif /* CONFIG_SERIAL_8250_LEGACY_CONSOLE */

static int univ8250_console_setup(struct console *co, char *options)
{


static struct console univ8250_console = {
	.name		= "ttyS",
#ifdef CONFIG_SERIAL_8250_LEGACY_CONSOLE
	.write		= univ8250_console_write,
	.flags		= CON_PRINTBUFFER | CON_ANYTIME,
#else
	.write_atomic	= univ8250_console_write_atomic,
	.write_thread	= univ8250_console_write_thread,
	.driver_enter	= univ8250_console_driver_enter,
	.driver_exit	= univ8250_console_driver_exit,
	.flags		= CON_PRINTBUFFER | CON_ANYTIME | CON_NBCON,
#endif
	.device		= uart_console_device,
	.setup		= univ8250_console_setup,
	.exit		= univ8250_console_exit,
	.match		= univ8250_console_match,
	.flags		= CON_PRINTBUFFER | CON_ANYTIME,
	.index		= -1,
	.data		= &serial8250_reg,
};

Lines 550-555 static int serial8250_em485_init(struct uart_8250_port *p) Link Here

(-)a/drivers/tty/serial/8250/8250_port.c (-2 / +158 lines)
550	if (!p->em485)	550	if (!p->em485)
551	return -ENOMEM;	551	return -ENOMEM;
552		552
		553	#ifndef CONFIG_SERIAL_8250_LEGACY_CONSOLE
		554	if (uart_console(&p->port))
		555	dev_warn(p->port.dev, "no atomic printing for rs485 consoles\n");
		556	#endif
		557
553	hrtimer_init(&p->em485->stop_tx_timer, CLOCK_MONOTONIC,	558	hrtimer_init(&p->em485->stop_tx_timer, CLOCK_MONOTONIC,
554	HRTIMER_MODE_REL);	559	HRTIMER_MODE_REL);
555	hrtimer_init(&p->em485->start_tx_timer, CLOCK_MONOTONIC,	560	hrtimer_init(&p->em485->start_tx_timer, CLOCK_MONOTONIC,
Lines 702-708 static void serial8250_set_sleep(struct uart_8250_port *p, int sleep) Link Here
702	serial8250_rpm_put(p);	707	serial8250_rpm_put(p);
703	}	708	}
704		709
705	static void serial8250_clear_IER(struct uart_8250_port *up)	710	/*
		711	* Only to be used by write_atomic() and the legacy write(), which do not
		712	* require port lock.
		713	*/
		714	static void __serial8250_clear_IER(struct uart_8250_port *up)
706	{	715	{
707	if (up->capabilities & UART_CAP_UUE)	716	if (up->capabilities & UART_CAP_UUE)
708	serial_out(up, UART_IER, UART_IER_UUE);	717	serial_out(up, UART_IER, UART_IER_UUE);
Lines 710-715 static void serial8250_clear_IER(struct uart_8250_port *up) Link Here
710	serial_out(up, UART_IER, 0);	719	serial_out(up, UART_IER, 0);
711	}	720	}
712		721
		722	static inline void serial8250_clear_IER(struct uart_8250_port *up)
		723	{
		724	/* Port locked to synchronize UART_IER access against the console. */
		725	lockdep_assert_held_once(&up->port.lock);
		726
		727	__serial8250_clear_IER(up);
		728	}
		729
713	#ifdef CONFIG_SERIAL_8250_RSA	730	#ifdef CONFIG_SERIAL_8250_RSA
714	/*	731	/*
715	* Attempts to turn on the RSA FIFO. Returns zero on failure.	732	* Attempts to turn on the RSA FIFO. Returns zero on failure.
Lines 3276-3281 void serial8250_init_port(struct uart_8250_port *up) Link Here
3276	struct uart_port *port = &up->port;	3293	struct uart_port *port = &up->port;
3277		3294
3278	spin_lock_init(&port->lock);	3295	spin_lock_init(&port->lock);
		3296	port->nbcon_locked_port = false;
3279	port->ctrl_id = 0;	3297	port->ctrl_id = 0;
3280	port->pm = NULL;	3298	port->pm = NULL;
3281	port->ops = &serial8250_pops;	3299	port->ops = &serial8250_pops;
Lines 3320-3325 static void serial8250_console_putchar(struct uart_port *port, unsigned char ch) Link Here
3320		3338
3321	wait_for_xmitr(up, UART_LSR_THRE);	3339	wait_for_xmitr(up, UART_LSR_THRE);
3322	serial_port_out(port, UART_TX, ch);	3340	serial_port_out(port, UART_TX, ch);
		3341
		3342	if (ch == '\n')
		3343	up->console_newline_needed = false;
		3344	else
		3345	up->console_newline_needed = true;
3323	}	3346	}
3324		3347
3325	/*	3348	/*
Lines 3348-3353 static void serial8250_console_restore(struct uart_8250_port *up) Link Here
3348	serial8250_out_MCR(up, up->mcr \| UART_MCR_DTR \| UART_MCR_RTS);	3371	serial8250_out_MCR(up, up->mcr \| UART_MCR_DTR \| UART_MCR_RTS);
3349	}	3372	}
3350		3373
		3374	#ifdef CONFIG_SERIAL_8250_LEGACY_CONSOLE
3351	/*	3375	/*
3352	* Print a string to the serial port using the device FIFO	3376	* Print a string to the serial port using the device FIFO
3353	*	3377	*
Lines 3406-3412 void serial8250_console_write(struct uart_8250_port up, const char s, Link Here
3406	* First save the IER then disable the interrupts	3430	* First save the IER then disable the interrupts
3407	*/	3431	*/
3408	ier = serial_port_in(port, UART_IER);	3432	ier = serial_port_in(port, UART_IER);
3409	serial8250_clear_IER(up);	3433	__serial8250_clear_IER(up);
3410		3434
3411	/* check scratch reg to see if port powered off during system sleep */	3435	/* check scratch reg to see if port powered off during system sleep */
3412	if (up->canary && (up->canary != serial_port_in(port, UART_SCR))) {	3436	if (up->canary && (up->canary != serial_port_in(port, UART_SCR))) {
Lines 3472-3477 void serial8250_console_write(struct uart_8250_port up, const char s, Link Here
3472	if (locked)	3496	if (locked)
3473	uart_port_unlock_irqrestore(port, flags);	3497	uart_port_unlock_irqrestore(port, flags);
3474	}	3498	}
		3499	#else
		3500	bool serial8250_console_write_thread(struct uart_8250_port *up,
		3501	struct nbcon_write_context *wctxt)
		3502	{
		3503	struct uart_8250_em485 *em485 = up->em485;
		3504	struct uart_port *port = &up->port;
		3505	bool done = false;
		3506	unsigned int ier;
		3507
		3508	touch_nmi_watchdog();
		3509
		3510	if (!nbcon_enter_unsafe(wctxt))
		3511	return false;
		3512
		3513	/* First save IER then disable the interrupts. */
		3514	ier = serial_port_in(port, UART_IER);
		3515	serial8250_clear_IER(up);
		3516
		3517	/* Check scratch reg if port powered off during system sleep. */
		3518	if (up->canary && (up->canary != serial_port_in(port, UART_SCR))) {
		3519	serial8250_console_restore(up);
		3520	up->canary = 0;
		3521	}
		3522
		3523	if (em485) {
		3524	if (em485->tx_stopped)
		3525	up->rs485_start_tx(up);
		3526	mdelay(port->rs485.delay_rts_before_send);
		3527	}
		3528
		3529	if (nbcon_exit_unsafe(wctxt)) {
		3530	int len = READ_ONCE(wctxt->len);
		3531	int i;
		3532
		3533	/*
		3534	* Write out the message. Toggle unsafe for each byte in order
		3535	* to give another (higher priority) context the opportunity
		3536	* for a friendly takeover. If such a takeover occurs, this
		3537	* context must reacquire ownership in order to perform final
		3538	* actions (such as re-enabling the interrupts).
		3539	*
		3540	* IMPORTANT: wctxt->outbuf and wctxt->len are no longer valid
		3541	* after a reacquire so writing the message must be
		3542	* aborted.
		3543	*/
		3544	for (i = 0; i < len; i++) {
		3545	if (!nbcon_enter_unsafe(wctxt)) {
		3546	nbcon_reacquire(wctxt);
		3547	break;
		3548	}
		3549
		3550	uart_console_write(port, wctxt->outbuf + i, 1, serial8250_console_putchar);
		3551
		3552	if (!nbcon_exit_unsafe(wctxt)) {
		3553	nbcon_reacquire(wctxt);
		3554	break;
		3555	}
		3556	}
		3557	done = (i == len);
		3558	} else {
		3559	nbcon_reacquire(wctxt);
		3560	}
		3561
		3562	while (!nbcon_enter_unsafe(wctxt))
3563	nbcon_reacquire(wctxt);
3564
3565	/* Finally, wait for transmitter to become empty and restore IER. */
3566	wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
3567	if (em485) {
3568	mdelay(port->rs485.delay_rts_after_send);
3569	if (em485->tx_stopped)
3570	up->rs485_stop_tx(up);
3571	}
3572	serial_port_out(port, UART_IER, ier);
3573
3574	/*
3575	* The receive handling will happen properly because the receive ready
3576	* bit will still be set; it is not cleared on read. However, modem
3577	* control will not, we must call it if we have saved something in the
3578	* saved flags while processing with interrupts off.
3579	*/
3580	if (up->msr_saved_flags)
3581	serial8250_modem_status(up);
3582
3583	/* Success if no handover/takeover and message fully printed. */
3584	return (nbcon_exit_unsafe(wctxt) && done);
3585	}
3586
3587	bool serial8250_console_write_atomic(struct uart_8250_port *up,
3588	struct nbcon_write_context *wctxt)
3589	{
3590	struct uart_port *port = &up->port;
3591	unsigned int ier;
3592
3593	/* Atomic console not supported for rs485 mode. */
3594	if (up->em485)
3595	return false;
3596
3597	touch_nmi_watchdog();
3598
3599	if (!nbcon_enter_unsafe(wctxt))
3600	return false;
3601
3602	/*
3603	* First save IER then disable the interrupts. The special variant to
3604	* clear IER is used because atomic printing may occur without holding
3605	* the port lock.
3606	*/
3607	ier = serial_port_in(port, UART_IER);
3608	__serial8250_clear_IER(up);
3609
3610	/* Check scratch reg if port powered off during system sleep. */
3611	if (up->canary && (up->canary != serial_port_in(port, UART_SCR))) {
3612	serial8250_console_restore(up);
3613	up->canary = 0;
3614	}
3615
3616	if (up->console_newline_needed)
3617	uart_console_write(port, "\n", 1, serial8250_console_putchar);
3618	uart_console_write(port, wctxt->outbuf, wctxt->len, serial8250_console_putchar);
3619
3620	/* Finally, wait for transmitter to become empty and restore IER. */
3621	wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
3622	serial_port_out(port, UART_IER, ier);
3623
3624	/* Success if no handover/takeover. */
3625	return nbcon_exit_unsafe(wctxt);
3626	}
3627	#endif /* CONFIG_SERIAL_8250_LEGACY_CONSOLE */
3475		3628
3476	static unsigned int probe_baud(struct uart_port *port)	3629	static unsigned int probe_baud(struct uart_port *port)
3477	{	3630	{
Lines 3490-3495 static unsigned int probe_baud(struct uart_port *port) Link Here
3490		3643
3491	int serial8250_console_setup(struct uart_port port, char options, bool probe)	3644	int serial8250_console_setup(struct uart_port port, char options, bool probe)
3492	{	3645	{
		3646	struct uart_8250_port *up = up_to_u8250p(port);
3493	int baud = 9600;	3647	int baud = 9600;
3494	int bits = 8;	3648	int bits = 8;
3495	int parity = 'n';	3649	int parity = 'n';
Lines 3499-3504 int serial8250_console_setup(struct uart_port port, char options, bool probe) Link Here
3499	if (!port->iobase && !port->membase)	3653	if (!port->iobase && !port->membase)
3500	return -ENODEV;	3654	return -ENODEV;
3501		3655
		3656	up->console_newline_needed = false;
		3657
3502	if (options)	3658	if (options)
3503	uart_parse_options(options, &baud, &parity, &bits, &flow);	3659	uart_parse_options(options, &baud, &parity, &bits, &flow);
3504	else if (probe)	3660	else if (probe)




				flag = TTY_FRAME;
		}

		sysrq = uart_prepare_sysrq_char(&uap->port, ch & 255);
		sysrq = uart_handle_sysrq_char(&uap->port, ch & 255);
		uart_port_lock(&uap->port);

		if (!sysrq)
			uart_insert_char(&uap->port, ch, UART011_DR_OE, ch, flag);
	}

	ret = pl011_dma_rx_trigger_dma(uap);

	pl011_dma_rx_chars(uap, pending, lastbuf, false);
	uart_unlock_and_check_sysrq(&uap->port);
	/*
	 * Do this check after we picked the DMA chars so we don't
	 * get some IRQ immediately from RX.

static irqreturn_t pl011_int(int irq, void *dev_id)
{
	struct uart_amba_port *uap = dev_id;
	unsigned long flags;
	unsigned int status, pass_counter = AMBA_ISR_PASS_LIMIT;
	int handled = 0;

	uart_port_lock(&uap->port);
	status = pl011_read(uap, REG_RIS) & uap->im;
	if (status) {
		do {

		handled = 1;
	}

	uart_unlock_and_check_sysrq(&uap->port);

	return IRQ_RETVAL(handled);
}


	clk_enable(uap->clk);

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&uap->port, &flags);
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&uap->port);
	else
		uart_port_lock_irqsave(&uap->port, &flags);

	/*
	 *	First save the CR then disable the interrupts

		pl011_write(old_cr, uap, REG_CR);

	if (locked)
		uart_port_unlock_irqrestore(&uap->port, flags);
	local_irq_restore(flags);

	clk_disable(uap->clk);
}




		up->port.icount.rx++;
		ch = rdata & AR933X_UART_DATA_TX_RX_MASK;

		if (uart_prepare_sysrq_char(&up->port, ch))
			continue;

		if ((up->port.ignore_status_mask & AR933X_DUMMY_STATUS_RD) == 0)

		ar933x_uart_tx_chars(up);
	}

	uart_unlock_and_check_sysrq(&up->port);

	return IRQ_HANDLED;
}

	unsigned int int_en;
	int locked = 1;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&up->port, &flags);
	if (up->port.sysrq)
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&up->port);
	else
		uart_port_lock_irqsave(&up->port, &flags);

	/*
	 * First save the IER then disable the interrupts

	ar933x_uart_write(up, AR933X_UART_INT_REG, AR933X_UART_INT_ALLINTS);

	if (locked)
		uart_port_unlock_irqrestore(&up->port, flags);

	local_irq_restore(flags);
}

static int ar933x_uart_console_setup(struct console *co, char *options)




				flag = TTY_PARITY;
		}

		if (uart_prepare_sysrq_char(port, c))
			continue;


		if ((cstat & port->ignore_status_mask) == 0)
			tty_insert_flip_char(tty_port, c, flag);


					       estat & UART_EXTINP_DCD_MASK);
	}

	uart_unlock_and_check_sysrq(port);
	return IRQ_HANDLED;
}


{
	struct uart_port *port;
	unsigned long flags;
	int locked = 1;

	port = &ports[co->index];

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(port, &flags);
	else
		uart_port_lock_irqsave(port, &flags);
	} else if (oops_in_progress) {
		locked = uart_port_trylock(port);
	} else {
		uart_port_lock(port);
		locked = 1;
	}

	/* call helper to deal with \r\n */
	uart_console_write(port, s, count, bcm_console_putchar);

	wait_for_xmitr(port);

	if (locked)
		uart_port_unlock_irqrestore(port, flags);
	local_irq_restore(flags);
}

/*




	int locked = 1;

	touch_nmi_watchdog();
	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&up->port, &flags);
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&up->port);
	else
		uart_port_lock_irqsave(&up->port, &flags);

	uart_console_write(&up->port, s, count, lpc32xx_hsuart_console_putchar);
	wait_for_xmit_empty(&up->port);

	if (locked)
		uart_port_unlock_irqrestore(&up->port, flags);
	local_irq_restore(flags);
}

static int __init lpc32xx_hsuart_console_setup(struct console *co,

			tty_insert_flip_char(tport, 0, TTY_FRAME);
		}

		if (!uart_prepare_sysrq_char(port, tmp & 0xff))
			tty_insert_flip_char(tport, (tmp & 0xFF), flag);

		tmp = readl(LPC32XX_HSUART_FIFO(port->membase));
	}

		__serial_lpc32xx_tx(port);
	}

	uart_unlock_and_check_sysrq(port);

	return IRQ_HANDLED;
}




				continue;
		}

		if (uart_prepare_sysrq_char(port, ch))
			continue;

		if ((status & port->ignore_status_mask) == 0)

			meson_uart_start_tx(port);
	}

	uart_unlock_and_check_sysrq(port);

	return IRQ_HANDLED;
}

				    u_int count)
{
	unsigned long flags;
	int locked = 1;
	u32 val, tmp;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(port, &flags);
	else
		uart_port_lock_irqsave(port, &flags);
		locked = uart_port_trylock(port);
	} else {
		uart_port_lock(port);
		locked = 1;
	}

	val = readl(port->membase + AML_UART_CONTROL);
	tmp = val & ~(AML_UART_TX_INT_EN | AML_UART_RX_INT_EN);

	writel(val, port->membase + AML_UART_CONTROL);

	if (locked)
		uart_port_unlock_irqrestore(port, flags);
	local_irq_restore(flags);
}

static void meson_serial_console_write(struct console *co, const char *s,




		if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))
			flag = TTY_NORMAL;

		sysrq = uart_prepare_sysrq_char(port, dma->virt[i]);
		sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
		uart_port_lock_irqsave(port, &flags);
		if (!sysrq)
			tty_insert_flip_char(tport, dma->virt[i], flag);
	}

	msm_start_rx_dma(msm_port);
done:
	uart_unlock_and_check_sysrq_irqrestore(port, flags);

	if (count)
		tty_flip_buffer_push(tport);

			if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))
				flag = TTY_NORMAL;

			sysrq = uart_prepare_sysrq_char(port, buf[i]);
			sysrq = uart_handle_sysrq_char(port, buf[i]);
			uart_port_lock(port);
			if (!sysrq)
				tty_insert_flip_char(tport, buf[i], flag);
		}

		else if (sr & MSM_UART_SR_PAR_FRAME_ERR)
			flag = TTY_FRAME;

		sysrq = uart_prepare_sysrq_char(port, c);
		sysrq = uart_handle_sysrq_char(port, c);
		uart_port_lock(port);
		if (!sysrq)
			tty_insert_flip_char(tport, c, flag);
	}

	struct uart_port *port = dev_id;
	struct msm_port *msm_port = to_msm_port(port);
	struct msm_dma *dma = &msm_port->rx_dma;
	unsigned long flags;
	unsigned int misr;
	u32 val;

	uart_port_lock(port);
	misr = msm_read(port, MSM_UART_MISR);
	msm_write(port, 0, MSM_UART_IMR); /* disable interrupt */


		msm_handle_delta_cts(port);

	msm_write(port, msm_port->imr, MSM_UART_IMR); /* restore interrupt */
	uart_unlock_and_check_sysrq(port);

	return IRQ_HANDLED;
}

			num_newlines++;
	count += num_newlines;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(port, &flags);
	if (port->sysrq)
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(port);
	else
		uart_port_lock_irqsave(port, &flags);

	if (is_uartdm)
		msm_reset_dm_count(port, count);

	}

	if (locked)
		uart_port_unlock_irqrestore(port, flags);

	local_irq_restore(flags);
}

static void msm_console_write(struct console *co, const char *s,





	up->port.icount.rx++;

	if (uart_prepare_sysrq_char(&up->port, ch))
		return;

	uart_insert_char(&up->port, lsr, UART_LSR_OE, ch, TTY_NORMAL);

		}
	} while (max_count--);

	uart_unlock_and_check_sysrq(&up->port);

	tty_flip_buffer_push(&up->port.state->port);


	unsigned int ier;
	int locked = 1;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&up->port, &flags);
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&up->port);
	else
		uart_port_lock_irqsave(&up->port, &flags);

	/*
	 * First save the IER then disable the interrupts

		check_modem_status(up);

	if (locked)
		uart_port_unlock_irqrestore(&up->port, flags);
	local_irq_restore(flags);
}

static int __init

Lines 199-204 static void owl_uart_receive_chars(struct uart_port *port) Link Here

(-)a/drivers/tty/serial/owl-uart.c (-18 / +12 lines)
199	stat = owl_uart_read(port, OWL_UART_STAT);	199	stat = owl_uart_read(port, OWL_UART_STAT);
200	while (!(stat & OWL_UART_STAT_RFEM)) {	200	while (!(stat & OWL_UART_STAT_RFEM)) {
201	char flag = TTY_NORMAL;	201	char flag = TTY_NORMAL;
		202	bool sysrq;
202		203
203	if (stat & OWL_UART_STAT_RXER)	204	if (stat & OWL_UART_STAT_RXER)
204	port->icount.overrun++;	205	port->icount.overrun++;
Lines 217-223 static void owl_uart_receive_chars(struct uart_port *port) Link Here
217	val = owl_uart_read(port, OWL_UART_RXDAT);	218	val = owl_uart_read(port, OWL_UART_RXDAT);
218	val &= 0xff;	219	val &= 0xff;
219		220
220	if ((stat & port->ignore_status_mask) == 0)	221	sysrq = uart_prepare_sysrq_char(port, val);
		222
		223	if (!sysrq && (stat & port->ignore_status_mask) == 0)
221	tty_insert_flip_char(&port->state->port, val, flag);	224	tty_insert_flip_char(&port->state->port, val, flag);
222		225
223	stat = owl_uart_read(port, OWL_UART_STAT);	226	stat = owl_uart_read(port, OWL_UART_STAT);
Lines 229-238 static void owl_uart_receive_chars(struct uart_port *port) Link Here
229	static irqreturn_t owl_uart_irq(int irq, void *dev_id)	232	static irqreturn_t owl_uart_irq(int irq, void *dev_id)
230	{	233	{
231	struct uart_port *port = dev_id;	234	struct uart_port *port = dev_id;
232	unsigned long flags;
233	u32 stat;	235	u32 stat;
234		236
235	uart_port_lock_irqsave(port, &flags);	237	uart_port_lock(port);
236		238
237	stat = owl_uart_read(port, OWL_UART_STAT);	239	stat = owl_uart_read(port, OWL_UART_STAT);
238		240
Lines 246-252 static irqreturn_t owl_uart_irq(int irq, void *dev_id) Link Here
246	stat \|= OWL_UART_STAT_RIP \| OWL_UART_STAT_TIP;	248	stat \|= OWL_UART_STAT_RIP \| OWL_UART_STAT_TIP;
247	owl_uart_write(port, stat, OWL_UART_STAT);	249	owl_uart_write(port, stat, OWL_UART_STAT);
248		250
249	uart_port_unlock_irqrestore(port, flags);	251	uart_unlock_and_check_sysrq(port);
250		252
251	return IRQ_HANDLED;	253	return IRQ_HANDLED;
252	}	254	}
Lines 508-525 static void owl_uart_port_write(struct uart_port port, const char s, Link Here
508	{	510	{
509	u32 old_ctl, val;	511	u32 old_ctl, val;
510	unsigned long flags;	512	unsigned long flags;
511	int locked;	513	int locked = 1;
512		514
513	local_irq_save(flags);	515	if (oops_in_progress)
514		516	locked = uart_port_trylock_irqsave(port, &flags);
515	if (port->sysrq)	517	else
516	locked = 0;	518	uart_port_lock_irqsave(port, &flags);
517	else if (oops_in_progress)
518	locked = uart_port_trylock(port);
519	else {
520	uart_port_lock(port);
521	locked = 1;
522	}
523		519
524	old_ctl = owl_uart_read(port, OWL_UART_CTL);	520	old_ctl = owl_uart_read(port, OWL_UART_CTL);
525	val = old_ctl \| OWL_UART_CTL_TRFS_TX;	521	val = old_ctl \| OWL_UART_CTL_TRFS_TX;
Lines 541-549 static void owl_uart_port_write(struct uart_port port, const char s, Link Here
541	owl_uart_write(port, old_ctl, OWL_UART_CTL);	537	owl_uart_write(port, old_ctl, OWL_UART_CTL);
542		538
543	if (locked)	539	if (locked)
544	uart_port_unlock(port);	540	uart_port_unlock_irqrestore(port, flags);
545
546	local_irq_restore(flags);
547	}	541	}
548		542
549	static void owl_uart_console_write(struct console co, const char s,	543	static void owl_uart_console_write(struct console co, const char s,





#define IRQ_NAME_SIZE 17
	char				irq_name[IRQ_NAME_SIZE];

	/* protect the eg20t_port private structure and io access to membase */
	spinlock_t lock;
};

/**

			if (uart_handle_break(port))
				continue;
		}
		if (uart_prepare_sysrq_char(port, rbr))
			continue;

		buf[i++] = rbr;

	iowrite8(lcr, priv->membase + UART_LCR);
}

static void push_rx(struct eg20t_port *priv, const unsigned char *buf,
		    int size)
{
	struct uart_port *port = &priv->port;
	struct tty_port *tport = &port->state->port;

	tty_insert_flip_string(tport, buf, size);
	tty_flip_buffer_push(tport);

	return 0;
}

static int dma_push_rx(struct eg20t_port *priv, int size)

{
	struct pch_uart_buffer *buf;
	int rx_size;

	if (!priv->start_rx) {
		pch_uart_hal_disable_interrupt(priv, PCH_UART_HAL_RX_INT |
						     PCH_UART_HAL_RX_ERR_INT);

	buf = &priv->rxbuf;
	do {
		rx_size = pch_uart_hal_read(priv, buf->buf, buf->size);
		push_rx(priv, buf->buf, rx_size);
		if (ret)
			return 0;
	} while (rx_size == buf->size);

	return PCH_UART_HANDLED_RX_INT;
}

static int handle_rx(struct eg20t_port *priv)
{
	return handle_rx_to(priv);
}

static int dma_handle_rx(struct eg20t_port *priv)
{
	struct uart_port *port = &priv->port;

	u8 lsr;
	int ret = 0;
	unsigned char iid;
	unsigned long flags;
	int next = 1;
	u8 msr;

	uart_port_lock(&priv->port);
	handled = 0;
	while (next) {
		iid = pch_uart_hal_get_iid(priv);

						PCH_UART_HAL_RX_INT |
						PCH_UART_HAL_RX_ERR_INT);
			} else {
				ret = handle_rx_to(priv);
			}
			break;
		case PCH_UART_IID_RDR_TO:	/* Received Data Ready

		handled |= (unsigned int)ret;
	}

	uart_unlock_and_check_sysrq(&priv->port);
	return IRQ_RETVAL(handled);
}


	unsigned long flags;

	priv = container_of(port, struct eg20t_port, port);
	uart_port_lock_irqsave(&priv->port, &flags);
	pch_uart_hal_set_break(priv, ctl);
	uart_port_unlock_irqrestore(&priv->port, flags);
}

/* Grab any interrupt resources and initialise any low level driver state. */


	baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);

	uart_port_lock_irqsave(port, &flags);
	uart_port_lock(port);

	uart_update_timeout(port, termios->c_cflag, baud);
	rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);

		tty_termios_encode_baud_rate(termios, baud, baud);

out:
	uart_port_unlock_irqrestore(port, flags);
	spin_unlock_irqrestore(&priv->lock, flags);
}

static const char *pch_uart_type(struct uart_port *port)

{
	struct eg20t_port *priv;
	unsigned long flags;
	int locked = 1;
	int port_locked = 1;
	u8 ier;

	priv = pch_uart_ports[co->index];

	touch_nmi_watchdog();

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&priv->port, &flags);
	else
		uart_port_lock_irqsave(&priv->port, &flags);
		/* serial8250_handle_port() already took the port lock */
		port_locked = 0;
	} else if (oops_in_progress) {
		priv_locked = spin_trylock(&priv->lock);
		port_locked = uart_port_trylock(&priv->port);
	} else {
		spin_lock(&priv->lock);
		uart_port_lock(&priv->port);
	}

	/*
	 *	First save the IER then disable the interrupts

	wait_for_xmitr(priv, UART_LSR_BOTH_EMPTY);
	iowrite8(ier, priv->membase + UART_IER);

	if (locked)
		uart_port_unlock_irqrestore(&priv->port, flags);
	if (priv_locked)
		spin_unlock(&priv->lock);
	local_irq_restore(flags);
}

static int __init pch_console_setup(struct console *co, char *options)

	pci_enable_msi(pdev);
	pci_set_master(pdev);

	spin_lock_init(&priv->lock);

	iobase = pci_resource_start(pdev, 0);
	mapbase = pci_resource_start(pdev, 1);
	priv->mapbase = mapbase;

		 KBUILD_MODNAME ":" PCH_UART_DRIVER_DEVICE "%d",
		 priv->port.line);

	spin_lock_init(&priv->port.lock);

	pci_set_drvdata(pdev, priv);
	priv->trigger_level = 1;
	priv->fcr = 0;




				flag = TTY_FRAME;
		}

		if (uart_prepare_sysrq_char(&up->port, ch))
			goto ignore_char;

		uart_insert_char(&up->port, *status, UART_LSR_OE, ch, flag);

	check_modem_status(up);
	if (lsr & UART_LSR_THRE)
		transmit_chars(up);
	uart_unlock_and_check_sysrq(&up->port);
	return IRQ_HANDLED;
}


	int locked = 1;

	clk_enable(up->clk);
	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&up->port, &flags);
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&up->port);
	else
		uart_port_lock_irqsave(&up->port, &flags);

	/*
	 *	First save the IER then disable the interrupts

	serial_out(up, UART_IER, ier);

	if (locked)
		uart_port_unlock_irqrestore(&up->port, flags);
	local_irq_restore(flags);
	clk_disable(up->clk);

}

#ifdef CONFIG_CONSOLE_POLL




		val &= 0xff;

		port->icount.rx++;
		if (!uart_prepare_sysrq_char(port, val))
			tty_insert_flip_char(&port->state->port, val, flag);

		status = rda_uart_read(port, RDA_UART_STATUS);
	}

static irqreturn_t rda_interrupt(int irq, void *dev_id)
{
	struct uart_port *port = dev_id;
	unsigned long flags;
	u32 val, irq_mask;

	uart_port_lock(port);

	/* Clear IRQ cause */
	val = rda_uart_read(port, RDA_UART_IRQ_CAUSE);

		rda_uart_send_chars(port);
	}

	uart_unlock_and_check_sysrq(port);

	return IRQ_HANDLED;
}

{
	u32 old_irq_mask;
	unsigned long flags;
	int locked = 1;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(port, &flags);
	else
		uart_port_lock_irqsave(port, &flags);
	} else if (oops_in_progress) {
		locked = uart_port_trylock(port);
	} else {
		uart_port_lock(port);
		locked = 1;
	}

	old_irq_mask = rda_uart_read(port, RDA_UART_IRQ_MASK);
	rda_uart_write(port, 0, RDA_UART_IRQ_MASK);

	rda_uart_write(port, old_irq_mask, RDA_UART_IRQ_MASK);

	if (locked)
		uart_port_unlock_irqrestore(port, flags);

	local_irq_restore(flags);
}

static void rda_uart_console_write(struct console *co, const char *s,




			break;

		ssp->port.icount.rx++;
		if (!uart_prepare_sysrq_char(&ssp->port, ch))
			uart_insert_char(&ssp->port, 0, 0, ch, TTY_NORMAL);
	}

	tty_flip_buffer_push(&ssp->port.state->port);

	if (ip & SIFIVE_SERIAL_IP_TXWM_MASK)
		__ssp_transmit_chars(ssp);

	uart_unlock_and_check_sysrq(&ssp->port);

	return IRQ_HANDLED;
}

	if (!ssp)
		return;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&ssp->port, &flags);
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&ssp->port);
	else
		uart_port_lock_irqsave(&ssp->port, &flags);

	ier = __ssp_readl(ssp, SIFIVE_SERIAL_IE_OFFS);
	__ssp_writel(0, SIFIVE_SERIAL_IE_OFFS, ssp);

	__ssp_writel(ier, SIFIVE_SERIAL_IE_OFFS, ssp);

	if (locked)
		uart_port_unlock_irqrestore(&ssp->port, flags);
	local_irq_restore(flags);
}

static int sifive_serial_console_setup(struct console *co, char *options)




		if (port->ignore_status_mask & SUP_DUMMY_READ)
			goto ignore_char;

		if (uart_prepare_sysrq_char(port, ch))
			goto ignore_char;

		uart_insert_char(port, lsr, SUP_UART_LSR_OE, ch, flag);

	if (isc & SUP_UART_ISC_TX)
		transmit_chars(port);

	uart_unlock_and_check_sysrq(port);

	return IRQ_HANDLED;
}

	unsigned long flags;
	int locked = 1;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&sunplus_console_ports[co->index]->port, &flags);
	if (sunplus_console_ports[co->index]->port.sysrq)
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&sunplus_console_ports[co->index]->port);
	else
		uart_port_lock_irqsave(&sunplus_console_ports[co->index]->port, &flags);

	uart_console_write(&sunplus_console_ports[co->index]->port, s, count,
			   sunplus_uart_console_putchar);

	if (locked)
		uart_port_unlock_irqrestore(&sunplus_console_ports[co->index]->port, flags);

	local_irq_restore(flags);
}

static int __init sunplus_console_setup(struct console *co, char *options)

Lines 3567-3574 static ssize_t show_cons_active(struct device *dev, Link Here

(-)a/drivers/tty/tty_io.c (-2 / +7 lines)
3567	for_each_console(c) {	3567	for_each_console(c) {
3568	if (!c->device)	3568	if (!c->device)
3569	continue;	3569	continue;
3570	if (!c->write)	3570	if (c->flags & CON_NBCON) {
3571	continue;	3571	if (!c->write_atomic && !c->write_thread)
		3572	continue;
		3573	} else {
		3574	if (!c->write)
		3575	continue;
		3576	}
3572	if ((c->flags & CON_ENABLED) == 0)	3577	if ((c->flags & CON_ENABLED) == 0)
3573	continue;	3578	continue;
3574	cs[i++] = c;	3579	cs[i++] = c;




		{ CON_ENABLED,		'E' },
		{ CON_CONSDEV,		'C' },
		{ CON_BOOT,		'B' },
		{ CON_NBCON,		'N' },
		{ CON_PRINTBUFFER,	'p' },
		{ CON_BRL,		'b' },
		{ CON_ANYTIME,		'a' },
	};
	char flags[ARRAY_SIZE(con_flags) + 1];
	struct console *con = v;
	char con_write = '-';
	unsigned int a;
	dev_t dev = 0;


	seq_setwidth(m, 21 - 1);
	seq_printf(m, "%s%d", con->name, con->index);
	seq_pad(m, ' ');
	if (con->flags & CON_NBCON) {
		if (con->write_atomic || con->write_thread)
			con_write = 'W';
	} else {
		if (con->write)
			con_write = 'W';
	}
	seq_printf(m, "%c%c%c (%s)", con->read ? 'R' : '-', con_write,
		   con->unblank ? 'U' : '-', flags);
	if (dev)
		seq_printf(m, " %4d:%d", MAJOR(dev), MINOR(dev));


Lines 35-42 static inline void local_bh_enable(void) Link Here

(-)a/include/linux/bottom_half.h (+2 lines)
35		35
36	#ifdef CONFIG_PREEMPT_RT	36	#ifdef CONFIG_PREEMPT_RT
37	extern bool local_bh_blocked(void);	37	extern bool local_bh_blocked(void);
		38	extern void softirq_preempt(void);
38	#else	39	#else
39	static inline bool local_bh_blocked(void) { return false; }	40	static inline bool local_bh_blocked(void) { return false; }
		41	static inline void softirq_preempt(void) { }
40	#endif	42	#endif
41		43
42	#endif /* _LINUX_BH_H */	44	#endif /* _LINUX_BH_H */





#include <linux/atomic.h>
#include <linux/bits.h>
#include <linux/irq_work.h>
#include <linux/rculist.h>
#include <linux/rcuwait.h>
#include <linux/types.h>

struct vc_data;

 */

/**
 * enum cons_flags - General console flags
 * @CON_PRINTBUFFER:	Used by newly registered consoles to avoid duplicate
 *			output of messages that were already shown by boot
 *			consoles or read by userspace via syslog() syscall.

static_assert(sizeof(struct nbcon_state) <= sizeof(int));

/**
 * enum nbcon_prio - console owner priority for nbcon consoles
 * @NBCON_PRIO_NONE:		Unused
 * @NBCON_PRIO_NORMAL:		Normal (non-emergency) usage
 * @NBCON_PRIO_EMERGENCY:	Emergency output (WARN/OOPS...)

/**
 * struct console - The console descriptor structure
 * @name:		The name of the console driver
 * @write:		Legacy write callback to output messages (Optional)
 * @read:		Read callback for console input (Optional)
 * @device:		The underlying TTY device driver (Optional)
 * @unblank:		Callback to unblank the console (Optional)

 * @data:		Driver private data
 * @node:		hlist node for the console list
 *
 * @write_atomic:	Write callback for atomic context
 * @nbcon_state:	State for nbcon consoles
 * @nbcon_seq:		Sequence number of the next record for nbcon to print
 * @pbufs:		Pointer to nbcon private buffer
 * @kthread:		Printer kthread for this console
 * @rcuwait:		RCU-safe wait object for @kthread waking
 * @irq_work:		Defer @kthread waking to IRQ work context
 */
struct console {
	char			name[16];

	struct hlist_node	node;

	/* nbcon console specific members */

	/**
	 * @write_atomic:
	 *
	 * NBCON callback to write out text in any context. (Optional)
	 *
	 * This callback is called with the console already acquired. The
	 * callback can use nbcon_can_proceed() at any time to verify that
	 * it is still the owner of the console. In the case that it has
	 * lost ownership, it is no longer allowed to go forward. In this
	 * case it must back out immediately and carefully. The buffer
	 * content is also no longer trusted since it no longer belongs to
	 * the context.
	 *
	 * If the callback needs to perform actions where ownership is not
	 * allowed to be taken over, nbcon_enter_unsafe() and
	 * nbcon_exit_unsafe() can be used to mark such sections. These
	 * functions are also points of possible ownership transfer. If
	 * either function returns false, ownership has been lost.
	 *
	 * If the driver must reacquire ownership in order to finalize or
	 * revert hardware changes, nbcon_reacquire() can be used. However,
	 * on reacquire the buffer content is no longer available. A
	 * reacquire cannot be used to resume printing.
	 *
	 * This callback can be called from any context (including NMI).
	 * Therefore it must avoid usage of any locking and instead rely
	 * on the console ownership for synchronization.
	 *
	 * Returns true if all text was successfully written out and
	 * ownership was never lost, otherwise false.
	 */
	bool (*write_atomic)(struct console *con, struct nbcon_write_context *wctxt);

	/**
	 * @write_thread:
	 *
	 * NBCON callback to write out text in task context. (Optional)
	 *
	 * This callback is called with the console already acquired. Any
	 * additional driver synchronization should have been performed by
	 * driver_enter().
	 *
	 * This callback is always called from task context but with migration
	 * disabled.
	 *
	 * The same criteria for console ownership verification and unsafe
	 * sections applies as with write_atomic(). The difference between
	 * this callback and write_atomic() is that this callback is used
	 * during normal operation and is always called from task context.
	 * This provides drivers with a relatively relaxed locking context
	 * for synchronizing output to the hardware.
	 *
	 * Returns true if all text was successfully written out, otherwise
	 * false.
	 */
	bool (*write_thread)(struct console *con, struct nbcon_write_context *wctxt);

	/**
	 * @driver_enter:
	 *
	 * NBCON callback to begin synchronization with driver code.
	 * (Required for NBCON if write_thread is provided)
	 *
	 * Console drivers typically must deal with access to the hardware
	 * via user input/output (such as an interactive login shell) and
	 * output of kernel messages via printk() calls. This callback is
	 * called before the kernel begins output via the write_thread()
	 * callback due to printk() calls. The driver can use this
	 * callback to acquire some driver lock in order to synchronize
	 * against user input/output (or any other driver functionality).
	 *
	 * This callback is always called from task context. It may use any
	 * synchronization method required by the driver. BUT this callback
	 * MUST disable migration. The console driver may be using a
	 * sychronization mechanism that already takes care of this (such as
	 * spinlocks). Otherwise this function must explicitly call
	 * migrate_disable().
	 *
	 * The flags argument is provided as a convenience to the driver. It
	 * will be passed again to driver_exit() when printing is completed
	 * (for example, if spin_lock_irqsave() was used). It can be ignored
	 * if the driver does not need it.
	 */
	void (*driver_enter)(struct console *con, unsigned long *flags);

	/**
	 * @driver_exit:
	 *
	 * NBCON callback to finish synchronization with driver code.
	 * (Required for NBCON if write_thread is provided)
	 *
	 * This callback is called after the kernel has finished printing a
	 * printk message. It is the counterpart to driver_enter().
	 *
	 * This callback is always called from task context. It must
	 * appropriately re-enable migration (depending on how driver_enter()
	 * disabled migration).
	 *
	 * The flags argument is the value of the same variable that was
	 * passed to driver_enter().
	 */
	void (*driver_exit)(struct console *con, unsigned long flags);

	atomic_t		__private nbcon_state;
	atomic_long_t		__private nbcon_seq;
	struct printk_buffers	*pbufs;
	struct task_struct	*kthread;
	struct rcuwait		rcuwait;
	struct irq_work		irq_work;
};

#ifdef CONFIG_LOCKDEP

	hlist_for_each_entry(con, &console_list, node)

#ifdef CONFIG_PRINTK
extern void nbcon_cpu_emergency_enter(void);
extern void nbcon_cpu_emergency_exit(void);
extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt);
extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt);
extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt);
extern void nbcon_reacquire(struct nbcon_write_context *wctxt);
#else
static inline void nbcon_cpu_emergency_enter(void) { }
static inline void nbcon_cpu_emergency_exit(void) { }
static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; }
static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; }
static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; }
static inline void nbcon_reacquire(struct nbcon_write_context *wctxt) { }
#endif

extern int console_set_on_cmdline;

Lines 65-71 Link Here

(-)a/include/linux/entry-common.h (-1 / +1 lines)
65	#define EXIT_TO_USER_MODE_WORK \	65	#define EXIT_TO_USER_MODE_WORK \
66	(_TIF_SIGPENDING \| _TIF_NOTIFY_RESUME \| _TIF_UPROBE \| \	66	(_TIF_SIGPENDING \| _TIF_NOTIFY_RESUME \| _TIF_UPROBE \| \
67	_TIF_NEED_RESCHED \| _TIF_PATCH_PENDING \| _TIF_NOTIFY_SIGNAL \| \	67	_TIF_NEED_RESCHED \| _TIF_PATCH_PENDING \| _TIF_NOTIFY_SIGNAL \| \
68	ARCH_EXIT_TO_USER_MODE_WORK)	68	_TIF_NEED_RESCHED_LAZY \| ARCH_EXIT_TO_USER_MODE_WORK)
69		69
70	/**	70	/**
71	* arch_enter_from_user_mode - Architecture specific sanity check for user mode regs	71	* arch_enter_from_user_mode - Architecture specific sanity check for user mode regs

Lines 18-24 Link Here

(-)a/include/linux/entry-kvm.h (-1 / +1 lines)
18		18
19	#define XFER_TO_GUEST_MODE_WORK \	19	#define XFER_TO_GUEST_MODE_WORK \
20	(_TIF_NEED_RESCHED \| _TIF_SIGPENDING \| _TIF_NOTIFY_SIGNAL \| \	20	(_TIF_NEED_RESCHED \| _TIF_SIGPENDING \| _TIF_NOTIFY_SIGNAL \| \
21	_TIF_NOTIFY_RESUME \| ARCH_XFER_TO_GUEST_MODE_WORK)	21	_TIF_NOTIFY_RESUME \| _TIF_NEED_RESCHED_LAZY \| ARCH_XFER_TO_GUEST_MODE_WORK)
22		22
23	struct kvm_vcpu;	23	struct kvm_vcpu;
24		24




extern void raise_softirq_irqoff(unsigned int nr);
extern void raise_softirq(unsigned int nr);

#ifdef CONFIG_PREEMPT_RT
DECLARE_PER_CPU(struct task_struct *, timersd);
DECLARE_PER_CPU(unsigned long, pending_timer_softirq);

extern void raise_timer_softirq(void);
extern void raise_hrtimer_softirq(void);

static inline unsigned int local_pending_timers(void)
{
        return __this_cpu_read(pending_timer_softirq);
}

#else
static inline void raise_timer_softirq(void)
{
	raise_softirq(TIMER_SOFTIRQ);
}

static inline void raise_hrtimer_softirq(void)
{
	raise_softirq_irqoff(HRTIMER_SOFTIRQ);
}

static inline unsigned int local_pending_timers(void)
{
        return local_softirq_pending();
}
#endif

DECLARE_PER_CPU(struct task_struct *, ksoftirqd);

static inline struct task_struct *this_cpu_ksoftirqd(void)

Lines 3365-3370 static inline void dev_xmit_recursion_dec(void) Link Here

(-)a/include/linux/netdevice.h (+1 lines)
3365	__this_cpu_dec(softnet_data.xmit.recursion);	3365	__this_cpu_dec(softnet_data.xmit.recursion);
3366	}	3366	}
3367		3367
		3368	void kick_defer_list_purge(struct softnet_data *sd, unsigned int cpu);
3368	void __netif_schedule(struct Qdisc *q);	3369	void __netif_schedule(struct Qdisc *q);
3369	void netif_schedule_queue(struct netdev_queue *txq);	3370	void netif_schedule_queue(struct netdev_queue *txq);
3370		3371




	unsigned int			pending_wakeup;
	unsigned int			pending_kill;
	unsigned int			pending_disable;
	unsigned int			pending_sigtrap;
	unsigned long			pending_addr;	/* SIGTRAP */
	struct irq_work			pending_irq;
	struct irq_work			pending_disable_irq;
	struct callback_head		pending_task;
	unsigned int			pending_work;


	struct rcu_head			rcu_head;

	/*
	 * Sum (event->pending_work + event->pending_work)
	 *
	 * The SIGTRAP is targeted at ctx->task, as such it won't do changing
	 * that until the signal is delivered.

Lines 9-14 Link Here

(-)a/include/linux/printk.h (-2 / +30 lines)
9	#include <linux/ratelimit_types.h>	9	#include <linux/ratelimit_types.h>
10	#include <linux/once_lite.h>	10	#include <linux/once_lite.h>
11		11
		12	struct uart_port;
		13
12	extern const char linux_banner[];	14	extern const char linux_banner[];
13	extern const char linux_proc_banner[];	15	extern const char linux_proc_banner[];
14		16
Lines 159-171 __printf(1, 2) __cold int _printk_deferred(const char *fmt, ...); Link Here
159		161
160	extern void __printk_safe_enter(void);	162	extern void __printk_safe_enter(void);
161	extern void __printk_safe_exit(void);	163	extern void __printk_safe_exit(void);
		164	extern void __printk_deferred_enter(void);
		165	extern void __printk_deferred_exit(void);
		166
162	/*	167	/*
163	* The printk_deferred_enter/exit macros are available only as a hack for	168	* The printk_deferred_enter/exit macros are available only as a hack for
164	* some code paths that need to defer all printk console printing. Interrupts	169	* some code paths that need to defer all printk console printing. Interrupts
165	* must be disabled for the deferred duration.	170	* must be disabled for the deferred duration.
166	*/	171	*/
167	#define printk_deferred_enter __printk_safe_enter	172	#define printk_deferred_enter() __printk_deferred_enter()
168	#define printk_deferred_exit __printk_safe_exit	173	#define printk_deferred_exit() __printk_deferred_exit()
169		174
170	/*	175	/*
171	* Please don't use printk_ratelimit(), because it shares ratelimiting state	176	* Please don't use printk_ratelimit(), because it shares ratelimiting state
Lines 192-197 void show_regs_print_info(const char *log_lvl); Link Here
192	extern asmlinkage void dump_stack_lvl(const char *log_lvl) __cold;	197	extern asmlinkage void dump_stack_lvl(const char *log_lvl) __cold;
193	extern asmlinkage void dump_stack(void) __cold;	198	extern asmlinkage void dump_stack(void) __cold;
194	void printk_trigger_flush(void);	199	void printk_trigger_flush(void);
		200	void printk_legacy_allow_panic_sync(void);
		201	extern void uart_nbcon_acquire(struct uart_port *up);
		202	extern void uart_nbcon_release(struct uart_port *up);
		203	void nbcon_atomic_flush_unsafe(void);
195	#else	204	#else
196	static inline __printf(1, 0)	205	static inline __printf(1, 0)
197	int vprintk(const char *s, va_list args)	206	int vprintk(const char *s, va_list args)
Lines 271-278 static inline void dump_stack(void) Link Here
271	static inline void printk_trigger_flush(void)	280	static inline void printk_trigger_flush(void)
272	{	281	{
273	}	282	}
		283
		284	static inline void printk_legacy_allow_panic_sync(void)
		285	{
		286	}
		287
		288	static inline void uart_nbcon_acquire(struct uart_port *up)
		289	{
		290	}
		291
		292	static inline void uart_nbcon_release(struct uart_port *up)
		293	{
		294	}
		295
		296	static inline void nbcon_atomic_flush_unsafe(void)
		297	{
		298	}
		299
274	#endif	300	#endif
275		301
		302	bool this_cpu_in_panic(void);
		303
276	#ifdef CONFIG_SMP	304	#ifdef CONFIG_SMP
277	extern int __printk_cpu_sync_try_get(void);	305	extern int __printk_cpu_sync_try_get(void);
278	extern void __printk_cpu_sync_wait(void);	306	extern void __printk_cpu_sync_wait(void);




}
#endif

extern bool task_is_pi_boosted(const struct task_struct *p);
extern int yield_to(struct task_struct *p, bool preempt);
extern void set_user_nice(struct task_struct *p, long nice);
extern int task_prio(const struct task_struct *p);

	update_ti_thread_flag(task_thread_info(tsk), flag, value);
}

static inline bool test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
{
	return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
}

static inline bool test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
{
	return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
}

static inline bool test_tsk_thread_flag(struct task_struct *tsk, int flag)
{
	return test_ti_thread_flag(task_thread_info(tsk), flag);
}

static inline void clear_tsk_need_resched(struct task_struct *tsk)
{
	clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
	if (IS_ENABLED(CONFIG_PREEMPT_BUILD_AUTO))
		clear_tsk_thread_flag(tsk, TIF_NEED_RESCHED_LAZY);
}

static inline bool test_tsk_need_resched(struct task_struct *tsk)
{
	return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
}


static __always_inline bool need_resched(void)
{
	return unlikely(tif_need_resched_lazy() || tif_need_resched());
}

/*




	 */
	smp_mb__after_atomic();

	return unlikely(need_resched());
}

static __always_inline bool __must_check current_clr_polling_and_test(void)

	 */
	smp_mb__after_atomic();

	return unlikely(need_resched());
}

#else


static inline bool __must_check current_set_polling_and_test(void)
{
	return unlikely(need_resched());
}
static inline bool __must_check current_clr_polling_and_test(void)
{
	return unlikely(need_resched());
}
#endif


Lines 153-158 struct uart_8250_port { Link Here

(-)a/include/linux/serial_8250.h (+6 lines)
153	#define MSR_SAVE_FLAGS UART_MSR_ANY_DELTA	153	#define MSR_SAVE_FLAGS UART_MSR_ANY_DELTA
154	unsigned char msr_saved_flags;	154	unsigned char msr_saved_flags;
155		155
		156	bool console_newline_needed;
		157
156	struct uart_8250_dma *dma;	158	struct uart_8250_dma *dma;
157	const struct uart_8250_ops *ops;	159	const struct uart_8250_ops *ops;
158		160
Lines 204-209 void serial8250_init_port(struct uart_8250_port *up); Link Here
204	void serial8250_set_defaults(struct uart_8250_port *up);	206	void serial8250_set_defaults(struct uart_8250_port *up);
205	void serial8250_console_write(struct uart_8250_port up, const char s,	207	void serial8250_console_write(struct uart_8250_port up, const char s,
206	unsigned int count);	208	unsigned int count);
		209	bool serial8250_console_write_atomic(struct uart_8250_port *up,
		210	struct nbcon_write_context *wctxt);
		211	bool serial8250_console_write_thread(struct uart_8250_port *up,
		212	struct nbcon_write_context *wctxt);
207	int serial8250_console_setup(struct uart_port port, char options, bool probe);	213	int serial8250_console_setup(struct uart_port port, char options, bool probe);
208	int serial8250_console_exit(struct uart_port *port);	214	int serial8250_console_exit(struct uart_port *port);
209		215




	struct uart_icount	icount;			/* statistics */

	struct console		*cons;			/* struct console, if any */
	bool			nbcon_locked_port;	/* True, if the port is locked by nbcon */
	/* flags must be updated while holding port mutex */
	upf_t			flags;


static inline void uart_port_lock(struct uart_port *up)
{
	spin_lock(&up->lock);
	uart_nbcon_acquire(up);
}

/**

static inline void uart_port_lock_irq(struct uart_port *up)
{
	spin_lock_irq(&up->lock);
	uart_nbcon_acquire(up);
}

/**

static inline void uart_port_lock_irqsave(struct uart_port *up, unsigned long *flags)
{
	spin_lock_irqsave(&up->lock, *flags);
	uart_nbcon_acquire(up);
}

/**

 */
static inline bool uart_port_trylock(struct uart_port *up)
{
	if (!spin_trylock(&up->lock))
		return false;

	uart_nbcon_acquire(up);
	return true;
}

/**

 */
static inline bool uart_port_trylock_irqsave(struct uart_port *up, unsigned long *flags)
{
	if (!spin_trylock_irqsave(&up->lock, *flags))
		return false;

	uart_nbcon_acquire(up);
	return true;
}

/**

 */
static inline void uart_port_unlock(struct uart_port *up)
{
	uart_nbcon_release(up);
	spin_unlock(&up->lock);
}


 */
static inline void uart_port_unlock_irq(struct uart_port *up)
{
	uart_nbcon_release(up);
	spin_unlock_irq(&up->lock);
}


 * @flags:	The saved interrupt flags for restore
 */
static inline void uart_port_unlock_irqrestore(struct uart_port *up, unsigned long flags)
{
	uart_nbcon_release(up);
	spin_unlock_irqrestore(&up->lock, flags);
}

/* Only for use in the console->driver_enter() callback. */
static inline void __uart_port_lock_irqsave(struct uart_port *up, unsigned long *flags)
{
	spin_lock_irqsave(&up->lock, *flags);
}

/* Only for use in the console->driver_exit() callback. */
static inline void __uart_port_unlock_irqrestore(struct uart_port *up, unsigned long flags)
{
	spin_unlock_irqrestore(&up->lock, flags);
}

Lines 59-64 enum syscall_work_bit { Link Here

(-)a/include/linux/thread_info.h (+24 lines)
59		59
60	#include <asm/thread_info.h>	60	#include <asm/thread_info.h>
61		61
		62	#ifdef CONFIG_PREEMPT_BUILD_AUTO
		63	# define TIF_NEED_RESCHED_LAZY TIF_ARCH_RESCHED_LAZY
		64	# define _TIF_NEED_RESCHED_LAZY _TIF_ARCH_RESCHED_LAZY
		65	# define TIF_NEED_RESCHED_LAZY_OFFSET (TIF_NEED_RESCHED_LAZY - TIF_NEED_RESCHED)
		66	#else
		67	# define TIF_NEED_RESCHED_LAZY TIF_NEED_RESCHED
		68	# define _TIF_NEED_RESCHED_LAZY _TIF_NEED_RESCHED
		69	# define TIF_NEED_RESCHED_LAZY_OFFSET 0
		70	#endif
		71
62	#ifdef __KERNEL__	72	#ifdef __KERNEL__
63		73
64	#ifndef arch_set_restart_data	74	#ifndef arch_set_restart_data
Lines 185-190 static __always_inline bool tif_need_resched(void) Link Here
185	(unsigned long *)(&current_thread_info()->flags));	195	(unsigned long *)(&current_thread_info()->flags));
186	}	196	}
187		197
		198	static __always_inline bool tif_need_resched_lazy(void)
		199	{
		200	return IS_ENABLED(CONFIG_PREEMPT_BUILD_AUTO) &&
		201	arch_test_bit(TIF_NEED_RESCHED_LAZY,
		202	(unsigned long *)(&current_thread_info()->flags));
		203	}
		204
188	#else	205	#else
189		206
190	static __always_inline bool tif_need_resched(void)	207	static __always_inline bool tif_need_resched(void)
Lines 193-198 static __always_inline bool tif_need_resched(void) Link Here
193	(unsigned long *)(&current_thread_info()->flags));	210	(unsigned long *)(&current_thread_info()->flags));
194	}	211	}
195		212
		213	static __always_inline bool tif_need_resched_lazy(void)
		214	{
		215	return IS_ENABLED(CONFIG_PREEMPT_BUILD_AUTO) &&
		216	test_bit(TIF_NEED_RESCHED_LAZY,
		217	(unsigned long *)(&current_thread_info()->flags));
		218	}
		219
196	#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */	220	#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */
197		221
198	#ifndef CONFIG_HAVE_ARCH_WITHIN_STACK_FRAMES	222	#ifndef CONFIG_HAVE_ARCH_WITHIN_STACK_FRAMES





enum trace_flag_type {
	TRACE_FLAG_IRQS_OFF		= 0x01,
	TRACE_FLAG_NEED_RESCHED		= 0x02,
	TRACE_FLAG_NEED_RESCHED_LAZY	= 0x04,
	TRACE_FLAG_HARDIRQ		= 0x08,
	TRACE_FLAG_SOFTIRQ		= 0x10,
	TRACE_FLAG_PREEMPT_RESCHED	= 0x20,


static inline unsigned int tracing_gen_ctx_flags(unsigned long irqflags)
{
	return tracing_gen_ctx_irq_test(0);
}
static inline unsigned int tracing_gen_ctx(void)
{
	return tracing_gen_ctx_irq_test(0);
}
#endif


Lines 11-16 config PREEMPT_BUILD Link Here

(-)a/kernel/Kconfig.preempt (-1 / +16 lines)
11	select PREEMPTION	11	select PREEMPTION
12	select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK	12	select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK
13		13
		14	config PREEMPT_BUILD_AUTO
		15	bool
		16	select PREEMPT_BUILD
		17
		18	config HAVE_PREEMPT_AUTO
		19	bool
		20
14	choice	21	choice
15	prompt "Preemption Model"	22	prompt "Preemption Model"
16	default PREEMPT_NONE	23	default PREEMPT_NONE
Lines 67-75 config PREEMPT Link Here
67	embedded system with latency requirements in the milliseconds	74	embedded system with latency requirements in the milliseconds
68	range.	75	range.
69		76
		77	config PREEMPT_AUTO
		78	bool "Automagic preemption mode with runtime tweaking support"
		79	depends on HAVE_PREEMPT_AUTO
		80	select PREEMPT_BUILD_AUTO
		81	help
		82	Add some sensible blurb here
		83
70	config PREEMPT_RT	84	config PREEMPT_RT
71	bool "Fully Preemptible Kernel (Real-Time)"	85	bool "Fully Preemptible Kernel (Real-Time)"
72	depends on EXPERT && ARCH_SUPPORTS_RT	86	depends on EXPERT && ARCH_SUPPORTS_RT
		87	select PREEMPT_BUILD_AUTO if HAVE_PREEMPT_AUTO
73	select PREEMPTION	88	select PREEMPTION
74	help	89	help
75	This option turns the kernel into a real-time kernel by replacing	90	This option turns the kernel into a real-time kernel by replacing
Lines 95-101 config PREEMPTION Link Here
95		110
96	config PREEMPT_DYNAMIC	111	config PREEMPT_DYNAMIC
97	bool "Preemption behaviour defined on boot"	112	bool "Preemption behaviour defined on boot"
98	depends on HAVE_PREEMPT_DYNAMIC && !PREEMPT_RT	113	depends on HAVE_PREEMPT_DYNAMIC && !PREEMPT_RT && !PREEMPT_AUTO
99	select JUMP_LABEL if HAVE_PREEMPT_DYNAMIC_KEY	114	select JUMP_LABEL if HAVE_PREEMPT_DYNAMIC_KEY
100	select PREEMPT_BUILD	115	select PREEMPT_BUILD
101	default y if HAVE_PREEMPT_DYNAMIC_CALL	116	default y if HAVE_PREEMPT_DYNAMIC_CALL

Lines 92-98 __always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs, Link Here

(-)a/kernel/entry/common.c (-2 / +2 lines)
92		92
93	local_irq_enable_exit_to_user(ti_work);	93	local_irq_enable_exit_to_user(ti_work);
94		94
95	if (ti_work & _TIF_NEED_RESCHED)	95	if (ti_work & (_TIF_NEED_RESCHED \| _TIF_NEED_RESCHED_LAZY))
96	schedule();	96	schedule();
97		97
98	if (ti_work & _TIF_UPROBE)	98	if (ti_work & _TIF_UPROBE)
Lines 301-307 void raw_irqentry_exit_cond_resched(void) Link Here
301	rcu_irq_exit_check_preempt();	301	rcu_irq_exit_check_preempt();
302	if (IS_ENABLED(CONFIG_DEBUG_ENTRY))	302	if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
303	WARN_ON_ONCE(!on_thread_stack());	303	WARN_ON_ONCE(!on_thread_stack());
304	if (need_resched())	304	if (test_tsk_need_resched(current))
305	preempt_schedule_irq();	305	preempt_schedule_irq();
306	}	306	}
307	}	307	}

Lines 13-19 static int xfer_to_guest_mode_work(struct kvm_vcpu *vcpu, unsigned long ti_work) Link Here

(-)a/kernel/entry/kvm.c (-1 / +1 lines)
13	return -EINTR;	13	return -EINTR;
14	}	14	}
15		15
16	if (ti_work & _TIF_NEED_RESCHED)	16	if (ti_work & (_TIF_NEED_RESCHED \| TIF_NEED_RESCHED_LAZY))
17	schedule();	17	schedule();
18		18
19	if (ti_work & _TIF_NOTIFY_RESUME)	19	if (ti_work & _TIF_NOTIFY_RESUME)




		state = PERF_EVENT_STATE_OFF;
	}

	if (event->pending_sigtrap) {
		bool dec = true;

		event->pending_sigtrap = 0;
		if (state != PERF_EVENT_STATE_OFF &&
		    !event->pending_work) {
			event->pending_work = 1;
			dec = false;
			WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));
			task_work_add(current, &event->pending_task, TWA_RESUME);
		}
		if (dec)
			local_dec(&event->ctx->nr_pending);
	}

	perf_event_set_state(event, state);

	if (!is_software_event(event))

 * hold the top-level event's child_mutex, so any descendant that
 * goes to exit will block in perf_event_exit_event().
 *
 * When called from perf_pending_disable it's OK because event->ctx
 * is the current context on this CPU and preemption is disabled,
 * hence we can't get into perf_event_task_sched_out for this context.
 */

void perf_event_disable_inatomic(struct perf_event *event)
{
	event->pending_disable = 1;
	irq_work_queue(&event->pending_disable_irq);
}

#define MAX_INTERRUPTS (~0ULL)

static void _free_event(struct perf_event *event)
{
	irq_work_sync(&event->pending_irq);
	irq_work_sync(&event->pending_disable_irq);

	unaccount_event(event);


/*
 * Deliver the pending work in-event-context or follow the context.
 */
static void __perf_pending_disable(struct perf_event *event)
{
	int cpu = READ_ONCE(event->oncpu);


	 * Yay, we hit home and are in the context of the event.
	 */
	if (cpu == smp_processor_id()) {
		if (event->pending_sigtrap) {
			event->pending_sigtrap = 0;
			perf_sigtrap(event);
			local_dec(&event->ctx->nr_pending);
		}
		if (event->pending_disable) {
			event->pending_disable = 0;
			perf_event_disable_local(event);

	 *				  irq_work_queue(); // FAILS
	 *
	 *  irq_work_run()
	 *    perf_pending_disable()
	 *
	 * But the event runs on CPU-B and wants disabling there.
	 */
	irq_work_queue_on(&event->pending_disable_irq, cpu);
}

static void perf_pending_disable(struct irq_work *entry)
{
	struct perf_event *event = container_of(entry, struct perf_event, pending_disable_irq);
	int rctx;

	/*
	 * If we 'fail' here, that's OK, it means recursion is already disabled
	 * and we won't recurse 'further'.
	 */
	rctx = perf_swevent_get_recursion_context();
	__perf_pending_disable(event);
	if (rctx >= 0)
		perf_swevent_put_recursion_context(rctx);
}

static void perf_pending_irq(struct irq_work *entry)

		perf_event_wakeup(event);
	}

	__perf_pending_irq(event);

	if (rctx >= 0)
		perf_swevent_put_recursion_context(rctx);
}

static void perf_pending_task(struct callback_head *head)
{
	struct perf_event *event = container_of(head, struct perf_event, pending_task);
	int rctx;

	/*
	 * If we 'fail' here, that's OK, it means recursion is already disabled
	 * and we won't recurse 'further'.
	 */
	preempt_disable_notrace();
	rctx = perf_swevent_get_recursion_context();

	if (event->pending_work) {
		event->pending_work = 0;

		local_dec(&event->ctx->nr_pending);
	}

	if (rctx >= 0)
		perf_swevent_put_recursion_context(rctx);
	preempt_enable_notrace();

	put_event(event);
}



		if (regs)
			pending_id = hash32_ptr((void *)instruction_pointer(regs)) ?: 1;
		if (!event->pending_work) {
			event->pending_work = pending_id;
			local_inc(&event->ctx->nr_pending);
			WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));
			task_work_add(current, &event->pending_task, TWA_RESUME);
			/*
			 * The NMI path returns directly to userland. The
			 * irq_work is raised as a dummy interrupt to ensure
			 * regular return path to user is taken and task_work
			 * is processed.
			 */
			if (in_nmi())
				irq_work_queue(&event->pending_disable_irq);
		} else if (event->attr.exclude_kernel && valid_sample) {
			/*
			 * Should not be able to return to user space without
			 * consuming pending_work; with exceptions:
			 *
			 *  1. Where !exclude_kernel, events can overflow again
			 *     in the kernel without returning to user space.

			 *     To approximate progress (with false negatives),
			 *     check 32-bit hash of the current IP.
			 */
			WARN_ON_ONCE(event->pending_work != pending_id);
		}

		event->pending_addr = 0;
		if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR))
			event->pending_addr = data->addr;
		irq_work_queue(&event->pending_irq);
	}

	READ_ONCE(event->overflow_handler)(event, data, regs);


	init_waitqueue_head(&event->waitq);
	init_irq_work(&event->pending_irq, perf_pending_irq);
	event->pending_disable_irq = IRQ_WORK_INIT_HARD(perf_pending_disable);
	init_task_work(&event->pending_task, perf_pending_task);

	mutex_init(&event->mmap_mutex);

		     &parent_event->child_total_time_running);
}

static bool task_work_cb_match(struct callback_head *cb, void *data)
{
	struct perf_event *event = container_of(cb, struct perf_event, pending_task);

	return event == data;
}

static void
perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx)
{

		 * Kick perf_poll() for is_event_hup();
		 */
		perf_event_wakeup(parent_event);
		/*
		 * Cancel pending task_work and update counters if it has not
		 * yet been delivered to userland. free_event() expects the
		 * reference counter at 1 and keeping the event around until the
		 * task return to userland will be a unexpected.
		 */
		if (event->pending_work &&
		    task_work_cancel_match(current, task_work_cb_match, event)) {
			put_event(event);
			local_dec(&event->ctx->nr_pending);
		}
		free_event(event);
		put_event(parent_event);
		return;

Lines 179-184 KERNEL_ATTR_RO(crash_elfcorehdr_size); Link Here

(-)a/kernel/ksysfs.c (+12 lines)
179		179
180	#endif /* CONFIG_CRASH_CORE */	180	#endif /* CONFIG_CRASH_CORE */
181		181
		182	#if defined(CONFIG_PREEMPT_RT)
		183	static ssize_t realtime_show(struct kobject *kobj,
		184	struct kobj_attribute attr, char buf)
		185	{
		186	return sprintf(buf, "%d\n", 1);
		187	}
		188	KERNEL_ATTR_RO(realtime);
		189	#endif
		190
182	/* whether file capabilities are enabled */	191	/* whether file capabilities are enabled */
183	static ssize_t fscaps_show(struct kobject *kobj,	192	static ssize_t fscaps_show(struct kobject *kobj,
184	struct kobj_attribute attr, char buf)	193	struct kobj_attribute attr, char buf)
Lines 274-279 static struct attribute * kernel_attrs[] = { Link Here
274	#ifndef CONFIG_TINY_RCU	283	#ifndef CONFIG_TINY_RCU
275	&rcu_expedited_attr.attr,	284	&rcu_expedited_attr.attr,
276	&rcu_normal_attr.attr,	285	&rcu_normal_attr.attr,
		286	#endif
		287	#ifdef CONFIG_PREEMPT_RT
		288	&realtime_attr.attr,
277	#endif	289	#endif
278	NULL	290	NULL
279	};	291	};

Lines 56-61 Link Here

(-)a/kernel/locking/lockdep.c (+5 lines)
56	#include <linux/kprobes.h>	56	#include <linux/kprobes.h>
57	#include <linux/lockdep.h>	57	#include <linux/lockdep.h>
58	#include <linux/context_tracking.h>	58	#include <linux/context_tracking.h>
		59	#include <linux/console.h>
59		60
60	#include <asm/sections.h>	61	#include <asm/sections.h>
61		62
Lines 3971-3976 print_usage_bug(struct task_struct curr, struct held_lock this, Link Here
3971	if (!debug_locks_off() \|\| debug_locks_silent)	3972	if (!debug_locks_off() \|\| debug_locks_silent)
3972	return;	3973	return;
3973		3974
		3975	nbcon_cpu_emergency_enter();
		3976
3974	pr_warn("\n");	3977	pr_warn("\n");
3975	pr_warn("================================\n");	3978	pr_warn("================================\n");
3976	pr_warn("WARNING: inconsistent lock state\n");	3979	pr_warn("WARNING: inconsistent lock state\n");
Lines 3999-4004 print_usage_bug(struct task_struct curr, struct held_lock this, Link Here
3999		4002
4000	pr_warn("\nstack backtrace:\n");	4003	pr_warn("\nstack backtrace:\n");
4001	dump_stack();	4004	dump_stack();
		4005
		4006	nbcon_cpu_emergency_exit();
4002	}	4007	}
4003		4008
4004	/*	4009	/*

Lines 370-375 void panic(const char *fmt, ...) Link Here

(-)a/kernel/panic.c (+17 lines)
370	*/	370	*/
371	atomic_notifier_call_chain(&panic_notifier_list, 0, buf);	371	atomic_notifier_call_chain(&panic_notifier_list, 0, buf);
372		372
		373	printk_legacy_allow_panic_sync();
		374
373	panic_print_sys_info(false);	375	panic_print_sys_info(false);
374		376
375	kmsg_dump(KMSG_DUMP_PANIC);	377	kmsg_dump(KMSG_DUMP_PANIC);
Lines 446-451 void panic(const char *fmt, ...) Link Here
446		448
447	/* Do not scroll important messages printed above */	449	/* Do not scroll important messages printed above */
448	suppress_printk = 1;	450	suppress_printk = 1;
		451
		452	/*
		453	* The final messages may not have been printed if in a context that
		454	* defers printing (such as NMI) and irq_work is not available.
		455	* Explicitly flush the kernel log buffer one last time.
		456	*/
		457	console_flush_on_panic(CONSOLE_FLUSH_PENDING);
		458	nbcon_atomic_flush_unsafe();
		459
449	local_irq_enable();	460	local_irq_enable();
450	for (i = 0; ; i += PANIC_TIMER_STEP) {	461	for (i = 0; ; i += PANIC_TIMER_STEP) {
451	touch_softlockup_watchdog();	462	touch_softlockup_watchdog();
Lines 623-628 bool oops_may_print(void) Link Here
623	*/	634	*/
624	void oops_enter(void)	635	void oops_enter(void)
625	{	636	{
		637	nbcon_cpu_emergency_enter();
626	tracing_off();	638	tracing_off();
627	/* can't trust the integrity of the kernel anymore: */	639	/* can't trust the integrity of the kernel anymore: */
628	debug_locks_off();	640	debug_locks_off();
Lines 645-650 void oops_exit(void) Link Here
645	{	657	{
646	do_oops_enter_exit();	658	do_oops_enter_exit();
647	print_oops_end_marker();	659	print_oops_end_marker();
		660	nbcon_cpu_emergency_exit();
648	kmsg_dump(KMSG_DUMP_OOPS);	661	kmsg_dump(KMSG_DUMP_OOPS);
649	}	662	}
650		663
Lines 656-661 struct warn_args { Link Here
656	void __warn(const char file, int line, void caller, unsigned taint,	669	void __warn(const char file, int line, void caller, unsigned taint,
657	struct pt_regs regs, struct warn_args args)	670	struct pt_regs regs, struct warn_args args)
658	{	671	{
		672	nbcon_cpu_emergency_enter();
		673
659	disable_trace_on_warning();	674	disable_trace_on_warning();
660		675
661	if (file)	676	if (file)
Lines 686-691 void __warn(const char file, int line, void caller, unsigned taint, Link Here
686		701
687	/* Just a warning, don't kill lockdep. */	702	/* Just a warning, don't kill lockdep. */
688	add_taint(taint, LOCKDEP_STILL_OK);	703	add_taint(taint, LOCKDEP_STILL_OK);
		704
		705	nbcon_cpu_emergency_exit();
689	}	706	}
690		707
691	#ifdef CONFIG_BUG	708	#ifdef CONFIG_BUG

Lines 4-9 Link Here

(-)a/kernel/printk/internal.h (+98 lines)
4	*/	4	*/
5	#include <linux/percpu.h>	5	#include <linux/percpu.h>
6	#include <linux/console.h>	6	#include <linux/console.h>
		7	#include <linux/jump_label.h>
7	#include "printk_ringbuffer.h"	8	#include "printk_ringbuffer.h"
8		9
9	#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)	10	#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)
Lines 20-25 int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write, Link Here
20	(con->flags & CON_BOOT) ? "boot" : "", \	21	(con->flags & CON_BOOT) ? "boot" : "", \
21	con->name, con->index, ##__VA_ARGS__)	22	con->name, con->index, ##__VA_ARGS__)
22		23
		24	#ifdef CONFIG_PREEMPT_RT
		25	# define force_printkthreads() (true)
		26	#else
		27	DECLARE_STATIC_KEY_FALSE(force_printkthreads_key);
		28	# define force_printkthreads() (static_branch_unlikely(&force_printkthreads_key))
		29	#endif
		30
23	#ifdef CONFIG_PRINTK	31	#ifdef CONFIG_PRINTK
24		32
25	#ifdef CONFIG_PRINTK_CALLER	33	#ifdef CONFIG_PRINTK_CALLER
Lines 44-49 enum printk_info_flags { Link Here
44	};	52	};
45		53
46	extern struct printk_ringbuffer *prb;	54	extern struct printk_ringbuffer *prb;
		55	extern bool printk_threads_enabled;
		56	extern bool have_legacy_console;
		57	extern bool have_boot_console;
		58
		59	/*
		60	* Specifies if the console lock/unlock dance is needed for console
		61	* printing. If @have_boot_console is true, the nbcon consoles will
		62	* be printed serially along with the legacy consoles because nbcon
		63	* consoles cannot print simultaneously with boot consoles.
		64	*/
		65	#define printing_via_unlock (have_legacy_console \|\| have_boot_console)
47		66
48	__printf(4, 0)	67	__printf(4, 0)
49	int vprintk_store(int facility, int level,	68	int vprintk_store(int facility, int level,
Lines 71-82 void defer_console_output(void); Link Here
71		90
72	u16 printk_parse_prefix(const char text, int level,	91	u16 printk_parse_prefix(const char text, int level,
73	enum printk_info_flags *flags);	92	enum printk_info_flags *flags);
		93	void console_lock_spinning_enable(void);
		94	int console_lock_spinning_disable_and_check(int cookie);
74		95
75	u64 nbcon_seq_read(struct console *con);	96	u64 nbcon_seq_read(struct console *con);
76	void nbcon_seq_force(struct console *con, u64 seq);	97	void nbcon_seq_force(struct console *con, u64 seq);
77	bool nbcon_alloc(struct console *con);	98	bool nbcon_alloc(struct console *con);
78	void nbcon_init(struct console *con);	99	void nbcon_init(struct console *con);
79	void nbcon_free(struct console *con);	100	void nbcon_free(struct console *con);
		101	enum nbcon_prio nbcon_get_default_prio(void);
		102	void nbcon_atomic_flush_all(void);
		103	bool nbcon_legacy_emit_next_record(struct console con, bool handover,
		104	int cookie, bool use_atomic);
		105	void nbcon_kthread_create(struct console *con);
		106	void nbcon_wake_threads(void);
		107	void nbcon_legacy_kthread_create(void);
		108
		109	/*
		110	* Check if the given console is currently capable and allowed to print
		111	* records. Note that this function does not consider the current context,
		112	* which can also play a role in deciding if @con can be used to print
		113	* records.
		114	*/
		115	static inline bool console_is_usable(struct console *con, short flags, bool use_atomic)
		116	{
		117	if (!(flags & CON_ENABLED))
		118	return false;
		119
		120	if ((flags & CON_SUSPENDED))
		121	return false;
		122
		123	if (flags & CON_NBCON) {
		124	if (use_atomic) {
		125	if (!con->write_atomic)
		126	return false;
		127	} else {
		128	if (!con->write_thread)
		129	return false;
		130	}
		131	} else {
		132	if (!con->write)
		133	return false;
		134	}
		135
		136	/*
		137	* Console drivers may assume that per-cpu resources have been
		138	* allocated. So unless they're explicitly marked as being able to
		139	* cope (CON_ANYTIME) don't call them until this CPU is officially up.
		140	*/
		141	if (!cpu_online(raw_smp_processor_id()) && !(flags & CON_ANYTIME))
		142	return false;
		143
		144	return true;
		145	}
		146
		147	/**
		148	* nbcon_kthread_wake - Wake up a printk thread
		149	* @con: Console to operate on
		150	*/
		151	static inline void nbcon_kthread_wake(struct console *con)
		152	{
		153	/*
		154	* Guarantee any new records can be seen by tasks preparing to wait
		155	* before this context checks if the rcuwait is empty.
		156	*
		157	* The full memory barrier in rcuwait_wake_up() pairs with the full
		158	* memory barrier within set_current_state() of
		159	* ___rcuwait_wait_event(), which is called after prepare_to_rcuwait()
		160	* adds the waiter but before it has checked the wait condition.
		161	*
		162	* This pairs with nbcon_kthread_func:A.
		163	*/
		164	rcuwait_wake_up(&con->rcuwait); /* LMM(nbcon_kthread_wake:A) */
165	}
80		166
81	#else	167	#else
82		168
Lines 84-89 void nbcon_free(struct console *con); Link Here
84	#define PRINTK_MESSAGE_MAX 0	170	#define PRINTK_MESSAGE_MAX 0
85	#define PRINTKRB_RECORD_MAX 0	171	#define PRINTKRB_RECORD_MAX 0
86		172
		173	static inline void nbcon_kthread_wake(struct console *con) { }
		174	static inline void nbcon_kthread_create(struct console *con) { }
		175	#define printk_threads_enabled (false)
		176	#define printing_via_unlock (false)
		177
87	/*	178	/*
88	* In !PRINTK builds we still export console_sem	179	* In !PRINTK builds we still export console_sem
89	* semaphore and some of console functions (console_unlock()/etc.), so	180	* semaphore and some of console functions (console_unlock()/etc.), so
Lines 98-103 static inline void nbcon_seq_force(struct console *con, u64 seq) { } Link Here
98	static inline bool nbcon_alloc(struct console *con) { return false; }	189	static inline bool nbcon_alloc(struct console *con) { return false; }
99	static inline void nbcon_init(struct console *con) { }	190	static inline void nbcon_init(struct console *con) { }
100	static inline void nbcon_free(struct console *con) { }	191	static inline void nbcon_free(struct console *con) { }
		192	static inline enum nbcon_prio nbcon_get_default_prio(void) { return NBCON_PRIO_NONE; }
		193	static inline void nbcon_atomic_flush_all(void) { }
		194	static inline bool nbcon_legacy_emit_next_record(struct console con, bool handover,
		195	int cookie, bool use_atomic) { return false; }
		196
		197	static inline bool console_is_usable(struct console *con, short flags,
		198	bool use_atomic) { return false; }
101		199
102	#endif /* CONFIG_PRINTK */	200	#endif /* CONFIG_PRINTK */
103		201




#include <linux/kernel.h>
#include <linux/console.h>
#include <linux/delay.h>
#include <linux/kthread.h>
#include <linux/slab.h>
#include <linux/serial_core.h>
#include <linux/syscore_ops.h>
#include "printk_ringbuffer.h"
#include "internal.h"
/*
 * Printk console printing implementation for consoles which does not depend

	return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom);
}

#ifdef CONFIG_64BIT

#define __seq_to_nbcon_seq(seq) (seq)
#define __nbcon_seq_to_seq(seq) (seq)

#else /* CONFIG_64BIT */

#define __seq_to_nbcon_seq(seq) ((u32)seq)

static inline u64 __nbcon_seq_to_seq(u32 nbcon_seq)
{
	u64 seq;
	u64 rb_next_seq;

	/*
	 * The provided sequence is only the lower 32 bits of the ringbuffer
	 * sequence. It needs to be expanded to 64bit. Get the next sequence
	 * number from the ringbuffer and fold it.
	 *
	 * Having a 32bit representation in the console is sufficient.
	 * If a console ever gets more than 2^31 records behind
	 * the ringbuffer then this is the least of the problems.
	 *
	 * Also the access to the ring buffer is always safe.
	 */
	rb_next_seq = prb_next_seq(prb);
	seq = rb_next_seq - ((u32)rb_next_seq - nbcon_seq);

	return seq;
}

#endif /* CONFIG_64BIT */

/**
 * nbcon_seq_read - Read the current console sequence
 * @con:	Console to read the sequence of

{
	unsigned long nbcon_seq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_seq));

	return __ulseq_to_u64seq(prb, nbcon_seq);
}

/**

	 */
	u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb));

	atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __u64seq_to_ulseq(valid_seq));

	/* Clear con->seq since nbcon consoles use con->nbcon_seq instead. */
	con->seq = 0;

 */
static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq)
{
	unsigned long nbcon_seq = __u64seq_to_ulseq(ctxt->seq);
	struct console *con = ctxt->console;

	if (atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_seq), &nbcon_seq,
				    __u64seq_to_ulseq(new_seq))) {
		ctxt->seq = new_seq;
	} else {
		ctxt->seq = nbcon_seq_read(con);
	}
}

bool printk_threads_enabled __ro_after_init;

/**
 * nbcon_context_try_acquire_direct - Try to acquire directly
 * @ctxt:	The context of the caller

 * nbcon_context_try_acquire - Try to acquire nbcon console
 * @ctxt:	The context of the caller
 *
 * Context:	Any context which could not be migrated to another CPU.
 * Return:	True if the console was acquired. False otherwise.
 *
 * If the caller allowed an unsafe hostile takeover, on success the

 * in an unsafe state. Otherwise, on success the caller may assume
 * the console is not in an unsafe state.
 */
__maybe_unused
static bool nbcon_context_try_acquire(struct nbcon_context *ctxt)
{
	unsigned int cpu = smp_processor_id();

}
EXPORT_SYMBOL_GPL(nbcon_exit_unsafe);

/**
 * nbcon_reacquire - Reacquire a console after losing ownership
 * @wctxt:	The write context that was handed to the write function
 *
 * Since ownership can be lost at any time due to handover or takeover, a
 * printing context _should_ be prepared to back out immediately and
 * carefully. However, there are many scenarios where the context _must_
 * reacquire ownership in order to finalize or revert hardware changes.
 *
 * This function allows a context to reacquire ownership using the same
 * priority as its previous ownership.
 *
 * Note that for printing contexts, after a successful reacquire the
 * context will have no output buffer because that has been lost. This
 * function cannot be used to resume printing.
 */
void nbcon_reacquire(struct nbcon_write_context *wctxt)
{
	struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
	struct console *con = ctxt->console;
	struct nbcon_state cur;

	while (!nbcon_context_try_acquire(ctxt))
		cpu_relax();

	wctxt->outbuf = NULL;
	wctxt->len = 0;
	nbcon_state_read(con, &cur);
	wctxt->unsafe_takeover = cur.unsafe_takeover;
}
EXPORT_SYMBOL_GPL(nbcon_reacquire);

/**
 * nbcon_emit_next_record - Emit a record in the acquired context
 * @wctxt:	The write context that will be handed to the write function
 * @use_atomic:	True if the write_atomic callback is to be used
 *
 * Return:	True if this context still owns the console. False if
 *		ownership was handed over or taken.

 * When true is returned, @wctxt->ctxt.backlog indicates whether there are
 * still records pending in the ringbuffer,
 */
static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt, bool use_atomic)
static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt)
{
	struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
	struct console *con = ctxt->console;

	unsigned long con_dropped;
	struct nbcon_state cur;
	unsigned long dropped;
	bool done = false;

	/*
	 * The printk buffers are filled within an unsafe section. This

	nbcon_state_read(con, &cur);
	wctxt->unsafe_takeover = cur.unsafe_takeover;

	if (use_atomic &&
	    con->write_atomic) {
		done = con->write_atomic(con, wctxt);

	} else if (!use_atomic &&
		   con->write_thread) {
		done = con->write_thread(con, wctxt);
	}

	if (!done) {
		/*
		 * The emit was aborted, probably due to a loss of ownership.
		 * Ensure ownership was lost or released before reporting the
		 * loss.
		 */
		nbcon_context_release(ctxt);
		return false;
	}

	/*
	 * Since any dropped message was successfully output, reset the

	return nbcon_context_exit_unsafe(ctxt);
}

/**
 * nbcon_kthread_should_wakeup - Check whether a printer thread should wakeup
 * @con:	Console to operate on
 * @ctxt:	The acquire context that contains the state
 *		at console_acquire()
 *
 * Return:	True if the thread should shutdown or if the console is
 *		allowed to print and a record is available. False otherwise.
 *
 * After the thread wakes up, it must first check if it should shutdown before
 * attempting any printing.
 */
static bool nbcon_kthread_should_wakeup(struct console *con, struct nbcon_context *ctxt)
{
	bool is_usable;
	short flags;
	int cookie;

	if (kthread_should_stop())
		return true;

	cookie = console_srcu_read_lock();
	flags = console_srcu_read_flags(con);
	is_usable = console_is_usable(con, flags, false);
	console_srcu_read_unlock(cookie);

	if (!is_usable)
		return false;

	/* Bring the sequence in @ctxt up to date */
	ctxt->seq = nbcon_seq_read(con);

	return prb_read_valid(prb, ctxt->seq, NULL);
}

/**
 * nbcon_kthread_func - The printer thread function
 * @__console:	Console to operate on
 */
static int nbcon_kthread_func(void *__console)
{
	struct console *con = __console;
	struct nbcon_write_context wctxt = {
		.ctxt.console	= con,
		.ctxt.prio	= NBCON_PRIO_NORMAL,
	};
	struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
	unsigned long flags;
	short con_flags;
	bool backlog;
	int cookie;
	int ret;

wait_for_event:
	/*
	 * Guarantee this task is visible on the rcuwait before
	 * checking the wake condition.
	 *
	 * The full memory barrier within set_current_state() of
	 * ___rcuwait_wait_event() pairs with the full memory
	 * barrier within rcuwait_has_sleeper().
	 *
	 * This pairs with rcuwait_has_sleeper:A and nbcon_kthread_wake:A.
	 */
	ret = rcuwait_wait_event(&con->rcuwait,
				 nbcon_kthread_should_wakeup(con, ctxt),
				 TASK_INTERRUPTIBLE); /* LMM(nbcon_kthread_func:A) */

	if (kthread_should_stop())
		return 0;

	/* Wait was interrupted by a spurious signal, go back to sleep. */
	if (ret)
		goto wait_for_event;

	do {
		backlog = false;

		cookie = console_srcu_read_lock();

		con_flags = console_srcu_read_flags(con);

		if (console_is_usable(con, con_flags, false)) {
			con->driver_enter(con, &flags);

			/*
			 * Ensure this stays on the CPU to make handover and
			 * takeover possible.
			 */
			cant_migrate();

			if (nbcon_context_try_acquire(ctxt)) {
				/*
				 * If the emit fails, this context is no
				 * longer the owner.
				 */
				if (nbcon_emit_next_record(&wctxt, false)) {
					nbcon_context_release(ctxt);
					backlog = ctxt->backlog;
				}
			}

			con->driver_exit(con, flags);
		}

		console_srcu_read_unlock(cookie);

	} while (backlog);

	goto wait_for_event;
}

/**
 * nbcon_irq_work - irq work to wake printk thread
 * @irq_work:	The irq work to operate on
 */
static void nbcon_irq_work(struct irq_work *irq_work)
{
	struct console *con = container_of(irq_work, struct console, irq_work);

	nbcon_kthread_wake(con);
}

static inline bool rcuwait_has_sleeper(struct rcuwait *w)
{
	bool has_sleeper;

	rcu_read_lock();
	/*
	 * Guarantee any new records can be seen by tasks preparing to wait
	 * before this context checks if the rcuwait is empty.
	 *
	 * This full memory barrier pairs with the full memory barrier within
	 * set_current_state() of ___rcuwait_wait_event(), which is called
	 * after prepare_to_rcuwait() adds the waiter but before it has
	 * checked the wait condition.
	 *
	 * This pairs with nbcon_kthread_func:A.
	 */
	smp_mb(); /* LMM(rcuwait_has_sleeper:A) */
	has_sleeper = !!rcu_dereference(w->task);
	rcu_read_unlock();

	return has_sleeper;
}

/**
 * nbcon_wake_threads - Wake up printing threads using irq_work
 */
void nbcon_wake_threads(void)
{
	struct console *con;
	int cookie;

	cookie = console_srcu_read_lock();
	for_each_console_srcu(con) {
		/*
		 * Only schedule irq_work if the printing thread is
		 * actively waiting. If not waiting, the thread will
		 * notice by itself that it has work to do.
		 */
		if (con->kthread && rcuwait_has_sleeper(&con->rcuwait))
			irq_work_queue(&con->irq_work);
	}
	console_srcu_read_unlock(cookie);
}

/* Track the nbcon emergency nesting per CPU. */
static DEFINE_PER_CPU(unsigned int, nbcon_pcpu_emergency_nesting);
static unsigned int early_nbcon_pcpu_emergency_nesting __initdata;

/**
 * nbcon_get_cpu_emergency_nesting - Get the per CPU emergency nesting pointer
 *
 * Return:	Either a pointer to the per CPU emergency nesting counter of
 *		the current CPU or to the init data during early boot.
 */
static __ref unsigned int *nbcon_get_cpu_emergency_nesting(void)
{
	/*
	 * The value of __printk_percpu_data_ready gets set in normal
	 * context and before SMP initialization. As a result it could
	 * never change while inside an nbcon emergency section.
	 */
	if (!printk_percpu_data_ready())
		return &early_nbcon_pcpu_emergency_nesting;

	return this_cpu_ptr(&nbcon_pcpu_emergency_nesting);
}

/**
 * nbcon_emit_one - Print one record for an nbcon console using the
 *			specified callback
 * @wctxt:	An initialized write context struct to use
 *		for this context
 * @use_atomic:	True if the write_atomic callback is to be used
 *
 * Return:	False if the given console could not print a record or there
 *		are no more records to print, otherwise true.
 *
 * This is an internal helper to handle the locking of the console before
 * calling nbcon_emit_next_record().
 */
static bool nbcon_emit_one(struct nbcon_write_context *wctxt, bool use_atomic)
{
	struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);

	if (!nbcon_context_try_acquire(ctxt))
		return false;

	/*
	 * nbcon_emit_next_record() returns false when the console was
	 * handed over or taken over. In both cases the context is no
	 * longer valid.
	 */
	if (!nbcon_emit_next_record(wctxt, use_atomic))
		return false;

	nbcon_context_release(ctxt);

	return ctxt->backlog;
}

/**
 * nbcon_get_default_prio - The appropriate nbcon priority to use for nbcon
 *				printing on the current CPU
 *
 * Context:	Any context which could not be migrated to another CPU.
 * Return:	The nbcon_prio to use for acquiring an nbcon console in this
 *		context for printing.
 */
enum nbcon_prio nbcon_get_default_prio(void)
{
	unsigned int *cpu_emergency_nesting;

	if (this_cpu_in_panic())
		return NBCON_PRIO_PANIC;

	cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
	if (*cpu_emergency_nesting)
		return NBCON_PRIO_EMERGENCY;

	return NBCON_PRIO_NORMAL;
}

/**
 * nbcon_legacy_emit_next_record - Print one record for an nbcon console
 *					in legacy contexts
 * @con:	The console to print on
 * @handover:	Will be set to true if a printk waiter has taken over the
 *		console_lock, in which case the caller is no longer holding
 *		both the console_lock and the SRCU read lock. Otherwise it
 *		is set to false.
 * @cookie:	The cookie from the SRCU read lock.
 * @use_atomic:	True if the write_atomic callback is to be used
 *
 * Context:	Any context which could not be migrated to another CPU.
 * Return:	True if a record could be printed, otherwise false.
 *
 * This function is meant to be called by console_flush_all() to print records
 * on nbcon consoles from legacy context (printing via console unlocking).
 * Essentially it is the nbcon version of console_emit_next_record().
 */
bool nbcon_legacy_emit_next_record(struct console *con, bool *handover,
				   int cookie, bool use_atomic)
{
	struct nbcon_write_context wctxt = { };
	struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
	bool progress = false;
	unsigned long flags;

	*handover = false;

	ctxt->console = con;

	if (use_atomic) {
		/* Use the same procedure as console_emit_next_record(). */
		printk_safe_enter_irqsave(flags);
		console_lock_spinning_enable();
		stop_critical_timings();

		ctxt->prio = nbcon_get_default_prio();
		progress = nbcon_emit_one(&wctxt, use_atomic);

		start_critical_timings();
		*handover = console_lock_spinning_disable_and_check(cookie);
		printk_safe_exit_irqrestore(flags);
	} else {
		con->driver_enter(con, &flags);
		cant_migrate();

		ctxt->prio = nbcon_get_default_prio();
		progress = nbcon_emit_one(&wctxt, use_atomic);

		con->driver_exit(con, flags);
	}

	return progress;
}

/**
 * __nbcon_atomic_flush_all - Flush all nbcon consoles using their
 *					write_atomic() callback
 * @stop_seq:			Flush up until this record
 * @allow_unsafe_takeover:	True, to allow unsafe hostile takeovers
 */
static void __nbcon_atomic_flush_all(u64 stop_seq, bool allow_unsafe_takeover)
{
	struct nbcon_write_context wctxt = { };
	struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
	struct console *con;
	bool any_progress;
	int cookie;

	do {
		any_progress = false;

		cookie = console_srcu_read_lock();
		for_each_console_srcu(con) {
			short flags = console_srcu_read_flags(con);
			unsigned long irq_flags;

			if (!(flags & CON_NBCON))
				continue;

			if (!console_is_usable(con, flags, true))
				continue;

			if (nbcon_seq_read(con) >= stop_seq)
				continue;

			memset(ctxt, 0, sizeof(*ctxt));
			ctxt->console			= con;
			ctxt->spinwait_max_us		= 2000;
			ctxt->allow_unsafe_takeover	= allow_unsafe_takeover;

			/*
			 * Atomic flushing does not use console driver
			 * synchronization (i.e. it does not hold the port
			 * lock for uart consoles). Therefore IRQs must be
			 * disabled to avoid being interrupted and then
			 * calling into a driver that will deadlock trying
			 * acquire console ownership.
			 *
			 * This also disables migration in order to get the
			 * current CPU priority.
			 */
			local_irq_save(irq_flags);

			ctxt->prio = nbcon_get_default_prio();

			any_progress |= nbcon_emit_one(&wctxt, true);

			local_irq_restore(irq_flags);
		}
		console_srcu_read_unlock(cookie);
	} while (any_progress);
}

/**
 * nbcon_atomic_flush_all - Flush all nbcon consoles using their
 *				write_atomic() callback
 *
 * Flush the backlog up through the currently newest record. Any new
 * records added while flushing will not be flushed. This is to avoid
 * one CPU printing unbounded because other CPUs continue to add records.
 */
void nbcon_atomic_flush_all(void)
{
	__nbcon_atomic_flush_all(prb_next_reserve_seq(prb), false);
}

/**
 * nbcon_atomic_flush_unsafe - Flush all nbcon consoles using their
 *	write_atomic() callback and allowing unsafe hostile takeovers
 *
 * Flush the backlog up through the currently newest record. Unsafe hostile
 * takeovers will be performed, if necessary.
 */
void nbcon_atomic_flush_unsafe(void)
{
	__nbcon_atomic_flush_all(prb_next_reserve_seq(prb), true);
}

/**
 * nbcon_cpu_emergency_enter - Enter an emergency section where printk()
 *	messages for that CPU are only stored
 *
 * Upon exiting the emergency section, all stored messages are flushed.
 *
 * Context:	Any context. Disables preemption.
 *
 * When within an emergency section, no printing occurs on that CPU. This
 * is to allow all emergency messages to be dumped into the ringbuffer before
 * flushing the ringbuffer. The actual printing occurs when exiting the
 * outermost emergency section.
 */
void nbcon_cpu_emergency_enter(void)
{
	unsigned int *cpu_emergency_nesting;

	preempt_disable();

	cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
	(*cpu_emergency_nesting)++;
}

/**
 * nbcon_cpu_emergency_exit - Exit an emergency section and flush the
 *	stored messages
 *
 * Flushing only occurs when exiting all nesting for the CPU.
 *
 * Context:	Any context. Enables preemption.
 */
void nbcon_cpu_emergency_exit(void)
{
	unsigned int *cpu_emergency_nesting;
	bool do_trigger_flush = false;

	cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();

	WARN_ON_ONCE(*cpu_emergency_nesting == 0);

	if (*cpu_emergency_nesting == 1)
		do_trigger_flush = true;

	/* Undo the nesting count of nbcon_cpu_emergency_enter(). */
	(*cpu_emergency_nesting)--;

	preempt_enable();

	if (do_trigger_flush)
		printk_trigger_flush();
}

/**
 * nbcon_kthread_stop - Stop a printer thread
 * @con:	Console to operate on
 */
static void nbcon_kthread_stop(struct console *con)
{
	lockdep_assert_console_list_lock_held();

	if (!con->kthread)
		return;

	kthread_stop(con->kthread);
	con->kthread = NULL;
}

/**
 * nbcon_kthread_create - Create a printer thread
 * @con:	Console to operate on
 *
 * If it fails, let the console proceed. The atomic part might
 * be usable and useful.
 */
void nbcon_kthread_create(struct console *con)
{
	struct task_struct *kt;

	lockdep_assert_console_list_lock_held();

	if (!(con->flags & CON_NBCON) || !con->write_thread)
		return;

	if (!printk_threads_enabled || con->kthread)
		return;

	/*
	 * Printer threads cannot be started as long as any boot console is
	 * registered because there is no way to synchronize the hardware
	 * registers between boot console code and regular console code.
	 */
	if (have_boot_console)
		return;

	kt = kthread_run(nbcon_kthread_func, con, "pr/%s%d", con->name, con->index);
	if (IS_ERR(kt)) {
		con_printk(KERN_ERR, con, "failed to start printing thread\n");
		return;
	}

	con->kthread = kt;

	/*
	 * It is important that console printing threads are scheduled
	 * shortly after a printk call and with generous runtime budgets.
	 */
	sched_set_normal(con->kthread, -20);
}

static int __init printk_setup_threads(void)
{
	struct console *con;

	console_list_lock();
	printk_threads_enabled = true;
	for_each_console(con)
		nbcon_kthread_create(con);
	if (force_printkthreads() && printing_via_unlock)
		nbcon_legacy_kthread_create();
	console_list_unlock();
	return 0;
}
early_initcall(printk_setup_threads);

/**
 * nbcon_alloc - Allocate buffers needed by the nbcon console
 * @con:	Console to allocate buffers for

	/* nbcon_alloc() must have been called and successful! */
	BUG_ON(!con->pbufs);

	rcuwait_init(&con->rcuwait);
	init_irq_work(&con->irq_work, nbcon_irq_work);
	nbcon_seq_force(con, con->seq);
	nbcon_state_set(con, &state);
	nbcon_kthread_create(con);
}

/**

{
	struct nbcon_state state = { };

	nbcon_kthread_stop(con);
	nbcon_state_set(con, &state);

	/* Boot consoles share global printk buffers. */


	con->pbufs = NULL;
}

static inline bool uart_is_nbcon(struct uart_port *up)
{
	int cookie;
	bool ret;

	if (!uart_console(up))
		return false;

	cookie = console_srcu_read_lock();
	ret = (console_srcu_read_flags(up->cons) & CON_NBCON);
	console_srcu_read_unlock(cookie);
	return ret;
}

/**
 * uart_nbcon_acquire - The second half of the port locking wrapper
 * @up:		The uart port whose @lock was locked
 *
 * The uart_port_lock() wrappers will first lock the spin_lock @up->lock.
 * Then this function is called to implement nbcon-specific processing.
 *
 * If @up is an nbcon console, this console will be acquired and marked as
 * unsafe. Otherwise this function does nothing.
 *
 * nbcon consoles acquired via the port lock wrapper always use priority
 * NBCON_PRIO_NORMAL.
 */
void uart_nbcon_acquire(struct uart_port *up)
{
	struct console *con = up->cons;
	struct nbcon_context ctxt;

	if (!uart_is_nbcon(up))
		return;

	WARN_ON_ONCE(up->nbcon_locked_port);

	do {
		do {
			memset(&ctxt, 0, sizeof(ctxt));
			ctxt.console	= con;
			ctxt.prio	= NBCON_PRIO_NORMAL;
		} while (!nbcon_context_try_acquire(&ctxt));

	} while (!nbcon_context_enter_unsafe(&ctxt));

	up->nbcon_locked_port = true;
}
EXPORT_SYMBOL_GPL(uart_nbcon_acquire);

/**
 * uart_nbcon_release - The first half of the port unlocking wrapper
 * @up:		The uart port whose @lock is about to be unlocked
 *
 * The uart_port_unlock() wrappers will first call this function to implement
 * nbcon-specific processing. Then afterwards the uart_port_unlock() wrappers
 * will unlock the spin_lock @up->lock.
 *
 * If @up is an nbcon console, the console will be marked as safe and
 * released. Otherwise this function does nothing.
 *
 * nbcon consoles acquired via the port lock wrapper always use priority
 * NBCON_PRIO_NORMAL.
 */
void uart_nbcon_release(struct uart_port *up)
{
	struct console *con = up->cons;
	struct nbcon_context ctxt = {
		.console	= con,
		.prio		= NBCON_PRIO_NORMAL,
	};

	if (!up->nbcon_locked_port)
		return;

	if (nbcon_context_exit_unsafe(&ctxt))
		nbcon_context_release(&ctxt);

	up->nbcon_locked_port = false;
}
EXPORT_SYMBOL_GPL(uart_nbcon_release);

/**
 * printk_kthread_shutdown - shutdown all threaded printers
 *
 * On system shutdown all threaded printers are stopped. This allows printk
 * to transition back to atomic printing, thus providing a robust mechanism
 * for the final shutdown/reboot messages to be output.
 */
static void printk_kthread_shutdown(void)
{
	struct console *con;

	console_list_lock();
	for_each_console(con) {
		if (con->flags & CON_NBCON)
			nbcon_kthread_stop(con);
	}
	console_list_unlock();
}

static struct syscore_ops printk_syscore_ops = {
	.shutdown = printk_kthread_shutdown,
};

static int __init printk_init_ops(void)
{
	register_syscore_ops(&printk_syscore_ops);
	return 0;
}
device_initcall(printk_init_ops);

Lines 195-200 static int __init control_devkmsg(char *str) Link Here

(-)a/kernel/printk/printk.c (-146 / +531 lines)
195	}	195	}
196	__setup("printk.devkmsg=", control_devkmsg);	196	__setup("printk.devkmsg=", control_devkmsg);
197		197
		198	#if !defined(CONFIG_PREEMPT_RT)
		199	DEFINE_STATIC_KEY_FALSE(force_printkthreads_key);
		200
		201	static int __init setup_forced_printkthreads(char *arg)
		202	{
		203	static_branch_enable(&force_printkthreads_key);
		204	return 0;
		205	}
		206	early_param("threadprintk", setup_forced_printkthreads);
		207	#endif
		208
198	char devkmsg_log_str[DEVKMSG_STR_MAX_SIZE] = "ratelimit";	209	char devkmsg_log_str[DEVKMSG_STR_MAX_SIZE] = "ratelimit";
199	#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)	210	#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)
200	int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write,	211	int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write,
Lines 282-287 EXPORT_SYMBOL(console_list_unlock); Link Here
282	* Return: A cookie to pass to console_srcu_read_unlock().	293	* Return: A cookie to pass to console_srcu_read_unlock().
283	*/	294	*/
284	int console_srcu_read_lock(void)	295	int console_srcu_read_lock(void)
		296	__acquires(&console_srcu)
285	{	297	{
286	return srcu_read_lock_nmisafe(&console_srcu);	298	return srcu_read_lock_nmisafe(&console_srcu);
287	}	299	}
Lines 295-300 EXPORT_SYMBOL(console_srcu_read_lock); Link Here
295	* Counterpart to console_srcu_read_lock()	307	* Counterpart to console_srcu_read_lock()
296	*/	308	*/
297	void console_srcu_read_unlock(int cookie)	309	void console_srcu_read_unlock(int cookie)
		310	__releases(&console_srcu)
298	{	311	{
299	srcu_read_unlock_nmisafe(&console_srcu, cookie);	312	srcu_read_unlock_nmisafe(&console_srcu, cookie);
300	}	313	}
Lines 347-352 static bool panic_in_progress(void) Link Here
347	return unlikely(atomic_read(&panic_cpu) != PANIC_CPU_INVALID);	360	return unlikely(atomic_read(&panic_cpu) != PANIC_CPU_INVALID);
348	}	361	}
349		362
		363	/* Return true if a panic is in progress on the current CPU. */
		364	bool this_cpu_in_panic(void)
		365	{
		366	/*
		367	* We can use raw_smp_processor_id() here because it is impossible for
		368	* the task to be migrated to the panic_cpu, or away from it. If
		369	* panic_cpu has already been set, and we're not currently executing on
		370	* that CPU, then we never will be.
		371	*/
		372	return unlikely(atomic_read(&panic_cpu) == raw_smp_processor_id());
		373	}
		374
		375	/*
		376	* Return true if a panic is in progress on a remote CPU.
		377	*
		378	* On true, the local CPU should immediately release any printing resources
		379	* that may be needed by the panic CPU.
		380	*/
		381	bool other_cpu_in_panic(void)
		382	{
		383	return (panic_in_progress() && !this_cpu_in_panic());
		384	}
		385
350	/*	386	/*
351	* This is used for debugging the mess that is the VT code by	387	* This is used for debugging the mess that is the VT code by
352	* keeping track if we have the console semaphore held. It's	388	* keeping track if we have the console semaphore held. It's
Lines 438-451 static int console_msg_format = MSG_FORMAT_DEFAULT; Link Here
438	/* syslog_lock protects syslog_* variables and write access to clear_seq. */	474	/* syslog_lock protects syslog_* variables and write access to clear_seq. */
439	static DEFINE_MUTEX(syslog_lock);	475	static DEFINE_MUTEX(syslog_lock);
440		476
441	#ifdef CONFIG_PRINTK
442	/*	477	/*
443	* During panic, heavy printk by other CPUs can delay the	478	* Specifies if a legacy console is registered. If legacy consoles are
444	* panic and risk deadlock on console resources.	479	* present, it is necessary to perform the console_lock/console_unlock dance
		480	* whenever console flushing should occur.
445	*/	481	*/
446	static int __read_mostly suppress_panic_printk;	482	bool have_legacy_console;
447		483
		484	/*
		485	* Specifies if an nbcon console is registered. If nbcon consoles are present,
		486	* synchronous printing of legacy consoles will not occur during panic until
		487	* the backtrace has been stored to the ringbuffer.
		488	*/
		489	bool have_nbcon_console;
		490
		491	/*
		492	* Specifies if a boot console is registered. If boot consoles are present,
		493	* nbcon consoles cannot print simultaneously and must be synchronized by
		494	* the console lock. This is because boot consoles and nbcon consoles may
		495	* have mapped the same hardware.
		496	*/
		497	bool have_boot_console;
		498
		499	#ifdef CONFIG_PRINTK
448	DECLARE_WAIT_QUEUE_HEAD(log_wait);	500	DECLARE_WAIT_QUEUE_HEAD(log_wait);
		501
		502	static DECLARE_WAIT_QUEUE_HEAD(legacy_wait);
		503
449	/* All 3 protected by @syslog_lock. */	504	/* All 3 protected by @syslog_lock. */
450	/* the next printk record to read by syslog(READ) or /proc/kmsg */	505	/* the next printk record to read by syslog(READ) or /proc/kmsg */
451	static u64 syslog_seq;	506	static u64 syslog_seq;
Lines 1844-1855 static bool console_waiter; Link Here
1844	* there may be a waiter spinning (like a spinlock). Also it must be	1899	* there may be a waiter spinning (like a spinlock). Also it must be
1845	* ready to hand over the lock at the end of the section.	1900	* ready to hand over the lock at the end of the section.
1846	*/	1901	*/
1847	static void console_lock_spinning_enable(void)	1902	void console_lock_spinning_enable(void)
1848	{	1903	{
		1904	/*
		1905	* Do not use spinning in panic(). The panic CPU wants to keep the lock.
		1906	* Non-panic CPUs abandon the flush anyway.
		1907	*
		1908	* Just keep the lockdep annotation. The panic-CPU should avoid
		1909	* taking console_owner_lock because it might cause a deadlock.
		1910	* This looks like the easiest way how to prevent false lockdep
		1911	* reports without handling races a lockless way.
		1912	*/
		1913	if (panic_in_progress())
		1914	goto lockdep;
		1915
1849	raw_spin_lock(&console_owner_lock);	1916	raw_spin_lock(&console_owner_lock);
1850	console_owner = current;	1917	console_owner = current;
1851	raw_spin_unlock(&console_owner_lock);	1918	raw_spin_unlock(&console_owner_lock);
1852		1919
		1920	lockdep:
1853	/* The waiter may spin on us after setting console_owner */	1921	/* The waiter may spin on us after setting console_owner */
1854	spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);	1922	spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
1855	}	1923	}
Lines 1870-1879 static void console_lock_spinning_enable(void) Link Here
1870	*	1938	*
1871	* Return: 1 if the lock rights were passed, 0 otherwise.	1939	* Return: 1 if the lock rights were passed, 0 otherwise.
1872	*/	1940	*/
1873	static int console_lock_spinning_disable_and_check(int cookie)	1941	int console_lock_spinning_disable_and_check(int cookie)
1874	{	1942	{
1875	int waiter;	1943	int waiter;
1876		1944
		1945	/*
		1946	* Ignore spinning waiters during panic() because they might get stopped
		1947	* or blocked at any time,
		1948	*
		1949	* It is safe because nobody is allowed to start spinning during panic
		1950	* in the first place. If there has been a waiter then non panic CPUs
		1951	* might stay spinning. They would get stopped anyway. The panic context
		1952	* will never start spinning and an interrupted spin on panic CPU will
		1953	* never continue.
		1954	*/
		1955	if (panic_in_progress()) {
		1956	/* Keep lockdep happy. */
		1957	spin_release(&console_owner_dep_map, _THIS_IP_);
		1958	return 0;
		1959	}
		1960
1877	raw_spin_lock(&console_owner_lock);	1961	raw_spin_lock(&console_owner_lock);
1878	waiter = READ_ONCE(console_waiter);	1962	waiter = READ_ONCE(console_waiter);
1879	console_owner = NULL;	1963	console_owner = NULL;
Lines 2259-2313 int vprintk_store(int facility, int level, Link Here
2259	return ret;	2343	return ret;
2260	}	2344	}
2261		2345
		2346	static bool legacy_allow_panic_sync;
		2347
		2348	/*
		2349	* This acts as a one-way switch to allow legacy consoles to print from
		2350	* the printk() caller context on a panic CPU.
		2351	*/
		2352	void printk_legacy_allow_panic_sync(void)
		2353	{
		2354	legacy_allow_panic_sync = true;
		2355	}
		2356
2262	asmlinkage int vprintk_emit(int facility, int level,	2357	asmlinkage int vprintk_emit(int facility, int level,
2263	const struct dev_printk_info *dev_info,	2358	const struct dev_printk_info *dev_info,
2264	const char *fmt, va_list args)	2359	const char *fmt, va_list args)
2265	{	2360	{
		2361	bool do_trylock_unlock = printing_via_unlock &&
		2362	!force_printkthreads();
2266	int printed_len;	2363	int printed_len;
2267	bool in_sched = false;
2268		2364
2269	/* Suppress unimportant messages after panic happens */	2365	/* Suppress unimportant messages after panic happens */
2270	if (unlikely(suppress_printk))	2366	if (unlikely(suppress_printk))
2271	return 0;	2367	return 0;
2272		2368
2273	if (unlikely(suppress_panic_printk) &&	2369	/*
2274	atomic_read(&panic_cpu) != raw_smp_processor_id())	2370	* The messages on the panic CPU are the most important. If
		2371	* non-panic CPUs are generating any messages, they will be
		2372	* silently dropped.
		2373	*/
		2374	if (other_cpu_in_panic())
2275	return 0;	2375	return 0;
2276		2376
2277	if (level == LOGLEVEL_SCHED) {	2377	if (level == LOGLEVEL_SCHED) {
2278	level = LOGLEVEL_DEFAULT;	2378	level = LOGLEVEL_DEFAULT;
2279	in_sched = true;	2379	/* If called from the scheduler, we can not call up(). */
		2380	do_trylock_unlock = false;
2280	}	2381	}
2281		2382
2282	printk_delay(level);	2383	printk_delay(level);
2283		2384
2284	printed_len = vprintk_store(facility, level, dev_info, fmt, args);	2385	printed_len = vprintk_store(facility, level, dev_info, fmt, args);
2285		2386
2286	/* If called from the scheduler, we can not call up(). */	2387	if (!have_boot_console && have_nbcon_console) {
2287	if (!in_sched) {	2388	bool is_panic_context = this_cpu_in_panic();
		2389
		2390	/*
		2391	* In panic, the legacy consoles are not allowed to print from
		2392	* the printk calling context unless explicitly allowed. This
		2393	* gives the safe nbcon consoles a chance to print out all the
		2394	* panic messages first. This restriction only applies if
		2395	* there are nbcon consoles registered.
		2396	*/
		2397	if (is_panic_context)
		2398	do_trylock_unlock &= legacy_allow_panic_sync;
		2399
		2400	/*
		2401	* There are situations where nbcon atomic printing should
		2402	* happen in the printk() caller context:
		2403	*
		2404	* - When this CPU is in panic.
		2405	*
		2406	* - When booting, before the printing threads have been
		2407	* started.
		2408	*
		2409	* - During shutdown, since the printing threads may not get
		2410	* a chance to print the final messages.
		2411	*
		2412	* Note that if boot consoles are registered, the
		2413	* console_lock/console_unlock dance must be relied upon
		2414	* instead because nbcon consoles cannot print simultaneously
		2415	* with boot consoles.
		2416	*/
		2417	if (is_panic_context \|\|
		2418	!printk_threads_enabled \|\|
		2419	(system_state > SYSTEM_RUNNING)) {
		2420	nbcon_atomic_flush_all();
		2421	}
		2422	}
		2423
		2424	nbcon_wake_threads();
		2425
		2426	if (do_trylock_unlock) {
2288	/*	2427	/*
2289	* The caller may be holding system-critical or	2428	* The caller may be holding system-critical or
2290	* timing-sensitive locks. Disable preemption during	2429	* timing-sensitive locks. Disable preemption during
2291	* printing of all remaining records to all consoles so that	2430	* printing of all remaining records to all consoles so that
2292	* this context can return as soon as possible. Hopefully	2431	* this context can return as soon as possible. Hopefully
2293	* another printk() caller will take over the printing.	2432	* another printk() caller will take over the printing.
		2433	*
		2434	* Also, nbcon_get_default_prio() requires migration disabled.
2294	*/	2435	*/
2295	preempt_disable();	2436	preempt_disable();
		2437
2296	/*	2438	/*
2297	* Try to acquire and then immediately release the console	2439	* Do not emit for EMERGENCY priority. The console will be
2298	* semaphore. The release will print out buffers. With the	2440	* explicitly flushed when exiting the emergency section.
2299	* spinning variant, this context tries to take over the
2300	* printing from another printing context.
2301	*/	2441	*/
2302	if (console_trylock_spinning())	2442	if (nbcon_get_default_prio() == NBCON_PRIO_EMERGENCY) {
2303	console_unlock();	2443	do_trylock_unlock = false;
		2444	} else {
		2445	/*
		2446	* Try to acquire and then immediately release the
		2447	* console semaphore. The release will print out
		2448	* buffers. With the spinning variant, this context
		2449	* tries to take over the printing from another
		2450	* printing context.
		2451	*/
		2452	if (console_trylock_spinning())
		2453	console_unlock();
		2454	}
		2455
2304	preempt_enable();	2456	preempt_enable();
2305	}	2457	}
2306		2458
2307	if (in_sched)	2459	if (do_trylock_unlock)
2308	defer_console_output();
2309	else
2310	wake_up_klogd();	2460	wake_up_klogd();
		2461	else
		2462	defer_console_output();
2311		2463
2312	return printed_len;	2464	return printed_len;
2313	}	2465	}
Lines 2335-2340 EXPORT_SYMBOL(_printk); Link Here
2335	static bool pr_flush(int timeout_ms, bool reset_on_progress);	2487	static bool pr_flush(int timeout_ms, bool reset_on_progress);
2336	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress);	2488	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress);
2337		2489
		2490	static struct task_struct *nbcon_legacy_kthread;
		2491
		2492	static inline void wake_up_legacy_kthread(void)
		2493	{
		2494	if (nbcon_legacy_kthread)
		2495	wake_up_interruptible(&legacy_wait);
		2496	}
		2497
2338	#else /* CONFIG_PRINTK */	2498	#else /* CONFIG_PRINTK */
2339		2499
2340	#define printk_time false	2500	#define printk_time false
Lines 2348-2353 static u64 syslog_seq; Link Here
2348	static bool pr_flush(int timeout_ms, bool reset_on_progress) { return true; }	2508	static bool pr_flush(int timeout_ms, bool reset_on_progress) { return true; }
2349	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; }	2509	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; }
2350		2510
		2511	static inline void nbcon_legacy_kthread_create(void) { }
		2512	static inline void wake_up_legacy_kthread(void) { }
2351	#endif /* CONFIG_PRINTK */	2513	#endif /* CONFIG_PRINTK */
2352		2514
2353	#ifdef CONFIG_EARLY_PRINTK	2515	#ifdef CONFIG_EARLY_PRINTK
Lines 2563-2568 void suspend_console(void) Link Here
2563	void resume_console(void)	2725	void resume_console(void)
2564	{	2726	{
2565	struct console *con;	2727	struct console *con;
		2728	short flags;
		2729	int cookie;
2566		2730
2567	if (!console_suspend_enabled)	2731	if (!console_suspend_enabled)
2568	return;	2732	return;
Lines 2579-2584 void resume_console(void) Link Here
2579	*/	2743	*/
2580	synchronize_srcu(&console_srcu);	2744	synchronize_srcu(&console_srcu);
2581		2745
		2746	/*
		2747	* Since this runs in task context, wake the threaded printers
		2748	* directly rather than scheduling irq_work to do it.
		2749	*/
		2750	cookie = console_srcu_read_lock();
		2751	for_each_console_srcu(con) {
		2752	flags = console_srcu_read_flags(con);
		2753	if (flags & CON_NBCON)
		2754	nbcon_kthread_wake(con);
		2755	}
		2756	console_srcu_read_unlock(cookie);
		2757
		2758	wake_up_legacy_kthread();
		2759
2582	pr_flush(1000, true);	2760	pr_flush(1000, true);
2583	}	2761	}
2584		2762
Lines 2593-2599 void resume_console(void) Link Here
2593	*/	2771	*/
2594	static int console_cpu_notify(unsigned int cpu)	2772	static int console_cpu_notify(unsigned int cpu)
2595	{	2773	{
2596	if (!cpuhp_tasks_frozen) {	2774	if (!cpuhp_tasks_frozen && printing_via_unlock &&
		2775	!force_printkthreads()) {
2597	/* If trylock fails, someone else is doing the printing */	2776	/* If trylock fails, someone else is doing the printing */
2598	if (console_trylock())	2777	if (console_trylock())
2599	console_unlock();	2778	console_unlock();
Lines 2601-2626 static int console_cpu_notify(unsigned int cpu) Link Here
2601	return 0;	2780	return 0;
2602	}	2781	}
2603		2782
2604	/*
2605	* Return true if a panic is in progress on a remote CPU.
2606	*
2607	* On true, the local CPU should immediately release any printing resources
2608	* that may be needed by the panic CPU.
2609	*/
2610	bool other_cpu_in_panic(void)
2611	{
2612	if (!panic_in_progress())
2613	return false;
2614
2615	/*
2616	* We can use raw_smp_processor_id() here because it is impossible for
2617	* the task to be migrated to the panic_cpu, or away from it. If
2618	* panic_cpu has already been set, and we're not currently executing on
2619	* that CPU, then we never will be.
2620	*/
2621	return atomic_read(&panic_cpu) != raw_smp_processor_id();
2622	}
2623
2624	/**	2783	/**
2625	* console_lock - block the console subsystem from printing	2784	* console_lock - block the console subsystem from printing
2626	*	2785	*
Lines 2670-2705 int is_console_locked(void) Link Here
2670	}	2829	}
2671	EXPORT_SYMBOL(is_console_locked);	2830	EXPORT_SYMBOL(is_console_locked);
2672		2831
2673	/*
2674	* Check if the given console is currently capable and allowed to print
2675	* records.
2676	*
2677	* Requires the console_srcu_read_lock.
2678	*/
2679	static inline bool console_is_usable(struct console *con)
2680	{
2681	short flags = console_srcu_read_flags(con);
2682
2683	if (!(flags & CON_ENABLED))
2684	return false;
2685
2686	if ((flags & CON_SUSPENDED))
2687	return false;
2688
2689	if (!con->write)
2690	return false;
2691
2692	/*
2693	* Console drivers may assume that per-cpu resources have been
2694	* allocated. So unless they're explicitly marked as being able to
2695	* cope (CON_ANYTIME) don't call them until this CPU is officially up.
2696	*/
2697	if (!cpu_online(raw_smp_processor_id()) && !(flags & CON_ANYTIME))
2698	return false;
2699
2700	return true;
2701	}
2702
2703	static void __console_unlock(void)	2832	static void __console_unlock(void)
2704	{	2833	{
2705	console_locked = 0;	2834	console_locked = 0;
Lines 2776-2783 void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) Link Here
2776	bool printk_get_next_message(struct printk_message *pmsg, u64 seq,	2905	bool printk_get_next_message(struct printk_message *pmsg, u64 seq,
2777	bool is_extended, bool may_suppress)	2906	bool is_extended, bool may_suppress)
2778	{	2907	{
2779	static int panic_console_dropped;
2780
2781	struct printk_buffers *pbufs = pmsg->pbufs;	2908	struct printk_buffers *pbufs = pmsg->pbufs;
2782	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);	2909	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
2783	const size_t outbuf_sz = sizeof(pbufs->outbuf);	2910	const size_t outbuf_sz = sizeof(pbufs->outbuf);
Lines 2805-2821 bool printk_get_next_message(struct printk_message *pmsg, u64 seq, Link Here
2805	pmsg->seq = r.info->seq;	2932	pmsg->seq = r.info->seq;
2806	pmsg->dropped = r.info->seq - seq;	2933	pmsg->dropped = r.info->seq - seq;
2807		2934
2808	/*
2809	* Check for dropped messages in panic here so that printk
2810	* suppression can occur as early as possible if necessary.
2811	*/
2812	if (pmsg->dropped &&
2813	panic_in_progress() &&
2814	panic_console_dropped++ > 10) {
2815	suppress_panic_printk = 1;
2816	pr_warn_once("Too many dropped messages. Suppress messages on non-panic CPUs to prevent livelock.\n");
2817	}
2818
2819	/* Skip record that has level above the console loglevel. */	2935	/* Skip record that has level above the console loglevel. */
2820	if (may_suppress && suppress_message_printing(r.info->level))	2936	if (may_suppress && suppress_message_printing(r.info->level))
2821	goto out;	2937	goto out;
Lines 2832-2837 bool printk_get_next_message(struct printk_message *pmsg, u64 seq, Link Here
2832	return true;	2948	return true;
2833	}	2949	}
2834		2950
		2951	/*
		2952	* Legacy console printing from printk() caller context does not respect
		2953	* raw_spinlock/spinlock nesting. For !PREEMPT_RT the lockdep warning is a
		2954	* false positive. For PREEMPT_RT the false positive condition does not
		2955	* occur.
		2956	*
		2957	* This map is used to establish LD_WAIT_SLEEP context for the console write
		2958	* callbacks when legacy printing to avoid false positive lockdep complaints,
		2959	* thus allowing lockdep to continue to function for real issues.
		2960	*/
		2961	#ifdef CONFIG_PREEMPT_RT
		2962	static inline void printk_legacy_lock_map_acquire_try(void) { }
		2963	static inline void printk_legacy_lock_map_release(void) { }
		2964	#else
		2965	DEFINE_WAIT_OVERRIDE_MAP(printk_legacy_map, LD_WAIT_SLEEP);
		2966
		2967	static inline void printk_legacy_lock_map_acquire_try(void)
		2968	{
		2969	lock_map_acquire_try(&printk_legacy_map);
		2970	}
		2971
		2972	static inline void printk_legacy_lock_map_release(void)
		2973	{
		2974	lock_map_release(&printk_legacy_map);
		2975	}
		2976	#endif /* CONFIG_PREEMPT_RT */
		2977
2835	/*	2978	/*
2836	* Used as the printk buffers for non-panic, serialized console printing.	2979	* Used as the printk buffers for non-panic, serialized console printing.
2837	* This is for legacy (!CON_NBCON) as well as all boot (CON_BOOT) consoles.	2980	* This is for legacy (!CON_NBCON) as well as all boot (CON_BOOT) consoles.
Lines 2881-2911 static bool console_emit_next_record(struct console con, bool handover, int co Link Here
2881	con->dropped = 0;	3024	con->dropped = 0;
2882	}	3025	}
2883		3026
2884	/*
2885	* While actively printing out messages, if another printk()
2886	* were to occur on another CPU, it may wait for this one to
2887	* finish. This task can not be preempted if there is a
2888	* waiter waiting to take over.
2889	*
2890	* Interrupts are disabled because the hand over to a waiter
2891	* must not be interrupted until the hand over is completed
2892	* (@console_waiter is cleared).
2893	*/
2894	printk_safe_enter_irqsave(flags);
2895	console_lock_spinning_enable();
2896
2897	/* Do not trace print latency. */
2898	stop_critical_timings();
2899
2900	/* Write everything out to the hardware. */	3027	/* Write everything out to the hardware. */
2901	con->write(con, outbuf, pmsg.outbuf_len);
2902		3028
2903	start_critical_timings();	3029	if (force_printkthreads()) {
		3030	/*
		3031	* With forced threading this function is either in a thread
		3032	* or panic context. So there is no need for concern about
		3033	* printk reentrance, handovers, or lockdep complaints.
		3034	*/
2904		3035
2905	con->seq = pmsg.seq + 1;	3036	con->write(con, outbuf, pmsg.outbuf_len);
		3037	con->seq = pmsg.seq + 1;
		3038	} else {
		3039	/*
		3040	* While actively printing out messages, if another printk()
		3041	* were to occur on another CPU, it may wait for this one to
		3042	* finish. This task can not be preempted if there is a
		3043	* waiter waiting to take over.
		3044	*
		3045	* Interrupts are disabled because the hand over to a waiter
		3046	* must not be interrupted until the hand over is completed
		3047	* (@console_waiter is cleared).
		3048	*/
		3049	printk_safe_enter_irqsave(flags);
		3050	console_lock_spinning_enable();
2906		3051
2907	*handover = console_lock_spinning_disable_and_check(cookie);	3052	/* Do not trace print latency. */
2908	printk_safe_exit_irqrestore(flags);	3053	stop_critical_timings();
		3054
		3055	printk_legacy_lock_map_acquire_try();
		3056	con->write(con, outbuf, pmsg.outbuf_len);
		3057	printk_legacy_lock_map_release();
		3058
		3059	start_critical_timings();
		3060
		3061	con->seq = pmsg.seq + 1;
		3062
		3063	*handover = console_lock_spinning_disable_and_check(cookie);
		3064	printk_safe_exit_irqrestore(flags);
		3065	}
2909	skip:	3066	skip:
2910	return true;	3067	return true;
2911	}	3068	}
Lines 2958-2970 static bool console_flush_all(bool do_cond_resched, u64 next_seq, bool handove Link Here
2958		3115
2959	cookie = console_srcu_read_lock();	3116	cookie = console_srcu_read_lock();
2960	for_each_console_srcu(con) {	3117	for_each_console_srcu(con) {
		3118	short flags = console_srcu_read_flags(con);
		3119	u64 printk_seq;
2961	bool progress;	3120	bool progress;
2962		3121
2963	if (!console_is_usable(con))	3122	/*
		3123	* console_flush_all() is only for legacy consoles,
		3124	* unless the nbcon console has no kthread printer.
		3125	*/
		3126	if ((flags & CON_NBCON) && con->kthread)
		3127	continue;
		3128
		3129	if (!console_is_usable(con, flags, !do_cond_resched))
2964	continue;	3130	continue;
2965	any_usable = true;	3131	any_usable = true;
2966		3132
2967	progress = console_emit_next_record(con, handover, cookie);	3133	if (flags & CON_NBCON) {
		3134	progress = nbcon_legacy_emit_next_record(con, handover, cookie,
		3135	!do_cond_resched);
		3136	printk_seq = nbcon_seq_read(con);
		3137	} else {
		3138	progress = console_emit_next_record(con, handover, cookie);
		3139
		3140	printk_seq = con->seq;
		3141	}
2968		3142
2969	/*	3143	/*
2970	* If a handover has occurred, the SRCU read lock	3144	* If a handover has occurred, the SRCU read lock
Lines 2974-2981 static bool console_flush_all(bool do_cond_resched, u64 next_seq, bool handove Link Here
2974	return false;	3148	return false;
2975		3149
2976	/* Track the next of the highest seq flushed. */	3150	/* Track the next of the highest seq flushed. */
2977	if (con->seq > *next_seq)	3151	if (printk_seq > *next_seq)
2978	*next_seq = con->seq;	3152	*next_seq = printk_seq;
2979		3153
2980	if (!progress)	3154	if (!progress)
2981	continue;	3155	continue;
Lines 2998-3016 static bool console_flush_all(bool do_cond_resched, u64 next_seq, bool handove Link Here
2998	return false;	3172	return false;
2999	}	3173	}
3000		3174
3001	/**	3175	static void console_flush_and_unlock(void)
3002	* console_unlock - unblock the console subsystem from printing
3003	*
3004	* Releases the console_lock which the caller holds to block printing of
3005	* the console subsystem.
3006	*
3007	* While the console_lock was held, console output may have been buffered
3008	* by printk(). If this is the case, console_unlock(); emits
3009	* the output prior to releasing the lock.
3010	*
3011	* console_unlock(); may be called from any context.
3012	*/
3013	void console_unlock(void)
3014	{	3176	{
3015	bool do_cond_resched;	3177	bool do_cond_resched;
3016	bool handover;	3178	bool handover;
Lines 3054-3059 void console_unlock(void) Link Here
3054	*/	3216	*/
3055	} while (prb_read_valid(prb, next_seq, NULL) && console_trylock());	3217	} while (prb_read_valid(prb, next_seq, NULL) && console_trylock());
3056	}	3218	}
		3219
		3220	/**
		3221	* console_unlock - unblock the console subsystem from printing
		3222	*
		3223	* Releases the console_lock which the caller holds to block printing of
		3224	* the console subsystem.
		3225	*
		3226	* While the console_lock was held, console output may have been buffered
		3227	* by printk(). If this is the case, console_unlock(); emits
		3228	* the output prior to releasing the lock.
		3229	*
		3230	* console_unlock(); may be called from any context.
		3231	*/
		3232	void console_unlock(void)
		3233	{
		3234	/*
		3235	* Forced threading relies on kthread and atomic consoles for
		3236	* printing. It never attempts to print from console_unlock().
		3237	*/
		3238	if (force_printkthreads()) {
		3239	__console_unlock();
		3240	return;
		3241	}
		3242
		3243	console_flush_and_unlock();
		3244	}
3057	EXPORT_SYMBOL(console_unlock);	3245	EXPORT_SYMBOL(console_unlock);
3058		3246
3059	/**	3247	/**
Lines 3187-3193 void console_flush_on_panic(enum con_flush_mode mode) Link Here
3187	console_srcu_read_unlock(cookie);	3375	console_srcu_read_unlock(cookie);
3188	}	3376	}
3189		3377
3190	console_flush_all(false, &next_seq, &handover);	3378	nbcon_atomic_flush_all();
		3379
		3380	if (printing_via_unlock)
		3381	console_flush_all(false, &next_seq, &handover);
3191	}	3382	}
3192		3383
3193	/*	3384	/*
Lines 3244-3256 EXPORT_SYMBOL(console_stop); Link Here
3244		3435
3245	void console_start(struct console *console)	3436	void console_start(struct console *console)
3246	{	3437	{
		3438	short flags;
		3439
3247	console_list_lock();	3440	console_list_lock();
3248	console_srcu_write_flags(console, console->flags \| CON_ENABLED);	3441	console_srcu_write_flags(console, console->flags \| CON_ENABLED);
		3442	flags = console->flags;
3249	console_list_unlock();	3443	console_list_unlock();
		3444
		3445	/*
		3446	* Ensure that all SRCU list walks have completed. The related
		3447	* printing context must be able to see it is enabled so that
		3448	* it is guaranteed to wake up and resume printing.
		3449	*/
		3450	synchronize_srcu(&console_srcu);
		3451
		3452	if (flags & CON_NBCON)
		3453	nbcon_kthread_wake(console);
		3454	else
		3455	wake_up_legacy_kthread();
		3456
3250	__pr_flush(console, 1000, true);	3457	__pr_flush(console, 1000, true);
3251	}	3458	}
3252	EXPORT_SYMBOL(console_start);	3459	EXPORT_SYMBOL(console_start);
3253		3460
		3461	#ifdef CONFIG_PRINTK
		3462	static bool printer_should_wake(void)
		3463	{
		3464	bool available = false;
		3465	struct console *con;
		3466	int cookie;
		3467
		3468	if (kthread_should_stop())
		3469	return true;
		3470
		3471	cookie = console_srcu_read_lock();
		3472	for_each_console_srcu(con) {
		3473	short flags = console_srcu_read_flags(con);
		3474	u64 printk_seq;
		3475
		3476	/*
		3477	* The legacy printer thread is only for legacy consoles,
		3478	* unless the nbcon console has no kthread printer.
		3479	*/
		3480	if ((flags & CON_NBCON) && con->kthread)
		3481	continue;
		3482
		3483	if (!console_is_usable(con, flags, true))
		3484	continue;
		3485
		3486	if (flags & CON_NBCON) {
		3487	printk_seq = nbcon_seq_read(con);
		3488	} else {
		3489	/*
		3490	* It is safe to read @seq because only this
		3491	* thread context updates @seq.
		3492	*/
		3493	printk_seq = con->seq;
		3494	}
		3495
		3496	if (prb_read_valid(prb, printk_seq, NULL)) {
		3497	available = true;
		3498	break;
		3499	}
		3500	}
		3501	console_srcu_read_unlock(cookie);
		3502
		3503	return available;
		3504	}
		3505
		3506	static int nbcon_legacy_kthread_func(void *unused)
		3507	{
		3508	int error;
		3509
		3510	for (;;) {
		3511	error = wait_event_interruptible(legacy_wait, printer_should_wake());
		3512
		3513	if (kthread_should_stop())
		3514	break;
		3515
		3516	if (error)
		3517	continue;
		3518
		3519	console_lock();
		3520	console_flush_and_unlock();
		3521	}
		3522
		3523	return 0;
		3524	}
3525
3526	void nbcon_legacy_kthread_create(void)
3527	{
3528	struct task_struct *kt;
3529
3530	lockdep_assert_held(&console_mutex);
3531
3532	if (!force_printkthreads())
3533	return;
3534
3535	if (!printk_threads_enabled \|\| nbcon_legacy_kthread)
3536	return;
3537
3538	kt = kthread_run(nbcon_legacy_kthread_func, NULL, "pr/legacy");
3539	if (IS_ERR(kt)) {
3540	pr_err("unable to start legacy printing thread\n");
3541	return;
3542	}
3543
3544	nbcon_legacy_kthread = kt;
3545
3546	/*
3547	* It is important that console printing threads are scheduled
3548	* shortly after a printk call and with generous runtime budgets.
3549	*/
3550	sched_set_normal(nbcon_legacy_kthread, -20);
3551	}
3552	#endif /* CONFIG_PRINTK */
3553
3254	static int __read_mostly keep_bootcon;	3554	static int __read_mostly keep_bootcon;
3255		3555
3256	static int __init keep_bootcon_setup(char *str)	3556	static int __init keep_bootcon_setup(char *str)
Lines 3382-3392 static void console_init_seq(struct console *newcon, bool bootcon_registered) Link Here
3382		3682
3383	newcon->seq = prb_next_seq(prb);	3683	newcon->seq = prb_next_seq(prb);
3384	for_each_console(con) {	3684	for_each_console(con) {
3385	if ((con->flags & CON_BOOT) &&	3685	u64 seq;
3386	(con->flags & CON_ENABLED) &&	3686
3387	con->seq < newcon->seq) {	3687	if (!((con->flags & CON_BOOT) &&
3388	newcon->seq = con->seq;	3688	(con->flags & CON_ENABLED))) {
		3689	continue;
3389	}	3690	}
		3691
		3692	if (con->flags & CON_NBCON)
		3693	seq = nbcon_seq_read(con);
		3694	else
		3695	seq = con->seq;
		3696
		3697	if (seq < newcon->seq)
		3698	newcon->seq = seq;
3390	}	3699	}
3391	}	3700	}
3392		3701
Lines 3503-3510 void register_console(struct console *newcon) Link Here
3503	newcon->dropped = 0;	3812	newcon->dropped = 0;
3504	console_init_seq(newcon, bootcon_registered);	3813	console_init_seq(newcon, bootcon_registered);
3505		3814
3506	if (newcon->flags & CON_NBCON)	3815	if (newcon->flags & CON_NBCON) {
		3816	have_nbcon_console = true;
3507	nbcon_init(newcon);	3817	nbcon_init(newcon);
		3818	} else {
		3819	have_legacy_console = true;
		3820	nbcon_legacy_kthread_create();
		3821	}
		3822
		3823	if (newcon->flags & CON_BOOT)
		3824	have_boot_console = true;
3508		3825
3509	/*	3826	/*
3510	* Put this console in the list - keep the	3827	* Put this console in the list - keep the
Lines 3558-3563 EXPORT_SYMBOL(register_console); Link Here
3558	/* Must be called under console_list_lock(). */	3875	/* Must be called under console_list_lock(). */
3559	static int unregister_console_locked(struct console *console)	3876	static int unregister_console_locked(struct console *console)
3560	{	3877	{
		3878	bool is_boot_con = (console->flags & CON_BOOT);
		3879	bool found_legacy_con = false;
		3880	bool found_nbcon_con = false;
		3881	bool found_boot_con = false;
		3882	struct console *c;
3561	int res;	3883	int res;
3562		3884
3563	lockdep_assert_console_list_lock_held();	3885	lockdep_assert_console_list_lock_held();
Lines 3605-3610 static int unregister_console_locked(struct console *console) Link Here
3605	if (console->exit)	3927	if (console->exit)
3606	res = console->exit(console);	3928	res = console->exit(console);
3607		3929
		3930	/*
		3931	* With this console gone, the global flags tracking registered
		3932	* console types may have changed. Update them.
		3933	*/
		3934	for_each_console(c) {
		3935	if (c->flags & CON_BOOT)
		3936	found_boot_con = true;
		3937
		3938	if (c->flags & CON_NBCON)
		3939	found_nbcon_con = true;
		3940	else
		3941	found_legacy_con = true;
		3942	}
		3943	if (!found_boot_con)
		3944	have_boot_console = false;
		3945	if (!found_legacy_con)
		3946	have_legacy_console = false;
		3947	if (!found_nbcon_con)
		3948	have_nbcon_console = false;
		3949
		3950	/*
		3951	* When the last boot console unregisters, start up the
		3952	* printing threads.
		3953	*/
		3954	if (is_boot_con && !have_boot_console) {
		3955	for_each_console(c)
		3956	nbcon_kthread_create(c);
		3957	}
		3958
		3959	#ifdef CONFIG_PRINTK
		3960	if (!printing_via_unlock && nbcon_legacy_kthread) {
		3961	kthread_stop(nbcon_legacy_kthread);
		3962	nbcon_legacy_kthread = NULL;
		3963	}
		3964	#endif
		3965
3608	return res;	3966	return res;
3609	}	3967	}
3610		3968
Lines 3755-3785 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre Link Here
3755	u64 last_diff = 0;	4113	u64 last_diff = 0;
3756	u64 printk_seq;	4114	u64 printk_seq;
3757	short flags;	4115	short flags;
		4116	bool locked;
3758	int cookie;	4117	int cookie;
3759	u64 diff;	4118	u64 diff;
3760	u64 seq;	4119	u64 seq;
3761		4120
3762	might_sleep();	4121	might_sleep();
3763		4122
3764	seq = prb_next_seq(prb);	4123	seq = prb_next_reserve_seq(prb);
3765		4124
3766	/* Flush the consoles so that records up to @seq are printed. */	4125	/*
3767	console_lock();	4126	* Flush the consoles so that records up to @seq are printed.
3768	console_unlock();	4127	* Otherwise this function will just wait for the threaded printers
		4128	* to print up to @seq.
		4129	*/
		4130	if (printing_via_unlock && !force_printkthreads()) {
		4131	console_lock();
		4132	console_unlock();
		4133	}
3769		4134
3770	for (;;) {	4135	for (;;) {
3771	unsigned long begin_jiffies;	4136	unsigned long begin_jiffies;
3772	unsigned long slept_jiffies;	4137	unsigned long slept_jiffies;
3773		4138
		4139	locked = false;
3774	diff = 0;	4140	diff = 0;
3775		4141
3776	/*	4142	if (printing_via_unlock) {
3777	* Hold the console_lock to guarantee safe access to	4143	/*
3778	* console->seq. Releasing console_lock flushes more	4144	* Hold the console_lock to guarantee safe access to
3779	* records in case @seq is still not printed on all	4145	* console->seq. Releasing console_lock flushes more
3780	* usable consoles.	4146	* records in case @seq is still not printed on all
3781	*/	4147	* usable consoles.
3782	console_lock();	4148	*/
		4149	console_lock();
		4150	locked = true;
		4151	}
3783		4152
3784	cookie = console_srcu_read_lock();	4153	cookie = console_srcu_read_lock();
3785	for_each_console_srcu(c) {	4154	for_each_console_srcu(c) {
Lines 3793-3804 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre Link Here
3793	* that they make forward progress, so only increment	4162	* that they make forward progress, so only increment
3794	* @diff for usable consoles.	4163	* @diff for usable consoles.
3795	*/	4164	*/
3796	if (!console_is_usable(c))	4165	if (!console_is_usable(c, flags, true) &&
		4166	!console_is_usable(c, flags, false)) {
3797	continue;	4167	continue;
		4168	}
3798		4169
3799	if (flags & CON_NBCON) {	4170	if (flags & CON_NBCON) {
3800	printk_seq = nbcon_seq_read(c);	4171	printk_seq = nbcon_seq_read(c);
3801	} else {	4172	} else {
		4173	WARN_ON_ONCE(!locked);
3802	printk_seq = c->seq;	4174	printk_seq = c->seq;
3803	}	4175	}
3804		4176
Lines 3810-3816 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre Link Here
3810	if (diff != last_diff && reset_on_progress)	4182	if (diff != last_diff && reset_on_progress)
3811	remaining_jiffies = timeout_jiffies;	4183	remaining_jiffies = timeout_jiffies;
3812		4184
3813	console_unlock();	4185	if (locked)
		4186	console_unlock();
3814		4187
3815	/* Note: @diff is 0 if there are no usable consoles. */	4188	/* Note: @diff is 0 if there are no usable consoles. */
3816	if (diff == 0 \|\| remaining_jiffies == 0)	4189	if (diff == 0 \|\| remaining_jiffies == 0)
Lines 3862-3870 static void wake_up_klogd_work_func(struct irq_work *irq_work) Link Here
3862	int pending = this_cpu_xchg(printk_pending, 0);	4235	int pending = this_cpu_xchg(printk_pending, 0);
3863		4236
3864	if (pending & PRINTK_PENDING_OUTPUT) {	4237	if (pending & PRINTK_PENDING_OUTPUT) {
3865	/* If trylock fails, someone else is doing the printing */	4238	if (force_printkthreads()) {
3866	if (console_trylock())	4239	wake_up_legacy_kthread();
3867	console_unlock();	4240	} else {
		4241	/*
		4242	* If trylock fails, some other context
		4243	* will do the printing.
		4244	*/
		4245	if (console_trylock())
		4246	console_unlock();
		4247	}
3868	}	4248	}
3869		4249
3870	if (pending & PRINTK_PENDING_WAKEUP)	4250	if (pending & PRINTK_PENDING_WAKEUP)
Lines 3932-3942 void defer_console_output(void) Link Here
3932	* New messages may have been added directly to the ringbuffer	4312	* New messages may have been added directly to the ringbuffer
3933	* using vprintk_store(), so wake any waiters as well.	4313	* using vprintk_store(), so wake any waiters as well.
3934	*/	4314	*/
3935	__wake_up_klogd(PRINTK_PENDING_WAKEUP \| PRINTK_PENDING_OUTPUT);	4315	int val = PRINTK_PENDING_WAKEUP;
		4316
		4317	if (printing_via_unlock)
		4318	val \|= PRINTK_PENDING_OUTPUT;
		4319	__wake_up_klogd(val);
3936	}	4320	}
3937		4321
3938	void printk_trigger_flush(void)	4322	void printk_trigger_flush(void)
3939	{	4323	{
		4324	nbcon_wake_threads();
3940	defer_console_output();	4325	defer_console_output();
3941	}	4326	}
3942		4327

Lines 6-11 Link Here

(-)a/kernel/printk/printk_ringbuffer.c (-54 / +283 lines)
6	#include <linux/errno.h>	6	#include <linux/errno.h>
7	#include <linux/bug.h>	7	#include <linux/bug.h>
8	#include "printk_ringbuffer.h"	8	#include "printk_ringbuffer.h"
		9	#include "internal.h"
9		10
10	/**	11	/**
11	* DOC: printk_ringbuffer overview	12	* DOC: printk_ringbuffer overview
Lines 303-308 Link Here
303	*	304	*
304	* desc_push_tail:B / desc_reserve:D	305	* desc_push_tail:B / desc_reserve:D
305	* set descriptor reusable (state), then push descriptor tail (id)	306	* set descriptor reusable (state), then push descriptor tail (id)
		307	*
		308	* desc_update_last_finalized:A / desc_last_finalized_seq:A
		309	* store finalized record, then set new highest finalized sequence number
306	*/	310	*/
307		311
308	#define DATA_SIZE(data_ring) _DATA_SIZE((data_ring)->size_bits)	312	#define DATA_SIZE(data_ring) _DATA_SIZE((data_ring)->size_bits)
Lines 1030-1038 static char data_alloc(struct printk_ringbuffer rb, unsigned int size, Link Here
1030	unsigned long next_lpos;	1034	unsigned long next_lpos;
1031		1035
1032	if (size == 0) {	1036	if (size == 0) {
1033	/* Specify a data-less block. */	1037	/*
1034	blk_lpos->begin = NO_LPOS;	1038	* Data blocks are not created for empty lines. Instead, the
1035	blk_lpos->next = NO_LPOS;	1039	* reader will recognize these special lpos values and handle
		1040	* it appropriately.
		1041	*/
		1042	blk_lpos->begin = EMPTY_LINE_LPOS;
		1043	blk_lpos->next = EMPTY_LINE_LPOS;
1036	return NULL;	1044	return NULL;
1037	}	1045	}
1038		1046
Lines 1210-1219 static const char get_data(struct prb_data_ring data_ring, Link Here
1210		1218
1211	/* Data-less data block description. */	1219	/* Data-less data block description. */
1212	if (BLK_DATALESS(blk_lpos)) {	1220	if (BLK_DATALESS(blk_lpos)) {
1213	if (blk_lpos->begin == NO_LPOS && blk_lpos->next == NO_LPOS) {	1221	/*
		1222	* Records that are just empty lines are also valid, even
		1223	* though they do not have a data block. For such records
		1224	* explicitly return empty string data to signify success.
		1225	*/
		1226	if (blk_lpos->begin == EMPTY_LINE_LPOS &&
		1227	blk_lpos->next == EMPTY_LINE_LPOS) {
1214	*data_size = 0;	1228	*data_size = 0;
1215	return "";	1229	return "";
1216	}	1230	}
		1231
		1232	/* Data lost, invalid, or otherwise unavailable. */
1217	return NULL;	1233	return NULL;
1218	}	1234	}
1219		1235
Lines 1441-1460 bool prb_reserve_in_last(struct prb_reserved_entry *e, struct printk_ringbuffer Link Here
1441	return false;	1457	return false;
1442	}	1458	}
1443		1459
		1460	/*
		1461	* @last_finalized_seq value guarantees that all records up to and including
		1462	* this sequence number are finalized and can be read. The only exception are
		1463	* too old records which have already been overwritten.
		1464	*
		1465	* It is also guaranteed that @last_finalized_seq only increases.
		1466	*
		1467	* Be aware that finalized records following non-finalized records are not
		1468	* reported because they are not yet available to the reader. For example,
		1469	* a new record stored via printk() will not be available to a printer if
		1470	* it follows a record that has not been finalized yet. However, once that
		1471	* non-finalized record becomes finalized, @last_finalized_seq will be
		1472	* appropriately updated and the full set of finalized records will be
		1473	* available to the printer. And since each printk() caller will either
		1474	* directly print or trigger deferred printing of all available unprinted
		1475	* records, all printk() messages will get printed.
		1476	*/
		1477	static u64 desc_last_finalized_seq(struct printk_ringbuffer *rb)
		1478	{
		1479	struct prb_desc_ring *desc_ring = &rb->desc_ring;
		1480	unsigned long ulseq;
		1481
		1482	/*
		1483	* Guarantee the sequence number is loaded before loading the
		1484	* associated record in order to guarantee that the record can be
		1485	* seen by this CPU. This pairs with desc_update_last_finalized:A.
		1486	*/
		1487	ulseq = atomic_long_read_acquire(&desc_ring->last_finalized_seq
		1488	); /* LMM(desc_last_finalized_seq:A) */
		1489
		1490	return __ulseq_to_u64seq(rb, ulseq);
		1491	}
		1492
		1493	static bool _prb_read_valid(struct printk_ringbuffer rb, u64 seq,
		1494	struct printk_record r, unsigned int line_count);
		1495
		1496	/*
		1497	* Check if there are records directly following @last_finalized_seq that are
		1498	* finalized. If so, update @last_finalized_seq to the latest of these
		1499	* records. It is not allowed to skip over records that are not yet finalized.
		1500	*/
		1501	static void desc_update_last_finalized(struct printk_ringbuffer *rb)
		1502	{
		1503	struct prb_desc_ring *desc_ring = &rb->desc_ring;
		1504	u64 old_seq = desc_last_finalized_seq(rb);
		1505	unsigned long oldval;
		1506	unsigned long newval;
		1507	u64 finalized_seq;
		1508	u64 try_seq;
		1509
		1510	try_again:
		1511	finalized_seq = old_seq;
		1512	try_seq = finalized_seq + 1;
		1513
		1514	/* Try to find later finalized records. */
		1515	while (_prb_read_valid(rb, &try_seq, NULL, NULL)) {
		1516	finalized_seq = try_seq;
		1517	try_seq++;
		1518	}
		1519
		1520	/* No update needed if no later finalized record was found. */
		1521	if (finalized_seq == old_seq)
		1522	return;
		1523
1524	oldval = __u64seq_to_ulseq(old_seq);
1525	newval = __u64seq_to_ulseq(finalized_seq);
1526
1527	/*
1528	* Set the sequence number of a later finalized record that has been
1529	* seen.
1530	*
1531	* Guarantee the record data is visible to other CPUs before storing
1532	* its sequence number. This pairs with desc_last_finalized_seq:A.
1533	*
1534	* Memory barrier involvement:
1535	*
1536	* If desc_last_finalized_seq:A reads from
1537	* desc_update_last_finalized:A, then desc_read:A reads from
1538	* _prb_commit:B.
1539	*
1540	* Relies on:
1541	*
1542	* RELEASE from _prb_commit:B to desc_update_last_finalized:A
1543	* matching
1544	* ACQUIRE from desc_last_finalized_seq:A to desc_read:A
1545	*
1546	* Note: _prb_commit:B and desc_update_last_finalized:A can be
1547	* different CPUs. However, the desc_update_last_finalized:A
1548	* CPU (which performs the release) must have previously seen
1549	* _prb_commit:B.
1550	*/
1551	if (!atomic_long_try_cmpxchg_release(&desc_ring->last_finalized_seq,
1552	&oldval, newval)) { /* LMM(desc_update_last_finalized:A) */
1553	old_seq = __ulseq_to_u64seq(rb, oldval);
1554	goto try_again;
1555	}
1556	}
1557
1444	/*	1558	/*
1445	* Attempt to finalize a specified descriptor. If this fails, the descriptor	1559	* Attempt to finalize a specified descriptor. If this fails, the descriptor
1446	* is either already final or it will finalize itself when the writer commits.	1560	* is either already final or it will finalize itself when the writer commits.
1447	*/	1561	*/
1448	static void desc_make_final(struct prb_desc_ring *desc_ring, unsigned long id)	1562	static void desc_make_final(struct printk_ringbuffer *rb, unsigned long id)
1449	{	1563	{
		1564	struct prb_desc_ring *desc_ring = &rb->desc_ring;
1450	unsigned long prev_state_val = DESC_SV(id, desc_committed);	1565	unsigned long prev_state_val = DESC_SV(id, desc_committed);
1451	struct prb_desc *d = to_desc(desc_ring, id);	1566	struct prb_desc *d = to_desc(desc_ring, id);
1452		1567
1453	atomic_long_cmpxchg_relaxed(&d->state_var, prev_state_val,	1568	if (atomic_long_try_cmpxchg_relaxed(&d->state_var, &prev_state_val,
1454	DESC_SV(id, desc_finalized)); /* LMM(desc_make_final:A) */	1569	DESC_SV(id, desc_finalized))) { /* LMM(desc_make_final:A) */
1455		1570	desc_update_last_finalized(rb);
1456	/* Best effort to remember the last finalized @id. */	1571	}
1457	atomic_long_set(&desc_ring->last_finalized_id, id);
1458	}	1572	}
1459		1573
1460	/**	1574	/**
Lines 1550-1556 bool prb_reserve(struct prb_reserved_entry e, struct printk_ringbuffer rb, Link Here
1550	* readers. (For seq==0 there is no previous descriptor.)	1664	* readers. (For seq==0 there is no previous descriptor.)
1551	*/	1665	*/
1552	if (info->seq > 0)	1666	if (info->seq > 0)
1553	desc_make_final(desc_ring, DESC_ID(id - 1));	1667	desc_make_final(rb, DESC_ID(id - 1));
1554		1668
1555	r->text_buf = data_alloc(rb, r->text_buf_size, &d->text_blk_lpos, id);	1669	r->text_buf = data_alloc(rb, r->text_buf_size, &d->text_blk_lpos, id);
1556	/* If text data allocation fails, a data-less record is committed. */	1670	/* If text data allocation fails, a data-less record is committed. */
Lines 1643-1649 void prb_commit(struct prb_reserved_entry *e) Link Here
1643	*/	1757	*/
1644	head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_commit:A) */	1758	head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_commit:A) */
1645	if (head_id != e->id)	1759	if (head_id != e->id)
1646	desc_make_final(desc_ring, e->id);	1760	desc_make_final(e->rb, e->id);
1647	}	1761	}
1648		1762
1649	/**	1763	/**
Lines 1663-1674 void prb_commit(struct prb_reserved_entry *e) Link Here
1663	*/	1777	*/
1664	void prb_final_commit(struct prb_reserved_entry *e)	1778	void prb_final_commit(struct prb_reserved_entry *e)
1665	{	1779	{
1666	struct prb_desc_ring *desc_ring = &e->rb->desc_ring;
1667
1668	_prb_commit(e, desc_finalized);	1780	_prb_commit(e, desc_finalized);
1669		1781
1670	/* Best effort to remember the last finalized @id. */	1782	desc_update_last_finalized(e->rb);
1671	atomic_long_set(&desc_ring->last_finalized_id, e->id);
1672	}	1783	}
1673		1784
1674	/*	1785	/*
Lines 1832-1838 static int prb_read(struct printk_ringbuffer *rb, u64 seq, Link Here
1832	}	1943	}
1833		1944
1834	/* Get the sequence number of the tail descriptor. */	1945	/* Get the sequence number of the tail descriptor. */
1835	static u64 prb_first_seq(struct printk_ringbuffer *rb)	1946	u64 prb_first_seq(struct printk_ringbuffer *rb)
1836	{	1947	{
1837	struct prb_desc_ring *desc_ring = &rb->desc_ring;	1948	struct prb_desc_ring *desc_ring = &rb->desc_ring;
1838	enum desc_state d_state;	1949	enum desc_state d_state;
Lines 1875-1886 static u64 prb_first_seq(struct printk_ringbuffer *rb) Link Here
1875	return seq;	1986	return seq;
1876	}	1987	}
1877		1988
1878	/*	1989	/**
1879	* Non-blocking read of a record. Updates @seq to the last finalized record	1990	* prb_next_reserve_seq() - Get the sequence number after the most recently
1880	* (which may have no data available).	1991	* reserved record.
1881	*	1992	*
1882	* See the description of prb_read_valid() and prb_read_valid_info()	1993	* @rb: The ringbuffer to get the sequence number from.
1883	* for details.	1994	*
		1995	* This is the public function available to readers to see what sequence
		1996	* number will be assigned to the next reserved record.
		1997	*
		1998	* Note that depending on the situation, this value can be equal to or
		1999	* higher than the sequence number returned by prb_next_seq().
		2000	*
		2001	* Context: Any context.
		2002	* Return: The sequence number that will be assigned to the next record
		2003	* reserved.
		2004	*/
		2005	u64 prb_next_reserve_seq(struct printk_ringbuffer *rb)
		2006	{
		2007	struct prb_desc_ring *desc_ring = &rb->desc_ring;
		2008	unsigned long last_finalized_id;
		2009	atomic_long_t *state_var;
		2010	u64 last_finalized_seq;
		2011	unsigned long head_id;
		2012	struct prb_desc desc;
		2013	unsigned long diff;
		2014	struct prb_desc *d;
		2015	int err;
		2016
		2017	/*
		2018	* It may not be possible to read a sequence number for @head_id.
		2019	* So the ID of @last_finailzed_seq is used to calculate what the
		2020	* sequence number of @head_id will be.
		2021	*/
		2022
		2023	try_again:
		2024	last_finalized_seq = desc_last_finalized_seq(rb);
		2025
		2026	/*
		2027	* @head_id is loaded after @last_finalized_seq to ensure that
		2028	* it points to the record with @last_finalized_seq or newer.
		2029	*
		2030	* Memory barrier involvement:
		2031	*
		2032	* If desc_last_finalized_seq:A reads from
		2033	* desc_update_last_finalized:A, then
		2034	* prb_next_reserve_seq:A reads from desc_reserve:D.
		2035	*
		2036	* Relies on:
		2037	*
		2038	* RELEASE from desc_reserve:D to desc_update_last_finalized:A
		2039	* matching
		2040	* ACQUIRE from desc_last_finalized_seq:A to prb_next_reserve_seq:A
		2041	*
		2042	* Note: desc_reserve:D and desc_update_last_finalized:A can be
		2043	* different CPUs. However, the desc_update_last_finalized:A CPU
		2044	* (which performs the release) must have previously seen
		2045	* desc_read:C, which implies desc_reserve:D can be seen.
		2046	*/
		2047	head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_next_reserve_seq:A) */
		2048
		2049	d = to_desc(desc_ring, last_finalized_seq);
		2050	state_var = &d->state_var;
		2051
		2052	/* Extract the ID, used to specify the descriptor to read. */
		2053	last_finalized_id = DESC_ID(atomic_long_read(state_var));
		2054
		2055	/* Ensure @last_finalized_id is correct. */
		2056	err = desc_read_finalized_seq(desc_ring, last_finalized_id, last_finalized_seq, &desc);
		2057
		2058	if (err == -EINVAL) {
2059	if (last_finalized_seq == 0) {
2060	/*
2061	* No record has been finalized or even reserved yet.
2062	*
2063	* The @head_id is initialized such that the first
2064	* increment will yield the first record (seq=0).
2065	* Handle it separately to avoid a negative @diff
2066	* below.
2067	*/
2068	if (head_id == DESC0_ID(desc_ring->count_bits))
2069	return 0;
2070
2071	/*
2072	* One or more descriptors are already reserved. Use
2073	* the descriptor ID of the first one (@seq=0) for
2074	* the @diff below.
2075	*/
2076	last_finalized_id = DESC0_ID(desc_ring->count_bits) + 1;
2077	} else {
2078	/* Record must have been overwritten. Try again. */
2079	goto try_again;
2080	}
2081	}
2082
2083	/* Diff of known descriptor IDs to compute related sequence numbers. */
2084	diff = head_id - last_finalized_id;
2085
2086	/*
2087	* @head_id points to the most recently reserved record, but this
2088	* function returns the sequence number that will be assigned to the
2089	* next (not yet reserved) record. Thus +1 is needed.
2090	*/
2091	return (last_finalized_seq + diff + 1);
2092	}
2093
2094	/*
2095	* Non-blocking read of a record.
2096	*
2097	* On success @seq is updated to the record that was read and (if provided)
2098	* @r and @line_count will contain the read/calculated data.
2099	*
2100	* On failure @seq is updated to a record that is not yet available to the
2101	* reader, but it will be the next record available to the reader.
2102	*
2103	* Note: When the current CPU is in panic, this function will skip over any
2104	* non-existent/non-finalized records in order to allow the panic CPU
2105	* to print any and all records that have been finalized.
1884	*/	2106	*/
1885	static bool _prb_read_valid(struct printk_ringbuffer rb, u64 seq,	2107	static bool _prb_read_valid(struct printk_ringbuffer rb, u64 seq,
1886	struct printk_record r, unsigned int line_count)	2108	struct printk_record r, unsigned int line_count)
Lines 1899-1910 static bool _prb_read_valid(struct printk_ringbuffer rb, u64 seq, Link Here
1899	*seq = tail_seq;	2121	*seq = tail_seq;
1900		2122
1901	} else if (err == -ENOENT) {	2123	} else if (err == -ENOENT) {
1902	/* Record exists, but no data available. Skip. */	2124	/* Record exists, but the data was lost. Skip. */
1903	(*seq)++;	2125	(*seq)++;
1904		2126
1905	} else {	2127	} else {
1906	/* Non-existent/non-finalized record. Must stop. */	2128	/*
1907	return false;	2129	* Non-existent/non-finalized record. Must stop.
		2130	*
		2131	* For panic situations it cannot be expected that
		2132	* non-finalized records will become finalized. But
		2133	* there may be other finalized records beyond that
		2134	* need to be printed for a panic situation. If this
		2135	* is the panic CPU, skip this
		2136	* non-existent/non-finalized record unless it is
		2137	* at or beyond the head, in which case it is not
		2138	* possible to continue.
		2139	*
		2140	* Note that new messages printed on panic CPU are
		2141	* finalized when we are here. The only exception
		2142	* might be the last message without trailing newline.
		2143	* But it would have the sequence number returned
		2144	* by "prb_next_reserve_seq() - 1".
		2145	*/
		2146	if (this_cpu_in_panic() && ((*seq + 1) < prb_next_reserve_seq(rb)))
		2147	(*seq)++;
		2148	else
		2149	return false;
1908	}	2150	}
1909	}	2151	}
1910		2152
Lines 1932-1938 static bool _prb_read_valid(struct printk_ringbuffer rb, u64 seq, Link Here
1932	* On success, the reader must check r->info.seq to see which record was	2174	* On success, the reader must check r->info.seq to see which record was
1933	* actually read. This allows the reader to detect dropped records.	2175	* actually read. This allows the reader to detect dropped records.
1934	*	2176	*
1935	* Failure means @seq refers to a not yet written record.	2177	* Failure means @seq refers to a record not yet available to the reader.
1936	*/	2178	*/
1937	bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq,	2179	bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq,
1938	struct printk_record *r)	2180	struct printk_record *r)
Lines 1962-1968 bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq, Link Here
1962	* On success, the reader must check info->seq to see which record meta data	2204	* On success, the reader must check info->seq to see which record meta data
1963	* was actually read. This allows the reader to detect dropped records.	2205	* was actually read. This allows the reader to detect dropped records.
1964	*	2206	*
1965	* Failure means @seq refers to a not yet written record.	2207	* Failure means @seq refers to a record not yet available to the reader.
1966	*/	2208	*/
1967	bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq,	2209	bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq,
1968	struct printk_info info, unsigned int line_count)	2210	struct printk_info info, unsigned int line_count)
Lines 2008-2014 u64 prb_first_valid_seq(struct printk_ringbuffer *rb) Link Here
2008	* newest sequence number available to readers will be.	2250	* newest sequence number available to readers will be.
2009	*	2251	*
2010	* This provides readers a sequence number to jump to if all currently	2252	* This provides readers a sequence number to jump to if all currently
2011	* available records should be skipped.	2253	* available records should be skipped. It is guaranteed that all records
		2254	* previous to the returned value have been finalized and are (or were)
		2255	* available to the reader.
2012	*	2256	*
2013	* Context: Any context.	2257	* Context: Any context.
2014	* Return: The sequence number of the next newest (not yet available) record	2258	* Return: The sequence number of the next newest (not yet available) record
Lines 2016-2049 u64 prb_first_valid_seq(struct printk_ringbuffer *rb) Link Here
2016	*/	2260	*/
2017	u64 prb_next_seq(struct printk_ringbuffer *rb)	2261	u64 prb_next_seq(struct printk_ringbuffer *rb)
2018	{	2262	{
2019	struct prb_desc_ring *desc_ring = &rb->desc_ring;
2020	enum desc_state d_state;
2021	unsigned long id;
2022	u64 seq;	2263	u64 seq;
2023		2264
2024	/* Check if the cached @id still points to a valid @seq. */	2265	seq = desc_last_finalized_seq(rb);
2025	id = atomic_long_read(&desc_ring->last_finalized_id);
2026	d_state = desc_read(desc_ring, id, NULL, &seq, NULL);
2027		2266
2028	if (d_state == desc_finalized \|\| d_state == desc_reusable) {	2267	/*
2029	/*	2268	* Begin searching after the last finalized record.
2030	* Begin searching after the last finalized record.	2269	*
2031	*	2270	* On 0, the search must begin at 0 because of hack#2
2032	* On 0, the search must begin at 0 because of hack#2	2271	* of the bootstrapping phase it is not known if a
2033	* of the bootstrapping phase it is not known if a	2272	* record at index 0 exists.
2034	* record at index 0 exists.	2273	*/
2035	*/	2274	if (seq != 0)
2036	if (seq != 0)	2275	seq++;
2037	seq++;
2038	} else {
2039	/*
2040	* The information about the last finalized sequence number
2041	* has gone. It should happen only when there is a flood of
2042	* new messages and the ringbuffer is rapidly recycled.
2043	* Give up and start from the beginning.
2044	*/
2045	seq = 0;
2046	}
2047		2276
2048	/*	2277	/*
2049	* The information about the last finalized @seq might be inaccurate.	2278	* The information about the last finalized @seq might be inaccurate.
Lines 2085-2091 void prb_init(struct printk_ringbuffer *rb, Link Here
2085	rb->desc_ring.infos = infos;	2314	rb->desc_ring.infos = infos;
2086	atomic_long_set(&rb->desc_ring.head_id, DESC0_ID(descbits));	2315	atomic_long_set(&rb->desc_ring.head_id, DESC0_ID(descbits));
2087	atomic_long_set(&rb->desc_ring.tail_id, DESC0_ID(descbits));	2316	atomic_long_set(&rb->desc_ring.tail_id, DESC0_ID(descbits));
2088	atomic_long_set(&rb->desc_ring.last_finalized_id, DESC0_ID(descbits));	2317	atomic_long_set(&rb->desc_ring.last_finalized_seq, 0);
2089		2318
2090	rb->text_data_ring.size_bits = textbits;	2319	rb->text_data_ring.size_bits = textbits;
2091	rb->text_data_ring.data = text_buf;	2320	rb->text_data_ring.data = text_buf;




	struct printk_info	*infos;
	atomic_long_t		head_id;
	atomic_long_t		tail_id;
	atomic_long_t		last_finalized_seq;
};

/*

#define DESC_SV(id, state)	(((unsigned long)state << DESC_FLAGS_SHIFT) | id)
#define DESC_ID_MASK		(~DESC_FLAGS_MASK)
#define DESC_ID(sv)		((sv) & DESC_ID_MASK)

/*
 * Special data block logical position values (for fields of
 * @prb_desc.text_blk_lpos).
 *
 * - Bit0 is used to identify if the record has no data block. (Implemented in
 *   the LPOS_DATALESS() macro.)
 *
 * - Bit1 specifies the reason for not having a data block.
 *
 * These special values could never be real lpos values because of the
 * meta data and alignment padding of data blocks. (See to_blk_size() for
 * details.)
 */
#define FAILED_LPOS		0x1
#define EMPTY_LINE_LPOS		0x3

#define FAILED_BLK_LPOS	\
{				\

		.infos		= &_##name##_infos[0],						\
		.head_id	= ATOMIC_INIT(DESC0_ID(descbits)),				\
		.tail_id	= ATOMIC_INIT(DESC0_ID(descbits)),				\
		.last_finalized_seq = ATOMIC_INIT(0),						\
	},											\
	.text_data_ring = {									\
		.size_bits	= (avgtextbits) + (descbits),					\

bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq,
			 struct printk_info *info, unsigned int *line_count);

u64 prb_first_seq(struct printk_ringbuffer *rb);
u64 prb_first_valid_seq(struct printk_ringbuffer *rb);
u64 prb_next_seq(struct printk_ringbuffer *rb);
u64 prb_next_reserve_seq(struct printk_ringbuffer *rb);

#ifdef CONFIG_64BIT

#define __u64seq_to_ulseq(u64seq) (u64seq)
#define __ulseq_to_u64seq(rb, ulseq) (ulseq)

#else /* CONFIG_64BIT */

#define __u64seq_to_ulseq(u64seq) ((u32)u64seq)

static inline u64 __ulseq_to_u64seq(struct printk_ringbuffer *rb, u32 ulseq)
{
	u64 rb_first_seq = prb_first_seq(rb);
	u64 seq;

	/*
	 * The provided sequence is only the lower 32 bits of the ringbuffer
	 * sequence. It needs to be expanded to 64bit. Get the first sequence
	 * number from the ringbuffer and fold it.
	 *
	 * Having a 32bit representation in the console is sufficient.
	 * If a console ever gets more than 2^31 records behind
	 * the ringbuffer then this is the least of the problems.
	 *
	 * Also the access to the ring buffer is always safe.
	 */
	seq = rb_first_seq - (s32)((u32)rb_first_seq - ulseq);

	return seq;
}

#endif /* CONFIG_64BIT */

#endif /* _KERNEL_PRINTK_RINGBUFFER_H */




	this_cpu_dec(printk_context);
}

void __printk_deferred_enter(void)
{
	cant_migrate();
	this_cpu_inc(printk_context);
}

void __printk_deferred_exit(void)
{
	cant_migrate();
	this_cpu_dec(printk_context);
}

asmlinkage int vprintk(const char *fmt, va_list args)
{
#ifdef CONFIG_KGDB_KDB

Lines 2409-2414 static int rcutorture_booster_init(unsigned int cpu) Link Here

(-)a/kernel/rcu/rcutorture.c (+6 lines)
2409	WARN_ON_ONCE(!t);	2409	WARN_ON_ONCE(!t);
2410	sp.sched_priority = 2;	2410	sp.sched_priority = 2;
2411	sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);	2411	sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
		2412	#ifdef CONFIG_PREEMPT_RT
		2413	t = per_cpu(timersd, cpu);
		2414	WARN_ON_ONCE(!t);
		2415	sp.sched_priority = 2;
		2416	sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
		2417	#endif
2412	}	2418	}
2413		2419
2414	/* Don't allow time recalculation while creating a new task. */	2420	/* Don't allow time recalculation while creating a new task. */

Lines 9-14 Link Here

(-)a/kernel/rcu/tree_stall.h (+5 lines)
9		9
10	#include <linux/kvm_para.h>	10	#include <linux/kvm_para.h>
11	#include <linux/rcu_notifier.h>	11	#include <linux/rcu_notifier.h>
		12	#include <linux/console.h>
12		13
13	//////////////////////////////////////////////////////////////////////////////	14	//////////////////////////////////////////////////////////////////////////////
14	//	15	//
Lines 604-609 static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) Link Here
604	if (rcu_stall_is_suppressed())	605	if (rcu_stall_is_suppressed())
605	return;	606	return;
606		607
		608	nbcon_cpu_emergency_enter();
		609
607	/*	610	/*
608	* OK, time to rat on our buddy...	611	* OK, time to rat on our buddy...
609	* See Documentation/RCU/stallwarn.rst for info on how to debug	612	* See Documentation/RCU/stallwarn.rst for info on how to debug
Lines 658-663 static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) Link Here
658	panic_on_rcu_stall();	661	panic_on_rcu_stall();
659		662
660	rcu_force_quiescent_state(); /* Kick them all. */	663	rcu_force_quiescent_state(); /* Kick them all. */
		664
		665	nbcon_cpu_emergency_exit();
661	}	666	}
662		667
663	static void print_cpu_stall(unsigned long gps)	668	static void print_cpu_stall(unsigned long gps)





#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
/*
 * Atomically set TIF_NEED_RESCHED[_LAZY] and test for TIF_POLLING_NRFLAG,
 * this avoids any races wrt polling state changes and thereby avoids
 * spurious IPIs.
 */
static inline bool set_nr_and_not_polling(struct task_struct *p, int tif_bit)
{
	struct thread_info *ti = task_thread_info(p);

	return !(fetch_or(&ti->flags, 1 << tif_bit) & _TIF_POLLING_NRFLAG);
}

/*

	do {
		if (!(val & _TIF_POLLING_NRFLAG))
			return false;
		if (val & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY))
			return true;
	} while (!try_cmpxchg(&ti->flags, &val, val | _TIF_NEED_RESCHED));


}

#else
static inline bool set_nr_and_not_polling(struct task_struct *p, int tif_bit)
{
	set_tsk_thread_flag(p, tif_bit);
	return true;
}


 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
static void __resched_curr(struct rq *rq, int lazy)
{
	int cpu, tif_bit = TIF_NEED_RESCHED + lazy;
	struct task_struct *curr = rq->curr;
	int cpu;

	lockdep_assert_rq_held(rq);

	if (unlikely(test_tsk_thread_flag(curr, tif_bit)))
		return;

	cpu = cpu_of(rq);

	if (cpu == smp_processor_id()) {
		set_tsk_thread_flag(curr, tif_bit);
		if (!lazy)
			set_preempt_need_resched();
		return;
	}

	if (set_nr_and_not_polling(curr, tif_bit)) {
		if (!lazy)
			smp_send_reschedule(cpu);
	} else {
		trace_sched_wake_idle_without_ipi(cpu);
	}
}

void resched_curr(struct rq *rq)
{
	__resched_curr(rq, 0);
}

void resched_curr_lazy(struct rq *rq)
{
	int lazy = IS_ENABLED(CONFIG_PREEMPT_BUILD_AUTO) && !sched_feat(FORCE_NEED_RESCHED) ?
		TIF_NEED_RESCHED_LAZY_OFFSET : 0;

	if (lazy && unlikely(test_tsk_thread_flag(rq->curr, TIF_NEED_RESCHED)))
		return;

	__resched_curr(rq, lazy);
}

void resched_cpu(int cpu)

	 * and testing of the above solutions didn't appear to report
	 * much benefits.
	 */
	if (set_nr_and_not_polling(rq->idle, TIF_NEED_RESCHED))
		smp_send_reschedule(cpu);
	else
		trace_sched_wake_idle_without_ipi(cpu);


#endif /* #ifdef CONFIG_PREEMPT_DYNAMIC */

/*
 * task_is_pi_boosted - Check if task has been PI boosted.
 * @p:	Task to check.
 *
 * Return true if task is subject to priority inheritance.
 */
bool task_is_pi_boosted(const struct task_struct *p)
{
	int prio = p->prio;

	if (!rt_prio(prio))
		return false;
	return prio != p->normal_prio;
}

/**
 * yield - yield the current processor to other threads.
 *




	.release	= seq_release,
};

static ssize_t sched_hog_write(struct file *filp, const char __user *ubuf,
			       size_t cnt, loff_t *ppos)
{
	unsigned long end = jiffies + 60 * HZ;

	for (; time_before(jiffies, end) && !signal_pending(current);)
		cpu_relax();

	return cnt;
}

static const struct file_operations sched_hog_fops = {
	.write		= sched_hog_write,
	.open		= simple_open,
	.llseek		= default_llseek,
};

static struct dentry *debugfs_sched;

static __init int sched_init_debug(void)


	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);

	debugfs_create_file("hog", 0200, debugfs_sched, NULL, &sched_hog_fops);

	return 0;
}
late_initcall(sched_init_debug);

Lines 975-982 static void clear_buddies(struct cfs_rq cfs_rq, struct sched_entity se); Link Here

(-)a/kernel/sched/fair.c (-15 / +31 lines)
975	* XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i	975	* XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i
976	* this is probably good enough.	976	* this is probably good enough.
977	*/	977	*/
978	static void update_deadline(struct cfs_rq cfs_rq, struct sched_entity se)	978	static void update_deadline(struct cfs_rq cfs_rq, struct sched_entity se, bool tick)
979	{	979	{
		980	struct rq *rq = rq_of(cfs_rq);
		981
980	if ((s64)(se->vruntime - se->deadline) < 0)	982	if ((s64)(se->vruntime - se->deadline) < 0)
981	return;	983	return;
982		984
Lines 995-1004 static void update_deadline(struct cfs_rq cfs_rq, struct sched_entity se) Link Here
995	/*	997	/*
996	* The task has consumed its request, reschedule.	998	* The task has consumed its request, reschedule.
997	*/	999	*/
998	if (cfs_rq->nr_running > 1) {	1000	if (cfs_rq->nr_running < 2)
999	resched_curr(rq_of(cfs_rq));	1001	return;
1000	clear_buddies(cfs_rq, se);	1002
		1003	if (!IS_ENABLED(CONFIG_PREEMPT_BUILD_AUTO) \|\| sched_feat(FORCE_NEED_RESCHED)) {
		1004	resched_curr(rq);
		1005	} else {
		1006	/* Did the task ignore the lazy reschedule request? */
		1007	if (tick && test_tsk_thread_flag(rq->curr, TIF_NEED_RESCHED_LAZY))
		1008	resched_curr(rq);
		1009	else
		1010	resched_curr_lazy(rq);
1001	}	1011	}
		1012	clear_buddies(cfs_rq, se);
1002	}	1013	}
1003		1014
1004	#include "pelt.h"	1015	#include "pelt.h"
Lines 1153-1159 s64 update_curr_common(struct rq *rq) Link Here
1153	/*	1164	/*
1154	* Update the current task's runtime statistics.	1165	* Update the current task's runtime statistics.
1155	*/	1166	*/
1156	static void update_curr(struct cfs_rq *cfs_rq)	1167	static void __update_curr(struct cfs_rq *cfs_rq, bool tick)
1157	{	1168	{
1158	struct sched_entity *curr = cfs_rq->curr;	1169	struct sched_entity *curr = cfs_rq->curr;
1159	s64 delta_exec;	1170	s64 delta_exec;
Lines 1411-1417 Link Here
1411	#else // !CONFIG_SCHED_BORE	1411	#else // !CONFIG_SCHED_BORE
1412	curr->vruntime += calc_delta_fair(delta_exec, curr);	1412	curr->vruntime += calc_delta_fair(delta_exec, curr);
1413	#endif // CONFIG_SCHED_BORE	1413	#endif // CONFIG_SCHED_BORE
1414	update_deadline(cfs_rq, curr);	1414	update_deadline(cfs_rq, curr, tick);
1415	update_min_vruntime(cfs_rq);	1415	update_min_vruntime(cfs_rq);
1416		1416
1417	if (entity_is_task(curr))	1417	if (entity_is_task(curr))
Lines 1175-1180 static void update_curr(struct cfs_rq *cfs_rq) Link Here
1175	account_cfs_rq_runtime(cfs_rq, delta_exec);	1186	account_cfs_rq_runtime(cfs_rq, delta_exec);
1176	}	1187	}
1177		1188
		1189	static inline void update_curr(struct cfs_rq *cfs_rq)
		1190	{
		1191	__update_curr(cfs_rq, false);
		1192	}
		1193
1178	static void update_curr_fair(struct rq *rq)	1194	static void update_curr_fair(struct rq *rq)
1179	{	1195	{
1180	update_curr(cfs_rq_of(&rq->curr->se));	1196	update_curr(cfs_rq_of(&rq->curr->se));
Lines 5493-5499 entity_tick(struct cfs_rq cfs_rq, struct sched_entity curr, int queued) Link Here
5493	/*	5509	/*
5494	* Update run-time statistics of the 'current'.	5510	* Update run-time statistics of the 'current'.
5495	*/	5511	*/
5496	update_curr(cfs_rq);	5512	__update_curr(cfs_rq, true);
5497		5513
5498	/*	5514	/*
5499	* Ensure that runnable average is periodically updated.	5515	* Ensure that runnable average is periodically updated.
Lines 5507-5513 entity_tick(struct cfs_rq cfs_rq, struct sched_entity curr, int queued) Link Here
5507	* validating it and just reschedule.	5523	* validating it and just reschedule.
5508	*/	5524	*/
5509	if (queued) {	5525	if (queued) {
5510	resched_curr(rq_of(cfs_rq));	5526	resched_curr_lazy(rq_of(cfs_rq));
5511	return;	5527	return;
5512	}	5528	}
5513	/*	5529	/*
Lines 5653-5659 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) Link Here
5653	* hierarchy can be throttled	5669	* hierarchy can be throttled
5654	*/	5670	*/
5655	if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))	5671	if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
5656	resched_curr(rq_of(cfs_rq));	5672	resched_curr_lazy(rq_of(cfs_rq));
5657	}	5673	}
5658		5674
5659	static __always_inline	5675	static __always_inline
Lines 5913-5919 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) Link Here
5913		5929
5914	/* Determine whether we need to wake up potentially idle CPU: */	5930	/* Determine whether we need to wake up potentially idle CPU: */
5915	if (rq->curr == rq->idle && rq->cfs.nr_running)	5931	if (rq->curr == rq->idle && rq->cfs.nr_running)
5916	resched_curr(rq);	5932	resched_curr_lazy(rq);
5917	}	5933	}
5918		5934
5919	#ifdef CONFIG_SMP	5935	#ifdef CONFIG_SMP
Lines 6628-6634 static void hrtick_start_fair(struct rq rq, struct task_struct p) Link Here
6628		6644
6629	if (delta < 0) {	6645	if (delta < 0) {
6630	if (task_current(rq, p))	6646	if (task_current(rq, p))
6631	resched_curr(rq);	6647	resched_curr_lazy(rq);
6632	return;	6648	return;
6633	}	6649	}
6634	hrtick_start(rq, delta);	6650	hrtick_start(rq, delta);
Lines 8298-8304 static void check_preempt_wakeup_fair(struct rq rq, struct task_struct p, int Link Here
8298	* prevents us from potentially nominating it as a false LAST_BUDDY	8314	* prevents us from potentially nominating it as a false LAST_BUDDY
8299	* below.	8315	* below.
8300	*/	8316	*/
8301	if (test_tsk_need_resched(curr))	8317	if (need_resched())
8302	return;	8318	return;
8303		8319
8304	/* Idle tasks are by definition preempted by non-idle tasks. */	8320	/* Idle tasks are by definition preempted by non-idle tasks. */
Lines 8340-8346 static void check_preempt_wakeup_fair(struct rq rq, struct task_struct p, int Link Here
8340	return;	8356	return;
8341		8357
8342	preempt:	8358	preempt:
8343	resched_curr(rq);	8359	resched_curr_lazy(rq);
8344	}	8360	}
8345		8361
8346	#ifdef CONFIG_SMP	8362	#ifdef CONFIG_SMP
Lines 12510-12516 static inline void task_tick_core(struct rq rq, struct task_struct curr) Link Here
12510	*/	12526	*/
12511	if (rq->core->core_forceidle_count && rq->cfs.nr_running == 1 &&	12527	if (rq->core->core_forceidle_count && rq->cfs.nr_running == 1 &&
12512	__entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE))	12528	__entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE))
12513	resched_curr(rq);	12529	resched_curr_lazy(rq);
12514	}	12530	}
12515		12531
12516	/*	12532	/*
Lines 12675-12681 prio_changed_fair(struct rq rq, struct task_struct p, int oldprio) Link Here
12675	*/	12691	*/
12676	if (task_current(rq, p)) {	12692	if (task_current(rq, p)) {
12677	if (p->prio > oldprio)	12693	if (p->prio > oldprio)
12678	resched_curr(rq);	12694	resched_curr_lazy(rq);
12679	} else	12695	} else
12680	wakeup_preempt(rq, p, 0);	12696	wakeup_preempt(rq, p, 0);
12681	}	12697	}

Lines 87-89 SCHED_FEAT(UTIL_EST, true) Link Here

(-)a/kernel/sched/features.h (+2 lines)
87	SCHED_FEAT(LATENCY_WARN, false)	87	SCHED_FEAT(LATENCY_WARN, false)
88		88
89	SCHED_FEAT(HZ_BW, true)	89	SCHED_FEAT(HZ_BW, true)
		90
		91	SCHED_FEAT(FORCE_NEED_RESCHED, false)

Lines 57-64 static noinline int __cpuidle cpu_idle_poll(void) Link Here

(-)a/kernel/sched/idle.c (-2 / +1 lines)
57	ct_cpuidle_enter();	57	ct_cpuidle_enter();
58		58
59	raw_local_irq_enable();	59	raw_local_irq_enable();
60	while (!tif_need_resched() &&	60	while (!need_resched() && (cpu_idle_force_poll \|\| tick_check_broadcast_expired()))
61	(cpu_idle_force_poll \|\| tick_check_broadcast_expired()))
62	cpu_relax();	61	cpu_relax();
63	raw_local_irq_disable();	62	raw_local_irq_disable();
64		63

Lines 2194-2201 static int rto_next_cpu(struct root_domain *rd) Link Here

(-)a/kernel/sched/rt.c (-1 / +4 lines)
2194		2194
2195	rd->rto_cpu = cpu;	2195	rd->rto_cpu = cpu;
2196		2196
2197	if (cpu < nr_cpu_ids)	2197	if (cpu < nr_cpu_ids) {
		2198	if (!has_pushable_tasks(cpu_rq(cpu)))
		2199	continue;
2198	return cpu;	2200	return cpu;
		2201	}
2199		2202
2200	rd->rto_cpu = -1;	2203	rd->rto_cpu = -1;
2201		2204

Lines 2463-2468 extern void init_sched_fair_class(void); Link Here

(-)a/kernel/sched/sched.h (+1 lines)
2463	extern void reweight_task(struct task_struct *p, int prio);	2463	extern void reweight_task(struct task_struct *p, int prio);
2464		2464
2465	extern void resched_curr(struct rq *rq);	2465	extern void resched_curr(struct rq *rq);
		2466	extern void resched_curr_lazy(struct rq *rq);
2466	extern void resched_cpu(int cpu);	2467	extern void resched_cpu(int cpu);
2467		2468
2468	extern struct rt_bandwidth def_rt_bandwidth;	2469	extern struct rt_bandwidth def_rt_bandwidth;

Lines 247-252 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) Link Here

(-)a/kernel/softirq.c (-1 / +94 lines)
247	}	247	}
248	EXPORT_SYMBOL(__local_bh_enable_ip);	248	EXPORT_SYMBOL(__local_bh_enable_ip);
249		249
		250	void softirq_preempt(void)
		251	{
		252	if (WARN_ON_ONCE(!preemptible()))
		253	return;
		254
		255	if (WARN_ON_ONCE(__this_cpu_read(softirq_ctrl.cnt) != SOFTIRQ_OFFSET))
		256	return;
		257
		258	__local_bh_enable(SOFTIRQ_OFFSET, true);
		259	/* preemption point */
		260	__local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
		261	}
		262
250	/*	263	/*
251	* Invoked from ksoftirqd_run() outside of the interrupt disabled section	264	* Invoked from ksoftirqd_run() outside of the interrupt disabled section
252	* to acquire the per CPU local lock for reentrancy protection.	265	* to acquire the per CPU local lock for reentrancy protection.
Lines 619-624 static inline void tick_irq_exit(void) Link Here
619	#endif	632	#endif
620	}	633	}
621		634
		635	#ifdef CONFIG_PREEMPT_RT
		636	DEFINE_PER_CPU(struct task_struct *, timersd);
		637	DEFINE_PER_CPU(unsigned long, pending_timer_softirq);
		638
		639	static void wake_timersd(void)
		640	{
		641	struct task_struct *tsk = __this_cpu_read(timersd);
		642
		643	if (tsk)
		644	wake_up_process(tsk);
		645	}
		646
		647	#else
		648
		649	static inline void wake_timersd(void) { }
		650
		651	#endif
		652
622	static inline void __irq_exit_rcu(void)	653	static inline void __irq_exit_rcu(void)
623	{	654	{
624	#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED	655	#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
Lines 631-636 static inline void __irq_exit_rcu(void) Link Here
631	if (!in_interrupt() && local_softirq_pending())	662	if (!in_interrupt() && local_softirq_pending())
632	invoke_softirq();	663	invoke_softirq();
633		664
		665	if (IS_ENABLED(CONFIG_PREEMPT_RT) && local_pending_timers() &&
		666	!(in_nmi() \| in_hardirq()))
		667	wake_timersd();
		668
634	tick_irq_exit();	669	tick_irq_exit();
635	}	670	}
636		671
Lines 963-974 static struct smp_hotplug_thread softirq_threads = { Link Here
963	.thread_comm = "ksoftirqd/%u",	998	.thread_comm = "ksoftirqd/%u",
964	};	999	};
965		1000
		1001	#ifdef CONFIG_PREEMPT_RT
		1002	static void timersd_setup(unsigned int cpu)
		1003	{
		1004	sched_set_fifo_low(current);
		1005	}
		1006
		1007	static int timersd_should_run(unsigned int cpu)
		1008	{
		1009	return local_pending_timers();
		1010	}
		1011
		1012	static void run_timersd(unsigned int cpu)
		1013	{
		1014	unsigned int timer_si;
		1015
		1016	ksoftirqd_run_begin();
		1017
		1018	timer_si = local_pending_timers();
		1019	__this_cpu_write(pending_timer_softirq, 0);
		1020	or_softirq_pending(timer_si);
		1021
		1022	__do_softirq();
		1023
		1024	ksoftirqd_run_end();
		1025	}
		1026
		1027	static void raise_ktimers_thread(unsigned int nr)
		1028	{
		1029	trace_softirq_raise(nr);
		1030	__this_cpu_or(pending_timer_softirq, 1 << nr);
		1031	}
		1032
		1033	void raise_hrtimer_softirq(void)
		1034	{
		1035	raise_ktimers_thread(HRTIMER_SOFTIRQ);
		1036	}
		1037
		1038	void raise_timer_softirq(void)
		1039	{
		1040	unsigned long flags;
		1041
		1042	local_irq_save(flags);
		1043	raise_ktimers_thread(TIMER_SOFTIRQ);
		1044	wake_timersd();
		1045	local_irq_restore(flags);
		1046	}
		1047
		1048	static struct smp_hotplug_thread timer_threads = {
		1049	.store = &timersd,
		1050	.setup = timersd_setup,
		1051	.thread_should_run = timersd_should_run,
		1052	.thread_fn = run_timersd,
		1053	.thread_comm = "ktimers/%u",
		1054	};
		1055	#endif
		1056
966	static __init int spawn_ksoftirqd(void)	1057	static __init int spawn_ksoftirqd(void)
967	{	1058	{
968	cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,	1059	cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
969	takeover_tasklets);	1060	takeover_tasklets);
970	BUG_ON(smpboot_register_percpu_thread(&softirq_threads));	1061	BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
971		1062	#ifdef CONFIG_PREEMPT_RT
		1063	BUG_ON(smpboot_register_percpu_thread(&timer_threads));
		1064	#endif
972	return 0;	1065	return 0;
973	}	1066	}
974	early_initcall(spawn_ksoftirqd);	1067	early_initcall(spawn_ksoftirqd);

Lines 1809-1815 void hrtimer_interrupt(struct clock_event_device *dev) Link Here

(-)a/kernel/time/hrtimer.c (-2 / +2 lines)
1809	if (!ktime_before(now, cpu_base->softirq_expires_next)) {	1809	if (!ktime_before(now, cpu_base->softirq_expires_next)) {
1810	cpu_base->softirq_expires_next = KTIME_MAX;	1810	cpu_base->softirq_expires_next = KTIME_MAX;
1811	cpu_base->softirq_activated = 1;	1811	cpu_base->softirq_activated = 1;
1812	raise_softirq_irqoff(HRTIMER_SOFTIRQ);	1812	raise_hrtimer_softirq();
1813	}	1813	}
1814		1814
1815	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);	1815	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
Lines 1922-1928 void hrtimer_run_queues(void) Link Here
1922	if (!ktime_before(now, cpu_base->softirq_expires_next)) {	1922	if (!ktime_before(now, cpu_base->softirq_expires_next)) {
1923	cpu_base->softirq_expires_next = KTIME_MAX;	1923	cpu_base->softirq_expires_next = KTIME_MAX;
1924	cpu_base->softirq_activated = 1;	1924	cpu_base->softirq_activated = 1;
1925	raise_softirq_irqoff(HRTIMER_SOFTIRQ);	1925	raise_hrtimer_softirq();
1926	}	1926	}
1927		1927
1928	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);	1928	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);

Lines 796-802 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) Link Here

(-)a/kernel/time/tick-sched.c (-1 / +1 lines)
796		796
797	static inline bool local_timer_softirq_pending(void)	797	static inline bool local_timer_softirq_pending(void)
798	{	798	{
799	return local_softirq_pending() & BIT(TIMER_SOFTIRQ);	799	return local_pending_timers() & BIT(TIMER_SOFTIRQ);
800	}	800	}
801		801
802	static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)	802	static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)

Lines 1470-1478 static inline void timer_base_unlock_expiry(struct timer_base *base) Link Here

(-)a/kernel/time/timer.c (-2 / +9 lines)
1470	*/	1470	*/
1471	static void timer_sync_wait_running(struct timer_base *base)	1471	static void timer_sync_wait_running(struct timer_base *base)
1472	{	1472	{
1473	if (atomic_read(&base->timer_waiters)) {	1473	bool need_preempt;
		1474
		1475	need_preempt = task_is_pi_boosted(current);
		1476	if (need_preempt \|\| atomic_read(&base->timer_waiters)) {
1474	raw_spin_unlock_irq(&base->lock);	1477	raw_spin_unlock_irq(&base->lock);
1475	spin_unlock(&base->expiry_lock);	1478	spin_unlock(&base->expiry_lock);
		1479
		1480	if (need_preempt)
		1481	softirq_preempt();
		1482
1476	spin_lock(&base->expiry_lock);	1483	spin_lock(&base->expiry_lock);
1477	raw_spin_lock_irq(&base->lock);	1484	raw_spin_lock_irq(&base->lock);
1478	}	1485	}
Lines 2070-2076 static void run_local_timers(void) Link Here
2070	if (time_before(jiffies, base->next_expiry))	2077	if (time_before(jiffies, base->next_expiry))
2071	return;	2078	return;
2072	}	2079	}
2073	raise_softirq(TIMER_SOFTIRQ);	2080	raise_timer_softirq();
2074	}	2081	}
2075		2082
2076	/*	2083	/*

Lines 2717-2722 unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status) Link Here

(-)a/kernel/trace/trace.c (+2 lines)
2717		2717
2718	if (tif_need_resched())	2718	if (tif_need_resched())
2719	trace_flags \|= TRACE_FLAG_NEED_RESCHED;	2719	trace_flags \|= TRACE_FLAG_NEED_RESCHED;
		2720	if (tif_need_resched_lazy())
		2721	trace_flags \|= TRACE_FLAG_NEED_RESCHED_LAZY;
2720	if (test_preempt_need_resched())	2722	if (test_preempt_need_resched())
2721	trace_flags \|= TRACE_FLAG_PREEMPT_RESCHED;	2723	trace_flags \|= TRACE_FLAG_PREEMPT_RESCHED;
2722	return (trace_flags << 16) \| (min_t(unsigned int, pc & 0xff, 0xf)) \|	2724	return (trace_flags << 16) \| (min_t(unsigned int, pc & 0xff, 0xf)) \|

Lines 460-476 int trace_print_lat_fmt(struct trace_seq s, struct trace_entry entry) Link Here

(-)a/kernel/trace/trace_output.c (-2 / +14 lines)
460	(entry->flags & TRACE_FLAG_IRQS_OFF && bh_off) ? 'D' :	460	(entry->flags & TRACE_FLAG_IRQS_OFF && bh_off) ? 'D' :
461	(entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :	461	(entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
462	bh_off ? 'b' :	462	bh_off ? 'b' :
463	(entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' :	463	!IS_ENABLED(CONFIG_TRACE_IRQFLAGS_SUPPORT) ? 'X' :
464	'.';	464	'.';
465		465
466	switch (entry->flags & (TRACE_FLAG_NEED_RESCHED \|	466	switch (entry->flags & (TRACE_FLAG_NEED_RESCHED \| TRACE_FLAG_NEED_RESCHED_LAZY \|
467	TRACE_FLAG_PREEMPT_RESCHED)) {	467	TRACE_FLAG_PREEMPT_RESCHED)) {
		468	case TRACE_FLAG_NEED_RESCHED \| TRACE_FLAG_NEED_RESCHED_LAZY \| TRACE_FLAG_PREEMPT_RESCHED:
		469	need_resched = 'B';
		470	break;
468	case TRACE_FLAG_NEED_RESCHED \| TRACE_FLAG_PREEMPT_RESCHED:	471	case TRACE_FLAG_NEED_RESCHED \| TRACE_FLAG_PREEMPT_RESCHED:
469	need_resched = 'N';	472	need_resched = 'N';
470	break;	473	break;
		474	case TRACE_FLAG_NEED_RESCHED_LAZY \| TRACE_FLAG_PREEMPT_RESCHED:
		475	need_resched = 'L';
		476	break;
		477	case TRACE_FLAG_NEED_RESCHED \| TRACE_FLAG_NEED_RESCHED_LAZY:
		478	need_resched = 'b';
		479	break;
471	case TRACE_FLAG_NEED_RESCHED:	480	case TRACE_FLAG_NEED_RESCHED:
472	need_resched = 'n';	481	need_resched = 'n';
473	break;	482	break;
		483	case TRACE_FLAG_NEED_RESCHED_LAZY:
		484	need_resched = 'l';
		485	break;
474	case TRACE_FLAG_PREEMPT_RESCHED:	486	case TRACE_FLAG_PREEMPT_RESCHED:
475	need_resched = 'p';	487	need_resched = 'p';
476	break;	488	break;

Lines 96-110 static void __dump_stack(const char *log_lvl) Link Here

(-)a/lib/dump_stack.c (-3 / +13 lines)
96	*/	96	*/
97	asmlinkage __visible void dump_stack_lvl(const char *log_lvl)	97	asmlinkage __visible void dump_stack_lvl(const char *log_lvl)
98	{	98	{
		99	bool in_panic = this_cpu_in_panic();
99	unsigned long flags;	100	unsigned long flags;
100		101
101	/*	102	/*
102	* Permit this cpu to perform nested stack dumps while serialising	103	* Permit this cpu to perform nested stack dumps while serialising
103	* against other CPUs	104	* against other CPUs, unless this CPU is in panic.
		105	*
		106	* When in panic, non-panic CPUs are not permitted to store new
		107	* printk messages so there is no need to synchronize the output.
		108	* This avoids potential deadlock in panic() if another CPU is
		109	* holding and unable to release the printk_cpu_sync.
104	*/	110	*/
105	printk_cpu_sync_get_irqsave(flags);	111	if (!in_panic)
		112	printk_cpu_sync_get_irqsave(flags);
		113
106	__dump_stack(log_lvl);	114	__dump_stack(log_lvl);
107	printk_cpu_sync_put_irqrestore(flags);	115
		116	if (!in_panic)
		117	printk_cpu_sync_put_irqrestore(flags);
108	}	118	}
109	EXPORT_SYMBOL(dump_stack_lvl);	119	EXPORT_SYMBOL(dump_stack_lvl);
110		120

Line 0 Link Here

(-)a/localversion-rt (+1 lines)
	1	-rt8

Lines 78-83 Link Here

(-)a/net/core/dev.c (-68 / +154 lines)
78	#include <linux/slab.h>	78	#include <linux/slab.h>
79	#include <linux/sched.h>	79	#include <linux/sched.h>
80	#include <linux/sched/mm.h>	80	#include <linux/sched/mm.h>
		81	#include <linux/smpboot.h>
81	#include <linux/mutex.h>	82	#include <linux/mutex.h>
82	#include <linux/rwsem.h>	83	#include <linux/rwsem.h>
83	#include <linux/string.h>	84	#include <linux/string.h>
Lines 216-250 static inline struct hlist_head dev_index_hash(struct net net, int ifindex) Link Here
216	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];	217	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
217	}	218	}
218		219
219	static inline void rps_lock_irqsave(struct softnet_data *sd,	220	#ifndef CONFIG_PREEMPT_RT
220	unsigned long *flags)	221
		222	static DEFINE_STATIC_KEY_FALSE(use_backlog_threads_key);
		223
		224	static int __init setup_backlog_napi_threads(char *arg)
221	{	225	{
222	if (IS_ENABLED(CONFIG_RPS))	226	static_branch_enable(&use_backlog_threads_key);
		227	return 0;
		228	}
		229	early_param("thread_backlog_napi", setup_backlog_napi_threads);
		230
		231	static bool use_backlog_threads(void)
		232	{
		233	return static_branch_unlikely(&use_backlog_threads_key);
		234	}
		235
		236	#else
		237
		238	static bool use_backlog_threads(void)
		239	{
		240	return true;
		241	}
		242
		243	#endif
		244
		245	static inline void backlog_lock_irq_save(struct softnet_data *sd,
		246	unsigned long *flags)
		247	{
		248	if (IS_ENABLED(CONFIG_RPS) \|\| use_backlog_threads())
223	spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);	249	spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
224	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))	250	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
225	local_irq_save(*flags);	251	local_irq_save(*flags);
226	}	252	}
227		253
228	static inline void rps_lock_irq_disable(struct softnet_data *sd)	254	static inline void backlog_lock_irq_disable(struct softnet_data *sd)
229	{	255	{
230	if (IS_ENABLED(CONFIG_RPS))	256	if (IS_ENABLED(CONFIG_RPS) \|\| use_backlog_threads())
231	spin_lock_irq(&sd->input_pkt_queue.lock);	257	spin_lock_irq(&sd->input_pkt_queue.lock);
232	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))	258	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
233	local_irq_disable();	259	local_irq_disable();
234	}	260	}
235		261
236	static inline void rps_unlock_irq_restore(struct softnet_data *sd,	262	static inline void backlog_unlock_irq_restore(struct softnet_data *sd,
237	unsigned long *flags)	263	unsigned long *flags)
238	{	264	{
239	if (IS_ENABLED(CONFIG_RPS))	265	if (IS_ENABLED(CONFIG_RPS) \|\| use_backlog_threads())
240	spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);	266	spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);
241	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))	267	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
242	local_irq_restore(*flags);	268	local_irq_restore(*flags);
243	}	269	}
244		270
245	static inline void rps_unlock_irq_enable(struct softnet_data *sd)	271	static inline void backlog_unlock_irq_enable(struct softnet_data *sd)
246	{	272	{
247	if (IS_ENABLED(CONFIG_RPS))	273	if (IS_ENABLED(CONFIG_RPS) \|\| use_backlog_threads())
248	spin_unlock_irq(&sd->input_pkt_queue.lock);	274	spin_unlock_irq(&sd->input_pkt_queue.lock);
249	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))	275	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
250	local_irq_enable();	276	local_irq_enable();
Lines 4420-4425 EXPORT_SYMBOL(__dev_direct_xmit); Link Here
4420	/*************************************************************************	4446	/*************************************************************************
4421	* Receiver routines	4447	* Receiver routines
4422	*************************************************************************/	4448	*************************************************************************/
		4449	static DEFINE_PER_CPU(struct task_struct *, backlog_napi);
4423		4450
4424	int netdev_max_backlog __read_mostly = 1000;	4451	int netdev_max_backlog __read_mostly = 1000;
4425	EXPORT_SYMBOL(netdev_max_backlog);	4452	EXPORT_SYMBOL(netdev_max_backlog);
Lines 4452-4469 static inline void ____napi_schedule(struct softnet_data *sd, Link Here
4452	*/	4479	*/
4453	thread = READ_ONCE(napi->thread);	4480	thread = READ_ONCE(napi->thread);
4454	if (thread) {	4481	if (thread) {
4455	/* Avoid doing set_bit() if the thread is in	4482	if (use_backlog_threads() && thread == raw_cpu_read(backlog_napi))
4456	* INTERRUPTIBLE state, cause napi_thread_wait()	4483	goto use_local_napi;
4457	* makes sure to proceed with napi polling	4484
4458	* if the thread is explicitly woken from here.	4485	set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4459	*/
4460	if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE)
4461	set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4462	wake_up_process(thread);	4486	wake_up_process(thread);
4463	return;	4487	return;
4464	}	4488	}
4465	}	4489	}
4466		4490
		4491	use_local_napi:
4467	list_add_tail(&napi->poll_list, &sd->poll_list);	4492	list_add_tail(&napi->poll_list, &sd->poll_list);
4468	WRITE_ONCE(napi->list_owner, smp_processor_id());	4493	WRITE_ONCE(napi->list_owner, smp_processor_id());
4469	/* If not called from net_rx_action()	4494	/* If not called from net_rx_action()
Lines 4709-4714 static void napi_schedule_rps(struct softnet_data *sd) Link Here
4709		4734
4710	#ifdef CONFIG_RPS	4735	#ifdef CONFIG_RPS
4711	if (sd != mysd) {	4736	if (sd != mysd) {
		4737	if (use_backlog_threads()) {
		4738	__napi_schedule_irqoff(&sd->backlog);
		4739	return;
		4740	}
		4741
4712	sd->rps_ipi_next = mysd->rps_ipi_list;	4742	sd->rps_ipi_next = mysd->rps_ipi_list;
4713	mysd->rps_ipi_list = sd;	4743	mysd->rps_ipi_list = sd;
4714		4744
Lines 4723-4728 static void napi_schedule_rps(struct softnet_data *sd) Link Here
4723	__napi_schedule_irqoff(&mysd->backlog);	4753	__napi_schedule_irqoff(&mysd->backlog);
4724	}	4754	}
4725		4755
		4756	void kick_defer_list_purge(struct softnet_data *sd, unsigned int cpu)
		4757	{
		4758	unsigned long flags;
		4759
		4760	if (use_backlog_threads()) {
		4761	backlog_lock_irq_save(sd, &flags);
		4762
		4763	if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
		4764	__napi_schedule_irqoff(&sd->backlog);
		4765
		4766	backlog_unlock_irq_restore(sd, &flags);
		4767
		4768	} else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) {
		4769	smp_call_function_single_async(cpu, &sd->defer_csd);
		4770	}
		4771	}
		4772
4726	#ifdef CONFIG_NET_FLOW_LIMIT	4773	#ifdef CONFIG_NET_FLOW_LIMIT
4727	int netdev_flow_limit_table_len __read_mostly = (1 << 12);	4774	int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4728	#endif	4775	#endif
Lines 4778-4784 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, Link Here
4778	reason = SKB_DROP_REASON_NOT_SPECIFIED;	4825	reason = SKB_DROP_REASON_NOT_SPECIFIED;
4779	sd = &per_cpu(softnet_data, cpu);	4826	sd = &per_cpu(softnet_data, cpu);
4780		4827
4781	rps_lock_irqsave(sd, &flags);	4828	backlog_lock_irq_save(sd, &flags);
4782	if (!netif_running(skb->dev))	4829	if (!netif_running(skb->dev))
4783	goto drop;	4830	goto drop;
4784	qlen = skb_queue_len(&sd->input_pkt_queue);	4831	qlen = skb_queue_len(&sd->input_pkt_queue);
Lines 4787-4793 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, Link Here
4787	enqueue:	4834	enqueue:
4788	__skb_queue_tail(&sd->input_pkt_queue, skb);	4835	__skb_queue_tail(&sd->input_pkt_queue, skb);
4789	input_queue_tail_incr_save(sd, qtail);	4836	input_queue_tail_incr_save(sd, qtail);
4790	rps_unlock_irq_restore(sd, &flags);	4837	backlog_unlock_irq_restore(sd, &flags);
4791	return NET_RX_SUCCESS;	4838	return NET_RX_SUCCESS;
4792	}	4839	}
4793		4840
Lines 4802-4808 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, Link Here
4802		4849
4803	drop:	4850	drop:
4804	sd->dropped++;	4851	sd->dropped++;
4805	rps_unlock_irq_restore(sd, &flags);	4852	backlog_unlock_irq_restore(sd, &flags);
4806		4853
4807	dev_core_stats_rx_dropped_inc(skb->dev);	4854	dev_core_stats_rx_dropped_inc(skb->dev);
4808	kfree_skb_reason(skb, reason);	4855	kfree_skb_reason(skb, reason);
Lines 5833-5839 static void flush_backlog(struct work_struct *work) Link Here
5833	local_bh_disable();	5880	local_bh_disable();
5834	sd = this_cpu_ptr(&softnet_data);	5881	sd = this_cpu_ptr(&softnet_data);
5835		5882
5836	rps_lock_irq_disable(sd);	5883	backlog_lock_irq_disable(sd);
5837	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {	5884	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
5838	if (skb->dev->reg_state == NETREG_UNREGISTERING) {	5885	if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5839	__skb_unlink(skb, &sd->input_pkt_queue);	5886	__skb_unlink(skb, &sd->input_pkt_queue);
Lines 5841-5847 static void flush_backlog(struct work_struct *work) Link Here
5841	input_queue_head_incr(sd);	5888	input_queue_head_incr(sd);
5842	}	5889	}
5843	}	5890	}
5844	rps_unlock_irq_enable(sd);	5891	backlog_unlock_irq_enable(sd);
5845		5892
5846	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {	5893	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
5847	if (skb->dev->reg_state == NETREG_UNREGISTERING) {	5894	if (skb->dev->reg_state == NETREG_UNREGISTERING) {
Lines 5859-5872 static bool flush_required(int cpu) Link Here
5859	struct softnet_data *sd = &per_cpu(softnet_data, cpu);	5906	struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5860	bool do_flush;	5907	bool do_flush;
5861		5908
5862	rps_lock_irq_disable(sd);	5909	backlog_lock_irq_disable(sd);
5863		5910
5864	/* as insertion into process_queue happens with the rps lock held,	5911	/* as insertion into process_queue happens with the rps lock held,
5865	* process_queue access may race only with dequeue	5912	* process_queue access may race only with dequeue
5866	*/	5913	*/
5867	do_flush = !skb_queue_empty(&sd->input_pkt_queue) \|\|	5914	do_flush = !skb_queue_empty(&sd->input_pkt_queue) \|\|
5868	!skb_queue_empty_lockless(&sd->process_queue);	5915	!skb_queue_empty_lockless(&sd->process_queue);
5869	rps_unlock_irq_enable(sd);	5916	backlog_unlock_irq_enable(sd);
5870		5917
5871	return do_flush;	5918	return do_flush;
5872	#endif	5919	#endif
Lines 5932-5938 static void net_rps_action_and_irq_enable(struct softnet_data *sd) Link Here
5932	#ifdef CONFIG_RPS	5979	#ifdef CONFIG_RPS
5933	struct softnet_data *remsd = sd->rps_ipi_list;	5980	struct softnet_data *remsd = sd->rps_ipi_list;
5934		5981
5935	if (remsd) {	5982	if (!use_backlog_threads() && remsd) {
5936	sd->rps_ipi_list = NULL;	5983	sd->rps_ipi_list = NULL;
5937		5984
5938	local_irq_enable();	5985	local_irq_enable();
Lines 5947-5953 static void net_rps_action_and_irq_enable(struct softnet_data *sd) Link Here
5947	static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)	5994	static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5948	{	5995	{
5949	#ifdef CONFIG_RPS	5996	#ifdef CONFIG_RPS
5950	return sd->rps_ipi_list != NULL;	5997	return !use_backlog_threads() && sd->rps_ipi_list;
5951	#else	5998	#else
5952	return false;	5999	return false;
5953	#endif	6000	#endif
Lines 5981-5987 static int process_backlog(struct napi_struct *napi, int quota) Link Here
5981		6028
5982	}	6029	}
5983		6030
5984	rps_lock_irq_disable(sd);	6031	backlog_lock_irq_disable(sd);
5985	if (skb_queue_empty(&sd->input_pkt_queue)) {	6032	if (skb_queue_empty(&sd->input_pkt_queue)) {
5986	/*	6033	/*
5987	* Inline a custom version of __napi_complete().	6034	* Inline a custom version of __napi_complete().
Lines 5991-6003 static int process_backlog(struct napi_struct *napi, int quota) Link Here
5991	* We can use a plain write instead of clear_bit(),	6038	* We can use a plain write instead of clear_bit(),
5992	* and we dont need an smp_mb() memory barrier.	6039	* and we dont need an smp_mb() memory barrier.
5993	*/	6040	*/
5994	napi->state = 0;	6041	napi->state &= NAPIF_STATE_THREADED;
5995	again = false;	6042	again = false;
5996	} else {	6043	} else {
5997	skb_queue_splice_tail_init(&sd->input_pkt_queue,	6044	skb_queue_splice_tail_init(&sd->input_pkt_queue,
5998	&sd->process_queue);	6045	&sd->process_queue);
5999	}	6046	}
6000	rps_unlock_irq_enable(sd);	6047	backlog_unlock_irq_enable(sd);
6001	}	6048	}
6002		6049
6003	return work;	6050	return work;
Lines 6654-6661 static int napi_poll(struct napi_struct n, struct list_head repoll) Link Here
6654		6701
6655	static int napi_thread_wait(struct napi_struct *napi)	6702	static int napi_thread_wait(struct napi_struct *napi)
6656	{	6703	{
6657	bool woken = false;
6658
6659	set_current_state(TASK_INTERRUPTIBLE);	6704	set_current_state(TASK_INTERRUPTIBLE);
6660		6705
6661	while (!kthread_should_stop()) {	6706	while (!kthread_should_stop()) {
Lines 6664-6678 static int napi_thread_wait(struct napi_struct *napi) Link Here
6664	* Testing SCHED bit is not enough because SCHED bit might be	6709	* Testing SCHED bit is not enough because SCHED bit might be
6665	* set by some other busy poll thread or by napi_disable().	6710	* set by some other busy poll thread or by napi_disable().
6666	*/	6711	*/
6667	if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state) \|\| woken) {	6712	if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) {
6668	WARN_ON(!list_empty(&napi->poll_list));	6713	WARN_ON(!list_empty(&napi->poll_list));
6669	__set_current_state(TASK_RUNNING);	6714	__set_current_state(TASK_RUNNING);
6670	return 0;	6715	return 0;
6671	}	6716	}
6672		6717
6673	schedule();	6718	schedule();
6674	/* woken being true indicates this thread owns this napi. */
6675	woken = true;
6676	set_current_state(TASK_INTERRUPTIBLE);	6719	set_current_state(TASK_INTERRUPTIBLE);
6677	}	6720	}
6678	__set_current_state(TASK_RUNNING);	6721	__set_current_state(TASK_RUNNING);
Lines 6701-6740 static void skb_defer_free_flush(struct softnet_data *sd) Link Here
6701	}	6744	}
6702	}	6745	}
6703		6746
		6747	static void napi_threaded_poll_loop(struct napi_struct *napi)
		6748	{
		6749	struct softnet_data *sd;
		6750
		6751	for (;;) {
		6752	bool repoll = false;
		6753	void *have;
		6754
		6755	local_bh_disable();
		6756	sd = this_cpu_ptr(&softnet_data);
		6757	sd->in_napi_threaded_poll = true;
		6758
		6759	have = netpoll_poll_lock(napi);
		6760	__napi_poll(napi, &repoll);
		6761	netpoll_poll_unlock(have);
		6762
		6763	sd->in_napi_threaded_poll = false;
		6764	barrier();
		6765
		6766	if (sd_has_rps_ipi_waiting(sd)) {
		6767	local_irq_disable();
		6768	net_rps_action_and_irq_enable(sd);
		6769	}
		6770	skb_defer_free_flush(sd);
		6771	local_bh_enable();
		6772
		6773	if (!repoll)
		6774	break;
		6775
		6776	cond_resched();
		6777	}
		6778	}
		6779
6704	static int napi_threaded_poll(void *data)	6780	static int napi_threaded_poll(void *data)
6705	{	6781	{
6706	struct napi_struct *napi = data;	6782	struct napi_struct *napi = data;
6707	struct softnet_data *sd;
6708	void *have;
6709		6783
6710	while (!napi_thread_wait(napi)) {	6784	while (!napi_thread_wait(napi))
6711	for (;;) {	6785	napi_threaded_poll_loop(napi);
6712	bool repoll = false;
6713		6786
6714	local_bh_disable();
6715	sd = this_cpu_ptr(&softnet_data);
6716	sd->in_napi_threaded_poll = true;
6717
6718	have = netpoll_poll_lock(napi);
6719	__napi_poll(napi, &repoll);
6720	netpoll_poll_unlock(have);
6721
6722	sd->in_napi_threaded_poll = false;
6723	barrier();
6724
6725	if (sd_has_rps_ipi_waiting(sd)) {
6726	local_irq_disable();
6727	net_rps_action_and_irq_enable(sd);
6728	}
6729	skb_defer_free_flush(sd);
6730	local_bh_enable();
6731
6732	if (!repoll)
6733	break;
6734
6735	cond_resched();
6736	}
6737	}
6738	return 0;	6787	return 0;
6739	}	6788	}
6740		6789
Lines 11333-11339 static int dev_cpu_dead(unsigned int oldcpu) Link Here
11333		11382
11334	list_del_init(&napi->poll_list);	11383	list_del_init(&napi->poll_list);
11335	if (napi->poll == process_backlog)	11384	if (napi->poll == process_backlog)
11336	napi->state = 0;	11385	napi->state &= NAPIF_STATE_THREADED;
11337	else	11386	else
11338	____napi_schedule(sd, napi);	11387	____napi_schedule(sd, napi);
11339	}	11388	}
Lines 11341-11352 static int dev_cpu_dead(unsigned int oldcpu) Link Here
11341	raise_softirq_irqoff(NET_TX_SOFTIRQ);	11390	raise_softirq_irqoff(NET_TX_SOFTIRQ);
11342	local_irq_enable();	11391	local_irq_enable();
11343		11392
		11393	if (!use_backlog_threads()) {
11344	#ifdef CONFIG_RPS	11394	#ifdef CONFIG_RPS
11345	remsd = oldsd->rps_ipi_list;	11395	remsd = oldsd->rps_ipi_list;
11346	oldsd->rps_ipi_list = NULL;	11396	oldsd->rps_ipi_list = NULL;
11347	#endif	11397	#endif
11348	/* send out pending IPI's on offline CPU */	11398	/* send out pending IPI's on offline CPU */
11349	net_rps_send_ipi(remsd);	11399	net_rps_send_ipi(remsd);
		11400	}
11350		11401
11351	/* Process offline CPU's input_pkt_queue */	11402	/* Process offline CPU's input_pkt_queue */
11352	while ((skb = __skb_dequeue(&oldsd->process_queue))) {	11403	while ((skb = __skb_dequeue(&oldsd->process_queue))) {
Lines 11665-11670 static void __init net_dev_struct_check(void) Link Here
11665	*	11716	*
11666	*/	11717	*/
11667		11718
		11719	static int backlog_napi_should_run(unsigned int cpu)
		11720	{
		11721	struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
		11722	struct napi_struct *napi = &sd->backlog;
		11723
		11724	return test_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
		11725	}
		11726
		11727	static void run_backlog_napi(unsigned int cpu)
		11728	{
		11729	struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
		11730
		11731	napi_threaded_poll_loop(&sd->backlog);
		11732	}
		11733
		11734	static void backlog_napi_setup(unsigned int cpu)
		11735	{
		11736	struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
		11737	struct napi_struct *napi = &sd->backlog;
		11738
		11739	napi->thread = this_cpu_read(backlog_napi);
		11740	set_bit(NAPI_STATE_THREADED, &napi->state);
		11741	}
		11742
		11743	static struct smp_hotplug_thread backlog_threads = {
		11744	.store = &backlog_napi,
		11745	.thread_should_run = backlog_napi_should_run,
		11746	.thread_fn = run_backlog_napi,
		11747	.thread_comm = "backlog_napi/%u",
		11748	.setup = backlog_napi_setup,
		11749	};
		11750
11668	/*	11751	/*
11669	* This is called single threaded during boot, so no need	11752	* This is called single threaded during boot, so no need
11670	* to take the rtnl semaphore.	11753	* to take the rtnl semaphore.
Lines 11717-11723 static int __init net_dev_init(void) Link Here
11717	init_gro_hash(&sd->backlog);	11800	init_gro_hash(&sd->backlog);
11718	sd->backlog.poll = process_backlog;	11801	sd->backlog.poll = process_backlog;
11719	sd->backlog.weight = weight_p;	11802	sd->backlog.weight = weight_p;
		11803	INIT_LIST_HEAD(&sd->backlog.poll_list);
11720	}	11804	}
		11805	if (use_backlog_threads())
		11806	smpboot_register_percpu_thread(&backlog_threads);
11721		11807
11722	dev_boot_phase = 0;	11808	dev_boot_phase = 0;
11723		11809

Lines 6921-6928 nodefer: __kfree_skb(skb); Link Here

(-)a/net/core/skbuff.c (-2 / +2 lines)
6921	/* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU	6921	/* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU
6922	* if we are unlucky enough (this seems very unlikely).	6922	* if we are unlucky enough (this seems very unlikely).
6923	*/	6923	*/
6924	if (unlikely(kick) && !cmpxchg(&sd->defer_ipi_scheduled, 0, 1))	6924	if (unlikely(kick))
6925	smp_call_function_single_async(cpu, &sd->defer_csd);	6925	kick_defer_list_purge(sd, cpu);
6926	}	6926	}
6927		6927
6928	static void skb_splice_csum_page(struct sk_buff skb, struct page page,	6928	static void skb_splice_csum_page(struct sk_buff skb, struct page page,

Return to bug 916954

Lines 1270-1275 config CHECKPOINT_RESTORE Link Here

(-)a/init/Kconfig (+19 lines)
1270		1270
1271	If unsure, say N here.	1271	If unsure, say N here.
1272		1272
		1273	config SCHED_BORE
		1274	bool "Burst-Oriented Response Enhancer"
		1275	default y
		1276	help
		1277	In Desktop and Mobile computing, one might prefer interactive
		1278	tasks to keep responsive no matter what they run in the background.
		1279
		1280	Enabling this kernel feature modifies the scheduler to discriminate
		1281	tasks by their burst time (runtime since it last went sleeping or
		1282	yielding state) and prioritize those that run less bursty.
		1283	Such tasks usually include window compositor, widgets backend,
		1284	terminal emulator, video playback, games and so on.
		1285	With a little impact to scheduling fairness, it may improve
		1286	responsiveness especially under heavy background workload.
		1287
		1288	You can turn it off by setting the sysctl kernel.sched_bore = 0.
		1289
		1290	If unsure, say Y here.
		1291
1273	config SCHED_AUTOGROUP	1292	config SCHED_AUTOGROUP
1274	bool "Automatic process group scheduling"	1293	bool "Automatic process group scheduling"
1275	select CGROUPS	1294	select CGROUPS

Lines 4507-4512 int wake_up_state(struct task_struct *p, unsigned int state) Link Here

(-)a/kernel/sched/core.c (+148 lines)
4507	return try_to_wake_up(p, state, 0);	4507	return try_to_wake_up(p, state, 0);
4508	}	4508	}
4509		4509
		4510	#ifdef CONFIG_SCHED_BORE
		4511	extern bool sched_bore;
		4512	extern u8 sched_burst_fork_atavistic;
		4513	extern uint sched_burst_cache_lifetime;
		4514
		4515	static void __init sched_init_bore(void) {
		4516	init_task.se.burst_time = 0;
		4517	init_task.se.prev_burst_penalty = 0;
		4518	init_task.se.curr_burst_penalty = 0;
		4519	init_task.se.burst_penalty = 0;
		4520	init_task.se.burst_score = 0;
		4521	init_task.se.on_cfs_rq = false;
		4522	init_task.se.child_burst_last_cached = 0;
		4523	init_task.se.burst_load = 0;
		4524	}
		4525
		4526	void inline sched_fork_bore(struct task_struct *p) {
		4527	p->se.burst_time = 0;
		4528	p->se.curr_burst_penalty = 0;
		4529	p->se.burst_score = 0;
		4530	p->se.on_cfs_rq = false;
		4531	p->se.child_burst_last_cached = 0;
		4532	p->se.burst_load = 0;
		4533	}
		4534
		4535	static u32 count_child_tasks(struct task_struct *p) {
		4536	struct task_struct *child;
		4537	u32 cnt = 0;
		4538	list_for_each_entry(child, &p->children, sibling) {cnt++;}
		4539	return cnt;
		4540	}
		4541
		4542	static inline bool task_is_inheritable(struct task_struct *p) {
		4543	return (p->sched_class == &fair_sched_class);
		4544	}
		4545
		4546	static inline bool child_burst_cache_expired(struct task_struct *p, u64 now) {
		4547	u64 expiration_time =
		4548	p->se.child_burst_last_cached + sched_burst_cache_lifetime;
		4549	return ((s64)(expiration_time - now) < 0);
		4550	}
		4551
		4552	static void __update_child_burst_cache(
		4553	struct task_struct *p, u32 cnt, u32 sum, u64 now) {
		4554	u8 avg = 0;
		4555	if (cnt) avg = sum / cnt;
		4556	p->se.child_burst = max(avg, p->se.burst_penalty);
		4557	p->se.child_burst_cnt = cnt;
		4558	p->se.child_burst_last_cached = now;
		4559	}
		4560
		4561	static inline void update_child_burst_direct(struct task_struct *p, u64 now) {
		4562	struct task_struct *child;
		4563	u32 cnt = 0;
		4564	u32 sum = 0;
		4565
		4566	list_for_each_entry(child, &p->children, sibling) {
		4567	if (!task_is_inheritable(child)) continue;
		4568	cnt++;
		4569	sum += child->se.burst_penalty;
		4570	}
		4571
		4572	__update_child_burst_cache(p, cnt, sum, now);
		4573	}
4574
4575	static inline u8 __inherit_burst_direct(struct task_struct *p, u64 now) {
4576	struct task_struct *parent = p->real_parent;
4577	if (child_burst_cache_expired(parent, now))
4578	update_child_burst_direct(parent, now);
4579
4580	return parent->se.child_burst;
4581	}
4582
4583	static void update_child_burst_topological(
4584	struct task_struct p, u64 now, u32 depth, u32 acnt, u32 *asum) {
4585	struct task_struct child, dec;
4586	u32 cnt = 0, dcnt = 0;
4587	u32 sum = 0;
4588
4589	list_for_each_entry(child, &p->children, sibling) {
4590	dec = child;
4591	while ((dcnt = count_child_tasks(dec)) == 1)
4592	dec = list_first_entry(&dec->children, struct task_struct, sibling);
4593
4594	if (!dcnt \|\| !depth) {
4595	if (!task_is_inheritable(dec)) continue;
4596	cnt++;
4597	sum += dec->se.burst_penalty;
4598	continue;
4599	}
4600	if (!child_burst_cache_expired(dec, now)) {
4601	cnt += dec->se.child_burst_cnt;
4602	sum += (u32)dec->se.child_burst * dec->se.child_burst_cnt;
4603	continue;
4604	}
4605	update_child_burst_topological(dec, now, depth - 1, &cnt, &sum);
4606	}
4607
4608	__update_child_burst_cache(p, cnt, sum, now);
4609	*acnt += cnt;
4610	*asum += sum;
4611	}
4612
4613	static inline u8 __inherit_burst_topological(struct task_struct *p, u64 now) {
4614	struct task_struct *anc = p->real_parent;
4615	u32 cnt = 0, sum = 0;
4616
4617	while (anc->real_parent != anc && count_child_tasks(anc) == 1)
4618	anc = anc->real_parent;
4619
4620	if (child_burst_cache_expired(anc, now))
4621	update_child_burst_topological(
4622	anc, now, sched_burst_fork_atavistic - 1, &cnt, &sum);
4623
4624	return anc->se.child_burst;
4625	}
4626
4627	static inline void inherit_burst(struct task_struct *p) {
4628	u8 burst_cache;
4629	u64 now = ktime_get_ns();
4630
4631	read_lock(&tasklist_lock);
4632	burst_cache = likely(sched_burst_fork_atavistic)?
4633	__inherit_burst_topological(p, now):
4634	__inherit_burst_direct(p, now);
4635	read_unlock(&tasklist_lock);
4636
4637	p->se.prev_burst_penalty = max(p->se.prev_burst_penalty, burst_cache);
4638	}
4639
4640	static void sched_post_fork_bore(struct task_struct *p) {
4641	if (p->sched_class == &fair_sched_class && likely(sched_bore))
4642	inherit_burst(p);
4643	p->se.burst_penalty = p->se.prev_burst_penalty;
4644	}
4645	#endif // CONFIG_SCHED_BORE
4646
4510	/*	4647	/*
4511	* Perform scheduler related setup for a newly forked process p.	4648	* Perform scheduler related setup for a newly forked process p.
4512	* p is forked by current.	4649	* p is forked by current.
Lines 4523-4528 static void __sched_fork(unsigned long clone_flags, struct task_struct *p) Link Here
4523	p->se.prev_sum_exec_runtime = 0;	4660	p->se.prev_sum_exec_runtime = 0;
4524	p->se.nr_migrations = 0;	4661	p->se.nr_migrations = 0;
4525	p->se.vruntime = 0;	4662	p->se.vruntime = 0;
		4663	#ifdef CONFIG_SCHED_BORE
		4664	sched_fork_bore(p);
		4665	#endif // CONFIG_SCHED_BORE
4526	p->se.vlag = 0;	4666	p->se.vlag = 0;
4527	p->se.slice = sysctl_sched_base_slice;	4667	p->se.slice = sysctl_sched_base_slice;
4528	INIT_LIST_HEAD(&p->se.group_node);	4668	INIT_LIST_HEAD(&p->se.group_node);
Lines 4839-4844 void sched_cgroup_fork(struct task_struct p, struct kernel_clone_args kargs) Link Here
4839		4979
4840	void sched_post_fork(struct task_struct *p)	4980	void sched_post_fork(struct task_struct *p)
4841	{	4981	{
		4982	#ifdef CONFIG_SCHED_BORE
		4983	sched_post_fork_bore(p);
		4984	#endif // CONFIG_SCHED_BORE
4842	uclamp_post_fork(p);	4985	uclamp_post_fork(p);
4843	}	4986	}
4844		4987
Lines 9910-9915 void __init sched_init(void) Link Here
9910	BUG_ON(&dl_sched_class != &stop_sched_class + 1);	10053	BUG_ON(&dl_sched_class != &stop_sched_class + 1);
9911	#endif	10054	#endif
9912		10055
		10056	#ifdef CONFIG_SCHED_BORE
		10057	sched_init_bore();
		10058	printk(KERN_INFO "BORE (Burst-Oriented Response Enhancer) CPU Scheduler modification 4.5.0 by Masahito Suzuki");
		10059	#endif // CONFIG_SCHED_BORE
		10060
9913	wait_bit_init();	10061	wait_bit_init();
9914		10062
9915	#ifdef CONFIG_FAIR_GROUP_SCHED	10063	#ifdef CONFIG_FAIR_GROUP_SCHED

Lines 6487-6492 Link Here

(-)a/Documentation/admin-guide/kernel-parameters.txt (+12 lines)
6487	Force threading of all interrupt handlers except those	6487	Force threading of all interrupt handlers except those
6488	marked explicitly IRQF_NO_THREAD.	6488	marked explicitly IRQF_NO_THREAD.
6489		6489
		6490	threadprintk [KNL]
		6491	Force threaded printing of all legacy consoles. Be
		6492	aware that with this option, the shutdown, reboot, and
		6493	panic messages may not be printed on the legacy
		6494	consoles. Also, earlycon/earlyprintk printing will be
		6495	delayed until a regular console or the kthread is
		6496	available.
		6497
		6498	Users can view /proc/consoles to see if their console
		6499	driver is legacy or not. Non-legacy (NBCON) console
		6500	drivers are already threaded and are shown with 'N'.
		6501
6490	topology= [S390]	6502	topology= [S390]
6491	Format: {off \| on}	6503	Format: {off \| on}
6492	Specify if the kernel should make use of the cpu	6504	Specify if the kernel should make use of the cpu

Lines 1303-1309 static void execlists_dequeue(struct intel_engine_cs *engine) Link Here

(-)a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c (-12 / +5 lines)
1303	* and context switches) submission.	1303	* and context switches) submission.
1304	*/	1304	*/
1305		1305
1306	spin_lock(&sched_engine->lock);	1306	spin_lock_irq(&sched_engine->lock);
1307		1307
1308	/*	1308	/*
1309	* If the queue is higher priority than the last	1309	* If the queue is higher priority than the last
Lines 1403-1409 static void execlists_dequeue(struct intel_engine_cs *engine) Link Here
1403	* Even if ELSP[1] is occupied and not worthy	1403	* Even if ELSP[1] is occupied and not worthy
1404	* of timeslices, our queue might be.	1404	* of timeslices, our queue might be.
1405	*/	1405	*/
1406	spin_unlock(&sched_engine->lock);	1406	spin_unlock_irq(&sched_engine->lock);
1407	return;	1407	return;
1408	}	1408	}
1409	}	1409	}
Lines 1429-1435 static void execlists_dequeue(struct intel_engine_cs *engine) Link Here
1429		1429
1430	if (last && !can_merge_rq(last, rq)) {	1430	if (last && !can_merge_rq(last, rq)) {
1431	spin_unlock(&ve->base.sched_engine->lock);	1431	spin_unlock(&ve->base.sched_engine->lock);
1432	spin_unlock(&engine->sched_engine->lock);	1432	spin_unlock_irq(&engine->sched_engine->lock);
1433	return; /* leave this for another sibling */	1433	return; /* leave this for another sibling */
1434	}	1434	}
1435		1435
Lines 1591-1597 static void execlists_dequeue(struct intel_engine_cs *engine) Link Here
1591	*/	1591	*/
1592	sched_engine->queue_priority_hint = queue_prio(sched_engine);	1592	sched_engine->queue_priority_hint = queue_prio(sched_engine);
1593	i915_sched_engine_reset_on_empty(sched_engine);	1593	i915_sched_engine_reset_on_empty(sched_engine);
1594	spin_unlock(&sched_engine->lock);	1594	spin_unlock_irq(&sched_engine->lock);
1595		1595
1596	/*	1596	/*
1597	* We can skip poking the HW if we ended up with exactly the same set	1597	* We can skip poking the HW if we ended up with exactly the same set
Lines 1617-1629 static void execlists_dequeue(struct intel_engine_cs *engine) Link Here
1617	}	1617	}
1618	}	1618	}
1619		1619
1620	static void execlists_dequeue_irq(struct intel_engine_cs *engine)
1621	{
1622	local_irq_disable(); /* Suspend interrupts across request submission */
1623	execlists_dequeue(engine);
1624	local_irq_enable(); /* flush irq_work (e.g. breadcrumb enabling) */
1625	}
1626
1627	static void clear_ports(struct i915_request **ports, int count)	1620	static void clear_ports(struct i915_request **ports, int count)
1628	{	1621	{
1629	memset_p((void **)ports, NULL, count);	1622	memset_p((void **)ports, NULL, count);
Lines 2478-2484 static void execlists_submission_tasklet(struct tasklet_struct *t) Link Here
2478	}	2471	}
2479		2472
2480	if (!engine->execlists.pending[0]) {	2473	if (!engine->execlists.pending[0]) {
2481	execlists_dequeue_irq(engine);	2474	execlists_dequeue(engine);
2482	start_timeslice(engine);	2475	start_timeslice(engine);
2483	}	2476	}
2484		2477

Lines 547-552 struct sched_entity { Link Here

(-)a/include/linux/sched.h (+12 lines)
547	u64 sum_exec_runtime;	547	u64 sum_exec_runtime;
548	u64 prev_sum_exec_runtime;	548	u64 prev_sum_exec_runtime;
549	u64 vruntime;	549	u64 vruntime;
		550	#ifdef CONFIG_SCHED_BORE
		551	u64 burst_time;
		552	u8 prev_burst_penalty;
		553	u8 curr_burst_penalty;
		554	u8 burst_penalty;
		555	u8 burst_score;
		556	u32 burst_load;
		557	bool on_cfs_rq;
		558	u8 child_burst;
		559	u32 child_burst_cnt;
		560	u64 child_burst_last_cached;
		561	#endif // CONFIG_SCHED_BORE
550	s64 vlag;	562	s64 vlag;
551	u64 slice;	563	u64 slice;
552		564

Lines 167-173 static const struct file_operations sched_feat_fops = { Link Here

(-)a/kernel/sched/debug.c (-4 / +63 lines)
167	};	167	};
168		168
169	#ifdef CONFIG_SMP	169	#ifdef CONFIG_SMP
		170	#ifdef CONFIG_SCHED_BORE
		171	static ssize_t sched_min_base_slice_write(struct file filp, const char __user ubuf,
		172	size_t cnt, loff_t *ppos)
		173	{
		174	char buf[16];
		175	unsigned int value;
		176
		177	if (cnt > 15)
		178	cnt = 15;
		179
		180	if (copy_from_user(&buf, ubuf, cnt))
		181	return -EFAULT;
		182	buf[cnt] = '\0';
		183
		184	if (kstrtouint(buf, 10, &value))
		185	return -EINVAL;
170		186
		187	if (!value)
		188	return -EINVAL;
		189
		190	sysctl_sched_min_base_slice = value;
		191	sched_update_min_base_slice();
		192
		193	*ppos += cnt;
		194	return cnt;
		195	}
		196
		197	static int sched_min_base_slice_show(struct seq_file m, void v)
		198	{
		199	seq_printf(m, "%d\n", sysctl_sched_min_base_slice);
		200	return 0;
		201	}
		202
		203	static int sched_min_base_slice_open(struct inode inode, struct file filp)
		204	{
		205	return single_open(filp, sched_min_base_slice_show, NULL);
		206	}
		207
		208	static const struct file_operations sched_min_base_slice_fops = {
		209	.open = sched_min_base_slice_open,
		210	.write = sched_min_base_slice_write,
		211	.read = seq_read,
		212	.llseek = seq_lseek,
		213	.release = single_release,
		214	};
		215	#else // !CONFIG_SCHED_BORE
171	static ssize_t sched_scaling_write(struct file filp, const char __user ubuf,	216	static ssize_t sched_scaling_write(struct file filp, const char __user ubuf,
172	size_t cnt, loff_t *ppos)	217	size_t cnt, loff_t *ppos)
173	{	218	{
Lines 213-219 static const struct file_operations sched_scaling_fops = { Link Here
213	.llseek = seq_lseek,	258	.llseek = seq_lseek,
214	.release = single_release,	259	.release = single_release,
215	};	260	};
216		261	#endif // CONFIG_SCHED_BORE
217	#endif /* SMP */	262	#endif /* SMP */
218		263
219	#ifdef CONFIG_PREEMPT_DYNAMIC	264	#ifdef CONFIG_PREEMPT_DYNAMIC
Lines 353-366 Link Here
353	debugfs_create_file("preempt", 0644, debugfs_sched, NULL, &sched_dynamic_fops);	353	debugfs_create_file("preempt", 0644, debugfs_sched, NULL, &sched_dynamic_fops);
354	#endif	354	#endif
355		355
356	debugfs_create_u32("base_slice_ns", 0644, debugfs_sched, &sysctl_sched_base_slice);	356	#ifdef CONFIG_SCHED_BORE
		357	debugfs_create_file("min_base_slice_ns", 0644, debugfs_sched, NULL, &sched_min_base_slice_fops);
		358	debugfs_create_u32("base_slice_ns", 0400, debugfs_sched, &sysctl_sched_base_slice);
		359	#else // !CONFIG_SCHED_BORE
		360	debugfs_create_u32("base_slice_ns", 0644, debugfs_sched, &sysctl_sched_base_slice);
		361	#endif // CONFIG_SCHED_BORE
357		362
358	#ifndef CONFIG_SCHED_ALT	363	#ifndef CONFIG_SCHED_ALT
359	debugfs_create_u32("latency_warn_ms", 0644, debugfs_sched, &sysctl_resched_latency_warn_ms);	364	debugfs_create_u32("latency_warn_ms", 0644, debugfs_sched, &sysctl_resched_latency_warn_ms);
360	debugfs_create_u32("latency_warn_once", 0644, debugfs_sched, &sysctl_resched_latency_warn_once);	365	debugfs_create_u33("latency_warn_once", 0644, debugfs_sched, &sysctl_resched_latency_warn_once);
361		366
362	#ifdef CONFIG_SMP	367	#ifdef CONFIG_SMP
363	debugfs_create_file("tunable_scaling", 0644, debugfs_sched, NULL, &sched_scaling_fops);	368	#if !defined(CONFIG_SCHED_BORE)
		369	debugfs_create_file("tunable_scaling", 0644, debugfs_sched, NULL, &sched_scaling_fops);
		370	#endif // CONFIG_SCHED_BORE
364	debugfs_create_u32("migration_cost_ns", 0644, debugfs_sched, &sysctl_sched_migration_cost);	371	debugfs_create_u32("migration_cost_ns", 0644, debugfs_sched, &sysctl_sched_migration_cost);
365	debugfs_create_u32("nr_migrate", 0644, debugfs_sched, &sysctl_sched_nr_migrate);	372	debugfs_create_u32("nr_migrate", 0644, debugfs_sched, &sysctl_sched_nr_migrate);
366		373
Lines 595-600 print_task(struct seq_file m, struct rq rq, struct task_struct *p) Link Here
595	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)),	647	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)),
596	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime)));	648	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime)));
597		649
		650	#ifdef CONFIG_SCHED_BORE
		651	SEQ_printf(m, " %2d", p->se.burst_score);
		652	#endif // CONFIG_SCHED_BORE
598	#ifdef CONFIG_NUMA_BALANCING	653	#ifdef CONFIG_NUMA_BALANCING
599	SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));	654	SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
600	#endif	655	#endif
Lines 1068-1073 void proc_sched_show_task(struct task_struct p, struct pid_namespace ns, Link Here
1068		1123
1069	P(se.load.weight);	1124	P(se.load.weight);
1070	#ifdef CONFIG_SMP	1125	#ifdef CONFIG_SMP
		1126	#ifdef CONFIG_SCHED_BORE
		1127	P(se.burst_load);
		1128	P(se.burst_score);
		1129	#endif // CONFIG_SCHED_BORE
1071	P(se.avg.load_sum);	1130	P(se.avg.load_sum);
1072	P(se.avg.runnable_sum);	1131	P(se.avg.runnable_sum);
1073	P(se.avg.util_sum);	1132	P(se.avg.util_sum);

Lines 36-41 config ARM Link Here

(-)a/arch/arm/Kconfig (-2 / +4 lines)
36	select ARCH_SUPPORTS_ATOMIC_RMW	36	select ARCH_SUPPORTS_ATOMIC_RMW
37	select ARCH_SUPPORTS_HUGETLBFS if ARM_LPAE	37	select ARCH_SUPPORTS_HUGETLBFS if ARM_LPAE
38	select ARCH_SUPPORTS_PER_VMA_LOCK	38	select ARCH_SUPPORTS_PER_VMA_LOCK
		39	select ARCH_SUPPORTS_RT if HAVE_POSIX_CPU_TIMERS_TASK_WORK
39	select ARCH_USE_BUILTIN_BSWAP	40	select ARCH_USE_BUILTIN_BSWAP
40	select ARCH_USE_CMPXCHG_LOCKREF	41	select ARCH_USE_CMPXCHG_LOCKREF
41	select ARCH_USE_MEMTEST	42	select ARCH_USE_MEMTEST
Lines 75-81 config ARM Link Here
75	select HAS_IOPORT	76	select HAS_IOPORT
76	select HAVE_ARCH_AUDITSYSCALL if AEABI && !OABI_COMPAT	77	select HAVE_ARCH_AUDITSYSCALL if AEABI && !OABI_COMPAT
77	select HAVE_ARCH_BITREVERSE if (CPU_32v7M \|\| CPU_32v7) && !CPU_32v6	78	select HAVE_ARCH_BITREVERSE if (CPU_32v7M \|\| CPU_32v7) && !CPU_32v6
78	select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU	79	select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU && !PREEMPT_RT
79	select HAVE_ARCH_KFENCE if MMU && !XIP_KERNEL	80	select HAVE_ARCH_KFENCE if MMU && !XIP_KERNEL
80	select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU	81	select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU
81	select HAVE_ARCH_KASAN if MMU && !XIP_KERNEL	82	select HAVE_ARCH_KASAN if MMU && !XIP_KERNEL
Lines 98-104 config ARM Link Here
98	select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE	99	select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE
99	select HAVE_EFFICIENT_UNALIGNED_ACCESS if (CPU_V6 \|\| CPU_V6K \|\| CPU_V7) && MMU	100	select HAVE_EFFICIENT_UNALIGNED_ACCESS if (CPU_V6 \|\| CPU_V6K \|\| CPU_V7) && MMU
100	select HAVE_EXIT_THREAD	101	select HAVE_EXIT_THREAD
101	select HAVE_FAST_GUP if ARM_LPAE	102	select HAVE_FAST_GUP if ARM_LPAE && !(PREEMPT_RT && HIGHPTE)
102	select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL	103	select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL
103	select HAVE_FUNCTION_ERROR_INJECTION	104	select HAVE_FUNCTION_ERROR_INJECTION
104	select HAVE_FUNCTION_GRAPH_TRACER	105	select HAVE_FUNCTION_GRAPH_TRACER
Lines 120-125 config ARM Link Here
120	select HAVE_PERF_EVENTS	121	select HAVE_PERF_EVENTS
121	select HAVE_PERF_REGS	122	select HAVE_PERF_REGS
122	select HAVE_PERF_USER_STACK_DUMP	123	select HAVE_PERF_USER_STACK_DUMP
		124	select HAVE_POSIX_CPU_TIMERS_TASK_WORK if !KVM
123	select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE	125	select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE
124	select HAVE_REGS_AND_STACK_ACCESS_API	126	select HAVE_REGS_AND_STACK_ACCESS_API
125	select HAVE_RSEQ	127	select HAVE_RSEQ

Lines 55-60 extern unsigned int VFP_arch_feroceon __alias(VFP_arch); Link Here

(-)a/arch/arm/vfp/vfpmodule.c (-21 / +53 lines)
55	*/	55	*/
56	union vfp_state *vfp_current_hw_state[NR_CPUS];	56	union vfp_state *vfp_current_hw_state[NR_CPUS];
57		57
		58	/*
		59	* Claim ownership of the VFP unit.
		60	*
		61	* The caller may change VFP registers until vfp_unlock() is called.
		62	*
		63	* local_bh_disable() is used to disable preemption and to disable VFP
		64	* processing in softirq context. On PREEMPT_RT kernels local_bh_disable() is
		65	* not sufficient because it only serializes soft interrupt related sections
		66	* via a local lock, but stays preemptible. Disabling preemption is the right
		67	* choice here as bottom half processing is always in thread context on RT
		68	* kernels so it implicitly prevents bottom half processing as well.
		69	*/
		70	static void vfp_lock(void)
		71	{
		72	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		73	local_bh_disable();
		74	else
		75	preempt_disable();
		76	}
		77
		78	static void vfp_unlock(void)
		79	{
		80	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		81	local_bh_enable();
		82	else
		83	preempt_enable();
		84	}
		85
58	/*	86	/*
59	* Is 'thread's most up to date state stored in this CPUs hardware?	87	* Is 'thread's most up to date state stored in this CPUs hardware?
60	* Must be called from non-preemptible context.	88	* Must be called from non-preemptible context.
Lines 240-246 static void vfp_panic(char *reason, u32 inst) Link Here
240	/*	268	/*
241	* Process bitmask of exception conditions.	269	* Process bitmask of exception conditions.
242	*/	270	*/
243	static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_regs *regs)	271	static int vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr)
244	{	272	{
245	int si_code = 0;	273	int si_code = 0;
246		274
Lines 248-255 static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_ Link Here
248		276
249	if (exceptions == VFP_EXCEPTION_ERROR) {	277	if (exceptions == VFP_EXCEPTION_ERROR) {
250	vfp_panic("unhandled bounce", inst);	278	vfp_panic("unhandled bounce", inst);
251	vfp_raise_sigfpe(FPE_FLTINV, regs);	279	return FPE_FLTINV;
252	return;
253	}	280	}
254		281
255	/*	282	/*
Lines 277-284 static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_ Link Here
277	RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF);	304	RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF);
278	RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV);	305	RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV);
279		306
280	if (si_code)	307	return si_code;
281	vfp_raise_sigfpe(si_code, regs);
282	}	308	}
283		309
284	/*	310	/*
Lines 324-329 static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs) Link Here
324	static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)	350	static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
325	{	351	{
326	u32 fpscr, orig_fpscr, fpsid, exceptions;	352	u32 fpscr, orig_fpscr, fpsid, exceptions;
		353	int si_code2 = 0;
		354	int si_code = 0;
327		355
328	pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);	356	pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);
329		357
Lines 369-376 static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) Link Here
369	* unallocated VFP instruction but with FPSCR.IXE set and not	397	* unallocated VFP instruction but with FPSCR.IXE set and not
370	* on VFP subarch 1.	398	* on VFP subarch 1.
371	*/	399	*/
372	vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr, regs);	400	si_code = vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr);
373	return;	401	goto exit;
374	}	402	}
375		403
376	/*	404	/*
Lines 394-407 static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) Link Here
394	*/	422	*/
395	exceptions = vfp_emulate_instruction(trigger, fpscr, regs);	423	exceptions = vfp_emulate_instruction(trigger, fpscr, regs);
396	if (exceptions)	424	if (exceptions)
397	vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);	425	si_code2 = vfp_raise_exceptions(exceptions, trigger, orig_fpscr);
398		426
399	/*	427	/*
400	* If there isn't a second FP instruction, exit now. Note that	428	* If there isn't a second FP instruction, exit now. Note that
401	* the FPEXC.FP2V bit is valid only if FPEXC.EX is 1.	429	* the FPEXC.FP2V bit is valid only if FPEXC.EX is 1.
402	*/	430	*/
403	if ((fpexc & (FPEXC_EX \| FPEXC_FP2V)) != (FPEXC_EX \| FPEXC_FP2V))	431	if ((fpexc & (FPEXC_EX \| FPEXC_FP2V)) != (FPEXC_EX \| FPEXC_FP2V))
404	return;	432	goto exit;
405		433
406	/*	434	/*
407	* The barrier() here prevents fpinst2 being read	435	* The barrier() here prevents fpinst2 being read
Lines 413-419 static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) Link Here
413	emulate:	441	emulate:
414	exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);	442	exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);
415	if (exceptions)	443	if (exceptions)
416	vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);	444	si_code = vfp_raise_exceptions(exceptions, trigger, orig_fpscr);
		445	exit:
		446	vfp_unlock();
		447	if (si_code2)
		448	vfp_raise_sigfpe(si_code2, regs);
		449	if (si_code)
		450	vfp_raise_sigfpe(si_code, regs);
417	}	451	}
418		452
419	static void vfp_enable(void *unused)	453	static void vfp_enable(void *unused)
Lines 512-522 static inline void vfp_pm_init(void) { } Link Here
512	*/	546	*/
513	void vfp_sync_hwstate(struct thread_info *thread)	547	void vfp_sync_hwstate(struct thread_info *thread)
514	{	548	{
515	unsigned int cpu = get_cpu();	549	vfp_lock();
516		550
517	local_bh_disable();	551	if (vfp_state_in_hw(raw_smp_processor_id(), thread)) {
518
519	if (vfp_state_in_hw(cpu, thread)) {
520	u32 fpexc = fmrx(FPEXC);	552	u32 fpexc = fmrx(FPEXC);
521		553
522	/*	554	/*
Lines 527-534 void vfp_sync_hwstate(struct thread_info *thread) Link Here
527	fmxr(FPEXC, fpexc);	559	fmxr(FPEXC, fpexc);
528	}	560	}
529		561
530	local_bh_enable();	562	vfp_unlock();
531	put_cpu();
532	}	563	}
533		564
534	/* Ensure that the thread reloads the hardware VFP state on the next use. */	565	/* Ensure that the thread reloads the hardware VFP state on the next use. */
Lines 683-689 static int vfp_support_entry(struct pt_regs *regs, u32 trigger) Link Here
683	if (!user_mode(regs))	714	if (!user_mode(regs))
684	return vfp_kmode_exception(regs, trigger);	715	return vfp_kmode_exception(regs, trigger);
685		716
686	local_bh_disable();	717	vfp_lock();
687	fpexc = fmrx(FPEXC);	718	fpexc = fmrx(FPEXC);
688		719
689	/*	720	/*
Lines 748-753 static int vfp_support_entry(struct pt_regs *regs, u32 trigger) Link Here
748	* replay the instruction that trapped.	779	* replay the instruction that trapped.
749	*/	780	*/
750	fmxr(FPEXC, fpexc);	781	fmxr(FPEXC, fpexc);
		782	vfp_unlock();
751	} else {	783	} else {
752	/* Check for synchronous or asynchronous exceptions */	784	/* Check for synchronous or asynchronous exceptions */
753	if (!(fpexc & (FPEXC_EX \| FPEXC_DEX))) {	785	if (!(fpexc & (FPEXC_EX \| FPEXC_DEX))) {
Lines 762-778 static int vfp_support_entry(struct pt_regs *regs, u32 trigger) Link Here
762	if (!(fpscr & FPSCR_IXE)) {	794	if (!(fpscr & FPSCR_IXE)) {
763	if (!(fpscr & FPSCR_LENGTH_MASK)) {	795	if (!(fpscr & FPSCR_LENGTH_MASK)) {
764	pr_debug("not VFP\n");	796	pr_debug("not VFP\n");
765	local_bh_enable();	797	vfp_unlock();
766	return -ENOEXEC;	798	return -ENOEXEC;
767	}	799	}
768	fpexc \|= FPEXC_DEX;	800	fpexc \|= FPEXC_DEX;
769	}	801	}
770	}	802	}
771	bounce: regs->ARM_pc += 4;	803	bounce: regs->ARM_pc += 4;
		804	/* VFP_bounce() will invoke vfp_unlock() */
772	VFP_bounce(trigger, fpexc, regs);	805	VFP_bounce(trigger, fpexc, regs);
773	}	806	}
774		807
775	local_bh_enable();
776	return 0;	808	return 0;
777	}	809	}
778		810
Lines 837-843 void kernel_neon_begin(void) Link Here
837	unsigned int cpu;	869	unsigned int cpu;
838	u32 fpexc;	870	u32 fpexc;
839		871
840	local_bh_disable();	872	vfp_lock();
841		873
842	/*	874	/*
843	* Kernel mode NEON is only allowed outside of hardirq context with	875	* Kernel mode NEON is only allowed outside of hardirq context with
Lines 868-874 void kernel_neon_end(void) Link Here
868	{	900	{
869	/* Disable the NEON/VFP unit. */	901	/* Disable the NEON/VFP unit. */
870	fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);	902	fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
871	local_bh_enable();	903	vfp_unlock();
872	}	904	}
873	EXPORT_SYMBOL(kernel_neon_end);	905	EXPORT_SYMBOL(kernel_neon_end);
874		906

Lines 49-54 config RISCV Link Here

(-)a/arch/riscv/Kconfig (+2 lines)
49	select ARCH_SUPPORTS_HUGETLBFS if MMU	49	select ARCH_SUPPORTS_HUGETLBFS if MMU
50	select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU	50	select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU
51	select ARCH_SUPPORTS_PER_VMA_LOCK if MMU	51	select ARCH_SUPPORTS_PER_VMA_LOCK if MMU
		52	select ARCH_SUPPORTS_RT
52	select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK	53	select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK
53	select ARCH_USE_MEMTEST	54	select ARCH_USE_MEMTEST
54	select ARCH_USE_QUEUED_RWLOCKS	55	select ARCH_USE_QUEUED_RWLOCKS
Lines 142-147 config RISCV Link Here
142	select HAVE_PERF_USER_STACK_DUMP	143	select HAVE_PERF_USER_STACK_DUMP
143	select HAVE_POSIX_CPU_TIMERS_TASK_WORK	144	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
144	select HAVE_PREEMPT_DYNAMIC_KEY if !XIP_KERNEL	145	select HAVE_PREEMPT_DYNAMIC_KEY if !XIP_KERNEL
		146	select HAVE_PREEMPT_AUTO
145	select HAVE_REGS_AND_STACK_ACCESS_API	147	select HAVE_REGS_AND_STACK_ACCESS_API
146	select HAVE_RETHOOK if !XIP_KERNEL	148	select HAVE_RETHOOK if !XIP_KERNEL
147	select HAVE_RSEQ	149	select HAVE_RSEQ

Lines 28-33 config X86_64 Link Here

(-)a/arch/x86/Kconfig (+3 lines)
28	select ARCH_HAS_GIGANTIC_PAGE	28	select ARCH_HAS_GIGANTIC_PAGE
29	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128	29	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
30	select ARCH_SUPPORTS_PER_VMA_LOCK	30	select ARCH_SUPPORTS_PER_VMA_LOCK
		31	select ARCH_SUPPORTS_RT
31	select HAVE_ARCH_SOFT_DIRTY	32	select HAVE_ARCH_SOFT_DIRTY
32	select MODULES_USE_ELF_RELA	33	select MODULES_USE_ELF_RELA
33	select NEED_DMA_MAP_STATE	34	select NEED_DMA_MAP_STATE
Lines 119-124 config X86 Link Here
119	select ARCH_USES_CFI_TRAPS if X86_64 && CFI_CLANG	120	select ARCH_USES_CFI_TRAPS if X86_64 && CFI_CLANG
120	select ARCH_SUPPORTS_LTO_CLANG	121	select ARCH_SUPPORTS_LTO_CLANG
121	select ARCH_SUPPORTS_LTO_CLANG_THIN	122	select ARCH_SUPPORTS_LTO_CLANG_THIN
		123	select ARCH_SUPPORTS_RT
122	select ARCH_USE_BUILTIN_BSWAP	124	select ARCH_USE_BUILTIN_BSWAP
123	select ARCH_USE_CMPXCHG_LOCKREF if X86_CMPXCHG64	125	select ARCH_USE_CMPXCHG_LOCKREF if X86_CMPXCHG64
124	select ARCH_USE_MEMTEST	126	select ARCH_USE_MEMTEST
Lines 275-280 config X86 Link Here
275	select HAVE_STATIC_CALL	277	select HAVE_STATIC_CALL
276	select HAVE_STATIC_CALL_INLINE if HAVE_OBJTOOL	278	select HAVE_STATIC_CALL_INLINE if HAVE_OBJTOOL
277	select HAVE_PREEMPT_DYNAMIC_CALL	279	select HAVE_PREEMPT_DYNAMIC_CALL
		280	select HAVE_PREEMPT_AUTO
278	select HAVE_RSEQ	281	select HAVE_RSEQ
279	select HAVE_RUST if X86_64	282	select HAVE_RUST if X86_64
280	select HAVE_SYSCALL_TRACEPOINTS	283	select HAVE_SYSCALL_TRACEPOINTS

Lines 81-88 struct thread_info { Link Here

(-)a/arch/x86/include/asm/thread_info.h (-2 / +4 lines)
81	#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */	81	#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */
82	#define TIF_SIGPENDING 2 /* signal pending */	82	#define TIF_SIGPENDING 2 /* signal pending */
83	#define TIF_NEED_RESCHED 3 /* rescheduling necessary */	83	#define TIF_NEED_RESCHED 3 /* rescheduling necessary */
84	#define TIF_SINGLESTEP 4 /* reenable singlestep on user return*/	84	#define TIF_ARCH_RESCHED_LAZY 4 /* Lazy rescheduling */
85	#define TIF_SSBD 5 /* Speculative store bypass disable */	85	#define TIF_SINGLESTEP 5 /* reenable singlestep on user return*/
		86	#define TIF_SSBD 6 /* Speculative store bypass disable */
86	#define TIF_SPEC_IB 9 /* Indirect branch speculation mitigation */	87	#define TIF_SPEC_IB 9 /* Indirect branch speculation mitigation */
87	#define TIF_SPEC_L1D_FLUSH 10 /* Flush L1D on mm switches (processes) */	88	#define TIF_SPEC_L1D_FLUSH 10 /* Flush L1D on mm switches (processes) */
88	#define TIF_USER_RETURN_NOTIFY 11 /* notify kernel of userspace return */	89	#define TIF_USER_RETURN_NOTIFY 11 /* notify kernel of userspace return */
Lines 104-109 struct thread_info { Link Here
104	#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)	105	#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
105	#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)	106	#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
106	#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)	107	#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
		108	#define _TIF_ARCH_RESCHED_LAZY (1 << TIF_ARCH_RESCHED_LAZY)
107	#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP)	109	#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP)
108	#define _TIF_SSBD (1 << TIF_SSBD)	110	#define _TIF_SSBD (1 << TIF_SSBD)
109	#define _TIF_SPEC_IB (1 << TIF_SPEC_IB)	111	#define _TIF_SPEC_IB (1 << TIF_SPEC_IB)

Lines 57-62 static void zram_free_page(struct zram *zram, size_t index); Link Here

(-)a/drivers/block/zram/zram_drv.c (+37 lines)
57	static int zram_read_page(struct zram zram, struct page page, u32 index,	57	static int zram_read_page(struct zram zram, struct page page, u32 index,
58	struct bio *parent);	58	struct bio *parent);
59		59
		60	#ifdef CONFIG_PREEMPT_RT
		61	static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages)
		62	{
		63	size_t index;
		64
		65	for (index = 0; index < num_pages; index++)
		66	spin_lock_init(&zram->table[index].lock);
		67	}
		68
		69	static int zram_slot_trylock(struct zram *zram, u32 index)
		70	{
		71	int ret;
		72
		73	ret = spin_trylock(&zram->table[index].lock);
		74	if (ret)
		75	__set_bit(ZRAM_LOCK, &zram->table[index].flags);
		76	return ret;
		77	}
		78
		79	static void zram_slot_lock(struct zram *zram, u32 index)
		80	{
		81	spin_lock(&zram->table[index].lock);
		82	__set_bit(ZRAM_LOCK, &zram->table[index].flags);
		83	}
		84
		85	static void zram_slot_unlock(struct zram *zram, u32 index)
		86	{
		87	__clear_bit(ZRAM_LOCK, &zram->table[index].flags);
		88	spin_unlock(&zram->table[index].lock);
		89	}
		90
		91	#else
		92
		93	static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages) { }
		94
60	static int zram_slot_trylock(struct zram *zram, u32 index)	95	static int zram_slot_trylock(struct zram *zram, u32 index)
61	{	96	{
62	return bit_spin_trylock(ZRAM_LOCK, &zram->table[index].flags);	97	return bit_spin_trylock(ZRAM_LOCK, &zram->table[index].flags);
Lines 71-76 static void zram_slot_unlock(struct zram *zram, u32 index) Link Here
71	{	106	{
72	bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags);	107	bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags);
73	}	108	}
		109	#endif
74		110
75	static inline bool init_done(struct zram *zram)	111	static inline bool init_done(struct zram *zram)
76	{	112	{
Lines 1241-1246 static bool zram_meta_alloc(struct zram *zram, u64 disksize) Link Here
1241		1277
1242	if (!huge_class_size)	1278	if (!huge_class_size)
1243	huge_class_size = zs_huge_class_size(zram->mem_pool);	1279	huge_class_size = zs_huge_class_size(zram->mem_pool);
		1280	zram_meta_init_table_locks(zram, num_pages);
1244	return true;	1281	return true;
1245	}	1282	}
1246		1283

Lines 580-586 void intel_pipe_update_start(struct intel_atomic_state *state, Link Here

(-)a/drivers/gpu/drm/i915/display/intel_crtc.c (-5 / +10 lines)
580	*/	580	*/
581	intel_psr_wait_for_idle_locked(new_crtc_state);	581	intel_psr_wait_for_idle_locked(new_crtc_state);
582		582
583	local_irq_disable();	583	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		584	local_irq_disable();
584		585
585	crtc->debug.min_vbl = min;	586	crtc->debug.min_vbl = min;
586	crtc->debug.max_vbl = max;	587	crtc->debug.max_vbl = max;
Lines 605-615 void intel_pipe_update_start(struct intel_atomic_state *state, Link Here
605	break;	606	break;
606	}	607	}
607		608
608	local_irq_enable();	609	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		610	local_irq_enable();
609		611
610	timeout = schedule_timeout(timeout);	612	timeout = schedule_timeout(timeout);
611		613
612	local_irq_disable();	614	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		615	local_irq_disable();
613	}	616	}
614		617
615	finish_wait(wq, &wait);	618	finish_wait(wq, &wait);
Lines 642-648 void intel_pipe_update_start(struct intel_atomic_state *state, Link Here
642	return;	645	return;
643		646
644	irq_disable:	647	irq_disable:
645	local_irq_disable();	648	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		649	local_irq_disable();
646	}	650	}
647		651
648	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)	652	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
Lines 744-750 void intel_pipe_update_end(struct intel_atomic_state *state, Link Here
744	*/	748	*/
745	intel_vrr_send_push(new_crtc_state);	749	intel_vrr_send_push(new_crtc_state);
746		750
747	local_irq_enable();	751	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		752	local_irq_enable();
748		753
749	if (intel_vgpu_active(dev_priv))	754	if (intel_vgpu_active(dev_priv))
750	goto out;	755	goto out;

Lines 275-280 int intel_crtc_scanline_to_hw(struct intel_crtc *crtc, int scanline) Link Here

(-)a/drivers/gpu/drm/i915/display/intel_vblank.c (-10 / +28 lines)
275	* all register accesses to the same cacheline to be serialized,	275	* all register accesses to the same cacheline to be serialized,
276	* otherwise they may hang.	276	* otherwise they may hang.
277	*/	277	*/
		278	static void intel_vblank_section_enter_irqsave(struct drm_i915_private i915, unsigned long flags)
		279	__acquires(i915->uncore.lock)
		280	{
		281	#ifdef I915
		282	spin_lock_irqsave(&i915->uncore.lock, *flags);
		283	#else
		284	*flags = 0;
		285	#endif
		286	}
		287
		288	static void intel_vblank_section_exit_irqrestore(struct drm_i915_private *i915, unsigned long flags)
		289	__releases(i915->uncore.lock)
		290	{
		291	#ifdef I915
		292	spin_unlock_irqrestore(&i915->uncore.lock, flags);
		293	#else
		294	if (flags)
		295	return;
		296	#endif
		297	}
278	static void intel_vblank_section_enter(struct drm_i915_private *i915)	298	static void intel_vblank_section_enter(struct drm_i915_private *i915)
279	__acquires(i915->uncore.lock)	299	__acquires(i915->uncore.lock)
280	{	300	{
Lines 332-341 static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc, Link Here
332	* timing critical raw register reads, potentially with	352	* timing critical raw register reads, potentially with
333	* preemption disabled, so the following code must not block.	353	* preemption disabled, so the following code must not block.
334	*/	354	*/
335	local_irq_save(irqflags);	355	intel_vblank_section_enter_irqsave(dev_priv, &irqflags);
336	intel_vblank_section_enter(dev_priv);
337		356
338	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */	357	if (IS_ENABLED(CONFIG_PREEMPT_RT))
		358	preempt_disable();
339		359
340	/* Get optional system timestamp before query. */	360	/* Get optional system timestamp before query. */
341	if (stime)	361	if (stime)
Lines 399-408 static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc, Link Here
399	if (etime)	419	if (etime)
400	*etime = ktime_get();	420	*etime = ktime_get();
401		421
402	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */	422	if (IS_ENABLED(CONFIG_PREEMPT_RT))
		423	preempt_enable();
403		424
404	intel_vblank_section_exit(dev_priv);	425	intel_vblank_section_exit_irqrestore(dev_priv, irqflags);
405	local_irq_restore(irqflags);
406		426
407	/*	427	/*
408	* While in vblank, position will be negative	428	* While in vblank, position will be negative
Lines 440-452 int intel_get_crtc_scanline(struct intel_crtc *crtc) Link Here
440	unsigned long irqflags;	460	unsigned long irqflags;
441	int position;	461	int position;
442		462
443	local_irq_save(irqflags);	463	intel_vblank_section_enter_irqsave(dev_priv, &irqflags);
444	intel_vblank_section_enter(dev_priv);
445		464
446	position = __intel_get_crtc_scanline(crtc);	465	position = __intel_get_crtc_scanline(crtc);
447		466
448	intel_vblank_section_exit(dev_priv);	467	intel_vblank_section_exit_irqrestore(dev_priv, irqflags);
449	local_irq_restore(irqflags);
450		468
451	return position;	469	return position;
452	}	470	}

Lines 592-597 serial8250_register_ports(struct uart_driver drv, struct device dev) Link Here

(-)a/drivers/tty/serial/8250/8250_core.c (-1 / +41 lines)
592		592
593	#ifdef CONFIG_SERIAL_8250_CONSOLE	593	#ifdef CONFIG_SERIAL_8250_CONSOLE
594		594
		595	#ifdef CONFIG_SERIAL_8250_LEGACY_CONSOLE
595	static void univ8250_console_write(struct console co, const char s,	596	static void univ8250_console_write(struct console co, const char s,
596	unsigned int count)	597	unsigned int count)
597	{	598	{
Lines 599-604 static void univ8250_console_write(struct console co, const char s, Link Here
599		600
600	serial8250_console_write(up, s, count);	601	serial8250_console_write(up, s, count);
601	}	602	}
		603	#else
		604	static bool univ8250_console_write_atomic(struct console *co,
		605	struct nbcon_write_context *wctxt)
		606	{
		607	struct uart_8250_port *up = &serial8250_ports[co->index];
		608
		609	return serial8250_console_write_atomic(up, wctxt);
		610	}
		611
		612	static bool univ8250_console_write_thread(struct console *co,
		613	struct nbcon_write_context *wctxt)
		614	{
		615	struct uart_8250_port *up = &serial8250_ports[co->index];
		616
		617	return serial8250_console_write_thread(up, wctxt);
		618	}
		619
		620	static void univ8250_console_driver_enter(struct console con, unsigned long flags)
		621	{
		622	struct uart_port *up = &serial8250_ports[con->index].port;
		623
		624	__uart_port_lock_irqsave(up, flags);
		625	}
		626
		627	static void univ8250_console_driver_exit(struct console *con, unsigned long flags)
		628	{
		629	struct uart_port *up = &serial8250_ports[con->index].port;
		630
		631	__uart_port_unlock_irqrestore(up, flags);
		632	}
		633	#endif /* CONFIG_SERIAL_8250_LEGACY_CONSOLE */
602		634
603	static int univ8250_console_setup(struct console co, char options)	635	static int univ8250_console_setup(struct console co, char options)
604	{	636	{
Lines 698-709 static int univ8250_console_match(struct console co, char name, int idx, Link Here
698		730
699	static struct console univ8250_console = {	731	static struct console univ8250_console = {
700	.name = "ttyS",	732	.name = "ttyS",
		733	#ifdef CONFIG_SERIAL_8250_LEGACY_CONSOLE
701	.write = univ8250_console_write,	734	.write = univ8250_console_write,
		735	.flags = CON_PRINTBUFFER \| CON_ANYTIME,
		736	#else
		737	.write_atomic = univ8250_console_write_atomic,
		738	.write_thread = univ8250_console_write_thread,
		739	.driver_enter = univ8250_console_driver_enter,
		740	.driver_exit = univ8250_console_driver_exit,
		741	.flags = CON_PRINTBUFFER \| CON_ANYTIME \| CON_NBCON,
		742	#endif
702	.device = uart_console_device,	743	.device = uart_console_device,
703	.setup = univ8250_console_setup,	744	.setup = univ8250_console_setup,
704	.exit = univ8250_console_exit,	745	.exit = univ8250_console_exit,
705	.match = univ8250_console_match,	746	.match = univ8250_console_match,
706	.flags = CON_PRINTBUFFER \| CON_ANYTIME,
707	.index = -1,	747	.index = -1,
708	.data = &serial8250_reg,	748	.data = &serial8250_reg,
709	};	749	};

Lines 348-357 static int pl011_fifo_to_tty(struct uart_amba_port *uap) Link Here

(-)a/drivers/tty/serial/amba-pl011.c (-16 / +8 lines)
348	flag = TTY_FRAME;	348	flag = TTY_FRAME;
349	}	349	}
350		350
351	uart_port_unlock(&uap->port);	351	sysrq = uart_prepare_sysrq_char(&uap->port, ch & 255);
352	sysrq = uart_handle_sysrq_char(&uap->port, ch & 255);
353	uart_port_lock(&uap->port);
354
355	if (!sysrq)	352	if (!sysrq)
356	uart_insert_char(&uap->port, ch, UART011_DR_OE, ch, flag);	353	uart_insert_char(&uap->port, ch, UART011_DR_OE, ch, flag);
357	}	354	}
Lines 1017-1023 static void pl011_dma_rx_callback(void *data) Link Here
1017	ret = pl011_dma_rx_trigger_dma(uap);	1014	ret = pl011_dma_rx_trigger_dma(uap);
1018		1015
1019	pl011_dma_rx_chars(uap, pending, lastbuf, false);	1016	pl011_dma_rx_chars(uap, pending, lastbuf, false);
1020	uart_port_unlock_irq(&uap->port);	1017	uart_unlock_and_check_sysrq(&uap->port);
1021	/*	1018	/*
1022	* Do this check after we picked the DMA chars so we don't	1019	* Do this check after we picked the DMA chars so we don't
1023	* get some IRQ immediately from RX.	1020	* get some IRQ immediately from RX.
Lines 1540-1550 static void check_apply_cts_event_workaround(struct uart_amba_port *uap) Link Here
1540	static irqreturn_t pl011_int(int irq, void *dev_id)	1537	static irqreturn_t pl011_int(int irq, void *dev_id)
1541	{	1538	{
1542	struct uart_amba_port *uap = dev_id;	1539	struct uart_amba_port *uap = dev_id;
1543	unsigned long flags;
1544	unsigned int status, pass_counter = AMBA_ISR_PASS_LIMIT;	1540	unsigned int status, pass_counter = AMBA_ISR_PASS_LIMIT;
1545	int handled = 0;	1541	int handled = 0;
1546		1542
1547	uart_port_lock_irqsave(&uap->port, &flags);	1543	uart_port_lock(&uap->port);
1548	status = pl011_read(uap, REG_RIS) & uap->im;	1544	status = pl011_read(uap, REG_RIS) & uap->im;
1549	if (status) {	1545	if (status) {
1550	do {	1546	do {
Lines 1573-1579 static irqreturn_t pl011_int(int irq, void *dev_id) Link Here
1573	handled = 1;	1569	handled = 1;
1574	}	1570	}
1575		1571
1576	uart_port_unlock_irqrestore(&uap->port, flags);	1572	uart_unlock_and_check_sysrq(&uap->port);
1577		1573
1578	return IRQ_RETVAL(handled);	1574	return IRQ_RETVAL(handled);
1579	}	1575	}
Lines 2322-2334 pl011_console_write(struct console co, const char s, unsigned int count) Link Here
2322		2318
2323	clk_enable(uap->clk);	2319	clk_enable(uap->clk);
2324		2320
2325	local_irq_save(flags);	2321	if (oops_in_progress)
2326	if (uap->port.sysrq)	2322	locked = uart_port_trylock_irqsave(&uap->port, &flags);
2327	locked = 0;
2328	else if (oops_in_progress)
2329	locked = uart_port_trylock(&uap->port);
2330	else	2323	else
2331	uart_port_lock(&uap->port);	2324	uart_port_lock_irqsave(&uap->port, &flags);
2332		2325
2333	/*	2326	/*
2334	* First save the CR then disable the interrupts	2327	* First save the CR then disable the interrupts
Lines 2354-2361 pl011_console_write(struct console co, const char s, unsigned int count) Link Here
2354	pl011_write(old_cr, uap, REG_CR);	2347	pl011_write(old_cr, uap, REG_CR);
2355		2348
2356	if (locked)	2349	if (locked)
2357	uart_port_unlock(&uap->port);	2350	uart_port_unlock_irqrestore(&uap->port, flags);
2358	local_irq_restore(flags);
2359		2351
2360	clk_disable(uap->clk);	2352	clk_disable(uap->clk);
2361	}	2353	}

Lines 378-384 static void ar933x_uart_rx_chars(struct ar933x_uart_port *up) Link Here

(-)a/drivers/tty/serial/ar933x_uart.c (-12 / +6 lines)
378	up->port.icount.rx++;	378	up->port.icount.rx++;
379	ch = rdata & AR933X_UART_DATA_TX_RX_MASK;	379	ch = rdata & AR933X_UART_DATA_TX_RX_MASK;
380		380
381	if (uart_handle_sysrq_char(&up->port, ch))	381	if (uart_prepare_sysrq_char(&up->port, ch))
382	continue;	382	continue;
383		383
384	if ((up->port.ignore_status_mask & AR933X_DUMMY_STATUS_RD) == 0)	384	if ((up->port.ignore_status_mask & AR933X_DUMMY_STATUS_RD) == 0)
Lines 468-474 static irqreturn_t ar933x_uart_interrupt(int irq, void *dev_id) Link Here
468	ar933x_uart_tx_chars(up);	468	ar933x_uart_tx_chars(up);
469	}	469	}
470		470
471	uart_port_unlock(&up->port);	471	uart_unlock_and_check_sysrq(&up->port);
472		472
473	return IRQ_HANDLED;	473	return IRQ_HANDLED;
474	}	474	}
Lines 627-640 static void ar933x_uart_console_write(struct console co, const char s, Link Here
627	unsigned int int_en;	627	unsigned int int_en;
628	int locked = 1;	628	int locked = 1;
629		629
630	local_irq_save(flags);	630	if (oops_in_progress)
631		631	locked = uart_port_trylock_irqsave(&up->port, &flags);
632	if (up->port.sysrq)
633	locked = 0;
634	else if (oops_in_progress)
635	locked = uart_port_trylock(&up->port);
636	else	632	else
637	uart_port_lock(&up->port);	633	uart_port_lock_irqsave(&up->port, &flags);
638		634
639	/*	635	/*
640	* First save the IER then disable the interrupts	636	* First save the IER then disable the interrupts
Lines 654-662 static void ar933x_uart_console_write(struct console co, const char s, Link Here
654	ar933x_uart_write(up, AR933X_UART_INT_REG, AR933X_UART_INT_ALLINTS);	650	ar933x_uart_write(up, AR933X_UART_INT_REG, AR933X_UART_INT_ALLINTS);
655		651
656	if (locked)	652	if (locked)
657	uart_port_unlock(&up->port);	653	uart_port_unlock_irqrestore(&up->port, flags);
658
659	local_irq_restore(flags);
660	}	654	}
661		655
662	static int ar933x_uart_console_setup(struct console co, char options)	656	static int ar933x_uart_console_setup(struct console co, char options)

Lines 285-294 static void bcm_uart_do_rx(struct uart_port *port) Link Here

(-)a/drivers/tty/serial/bcm63xx_uart.c (-16 / +8 lines)
285	flag = TTY_PARITY;	285	flag = TTY_PARITY;
286	}	286	}
287		287
288	if (uart_handle_sysrq_char(port, c))	288	if (uart_prepare_sysrq_char(port, c))
289	continue;	289	continue;
290		290
291
292	if ((cstat & port->ignore_status_mask) == 0)	291	if ((cstat & port->ignore_status_mask) == 0)
293	tty_insert_flip_char(tty_port, c, flag);	292	tty_insert_flip_char(tty_port, c, flag);
294		293
Lines 353-359 static irqreturn_t bcm_uart_interrupt(int irq, void *dev_id) Link Here
353	estat & UART_EXTINP_DCD_MASK);	352	estat & UART_EXTINP_DCD_MASK);
354	}	353	}
355		354
356	uart_port_unlock(port);	355	uart_unlock_and_check_sysrq(port);
357	return IRQ_HANDLED;	356	return IRQ_HANDLED;
358	}	357	}
359		358
Lines 703-722 static void bcm_console_write(struct console co, const char s, Link Here
703	{	702	{
704	struct uart_port *port;	703	struct uart_port *port;
705	unsigned long flags;	704	unsigned long flags;
706	int locked;	705	int locked = 1;
707		706
708	port = &ports[co->index];	707	port = &ports[co->index];
709		708
710	local_irq_save(flags);	709	if (oops_in_progress)
711	if (port->sysrq) {	710	locked = uart_port_trylock_irqsave(port, &flags);
712	/* bcm_uart_interrupt() already took the lock */	711	else
713	locked = 0;	712	uart_port_lock_irqsave(port, &flags);
714	} else if (oops_in_progress) {
715	locked = uart_port_trylock(port);
716	} else {
717	uart_port_lock(port);
718	locked = 1;
719	}
720		713
721	/* call helper to deal with \r\n */	714	/* call helper to deal with \r\n */
722	uart_console_write(port, s, count, bcm_console_putchar);	715	uart_console_write(port, s, count, bcm_console_putchar);
Lines 725-732 static void bcm_console_write(struct console co, const char s, Link Here
725	wait_for_xmitr(port);	718	wait_for_xmitr(port);
726		719
727	if (locked)	720	if (locked)
728	uart_port_unlock(port);	721	uart_port_unlock_irqrestore(port, flags);
729	local_irq_restore(flags);
730	}	722	}
731		723
732	/*	724	/*

Lines 136-155 static void lpc32xx_hsuart_console_write(struct console co, const char s, Link Here

(-)a/drivers/tty/serial/lpc32xx_hs.c (-10 / +7 lines)
136	int locked = 1;	136	int locked = 1;
137		137
138	touch_nmi_watchdog();	138	touch_nmi_watchdog();
139	local_irq_save(flags);	139	if (oops_in_progress)
140	if (up->port.sysrq)	140	locked = uart_port_trylock_irqsave(&up->port, &flags);
141	locked = 0;
142	else if (oops_in_progress)
143	locked = uart_port_trylock(&up->port);
144	else	141	else
145	uart_port_lock(&up->port);	142	uart_port_lock_irqsave(&up->port, &flags);
146		143
147	uart_console_write(&up->port, s, count, lpc32xx_hsuart_console_putchar);	144	uart_console_write(&up->port, s, count, lpc32xx_hsuart_console_putchar);
148	wait_for_xmit_empty(&up->port);	145	wait_for_xmit_empty(&up->port);
149		146
150	if (locked)	147	if (locked)
151	uart_port_unlock(&up->port);	148	uart_port_unlock_irqrestore(&up->port, flags);
152	local_irq_restore(flags);
153	}	149	}
154		150
155	static int __init lpc32xx_hsuart_console_setup(struct console *co,	151	static int __init lpc32xx_hsuart_console_setup(struct console *co,
Lines 268-274 static void __serial_lpc32xx_rx(struct uart_port *port) Link Here
268	tty_insert_flip_char(tport, 0, TTY_FRAME);	264	tty_insert_flip_char(tport, 0, TTY_FRAME);
269	}	265	}
270		266
271	tty_insert_flip_char(tport, (tmp & 0xFF), flag);	267	if (!uart_prepare_sysrq_char(port, tmp & 0xff))
		268	tty_insert_flip_char(tport, (tmp & 0xFF), flag);
272		269
273	tmp = readl(LPC32XX_HSUART_FIFO(port->membase));	270	tmp = readl(LPC32XX_HSUART_FIFO(port->membase));
274	}	271	}
Lines 333-339 static irqreturn_t serial_lpc32xx_interrupt(int irq, void *dev_id) Link Here
333	__serial_lpc32xx_tx(port);	330	__serial_lpc32xx_tx(port);
334	}	331	}
335		332
336	uart_port_unlock(port);	333	uart_unlock_and_check_sysrq(port);
337		334
338	return IRQ_HANDLED;	335	return IRQ_HANDLED;
339	}	336	}

Lines 220-226 static void meson_receive_chars(struct uart_port *port) Link Here

(-)a/drivers/tty/serial/meson_uart.c (-14 / +8 lines)
220	continue;	220	continue;
221	}	221	}
222		222
223	if (uart_handle_sysrq_char(port, ch))	223	if (uart_prepare_sysrq_char(port, ch))
224	continue;	224	continue;
225		225
226	if ((status & port->ignore_status_mask) == 0)	226	if ((status & port->ignore_status_mask) == 0)
Lines 248-254 static irqreturn_t meson_uart_interrupt(int irq, void *dev_id) Link Here
248	meson_uart_start_tx(port);	248	meson_uart_start_tx(port);
249	}	249	}
250		250
251	uart_port_unlock(port);	251	uart_unlock_and_check_sysrq(port);
252		252
253	return IRQ_HANDLED;	253	return IRQ_HANDLED;
254	}	254	}
Lines 556-573 static void meson_serial_port_write(struct uart_port port, const char s, Link Here
556	u_int count)	556	u_int count)
557	{	557	{
558	unsigned long flags;	558	unsigned long flags;
559	int locked;	559	int locked = 1;
560	u32 val, tmp;	560	u32 val, tmp;
561		561
562	local_irq_save(flags);	562	if (oops_in_progress)
563	if (port->sysrq) {	563	locked = uart_port_trylock_irqsave(port, &flags);
564	locked = 0;	564	else
565	} else if (oops_in_progress) {	565	uart_port_lock_irqsave(port, &flags);
566	locked = uart_port_trylock(port);
567	} else {
568	uart_port_lock(port);
569	locked = 1;
570	}
571		566
572	val = readl(port->membase + AML_UART_CONTROL);	567	val = readl(port->membase + AML_UART_CONTROL);
573	tmp = val & ~(AML_UART_TX_INT_EN \| AML_UART_RX_INT_EN);	568	tmp = val & ~(AML_UART_TX_INT_EN \| AML_UART_RX_INT_EN);
Lines 577-584 static void meson_serial_port_write(struct uart_port port, const char s, Link Here
577	writel(val, port->membase + AML_UART_CONTROL);	572	writel(val, port->membase + AML_UART_CONTROL);
578		573
579	if (locked)	574	if (locked)
580	uart_port_unlock(port);	575	uart_port_unlock_irqrestore(port, flags);
581	local_irq_restore(flags);
582	}	576	}
583		577
584	static void meson_serial_console_write(struct console co, const char s,	578	static void meson_serial_console_write(struct console co, const char s,

Lines 588-603 static void msm_complete_rx_dma(void *args) Link Here

(-)a/drivers/tty/serial/msm_serial.c (-23 / +10 lines)
588	if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))	588	if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))
589	flag = TTY_NORMAL;	589	flag = TTY_NORMAL;
590		590
591	uart_port_unlock_irqrestore(port, flags);	591	sysrq = uart_prepare_sysrq_char(port, dma->virt[i]);
592	sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
593	uart_port_lock_irqsave(port, &flags);
594	if (!sysrq)	592	if (!sysrq)
595	tty_insert_flip_char(tport, dma->virt[i], flag);	593	tty_insert_flip_char(tport, dma->virt[i], flag);
596	}	594	}
597		595
598	msm_start_rx_dma(msm_port);	596	msm_start_rx_dma(msm_port);
599	done:	597	done:
600	uart_port_unlock_irqrestore(port, flags);	598	uart_unlock_and_check_sysrq_irqrestore(port, flags);
601		599
602	if (count)	600	if (count)
603	tty_flip_buffer_push(tport);	601	tty_flip_buffer_push(tport);
Lines 763-771 static void msm_handle_rx_dm(struct uart_port *port, unsigned int misr) Link Here
763	if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))	761	if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))
764	flag = TTY_NORMAL;	762	flag = TTY_NORMAL;
765		763
766	uart_port_unlock(port);	764	sysrq = uart_prepare_sysrq_char(port, buf[i]);
767	sysrq = uart_handle_sysrq_char(port, buf[i]);
768	uart_port_lock(port);
769	if (!sysrq)	765	if (!sysrq)
770	tty_insert_flip_char(tport, buf[i], flag);	766	tty_insert_flip_char(tport, buf[i], flag);
771	}	767	}
Lines 825-833 static void msm_handle_rx(struct uart_port *port) Link Here
825	else if (sr & MSM_UART_SR_PAR_FRAME_ERR)	821	else if (sr & MSM_UART_SR_PAR_FRAME_ERR)
826	flag = TTY_FRAME;	822	flag = TTY_FRAME;
827		823
828	uart_port_unlock(port);	824	sysrq = uart_prepare_sysrq_char(port, c);
829	sysrq = uart_handle_sysrq_char(port, c);
830	uart_port_lock(port);
831	if (!sysrq)	825	if (!sysrq)
832	tty_insert_flip_char(tport, c, flag);	826	tty_insert_flip_char(tport, c, flag);
833	}	827	}
Lines 948-958 static irqreturn_t msm_uart_irq(int irq, void *dev_id) Link Here
948	struct uart_port *port = dev_id;	942	struct uart_port *port = dev_id;
949	struct msm_port *msm_port = to_msm_port(port);	943	struct msm_port *msm_port = to_msm_port(port);
950	struct msm_dma *dma = &msm_port->rx_dma;	944	struct msm_dma *dma = &msm_port->rx_dma;
951	unsigned long flags;
952	unsigned int misr;	945	unsigned int misr;
953	u32 val;	946	u32 val;
954		947
955	uart_port_lock_irqsave(port, &flags);	948	uart_port_lock(port);
956	misr = msm_read(port, MSM_UART_MISR);	949	misr = msm_read(port, MSM_UART_MISR);
957	msm_write(port, 0, MSM_UART_IMR); /* disable interrupt */	950	msm_write(port, 0, MSM_UART_IMR); /* disable interrupt */
958		951
Lines 984-990 static irqreturn_t msm_uart_irq(int irq, void *dev_id) Link Here
984	msm_handle_delta_cts(port);	977	msm_handle_delta_cts(port);
985		978
986	msm_write(port, msm_port->imr, MSM_UART_IMR); /* restore interrupt */	979	msm_write(port, msm_port->imr, MSM_UART_IMR); /* restore interrupt */
987	uart_port_unlock_irqrestore(port, flags);	980	uart_unlock_and_check_sysrq(port);
988		981
989	return IRQ_HANDLED;	982	return IRQ_HANDLED;
990	}	983	}
Lines 1621-1634 static void __msm_console_write(struct uart_port port, const char s, Link Here
1621	num_newlines++;	1614	num_newlines++;
1622	count += num_newlines;	1615	count += num_newlines;
1623		1616
1624	local_irq_save(flags);	1617	if (oops_in_progress)
1625		1618	locked = uart_port_trylock_irqsave(port, &flags);
1626	if (port->sysrq)
1627	locked = 0;
1628	else if (oops_in_progress)
1629	locked = uart_port_trylock(port);
1630	else	1619	else
1631	uart_port_lock(port);	1620	uart_port_lock_irqsave(port, &flags);
1632		1621
1633	if (is_uartdm)	1622	if (is_uartdm)
1634	msm_reset_dm_count(port, count);	1623	msm_reset_dm_count(port, count);
Lines 1667-1675 static void __msm_console_write(struct uart_port port, const char s, Link Here
1667	}	1656	}
1668		1657
1669	if (locked)	1658	if (locked)
1670	uart_port_unlock(port);	1659	uart_port_unlock_irqrestore(port, flags);
1671
1672	local_irq_restore(flags);
1673	}	1660	}
1674		1661
1675	static void msm_console_write(struct console co, const char s,	1662	static void msm_console_write(struct console co, const char s,

Lines 508-514 static void serial_omap_rdi(struct uart_omap_port *up, unsigned int lsr) Link Here

(-)a/drivers/tty/serial/omap-serial.c (-10 / +6 lines)
508		508
509	up->port.icount.rx++;	509	up->port.icount.rx++;
510		510
511	if (uart_handle_sysrq_char(&up->port, ch))	511	if (uart_prepare_sysrq_char(&up->port, ch))
512	return;	512	return;
513		513
514	uart_insert_char(&up->port, lsr, UART_LSR_OE, ch, TTY_NORMAL);	514	uart_insert_char(&up->port, lsr, UART_LSR_OE, ch, TTY_NORMAL);
Lines 563-569 static irqreturn_t serial_omap_irq(int irq, void *dev_id) Link Here
563	}	563	}
564	} while (max_count--);	564	} while (max_count--);
565		565
566	uart_port_unlock(&up->port);	566	uart_unlock_and_check_sysrq(&up->port);
567		567
568	tty_flip_buffer_push(&up->port.state->port);	568	tty_flip_buffer_push(&up->port.state->port);
569		569
Lines 1212-1224 serial_omap_console_write(struct console co, const char s, Link Here
1212	unsigned int ier;	1212	unsigned int ier;
1213	int locked = 1;	1213	int locked = 1;
1214		1214
1215	local_irq_save(flags);	1215	if (oops_in_progress)
1216	if (up->port.sysrq)	1216	locked = uart_port_trylock_irqsave(&up->port, &flags);
1217	locked = 0;
1218	else if (oops_in_progress)
1219	locked = uart_port_trylock(&up->port);
1220	else	1217	else
1221	uart_port_lock(&up->port);	1218	uart_port_lock_irqsave(&up->port, &flags);
1222		1219
1223	/*	1220	/*
1224	* First save the IER then disable the interrupts	1221	* First save the IER then disable the interrupts
Lines 1245-1252 serial_omap_console_write(struct console co, const char s, Link Here
1245	check_modem_status(up);	1242	check_modem_status(up);
1246		1243
1247	if (locked)	1244	if (locked)
1248	uart_port_unlock(&up->port);	1245	uart_port_unlock_irqrestore(&up->port, flags);
1249	local_irq_restore(flags);
1250	}	1246	}
1251		1247
1252	static int __init	1248	static int __init

Lines 237-245 struct eg20t_port { Link Here

(-)a/drivers/tty/serial/pch_uart.c (-51 / +19 lines)
237		237
238	#define IRQ_NAME_SIZE 17	238	#define IRQ_NAME_SIZE 17
239	char irq_name[IRQ_NAME_SIZE];	239	char irq_name[IRQ_NAME_SIZE];
240
241	/* protect the eg20t_port private structure and io access to membase */
242	spinlock_t lock;
243	};	240	};
244		241
245	/**	242	/**
Lines 567-573 static int pch_uart_hal_read(struct eg20t_port priv, unsigned char buf, Link Here
567	if (uart_handle_break(port))	564	if (uart_handle_break(port))
568	continue;	565	continue;
569	}	566	}
570	if (uart_handle_sysrq_char(port, rbr))	567	if (uart_prepare_sysrq_char(port, rbr))
571	continue;	568	continue;
572		569
573	buf[i++] = rbr;	570	buf[i++] = rbr;
Lines 599-614 static void pch_uart_hal_set_break(struct eg20t_port *priv, int on) Link Here
599	iowrite8(lcr, priv->membase + UART_LCR);	596	iowrite8(lcr, priv->membase + UART_LCR);
600	}	597	}
601		598
602	static int push_rx(struct eg20t_port priv, const unsigned char buf,	599	static void push_rx(struct eg20t_port priv, const unsigned char buf,
603	int size)	600	int size)
604	{	601	{
605	struct uart_port *port = &priv->port;	602	struct uart_port *port = &priv->port;
606	struct tty_port *tport = &port->state->port;	603	struct tty_port *tport = &port->state->port;
607		604
608	tty_insert_flip_string(tport, buf, size);	605	tty_insert_flip_string(tport, buf, size);
609	tty_flip_buffer_push(tport);	606	tty_flip_buffer_push(tport);
610
611	return 0;
612	}	607	}
613		608
614	static int dma_push_rx(struct eg20t_port *priv, int size)	609	static int dma_push_rx(struct eg20t_port *priv, int size)
Lines 761-767 static int handle_rx_to(struct eg20t_port *priv) Link Here
761	{	756	{
762	struct pch_uart_buffer *buf;	757	struct pch_uart_buffer *buf;
763	int rx_size;	758	int rx_size;
764	int ret;	759
765	if (!priv->start_rx) {	760	if (!priv->start_rx) {
766	pch_uart_hal_disable_interrupt(priv, PCH_UART_HAL_RX_INT \|	761	pch_uart_hal_disable_interrupt(priv, PCH_UART_HAL_RX_INT \|
767	PCH_UART_HAL_RX_ERR_INT);	762	PCH_UART_HAL_RX_ERR_INT);
Lines 770-788 static int handle_rx_to(struct eg20t_port *priv) Link Here
770	buf = &priv->rxbuf;	765	buf = &priv->rxbuf;
771	do {	766	do {
772	rx_size = pch_uart_hal_read(priv, buf->buf, buf->size);	767	rx_size = pch_uart_hal_read(priv, buf->buf, buf->size);
773	ret = push_rx(priv, buf->buf, rx_size);	768	push_rx(priv, buf->buf, rx_size);
774	if (ret)
775	return 0;
776	} while (rx_size == buf->size);	769	} while (rx_size == buf->size);
777		770
778	return PCH_UART_HANDLED_RX_INT;	771	return PCH_UART_HANDLED_RX_INT;
779	}	772	}
780		773
781	static int handle_rx(struct eg20t_port *priv)
782	{
783	return handle_rx_to(priv);
784	}
785
786	static int dma_handle_rx(struct eg20t_port *priv)	774	static int dma_handle_rx(struct eg20t_port *priv)
787	{	775	{
788	struct uart_port *port = &priv->port;	776	struct uart_port *port = &priv->port;
Lines 1019-1029 static irqreturn_t pch_uart_interrupt(int irq, void *dev_id) Link Here
1019	u8 lsr;	1007	u8 lsr;
1020	int ret = 0;	1008	int ret = 0;
1021	unsigned char iid;	1009	unsigned char iid;
1022	unsigned long flags;
1023	int next = 1;	1010	int next = 1;
1024	u8 msr;	1011	u8 msr;
1025		1012
1026	spin_lock_irqsave(&priv->lock, flags);	1013	uart_port_lock(&priv->port);
1027	handled = 0;	1014	handled = 0;
1028	while (next) {	1015	while (next) {
1029	iid = pch_uart_hal_get_iid(priv);	1016	iid = pch_uart_hal_get_iid(priv);
Lines 1051-1057 static irqreturn_t pch_uart_interrupt(int irq, void *dev_id) Link Here
1051	PCH_UART_HAL_RX_INT \|	1038	PCH_UART_HAL_RX_INT \|
1052	PCH_UART_HAL_RX_ERR_INT);	1039	PCH_UART_HAL_RX_ERR_INT);
1053	} else {	1040	} else {
1054	ret = handle_rx(priv);	1041	ret = handle_rx_to(priv);
1055	}	1042	}
1056	break;	1043	break;
1057	case PCH_UART_IID_RDR_TO: /* Received Data Ready	1044	case PCH_UART_IID_RDR_TO: /* Received Data Ready
Lines 1083-1089 static irqreturn_t pch_uart_interrupt(int irq, void *dev_id) Link Here
1083	handled \|= (unsigned int)ret;	1070	handled \|= (unsigned int)ret;
1084	}	1071	}
1085		1072
1086	spin_unlock_irqrestore(&priv->lock, flags);	1073	uart_unlock_and_check_sysrq(&priv->port);
1087	return IRQ_RETVAL(handled);	1074	return IRQ_RETVAL(handled);
1088	}	1075	}
1089		1076
Lines 1194-1202 static void pch_uart_break_ctl(struct uart_port *port, int ctl) Link Here
1194	unsigned long flags;	1181	unsigned long flags;
1195		1182
1196	priv = container_of(port, struct eg20t_port, port);	1183	priv = container_of(port, struct eg20t_port, port);
1197	spin_lock_irqsave(&priv->lock, flags);	1184	uart_port_lock_irqsave(&priv->port, &flags);
1198	pch_uart_hal_set_break(priv, ctl);	1185	pch_uart_hal_set_break(priv, ctl);
1199	spin_unlock_irqrestore(&priv->lock, flags);	1186	uart_port_unlock_irqrestore(&priv->port, flags);
1200	}	1187	}
1201		1188
1202	/* Grab any interrupt resources and initialise any low level driver state. */	1189	/* Grab any interrupt resources and initialise any low level driver state. */
Lines 1346-1353 static void pch_uart_set_termios(struct uart_port *port, Link Here
1346		1333
1347	baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);	1334	baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);
1348		1335
1349	spin_lock_irqsave(&priv->lock, flags);	1336	uart_port_lock_irqsave(port, &flags);
1350	uart_port_lock(port);
1351		1337
1352	uart_update_timeout(port, termios->c_cflag, baud);	1338	uart_update_timeout(port, termios->c_cflag, baud);
1353	rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);	1339	rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);
Lines 1360-1367 static void pch_uart_set_termios(struct uart_port *port, Link Here
1360	tty_termios_encode_baud_rate(termios, baud, baud);	1346	tty_termios_encode_baud_rate(termios, baud, baud);
1361		1347
1362	out:	1348	out:
1363	uart_port_unlock(port);	1349	uart_port_unlock_irqrestore(port, flags);
1364	spin_unlock_irqrestore(&priv->lock, flags);
1365	}	1350	}
1366		1351
1367	static const char pch_uart_type(struct uart_port port)	1352	static const char pch_uart_type(struct uart_port port)
Lines 1565-1591 pch_console_write(struct console co, const char s, unsigned int count) Link Here
1565	{	1550	{
1566	struct eg20t_port *priv;	1551	struct eg20t_port *priv;
1567	unsigned long flags;	1552	unsigned long flags;
1568	int priv_locked = 1;	1553	int locked = 1;
1569	int port_locked = 1;
1570	u8 ier;	1554	u8 ier;
1571		1555
1572	priv = pch_uart_ports[co->index];	1556	priv = pch_uart_ports[co->index];
1573		1557
1574	touch_nmi_watchdog();	1558	touch_nmi_watchdog();
1575		1559
1576	local_irq_save(flags);	1560	if (oops_in_progress)
1577	if (priv->port.sysrq) {	1561	locked = uart_port_trylock_irqsave(&priv->port, &flags);
1578	/* call to uart_handle_sysrq_char already took the priv lock */	1562	else
1579	priv_locked = 0;	1563	uart_port_lock_irqsave(&priv->port, &flags);
1580	/* serial8250_handle_port() already took the port lock */
1581	port_locked = 0;
1582	} else if (oops_in_progress) {
1583	priv_locked = spin_trylock(&priv->lock);
1584	port_locked = uart_port_trylock(&priv->port);
1585	} else {
1586	spin_lock(&priv->lock);
1587	uart_port_lock(&priv->port);
1588	}
1589		1564
1590	/*	1565	/*
1591	* First save the IER then disable the interrupts	1566	* First save the IER then disable the interrupts
Lines 1603-1613 pch_console_write(struct console co, const char s, unsigned int count) Link Here
1603	wait_for_xmitr(priv, UART_LSR_BOTH_EMPTY);	1578	wait_for_xmitr(priv, UART_LSR_BOTH_EMPTY);
1604	iowrite8(ier, priv->membase + UART_IER);	1579	iowrite8(ier, priv->membase + UART_IER);
1605		1580
1606	if (port_locked)	1581	if (locked)
1607	uart_port_unlock(&priv->port);	1582	uart_port_unlock_irqrestore(&priv->port, flags);
1608	if (priv_locked)
1609	spin_unlock(&priv->lock);
1610	local_irq_restore(flags);
1611	}	1583	}
1612		1584
1613	static int __init pch_console_setup(struct console co, char options)	1585	static int __init pch_console_setup(struct console co, char options)
Lines 1704-1711 static struct eg20t_port pch_uart_init_port(struct pci_dev pdev, Link Here
1704	pci_enable_msi(pdev);	1676	pci_enable_msi(pdev);
1705	pci_set_master(pdev);	1677	pci_set_master(pdev);
1706		1678
1707	spin_lock_init(&priv->lock);
1708
1709	iobase = pci_resource_start(pdev, 0);	1679	iobase = pci_resource_start(pdev, 0);
1710	mapbase = pci_resource_start(pdev, 1);	1680	mapbase = pci_resource_start(pdev, 1);
1711	priv->mapbase = mapbase;	1681	priv->mapbase = mapbase;
Lines 1735-1742 static struct eg20t_port pch_uart_init_port(struct pci_dev pdev, Link Here
1735	KBUILD_MODNAME ":" PCH_UART_DRIVER_DEVICE "%d",	1705	KBUILD_MODNAME ":" PCH_UART_DRIVER_DEVICE "%d",
1736	priv->port.line);	1706	priv->port.line);
1737		1707
1738	spin_lock_init(&priv->port.lock);
1739
1740	pci_set_drvdata(pdev, priv);	1708	pci_set_drvdata(pdev, priv);
1741	priv->trigger_level = 1;	1709	priv->trigger_level = 1;
1742	priv->fcr = 0;	1710	priv->fcr = 0;

Lines 151-157 static inline void receive_chars(struct uart_pxa_port up, int status) Link Here

(-)a/drivers/tty/serial/pxa.c (-11 / +6 lines)
151	flag = TTY_FRAME;	151	flag = TTY_FRAME;
152	}	152	}
153		153
154	if (uart_handle_sysrq_char(&up->port, ch))	154	if (uart_prepare_sysrq_char(&up->port, ch))
155	goto ignore_char;	155	goto ignore_char;
156		156
157	uart_insert_char(&up->port, *status, UART_LSR_OE, ch, flag);	157	uart_insert_char(&up->port, *status, UART_LSR_OE, ch, flag);
Lines 232-238 static inline irqreturn_t serial_pxa_irq(int irq, void *dev_id) Link Here
232	check_modem_status(up);	232	check_modem_status(up);
233	if (lsr & UART_LSR_THRE)	233	if (lsr & UART_LSR_THRE)
234	transmit_chars(up);	234	transmit_chars(up);
235	uart_port_unlock(&up->port);	235	uart_unlock_and_check_sysrq(&up->port);
236	return IRQ_HANDLED;	236	return IRQ_HANDLED;
237	}	237	}
238		238
Lines 604-616 serial_pxa_console_write(struct console co, const char s, unsigned int count) Link Here
604	int locked = 1;	604	int locked = 1;
605		605
606	clk_enable(up->clk);	606	clk_enable(up->clk);
607	local_irq_save(flags);	607	if (oops_in_progress)
608	if (up->port.sysrq)	608	locked = uart_port_trylock_irqsave(&up->port, &flags);
609	locked = 0;
610	else if (oops_in_progress)
611	locked = uart_port_trylock(&up->port);
612	else	609	else
613	uart_port_lock(&up->port);	610	uart_port_lock_irqsave(&up->port, &flags);
614		611
615	/*	612	/*
616	* First save the IER then disable the interrupts	613	* First save the IER then disable the interrupts
Lines 628-637 serial_pxa_console_write(struct console co, const char s, unsigned int count) Link Here
628	serial_out(up, UART_IER, ier);	625	serial_out(up, UART_IER, ier);
629		626
630	if (locked)	627	if (locked)
631	uart_port_unlock(&up->port);	628	uart_port_unlock_irqrestore(&up->port, flags);
632	local_irq_restore(flags);
633	clk_disable(up->clk);	629	clk_disable(up->clk);
634
635	}	630	}
636		631
637	#ifdef CONFIG_CONSOLE_POLL	632	#ifdef CONFIG_CONSOLE_POLL

Lines 394-400 static void rda_uart_receive_chars(struct uart_port *port) Link Here

(-)a/drivers/tty/serial/rda-uart.c (-18 / +10 lines)
394	val &= 0xff;	394	val &= 0xff;
395		395
396	port->icount.rx++;	396	port->icount.rx++;
397	tty_insert_flip_char(&port->state->port, val, flag);	397	if (!uart_prepare_sysrq_char(port, val))
		398	tty_insert_flip_char(&port->state->port, val, flag);
398		399
399	status = rda_uart_read(port, RDA_UART_STATUS);	400	status = rda_uart_read(port, RDA_UART_STATUS);
400	}	401	}
Lines 405-414 static void rda_uart_receive_chars(struct uart_port *port) Link Here
405	static irqreturn_t rda_interrupt(int irq, void *dev_id)	406	static irqreturn_t rda_interrupt(int irq, void *dev_id)
406	{	407	{
407	struct uart_port *port = dev_id;	408	struct uart_port *port = dev_id;
408	unsigned long flags;
409	u32 val, irq_mask;	409	u32 val, irq_mask;
410		410
411	uart_port_lock_irqsave(port, &flags);	411	uart_port_lock(port);
412		412
413	/* Clear IRQ cause */	413	/* Clear IRQ cause */
414	val = rda_uart_read(port, RDA_UART_IRQ_CAUSE);	414	val = rda_uart_read(port, RDA_UART_IRQ_CAUSE);
Lines 425-431 static irqreturn_t rda_interrupt(int irq, void *dev_id) Link Here
425	rda_uart_send_chars(port);	425	rda_uart_send_chars(port);
426	}	426	}
427		427
428	uart_port_unlock_irqrestore(port, flags);	428	uart_unlock_and_check_sysrq(port);
429		429
430	return IRQ_HANDLED;	430	return IRQ_HANDLED;
431	}	431	}
Lines 590-607 static void rda_uart_port_write(struct uart_port port, const char s, Link Here
590	{	590	{
591	u32 old_irq_mask;	591	u32 old_irq_mask;
592	unsigned long flags;	592	unsigned long flags;
593	int locked;	593	int locked = 1;
594		594
595	local_irq_save(flags);	595	if (oops_in_progress)
596		596	locked = uart_port_trylock_irqsave(port, &flags);
597	if (port->sysrq) {	597	else
598	locked = 0;	598	uart_port_lock_irqsave(port, &flags);
599	} else if (oops_in_progress) {
600	locked = uart_port_trylock(port);
601	} else {
602	uart_port_lock(port);
603	locked = 1;
604	}
605		599
606	old_irq_mask = rda_uart_read(port, RDA_UART_IRQ_MASK);	600	old_irq_mask = rda_uart_read(port, RDA_UART_IRQ_MASK);
607	rda_uart_write(port, 0, RDA_UART_IRQ_MASK);	601	rda_uart_write(port, 0, RDA_UART_IRQ_MASK);
Lines 615-623 static void rda_uart_port_write(struct uart_port port, const char s, Link Here
615	rda_uart_write(port, old_irq_mask, RDA_UART_IRQ_MASK);	609	rda_uart_write(port, old_irq_mask, RDA_UART_IRQ_MASK);
616		610
617	if (locked)	611	if (locked)
618	uart_port_unlock(port);	612	uart_port_unlock_irqrestore(port, flags);
619
620	local_irq_restore(flags);
621	}	613	}
622		614
623	static void rda_uart_console_write(struct console co, const char s,	615	static void rda_uart_console_write(struct console co, const char s,

Lines 412-418 static void __ssp_receive_chars(struct sifive_serial_port *ssp) Link Here

(-)a/drivers/tty/serial/sifive.c (-10 / +7 lines)
412	break;	412	break;
413		413
414	ssp->port.icount.rx++;	414	ssp->port.icount.rx++;
415	uart_insert_char(&ssp->port, 0, 0, ch, TTY_NORMAL);	415	if (!uart_prepare_sysrq_char(&ssp->port, ch))
		416	uart_insert_char(&ssp->port, 0, 0, ch, TTY_NORMAL);
416	}	417	}
417		418
418	tty_flip_buffer_push(&ssp->port.state->port);	419	tty_flip_buffer_push(&ssp->port.state->port);
Lines 534-540 static irqreturn_t sifive_serial_irq(int irq, void *dev_id) Link Here
534	if (ip & SIFIVE_SERIAL_IP_TXWM_MASK)	535	if (ip & SIFIVE_SERIAL_IP_TXWM_MASK)
535	__ssp_transmit_chars(ssp);	536	__ssp_transmit_chars(ssp);
536		537
537	uart_port_unlock(&ssp->port);	538	uart_unlock_and_check_sysrq(&ssp->port);
538		539
539	return IRQ_HANDLED;	540	return IRQ_HANDLED;
540	}	541	}
Lines 791-803 static void sifive_serial_console_write(struct console co, const char s, Link Here
791	if (!ssp)	792	if (!ssp)
792	return;	793	return;
793		794
794	local_irq_save(flags);	795	if (oops_in_progress)
795	if (ssp->port.sysrq)	796	locked = uart_port_trylock_irqsave(&ssp->port, &flags);
796	locked = 0;
797	else if (oops_in_progress)
798	locked = uart_port_trylock(&ssp->port);
799	else	797	else
800	uart_port_lock(&ssp->port);	798	uart_port_lock_irqsave(&ssp->port, &flags);
801		799
802	ier = __ssp_readl(ssp, SIFIVE_SERIAL_IE_OFFS);	800	ier = __ssp_readl(ssp, SIFIVE_SERIAL_IE_OFFS);
803	__ssp_writel(0, SIFIVE_SERIAL_IE_OFFS, ssp);	801	__ssp_writel(0, SIFIVE_SERIAL_IE_OFFS, ssp);
Lines 807-814 static void sifive_serial_console_write(struct console co, const char s, Link Here
807	__ssp_writel(ier, SIFIVE_SERIAL_IE_OFFS, ssp);	805	__ssp_writel(ier, SIFIVE_SERIAL_IE_OFFS, ssp);
808		806
809	if (locked)	807	if (locked)
810	uart_port_unlock(&ssp->port);	808	uart_port_unlock_irqrestore(&ssp->port, flags);
811	local_irq_restore(flags);
812	}	809	}
813		810
814	static int sifive_serial_console_setup(struct console co, char options)	811	static int sifive_serial_console_setup(struct console co, char options)

Lines 260-266 static void receive_chars(struct uart_port *port) Link Here

(-)a/drivers/tty/serial/sunplus-uart.c (-12 / +6 lines)
260	if (port->ignore_status_mask & SUP_DUMMY_READ)	260	if (port->ignore_status_mask & SUP_DUMMY_READ)
261	goto ignore_char;	261	goto ignore_char;
262		262
263	if (uart_handle_sysrq_char(port, ch))	263	if (uart_prepare_sysrq_char(port, ch))
264	goto ignore_char;	264	goto ignore_char;
265		265
266	uart_insert_char(port, lsr, SUP_UART_LSR_OE, ch, flag);	266	uart_insert_char(port, lsr, SUP_UART_LSR_OE, ch, flag);
Lines 287-293 static irqreturn_t sunplus_uart_irq(int irq, void *args) Link Here
287	if (isc & SUP_UART_ISC_TX)	287	if (isc & SUP_UART_ISC_TX)
288	transmit_chars(port);	288	transmit_chars(port);
289		289
290	uart_port_unlock(port);	290	uart_unlock_and_check_sysrq(port);
291		291
292	return IRQ_HANDLED;	292	return IRQ_HANDLED;
293	}	293	}
Lines 512-533 static void sunplus_console_write(struct console *co, Link Here
512	unsigned long flags;	512	unsigned long flags;
513	int locked = 1;	513	int locked = 1;
514		514
515	local_irq_save(flags);	515	if (oops_in_progress)
516		516	locked = uart_port_trylock_irqsave(&sunplus_console_ports[co->index]->port, &flags);
517	if (sunplus_console_ports[co->index]->port.sysrq)
518	locked = 0;
519	else if (oops_in_progress)
520	locked = uart_port_trylock(&sunplus_console_ports[co->index]->port);
521	else	517	else
522	uart_port_lock(&sunplus_console_ports[co->index]->port);	518	uart_port_lock_irqsave(&sunplus_console_ports[co->index]->port, &flags);
523		519
524	uart_console_write(&sunplus_console_ports[co->index]->port, s, count,	520	uart_console_write(&sunplus_console_ports[co->index]->port, s, count,
525	sunplus_uart_console_putchar);	521	sunplus_uart_console_putchar);
526		522
527	if (locked)	523	if (locked)
528	uart_port_unlock(&sunplus_console_ports[co->index]->port);	524	uart_port_unlock_irqrestore(&sunplus_console_ports[co->index]->port, flags);
529
530	local_irq_restore(flags);
531	}	525	}
532		526
533	static int __init sunplus_console_setup(struct console co, char options)	527	static int __init sunplus_console_setup(struct console co, char options)

Lines 21-32 static int show_console_dev(struct seq_file m, void v) Link Here

(-)a/fs/proc/consoles.c (-3 / +11 lines)
21	{ CON_ENABLED, 'E' },	21	{ CON_ENABLED, 'E' },
22	{ CON_CONSDEV, 'C' },	22	{ CON_CONSDEV, 'C' },
23	{ CON_BOOT, 'B' },	23	{ CON_BOOT, 'B' },
		24	{ CON_NBCON, 'N' },
24	{ CON_PRINTBUFFER, 'p' },	25	{ CON_PRINTBUFFER, 'p' },
25	{ CON_BRL, 'b' },	26	{ CON_BRL, 'b' },
26	{ CON_ANYTIME, 'a' },	27	{ CON_ANYTIME, 'a' },
27	};	28	};
28	char flags[ARRAY_SIZE(con_flags) + 1];	29	char flags[ARRAY_SIZE(con_flags) + 1];
29	struct console *con = v;	30	struct console *con = v;
		31	char con_write = '-';
30	unsigned int a;	32	unsigned int a;
31	dev_t dev = 0;	33	dev_t dev = 0;
32		34
Lines 57-65 static int show_console_dev(struct seq_file m, void v) Link Here
57	seq_setwidth(m, 21 - 1);	59	seq_setwidth(m, 21 - 1);
58	seq_printf(m, "%s%d", con->name, con->index);	60	seq_printf(m, "%s%d", con->name, con->index);
59	seq_pad(m, ' ');	61	seq_pad(m, ' ');
60	seq_printf(m, "%c%c%c (%s)", con->read ? 'R' : '-',	62	if (con->flags & CON_NBCON) {
61	con->write ? 'W' : '-', con->unblank ? 'U' : '-',	63	if (con->write_atomic \|\| con->write_thread)
62	flags);	64	con_write = 'W';
		65	} else {
		66	if (con->write)
		67	con_write = 'W';
		68	}
		69	seq_printf(m, "%c%c%c (%s)", con->read ? 'R' : '-', con_write,
		70	con->unblank ? 'U' : '-', flags);
63	if (dev)	71	if (dev)
64	seq_printf(m, " %4d:%d", MAJOR(dev), MINOR(dev));	72	seq_printf(m, " %4d:%d", MAJOR(dev), MINOR(dev));
65		73

Lines 16-22 Link Here

(-)a/include/linux/console.h (-6 / +121 lines)
16		16
17	#include <linux/atomic.h>	17	#include <linux/atomic.h>
18	#include <linux/bits.h>	18	#include <linux/bits.h>
		19	#include <linux/irq_work.h>
19	#include <linux/rculist.h>	20	#include <linux/rculist.h>
		21	#include <linux/rcuwait.h>
20	#include <linux/types.h>	22	#include <linux/types.h>
21		23
22	struct vc_data;	24	struct vc_data;
Lines 137-143 static inline int con_debug_leave(void) Link Here
137	*/	139	*/
138		140
139	/**	141	/**
140	* cons_flags - General console flags	142	* enum cons_flags - General console flags
141	* @CON_PRINTBUFFER: Used by newly registered consoles to avoid duplicate	143	* @CON_PRINTBUFFER: Used by newly registered consoles to avoid duplicate
142	* output of messages that were already shown by boot	144	* output of messages that were already shown by boot
143	* consoles or read by userspace via syslog() syscall.	145	* consoles or read by userspace via syslog() syscall.
Lines 218-224 struct nbcon_state { Link Here
218	static_assert(sizeof(struct nbcon_state) <= sizeof(int));	220	static_assert(sizeof(struct nbcon_state) <= sizeof(int));
219		221
220	/**	222	/**
221	* nbcon_prio - console owner priority for nbcon consoles	223	* enum nbcon_prio - console owner priority for nbcon consoles
222	* @NBCON_PRIO_NONE: Unused	224	* @NBCON_PRIO_NONE: Unused
223	* @NBCON_PRIO_NORMAL: Normal (non-emergency) usage	225	* @NBCON_PRIO_NORMAL: Normal (non-emergency) usage
224	* @NBCON_PRIO_EMERGENCY: Emergency output (WARN/OOPS...)	226	* @NBCON_PRIO_EMERGENCY: Emergency output (WARN/OOPS...)
Lines 285-291 struct nbcon_write_context { Link Here
285	/**	287	/**
286	* struct console - The console descriptor structure	288	* struct console - The console descriptor structure
287	* @name: The name of the console driver	289	* @name: The name of the console driver
288	* @write: Write callback to output messages (Optional)	290	* @write: Legacy write callback to output messages (Optional)
289	* @read: Read callback for console input (Optional)	291	* @read: Read callback for console input (Optional)
290	* @device: The underlying TTY device driver (Optional)	292	* @device: The underlying TTY device driver (Optional)
291	* @unblank: Callback to unblank the console (Optional)	293	* @unblank: Callback to unblank the console (Optional)
Lines 302-311 struct nbcon_write_context { Link Here
302	* @data: Driver private data	304	* @data: Driver private data
303	* @node: hlist node for the console list	305	* @node: hlist node for the console list
304	*	306	*
305	* @write_atomic: Write callback for atomic context
306	* @nbcon_state: State for nbcon consoles	307	* @nbcon_state: State for nbcon consoles
307	* @nbcon_seq: Sequence number of the next record for nbcon to print	308	* @nbcon_seq: Sequence number of the next record for nbcon to print
308	* @pbufs: Pointer to nbcon private buffer	309	* @pbufs: Pointer to nbcon private buffer
		310	* @kthread: Printer kthread for this console
		311	* @rcuwait: RCU-safe wait object for @kthread waking
		312	* @irq_work: Defer @kthread waking to IRQ work context
309	*/	313	*/
310	struct console {	314	struct console {
311	char name[16];	315	char name[16];
Lines 327-337 struct console { Link Here
327	struct hlist_node node;	331	struct hlist_node node;
328		332
329	/* nbcon console specific members */	333	/* nbcon console specific members */
330	bool (write_atomic)(struct console con,	334
331	struct nbcon_write_context *wctxt);	335	/**
		336	* @write_atomic:
		337	*
		338	* NBCON callback to write out text in any context. (Optional)
		339	*
		340	* This callback is called with the console already acquired. The
		341	* callback can use nbcon_can_proceed() at any time to verify that
		342	* it is still the owner of the console. In the case that it has
		343	* lost ownership, it is no longer allowed to go forward. In this
		344	* case it must back out immediately and carefully. The buffer
		345	* content is also no longer trusted since it no longer belongs to
		346	* the context.
		347	*
		348	* If the callback needs to perform actions where ownership is not
		349	* allowed to be taken over, nbcon_enter_unsafe() and
		350	* nbcon_exit_unsafe() can be used to mark such sections. These
		351	* functions are also points of possible ownership transfer. If
		352	* either function returns false, ownership has been lost.
		353	*
		354	* If the driver must reacquire ownership in order to finalize or
		355	* revert hardware changes, nbcon_reacquire() can be used. However,
		356	* on reacquire the buffer content is no longer available. A
		357	* reacquire cannot be used to resume printing.
		358	*
		359	* This callback can be called from any context (including NMI).
		360	* Therefore it must avoid usage of any locking and instead rely
		361	* on the console ownership for synchronization.
		362	*
		363	* Returns true if all text was successfully written out and
		364	* ownership was never lost, otherwise false.
		365	*/
		366	bool (write_atomic)(struct console con, struct nbcon_write_context *wctxt);
		367
		368	/**
		369	* @write_thread:
		370	*
		371	* NBCON callback to write out text in task context. (Optional)
		372	*
		373	* This callback is called with the console already acquired. Any
		374	* additional driver synchronization should have been performed by
		375	* driver_enter().
		376	*
		377	* This callback is always called from task context but with migration
		378	* disabled.
		379	*
		380	* The same criteria for console ownership verification and unsafe
		381	* sections applies as with write_atomic(). The difference between
		382	* this callback and write_atomic() is that this callback is used
		383	* during normal operation and is always called from task context.
		384	* This provides drivers with a relatively relaxed locking context
		385	* for synchronizing output to the hardware.
		386	*
		387	* Returns true if all text was successfully written out, otherwise
		388	* false.
		389	*/
		390	bool (write_thread)(struct console con, struct nbcon_write_context *wctxt);
		391
		392	/**
		393	* @driver_enter:
		394	*
		395	* NBCON callback to begin synchronization with driver code.
		396	* (Required for NBCON if write_thread is provided)
		397	*
		398	* Console drivers typically must deal with access to the hardware
		399	* via user input/output (such as an interactive login shell) and
400	* output of kernel messages via printk() calls. This callback is
401	* called before the kernel begins output via the write_thread()
402	* callback due to printk() calls. The driver can use this
403	* callback to acquire some driver lock in order to synchronize
404	* against user input/output (or any other driver functionality).
405	*
406	* This callback is always called from task context. It may use any
407	* synchronization method required by the driver. BUT this callback
408	* MUST disable migration. The console driver may be using a
409	* sychronization mechanism that already takes care of this (such as
410	* spinlocks). Otherwise this function must explicitly call
411	* migrate_disable().
412	*
413	* The flags argument is provided as a convenience to the driver. It
414	* will be passed again to driver_exit() when printing is completed
415	* (for example, if spin_lock_irqsave() was used). It can be ignored
416	* if the driver does not need it.
417	*/
418	void (driver_enter)(struct console con, unsigned long *flags);
419
420	/**
421	* @driver_exit:
422	*
423	* NBCON callback to finish synchronization with driver code.
424	* (Required for NBCON if write_thread is provided)
425	*
426	* This callback is called after the kernel has finished printing a
427	* printk message. It is the counterpart to driver_enter().
428	*
429	* This callback is always called from task context. It must
430	* appropriately re-enable migration (depending on how driver_enter()
431	* disabled migration).
432	*
433	* The flags argument is the value of the same variable that was
434	* passed to driver_enter().
435	*/
436	void (driver_exit)(struct console con, unsigned long flags);
437
332	atomic_t __private nbcon_state;	438	atomic_t __private nbcon_state;
333	atomic_long_t __private nbcon_seq;	439	atomic_long_t __private nbcon_seq;
334	struct printk_buffers *pbufs;	440	struct printk_buffers *pbufs;
		441	struct task_struct *kthread;
		442	struct rcuwait rcuwait;
		443	struct irq_work irq_work;
335	};	444	};
336		445
337	#ifdef CONFIG_LOCKDEP	446	#ifdef CONFIG_LOCKDEP
Lines 459-471 static inline bool console_is_registered(const struct console *con) Link Here
459	hlist_for_each_entry(con, &console_list, node)	568	hlist_for_each_entry(con, &console_list, node)
460		569
461	#ifdef CONFIG_PRINTK	570	#ifdef CONFIG_PRINTK
		571	extern void nbcon_cpu_emergency_enter(void);
		572	extern void nbcon_cpu_emergency_exit(void);
462	extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt);	573	extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt);
463	extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt);	574	extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt);
464	extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt);	575	extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt);
		576	extern void nbcon_reacquire(struct nbcon_write_context *wctxt);
465	#else	577	#else
		578	static inline void nbcon_cpu_emergency_enter(void) { }
		579	static inline void nbcon_cpu_emergency_exit(void) { }
466	static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; }	580	static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; }
467	static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; }	581	static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; }
468	static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; }	582	static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; }
		583	static inline void nbcon_reacquire(struct nbcon_write_context *wctxt) { }
469	#endif	584	#endif
470		585
471	extern int console_set_on_cmdline;	586	extern int console_set_on_cmdline;

Lines 609-614 extern void __raise_softirq_irqoff(unsigned int nr); Link Here

(-)a/include/linux/interrupt.h (+29 lines)
609	extern void raise_softirq_irqoff(unsigned int nr);	609	extern void raise_softirq_irqoff(unsigned int nr);
610	extern void raise_softirq(unsigned int nr);	610	extern void raise_softirq(unsigned int nr);
611		611
		612	#ifdef CONFIG_PREEMPT_RT
		613	DECLARE_PER_CPU(struct task_struct *, timersd);
		614	DECLARE_PER_CPU(unsigned long, pending_timer_softirq);
		615
		616	extern void raise_timer_softirq(void);
		617	extern void raise_hrtimer_softirq(void);
		618
		619	static inline unsigned int local_pending_timers(void)
		620	{
		621	return __this_cpu_read(pending_timer_softirq);
		622	}
		623
		624	#else
		625	static inline void raise_timer_softirq(void)
		626	{
		627	raise_softirq(TIMER_SOFTIRQ);
		628	}
		629
		630	static inline void raise_hrtimer_softirq(void)
		631	{
		632	raise_softirq_irqoff(HRTIMER_SOFTIRQ);
		633	}
		634
		635	static inline unsigned int local_pending_timers(void)
		636	{
		637	return local_softirq_pending();
		638	}
		639	#endif
		640
612	DECLARE_PER_CPU(struct task_struct *, ksoftirqd);	641	DECLARE_PER_CPU(struct task_struct *, ksoftirqd);
613		642
614	static inline struct task_struct *this_cpu_ksoftirqd(void)	643	static inline struct task_struct *this_cpu_ksoftirqd(void)

Lines 781-789 struct perf_event { Link Here

(-)a/include/linux/perf_event.h (-2 / +2 lines)
781	unsigned int pending_wakeup;	781	unsigned int pending_wakeup;
782	unsigned int pending_kill;	782	unsigned int pending_kill;
783	unsigned int pending_disable;	783	unsigned int pending_disable;
784	unsigned int pending_sigtrap;
785	unsigned long pending_addr; /* SIGTRAP */	784	unsigned long pending_addr; /* SIGTRAP */
786	struct irq_work pending_irq;	785	struct irq_work pending_irq;
		786	struct irq_work pending_disable_irq;
787	struct callback_head pending_task;	787	struct callback_head pending_task;
788	unsigned int pending_work;	788	unsigned int pending_work;
789		789
Lines 959-965 struct perf_event_context { Link Here
959	struct rcu_head rcu_head;	959	struct rcu_head rcu_head;
960		960
961	/*	961	/*
962	* Sum (event->pending_sigtrap + event->pending_work)	962	* Sum (event->pending_work + event->pending_work)
963	*	963	*
964	* The SIGTRAP is targeted at ctx->task, as such it won't do changing	964	* The SIGTRAP is targeted at ctx->task, as such it won't do changing
965	* that until the signal is delivered.	965	* that until the signal is delivered.

Lines 1791-1796 static inline int dl_task_check_affinity(struct task_struct *p, const struct cpu Link Here

(-)a/include/linux/sched.h (-5 / +8 lines)
1791	}	1791	}
1792	#endif	1792	#endif
1793		1793
		1794	extern bool task_is_pi_boosted(const struct task_struct *p);
1794	extern int yield_to(struct task_struct *p, bool preempt);	1795	extern int yield_to(struct task_struct *p, bool preempt);
1795	extern void set_user_nice(struct task_struct *p, long nice);	1796	extern void set_user_nice(struct task_struct *p, long nice);
1796	extern int task_prio(const struct task_struct *p);	1797	extern int task_prio(const struct task_struct *p);
Lines 1933-1949 static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag, Link Here
1933	update_ti_thread_flag(task_thread_info(tsk), flag, value);	1934	update_ti_thread_flag(task_thread_info(tsk), flag, value);
1934	}	1935	}
1935		1936
1936	static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)	1937	static inline bool test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
1937	{	1938	{
1938	return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);	1939	return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
1939	}	1940	}
1940		1941
1941	static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)	1942	static inline bool test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
1942	{	1943	{
1943	return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);	1944	return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
1944	}	1945	}
1945		1946
1946	static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)	1947	static inline bool test_tsk_thread_flag(struct task_struct *tsk, int flag)
1947	{	1948	{
1948	return test_ti_thread_flag(task_thread_info(tsk), flag);	1949	return test_ti_thread_flag(task_thread_info(tsk), flag);
1949	}	1950	}
Lines 1956-1964 static inline void set_tsk_need_resched(struct task_struct *tsk) Link Here
1956	static inline void clear_tsk_need_resched(struct task_struct *tsk)	1957	static inline void clear_tsk_need_resched(struct task_struct *tsk)
1957	{	1958	{
1958	clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);	1959	clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
		1960	if (IS_ENABLED(CONFIG_PREEMPT_BUILD_AUTO))
		1961	clear_tsk_thread_flag(tsk, TIF_NEED_RESCHED_LAZY);
1959	}	1962	}
1960		1963
1961	static inline int test_tsk_need_resched(struct task_struct *tsk)	1964	static inline bool test_tsk_need_resched(struct task_struct *tsk)
1962	{	1965	{
1963	return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));	1966	return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
1964	}	1967	}
Lines 2099-2105 static inline bool preempt_model_preemptible(void) Link Here
2099		2102
2100	static __always_inline bool need_resched(void)	2103	static __always_inline bool need_resched(void)
2101	{	2104	{
2102	return unlikely(tif_need_resched());	2105	return unlikely(tif_need_resched_lazy() \|\| tif_need_resched());
2103	}	2106	}
2104		2107
2105	/*	2108	/*

Lines 63-69 static __always_inline bool __must_check current_set_polling_and_test(void) Link Here

(-)a/include/linux/sched/idle.h (-4 / +4 lines)
63	*/	63	*/
64	smp_mb__after_atomic();	64	smp_mb__after_atomic();
65		65
66	return unlikely(tif_need_resched());	66	return unlikely(need_resched());
67	}	67	}
68		68
69	static __always_inline bool __must_check current_clr_polling_and_test(void)	69	static __always_inline bool __must_check current_clr_polling_and_test(void)
Lines 76-82 static __always_inline bool __must_check current_clr_polling_and_test(void) Link Here
76	*/	76	*/
77	smp_mb__after_atomic();	77	smp_mb__after_atomic();
78		78
79	return unlikely(tif_need_resched());	79	return unlikely(need_resched());
80	}	80	}
81		81
82	#else	82	#else
Lines 85-95 static inline void __current_clr_polling(void) { } Link Here
85		85
86	static inline bool __must_check current_set_polling_and_test(void)	86	static inline bool __must_check current_set_polling_and_test(void)
87	{	87	{
88	return unlikely(tif_need_resched());	88	return unlikely(need_resched());
89	}	89	}
90	static inline bool __must_check current_clr_polling_and_test(void)	90	static inline bool __must_check current_clr_polling_and_test(void)
91	{	91	{
92	return unlikely(tif_need_resched());	92	return unlikely(need_resched());
93	}	93	}
94	#endif	94	#endif
95		95

Lines 488-493 struct uart_port { Link Here

(-)a/include/linux/serial_core.h (-2 / +29 lines)
488	struct uart_icount icount; /* statistics */	488	struct uart_icount icount; /* statistics */
489		489
490	struct console cons; / struct console, if any */	490	struct console cons; / struct console, if any */
		491	bool nbcon_locked_port; /* True, if the port is locked by nbcon */
491	/* flags must be updated while holding port mutex */	492	/* flags must be updated while holding port mutex */
492	upf_t flags;	493	upf_t flags;
493		494
Lines 595-600 struct uart_port { Link Here
595	static inline void uart_port_lock(struct uart_port *up)	596	static inline void uart_port_lock(struct uart_port *up)
596	{	597	{
597	spin_lock(&up->lock);	598	spin_lock(&up->lock);
		599	uart_nbcon_acquire(up);
598	}	600	}
599		601
600	/**	602	/**
Lines 604-609 static inline void uart_port_lock(struct uart_port *up) Link Here
604	static inline void uart_port_lock_irq(struct uart_port *up)	606	static inline void uart_port_lock_irq(struct uart_port *up)
605	{	607	{
606	spin_lock_irq(&up->lock);	608	spin_lock_irq(&up->lock);
		609	uart_nbcon_acquire(up);
607	}	610	}
608		611
609	/**	612	/**
Lines 614-619 static inline void uart_port_lock_irq(struct uart_port *up) Link Here
614	static inline void uart_port_lock_irqsave(struct uart_port up, unsigned long flags)	617	static inline void uart_port_lock_irqsave(struct uart_port up, unsigned long flags)
615	{	618	{
616	spin_lock_irqsave(&up->lock, *flags);	619	spin_lock_irqsave(&up->lock, *flags);
		620	uart_nbcon_acquire(up);
617	}	621	}
618		622
619	/**	623	/**
Lines 624-630 static inline void uart_port_lock_irqsave(struct uart_port up, unsigned long f Link Here
624	*/	628	*/
625	static inline bool uart_port_trylock(struct uart_port *up)	629	static inline bool uart_port_trylock(struct uart_port *up)
626	{	630	{
627	return spin_trylock(&up->lock);	631	if (!spin_trylock(&up->lock))
		632	return false;
		633
		634	uart_nbcon_acquire(up);
		635	return true;
628	}	636	}
629		637
630	/**	638	/**
Lines 636-642 static inline bool uart_port_trylock(struct uart_port *up) Link Here
636	*/	644	*/
637	static inline bool uart_port_trylock_irqsave(struct uart_port up, unsigned long flags)	645	static inline bool uart_port_trylock_irqsave(struct uart_port up, unsigned long flags)
638	{	646	{
639	return spin_trylock_irqsave(&up->lock, *flags);	647	if (!spin_trylock_irqsave(&up->lock, *flags))
		648	return false;
		649
		650	uart_nbcon_acquire(up);
		651	return true;
640	}	652	}
641		653
642	/**	654	/**
Lines 645-650 static inline bool uart_port_trylock_irqsave(struct uart_port *up, unsigned long Link Here
645	*/	657	*/
646	static inline void uart_port_unlock(struct uart_port *up)	658	static inline void uart_port_unlock(struct uart_port *up)
647	{	659	{
		660	uart_nbcon_release(up);
648	spin_unlock(&up->lock);	661	spin_unlock(&up->lock);
649	}	662	}
650		663
Lines 654-659 static inline void uart_port_unlock(struct uart_port *up) Link Here
654	*/	667	*/
655	static inline void uart_port_unlock_irq(struct uart_port *up)	668	static inline void uart_port_unlock_irq(struct uart_port *up)
656	{	669	{
		670	uart_nbcon_release(up);
657	spin_unlock_irq(&up->lock);	671	spin_unlock_irq(&up->lock);
658	}	672	}
659		673
Lines 663-668 static inline void uart_port_unlock_irq(struct uart_port *up) Link Here
663	* @flags: The saved interrupt flags for restore	677	* @flags: The saved interrupt flags for restore
664	*/	678	*/
665	static inline void uart_port_unlock_irqrestore(struct uart_port *up, unsigned long flags)	679	static inline void uart_port_unlock_irqrestore(struct uart_port *up, unsigned long flags)
		680	{
		681	uart_nbcon_release(up);
		682	spin_unlock_irqrestore(&up->lock, flags);
		683	}
		684
		685	/* Only for use in the console->driver_enter() callback. */
		686	static inline void __uart_port_lock_irqsave(struct uart_port up, unsigned long flags)
		687	{
		688	spin_lock_irqsave(&up->lock, *flags);
		689	}
		690
		691	/* Only for use in the console->driver_exit() callback. */
		692	static inline void __uart_port_unlock_irqrestore(struct uart_port *up, unsigned long flags)
666	{	693	{
667	spin_unlock_irqrestore(&up->lock, flags);	694	spin_unlock_irqrestore(&up->lock, flags);
668	}	695	}