Attachment #889979 for bug #916954




#else // !CONFIG_SCHED_BORE
	curr->vruntime += calc_delta_fair(delta_exec, curr);
#endif // CONFIG_SCHED_BORE
	update_deadline(cfs_rq, curr, tick);
	update_min_vruntime(cfs_rq);

	if (entity_is_task(curr))
--
include/linux/sched.h   |  10 ++
init/Kconfig            |  17 ++++
kernel/sched/core.c     | 144 ++++++++++++++++++++++++++
kernel/sched/debug.c    |  60 ++++++++++-
kernel/sched/fair.c     | 218 +++++++++++++++++++++++++++++++++++++++-
kernel/sched/features.h |   4 +
kernel/sched/sched.h    |   7 ++
7 files changed, 457 insertions(+), 3 deletions(-)




	u64				sum_exec_runtime;
	u64				prev_sum_exec_runtime;
	u64				vruntime;
#ifdef CONFIG_SCHED_BORE
	u64				burst_time;
	u8				prev_burst_penalty;
	u8				curr_burst_penalty;
	u8				burst_penalty;
	u8				burst_score;
	u8				child_burst;
	u32				child_burst_cnt;
	u64				child_burst_last_cached;
#endif // CONFIG_SCHED_BORE
	s64				vlag;
	u64				slice;






	  If unsure, say N here.

config SCHED_BORE
	bool "Burst-Oriented Response Enhancer"
	default y
	help
	  In Desktop and Mobile computing, one might prefer interactive
	  tasks to keep responsive no matter what they run in the background.

	  Enabling this kernel feature modifies the scheduler to discriminate
	  tasks by their burst time (runtime since it last went sleeping or
	  yielding state) and prioritize those that run less bursty.
	  Such tasks usually include window compositor, widgets backend,
	  terminal emulator, video playback, games and so on.
	  With a little impact to scheduling fairness, it may improve
	  responsiveness especially under heavy background workload.

	  If unsure, say Y here.

config SCHED_AUTOGROUP
	bool "Automatic process group scheduling"
	select CGROUPS




	return try_to_wake_up(p, state, 0);
}

#ifdef CONFIG_SCHED_BORE
extern bool sched_bore;
extern u8   sched_burst_fork_atavistic;
extern uint sched_burst_cache_lifetime;

static void __init sched_init_bore(void) {
	init_task.se.burst_time = 0;
	init_task.se.prev_burst_penalty = 0;
	init_task.se.curr_burst_penalty = 0;
	init_task.se.burst_penalty = 0;
	init_task.se.burst_score = 0;
	init_task.se.child_burst_last_cached = 0;
}

void inline sched_fork_bore(struct task_struct *p) {
	p->se.burst_time = 0;
	p->se.curr_burst_penalty = 0;
	p->se.burst_score = 0;
	p->se.child_burst_last_cached = 0;
}

static u32 count_child_tasks(struct task_struct *p) {
	struct task_struct *child;
	u32 cnt = 0;
	list_for_each_entry(child, &p->children, sibling) {cnt++;}
	return cnt;
}

static inline bool task_is_inheritable(struct task_struct *p) {
	return (p->sched_class == &fair_sched_class);
}

static inline bool child_burst_cache_expired(struct task_struct *p, u64 now) {
	u64 expiration_time =
		p->se.child_burst_last_cached + sched_burst_cache_lifetime;
	return ((s64)(expiration_time - now) < 0);
}

static void __update_child_burst_cache(
	struct task_struct *p, u32 cnt, u32 sum, u64 now) {
	u8 avg = 0;
	if (cnt) avg = sum / cnt;
	p->se.child_burst = max(avg, p->se.burst_penalty);
	p->se.child_burst_cnt = cnt;
	p->se.child_burst_last_cached = now;
}

static inline void update_child_burst_direct(struct task_struct *p, u64 now) {
	struct task_struct *child;
	u32 cnt = 0;
	u32 sum = 0;

	list_for_each_entry(child, &p->children, sibling) {
		if (!task_is_inheritable(child)) continue;
		cnt++;
		sum += child->se.burst_penalty;
	}

	__update_child_burst_cache(p, cnt, sum, now);
}

static inline u8 __inherit_burst_direct(struct task_struct *p, u64 now) {
	struct task_struct *parent = p->real_parent;
	if (child_burst_cache_expired(parent, now))
		update_child_burst_direct(parent, now);

	return parent->se.child_burst;
}

static void update_child_burst_topological(
	struct task_struct *p, u64 now, u32 depth, u32 *acnt, u32 *asum) {
	struct task_struct *child, *dec;
	u32 cnt = 0, dcnt = 0;
	u32 sum = 0;

	list_for_each_entry(child, &p->children, sibling) {
		dec = child;
		while ((dcnt = count_child_tasks(dec)) == 1)
			dec = list_first_entry(&dec->children, struct task_struct, sibling);
		
		if (!dcnt || !depth) {
			if (!task_is_inheritable(dec)) continue;
			cnt++;
			sum += dec->se.burst_penalty;
			continue;
		}
		if (!child_burst_cache_expired(dec, now)) {
			cnt += dec->se.child_burst_cnt;
			sum += (u32)dec->se.child_burst * dec->se.child_burst_cnt;
			continue;
		}
		update_child_burst_topological(dec, now, depth - 1, &cnt, &sum);
	}

	__update_child_burst_cache(p, cnt, sum, now);
	*acnt += cnt;
	*asum += sum;
}

static inline u8 __inherit_burst_topological(struct task_struct *p, u64 now) {
	struct task_struct *anc = p->real_parent;
	u32 cnt = 0, sum = 0;

	while (anc->real_parent != anc && count_child_tasks(anc) == 1)
		anc = anc->real_parent;

	if (child_burst_cache_expired(anc, now))
		update_child_burst_topological(
			anc, now, sched_burst_fork_atavistic - 1, &cnt, &sum);

	return anc->se.child_burst;
}

static inline void inherit_burst(struct task_struct *p) {
	u8 burst_cache;
	u64 now = ktime_get_ns();

	read_lock(&tasklist_lock);
	burst_cache = likely(sched_burst_fork_atavistic)?
		__inherit_burst_topological(p, now):
		__inherit_burst_direct(p, now);
	read_unlock(&tasklist_lock);

	p->se.prev_burst_penalty = max(p->se.prev_burst_penalty, burst_cache);
}

static void sched_post_fork_bore(struct task_struct *p) {
	if (p->sched_class == &fair_sched_class)
		inherit_burst(p);
	p->se.burst_penalty = p->se.prev_burst_penalty;
}
#endif // CONFIG_SCHED_BORE

/*
 * Perform scheduler related setup for a newly forked process p.
 * p is forked by current.

	p->se.prev_sum_exec_runtime	= 0;
	p->se.nr_migrations		= 0;
	p->se.vruntime			= 0;
#ifdef CONFIG_SCHED_BORE
	sched_fork_bore(p);
#endif // CONFIG_SCHED_BORE
	p->se.vlag			= 0;
	p->se.slice			= sysctl_sched_base_slice;
	INIT_LIST_HEAD(&p->se.group_node);


void sched_post_fork(struct task_struct *p)
{
#ifdef CONFIG_SCHED_BORE
	sched_post_fork_bore(p);
#endif // CONFIG_SCHED_BORE
	uclamp_post_fork(p);
}


	BUG_ON(&dl_sched_class != &stop_sched_class + 1);
#endif

#ifdef CONFIG_SCHED_BORE
	sched_init_bore();
	printk(KERN_INFO "BORE (Burst-Oriented Response Enhancer) CPU Scheduler modification 5.0.3 by Masahito Suzuki");
#endif // CONFIG_SCHED_BORE

	wait_bit_init();

#ifdef CONFIG_FAIR_GROUP_SCHED




};

#ifdef CONFIG_SMP
#ifdef CONFIG_SCHED_BORE
static ssize_t sched_min_base_slice_write(struct file *filp, const char __user *ubuf,
				   size_t cnt, loff_t *ppos)
{
	char buf[16];
	unsigned int value;

	if (cnt > 15)
		cnt = 15;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;
	buf[cnt] = '\0';

	if (kstrtouint(buf, 10, &value))
		return -EINVAL;

	if (!value)
		return -EINVAL;

	sysctl_sched_min_base_slice = value;
	sched_update_min_base_slice();

	*ppos += cnt;
	return cnt;
}

static int sched_min_base_slice_show(struct seq_file *m, void *v)
{
	seq_printf(m, "%d\n", sysctl_sched_min_base_slice);
	return 0;
}

static int sched_min_base_slice_open(struct inode *inode, struct file *filp)
{
	return single_open(filp, sched_min_base_slice_show, NULL);
}

static const struct file_operations sched_min_base_slice_fops = {
	.open		= sched_min_base_slice_open,
	.write		= sched_min_base_slice_write,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};
#else // !CONFIG_SCHED_BORE
static ssize_t sched_scaling_write(struct file *filp, const char __user *ubuf,
				   size_t cnt, loff_t *ppos)
{

	.llseek		= seq_lseek,
	.release	= single_release,
};
#endif // CONFIG_SCHED_BORE
#endif /* SMP */

#ifdef CONFIG_PREEMPT_DYNAMIC

	debugfs_create_file("preempt", 0644, debugfs_sched, NULL, &sched_dynamic_fops);
#endif

#ifdef CONFIG_SCHED_BORE
	debugfs_create_file("min_base_slice_ns", 0644, debugfs_sched, NULL, &sched_min_base_slice_fops);
	debugfs_create_u32("base_slice_ns", 0400, debugfs_sched, &sysctl_sched_base_slice);
#else // !CONFIG_SCHED_BORE
 	debugfs_create_u32("base_slice_ns", 0644, debugfs_sched, &sysctl_sched_base_slice);
#endif // CONFIG_SCHED_BORE

#ifndef CONFIG_SCHED_ALT
	debugfs_create_u32("latency_warn_ms", 0644, debugfs_sched, &sysctl_resched_latency_warn_ms);
	debugfs_create_u32("latency_warn_once", 0644, debugfs_sched, &sysctl_resched_latency_warn_once);

#ifdef CONFIG_SMP
#if !defined(CONFIG_SCHED_BORE)
	debugfs_create_file("tunable_scaling", 0644, debugfs_sched, NULL, &sched_scaling_fops);
#endif // CONFIG_SCHED_BORE
	debugfs_create_u32("migration_cost_ns", 0644, debugfs_sched, &sysctl_sched_migration_cost);
	debugfs_create_u32("nr_migrate", 0644, debugfs_sched, &sysctl_sched_nr_migrate);


		SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)),
		SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime)));

#ifdef CONFIG_SCHED_BORE
	SEQ_printf(m, " %2d", p->se.burst_score);
#endif // CONFIG_SCHED_BORE
#ifdef CONFIG_NUMA_BALANCING
	SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
#endif


	P(se.load.weight);
#ifdef CONFIG_SMP
#ifdef CONFIG_SCHED_BORE
	P(se.burst_score);
#endif // CONFIG_SCHED_BORE
	P(se.avg.load_sum);
	P(se.avg.runnable_sum);
	P(se.avg.util_sum);

Lines 19-24 Link Here

(-)a/kernel/sched/fair.c (-2 / +216 lines)
19	*	19	*
20	* Adaptive scheduling granularity, math enhancements by Peter Zijlstra	20	* Adaptive scheduling granularity, math enhancements by Peter Zijlstra
21	* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra	21	* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
		22	*
		23	* Burst-Oriented Response Enhancer (BORE) CPU Scheduler
		24	* Copyright (C) 2021-2024 Masahito Suzuki <firelzrd@gmail.com>
22	*/	25	*/
23	#include <linux/energy_model.h>	26	#include <linux/energy_model.h>
24	#include <linux/mmap_lock.h>	27	#include <linux/mmap_lock.h>
Lines 64-83 Link Here
64	* SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)	67	* SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)
65	* SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus	68	* SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
66	*	69	*
67	* (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))	70	* (BORE default SCHED_TUNABLESCALING_NONE = *1 constant)
		71	* (EEVDF default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
68	*/	72	*/
		73	#ifdef CONFIG_SCHED_BORE
		74	unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_NONE;
		75	#else // !CONFIG_SCHED_BORE
69	unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;	76	unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;
		77	#endif // CONFIG_SCHED_BORE
70		78
71	/*	79	/*
72	* Minimal preemption granularity for CPU-bound tasks:	80	* Minimal preemption granularity for CPU-bound tasks:
73	*	81	*
74	* (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)	82	* (BORE default: max(1 sec / HZ, min_base_slice) constant, units: nanoseconds)
		83	* (EEVDF default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)
75	*/	84	*/
		85	#ifdef CONFIG_SCHED_BORE
		86	unsigned int sysctl_sched_base_slice = 1000000000ULL / HZ;
		87	static unsigned int configured_sched_base_slice = 1000000000ULL / HZ;
		88	unsigned int sysctl_sched_min_base_slice = 2000000ULL;
		89	#else // !CONFIG_SCHED_BORE
76	unsigned int sysctl_sched_base_slice = 750000ULL;	90	unsigned int sysctl_sched_base_slice = 750000ULL;
77	static unsigned int normalized_sysctl_sched_base_slice = 750000ULL;	91	static unsigned int normalized_sysctl_sched_base_slice = 750000ULL;
		92	#endif // CONFIG_SCHED_BORE
78		93
79	const_debug unsigned int sysctl_sched_migration_cost = 500000UL;	94	const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
80		95
		96	#ifdef CONFIG_SCHED_BORE
		97	u8 __read_mostly sched_bore = 1;
		98	u8 __read_mostly sched_burst_smoothness_long = 1;
		99	u8 __read_mostly sched_burst_smoothness_short = 0;
		100	u8 __read_mostly sched_burst_fork_atavistic = 2;
		101	u8 __read_mostly sched_burst_penalty_offset = 22;
		102	uint __read_mostly sched_burst_penalty_scale = 1280;
		103	uint __read_mostly sched_burst_cache_lifetime = 60000000;
		104	static int __maybe_unused thirty_two = 32;
		105	static int __maybe_unused sixty_four = 64;
		106	static int __maybe_unused maxval_12_bits = 4095;
		107
		108	#define MAX_BURST_PENALTY (39U <<2)
		109
		110	static inline u32 log2plus1_u64_u32f8(u64 v) {
		111	u32 msb = fls64(v);
		112	s32 excess_bits = msb - 9;
		113	u8 fractional = (0 <= excess_bits)? v >> excess_bits: v << -excess_bits;
		114	return msb << 8 \| fractional;
		115	}
		116
		117	static inline u32 calc_burst_penalty(u64 burst_time) {
		118	u32 greed, tolerance, penalty, scaled_penalty;
		119
		120	greed = log2plus1_u64_u32f8(burst_time);
		121	tolerance = sched_burst_penalty_offset << 8;
		122	penalty = max(0, (s32)greed - (s32)tolerance);
		123	scaled_penalty = penalty * sched_burst_penalty_scale >> 16;
		124
		125	return min(MAX_BURST_PENALTY, scaled_penalty);
		126	}
		127
		128	static inline u64 scale_slice(u64 delta, struct sched_entity *se) {
		129	return mul_u64_u32_shr(delta, sched_prio_to_wmult[se->burst_score], 22);
		130	}
		131
		132	static inline u64 __unscale_slice(u64 delta, u8 score) {
		133	return mul_u64_u32_shr(delta, sched_prio_to_weight[score], 10);
		134	}
		135
		136	static inline u64 unscale_slice(u64 delta, struct sched_entity *se) {
		137	return __unscale_slice(delta, se->burst_score);
		138	}
		139
		140	void reweight_task(struct task_struct *p, int prio);
		141
		142	static void update_burst_score(struct sched_entity *se) {
		143	if (!entity_is_task(se)) return;
		144	struct task_struct *p = task_of(se);
		145	u8 prio = p->static_prio - MAX_RT_PRIO;
		146	u8 prev_prio = min(39, prio + se->burst_score);
		147
		148	se->burst_score = se->burst_penalty >> 2;
		149
		150	u8 new_prio = min(39, prio + se->burst_score);
		151	if (new_prio != prev_prio)
		152	reweight_task(p, new_prio);
		153	}
		154
		155	static void update_burst_penalty(struct sched_entity *se) {
		156	se->curr_burst_penalty = calc_burst_penalty(se->burst_time);
		157	se->burst_penalty = max(se->prev_burst_penalty, se->curr_burst_penalty);
		158	update_burst_score(se);
		159	}
160
161	static inline u32 binary_smooth(u32 new, u32 old) {
162	int increment = new - old;
163	return (0 <= increment)?
164	old + ( increment >> (int)sched_burst_smoothness_long):
165	old - (-increment >> (int)sched_burst_smoothness_short);
166	}
167
168	static void restart_burst(struct sched_entity *se) {
169	se->burst_penalty = se->prev_burst_penalty =
170	binary_smooth(se->curr_burst_penalty, se->prev_burst_penalty);
171	se->curr_burst_penalty = 0;
172	se->burst_time = 0;
173	update_burst_score(se);
174	}
175
176	static void restart_burst_rescale_deadline(struct sched_entity *se) {
177	s64 vscaled, wremain, vremain = se->deadline - se->vruntime;
178	u8 prev_score = se->burst_score;
179	restart_burst(se);
180	if (prev_score > se->burst_score) {
181	wremain = __unscale_slice(abs(vremain), prev_score);
182	vscaled = scale_slice(wremain, se);
183	if (unlikely(vremain < 0))
184	vscaled = -vscaled;
185	se->deadline = se->vruntime + vscaled;
186	}
187	}
188	#endif // CONFIG_SCHED_BORE
189
81	int sched_thermal_decay_shift;	190	int sched_thermal_decay_shift;
82	static int __init setup_sched_thermal_decay_shift(char *str)	191	static int __init setup_sched_thermal_decay_shift(char *str)
83	{	192	{
Lines 137-142 static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; Link Here
137		246
138	#ifdef CONFIG_SYSCTL	247	#ifdef CONFIG_SYSCTL
139	static struct ctl_table sched_fair_sysctls[] = {	248	static struct ctl_table sched_fair_sysctls[] = {
		249	#ifdef CONFIG_SCHED_BORE
		250	{
		251	.procname = "sched_bore",
		252	.data = &sched_bore,
		253	.maxlen = sizeof(u8),
		254	.mode = 0644,
		255	.proc_handler = proc_dou8vec_minmax,
		256	.extra1 = SYSCTL_ONE,
		257	.extra2 = SYSCTL_ONE,
		258	},
		259	{
		260	.procname = "sched_burst_smoothness_long",
		261	.data = &sched_burst_smoothness_long,
		262	.maxlen = sizeof(u8),
		263	.mode = 0644,
		264	.proc_handler = proc_dou8vec_minmax,
		265	.extra1 = SYSCTL_ZERO,
		266	.extra2 = SYSCTL_ONE,
		267	},
		268	{
		269	.procname = "sched_burst_smoothness_short",
		270	.data = &sched_burst_smoothness_short,
		271	.maxlen = sizeof(u8),
		272	.mode = 0644,
		273	.proc_handler = proc_dou8vec_minmax,
		274	.extra1 = SYSCTL_ZERO,
		275	.extra2 = SYSCTL_ONE,
		276	},
		277	{
		278	.procname = "sched_burst_fork_atavistic",
		279	.data = &sched_burst_fork_atavistic,
		280	.maxlen = sizeof(u8),
		281	.mode = 0644,
		282	.proc_handler = proc_dou8vec_minmax,
		283	.extra1 = SYSCTL_ZERO,
		284	.extra2 = SYSCTL_THREE,
		285	},
		286	{
		287	.procname = "sched_burst_penalty_offset",
		288	.data = &sched_burst_penalty_offset,
		289	.maxlen = sizeof(u8),
		290	.mode = 0644,
		291	.proc_handler = proc_dou8vec_minmax,
		292	.extra1 = SYSCTL_ZERO,
		293	.extra2 = &sixty_four,
		294	},
		295	{
		296	.procname = "sched_burst_penalty_scale",
		297	.data = &sched_burst_penalty_scale,
		298	.maxlen = sizeof(uint),
		299	.mode = 0644,
		300	.proc_handler = proc_douintvec_minmax,
		301	.extra1 = SYSCTL_ZERO,
		302	.extra2 = &maxval_12_bits,
		303	},
		304	{
		305	.procname = "sched_burst_cache_lifetime",
		306	.data = &sched_burst_cache_lifetime,
		307	.maxlen = sizeof(uint),
		308	.mode = 0644,
		309	.proc_handler = proc_douintvec,
		310	},
		311	#endif // CONFIG_SCHED_BORE
140	#ifdef CONFIG_CFS_BANDWIDTH	312	#ifdef CONFIG_CFS_BANDWIDTH
141	{	313	{
142	.procname = "sched_cfs_bandwidth_slice_us",	314	.procname = "sched_cfs_bandwidth_slice_us",
Lines 195-200 static inline void update_load_set(struct load_weight *lw, unsigned long w) Link Here
195	*	367	*
196	* This idea comes from the SD scheduler of Con Kolivas:	368	* This idea comes from the SD scheduler of Con Kolivas:
197	*/	369	*/
		370	#ifdef CONFIG_SCHED_BORE
		371	static void update_sysctl(void) {
		372	sysctl_sched_base_slice =
		373	max(sysctl_sched_min_base_slice, configured_sched_base_slice);
		374	}
		375	void sched_update_min_base_slice(void) { update_sysctl(); }
		376	#else // !CONFIG_SCHED_BORE
198	static unsigned int get_update_sysctl_factor(void)	377	static unsigned int get_update_sysctl_factor(void)
199	{	378	{
200	unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8);	379	unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8);
Lines 225-230 static void update_sysctl(void) Link Here
225	SET_SYSCTL(sched_base_slice);	404	SET_SYSCTL(sched_base_slice);
226	#undef SET_SYSCTL	405	#undef SET_SYSCTL
227	}	406	}
		407	#endif // CONFIG_SCHED_BORE
228		408
229	void __init sched_init_granularity(void)	409	void __init sched_init_granularity(void)
230	{	410	{
Lines 704-709 static void update_entity_lag(struct cfs_rq cfs_rq, struct sched_entity se) Link Here
704	lag = avg_vruntime(cfs_rq) - se->vruntime;	884	lag = avg_vruntime(cfs_rq) - se->vruntime;
705		885
706	limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se);	886	limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se);
		887	#ifdef CONFIG_SCHED_BORE
		888	limit >>= 1;
		889	#endif // CONFIG_SCHED_BORE
707	se->vlag = clamp(lag, -limit, limit);	890	se->vlag = clamp(lag, -limit, limit);
708	}	891	}
709		892
Lines 955-960 struct sched_entity __pick_last_entity(struct cfs_rq cfs_rq) Link Here
955	* Scheduling class statistics methods:	1138	* Scheduling class statistics methods:
956	*/	1139	*/
957	#ifdef CONFIG_SMP	1140	#ifdef CONFIG_SMP
		1141	#if !defined(CONFIG_SCHED_BORE)
958	int sched_update_scaling(void)	1142	int sched_update_scaling(void)
959	{	1143	{
960	unsigned int factor = get_update_sysctl_factor();	1144	unsigned int factor = get_update_sysctl_factor();
Lines 966-971 int sched_update_scaling(void) Link Here
966		1150
967	return 0;	1151	return 0;
968	}	1152	}
		1153	#endif // CONFIG_SCHED_BORE
969	#endif	1154	#endif
970	#endif	1155	#endif
971		1156
Lines 1165-1171 static void update_curr(struct cfs_rq *cfs_rq) Link Here
1165	if (unlikely(delta_exec <= 0))	1350	if (unlikely(delta_exec <= 0))
1166	return;	1351	return;
1167		1352
		1353	#ifdef CONFIG_SCHED_BORE
		1354	curr->burst_time += delta_exec;
		1355	update_burst_penalty(curr);
		1356	curr->vruntime += max(1ULL, calc_delta_fair(delta_exec, curr));
		1357	#else // !CONFIG_SCHED_BORE
1168	curr->vruntime += calc_delta_fair(delta_exec, curr);	1358	curr->vruntime += calc_delta_fair(delta_exec, curr);
		1359	#endif // CONFIG_SCHED_BORE
1169	update_deadline(cfs_rq, curr);	1360	update_deadline(cfs_rq, curr);
1170	update_min_vruntime(cfs_rq);	1361	update_min_vruntime(cfs_rq);
1171		1362
Lines 5171-5176 place_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags) Link Here
5171	*	5362	*
5172	* EEVDF: placement strategy #1 / #2	5363	* EEVDF: placement strategy #1 / #2
5173	*/	5364	*/
		5365	#ifdef CONFIG_SCHED_BORE
		5366	if (se->vlag)
		5367	#endif // CONFIG_SCHED_BORE
5174	if (sched_feat(PLACE_LAG) && cfs_rq->nr_running) {	5368	if (sched_feat(PLACE_LAG) && cfs_rq->nr_running) {
5175	struct sched_entity *curr = cfs_rq->curr;	5369	struct sched_entity *curr = cfs_rq->curr;
5176	unsigned long load;	5370	unsigned long load;
Lines 6803-6808 static void dequeue_task_fair(struct rq rq, struct task_struct p, int flags) Link Here
6803	bool was_sched_idle = sched_idle_rq(rq);	6997	bool was_sched_idle = sched_idle_rq(rq);
6804		6998
6805	util_est_dequeue(&rq->cfs, p);	6999	util_est_dequeue(&rq->cfs, p);
		7000	#ifdef CONFIG_SCHED_BORE
		7001	if (task_sleep) {
		7002	cfs_rq = cfs_rq_of(se);
		7003	if (cfs_rq->curr == se)
		7004	update_curr(cfs_rq);
		7005	restart_burst(se);
		7006	}
		7007	#endif // CONFIG_SCHED_BORE
6806		7008
6807	for_each_sched_entity(se) {	7009	for_each_sched_entity(se) {
6808	cfs_rq = cfs_rq_of(se);	7010	cfs_rq = cfs_rq_of(se);
Lines 8552-8567 static void yield_task_fair(struct rq *rq) Link Here
8552	/*	8754	/*
8553	* Are we the only task in the tree?	8755	* Are we the only task in the tree?
8554	*/	8756	*/
		8757	#if !defined(CONFIG_SCHED_BORE)
8555	if (unlikely(rq->nr_running == 1))	8758	if (unlikely(rq->nr_running == 1))
8556	return;	8759	return;
8557		8760
8558	clear_buddies(cfs_rq, se);	8761	clear_buddies(cfs_rq, se);
		8762	#endif // CONFIG_SCHED_BORE
8559		8763
8560	update_rq_clock(rq);	8764	update_rq_clock(rq);
8561	/*	8765	/*
8562	* Update run-time statistics of the 'current'.	8766	* Update run-time statistics of the 'current'.
8563	*/	8767	*/
8564	update_curr(cfs_rq);	8768	update_curr(cfs_rq);
		8769	#ifdef CONFIG_SCHED_BORE
		8770	restart_burst_rescale_deadline(se);
		8771	if (unlikely(rq->nr_running == 1))
		8772	return;
		8773
		8774	clear_buddies(cfs_rq, se);
		8775	#endif // CONFIG_SCHED_BORE
8565	/*	8776	/*
8566	* Tell update_rq_clock() that we've just updated,	8777	* Tell update_rq_clock() that we've just updated,
8567	* so we don't do microscopic update in schedule()	8778	* so we don't do microscopic update in schedule()
Lines 12651-12656 static void task_fork_fair(struct task_struct *p) Link Here
12651	curr = cfs_rq->curr;	12862	curr = cfs_rq->curr;
12652	if (curr)	12863	if (curr)
12653	update_curr(cfs_rq);	12864	update_curr(cfs_rq);
		12865	#ifdef CONFIG_SCHED_BORE
		12866	update_burst_score(se);
		12867	#endif // CONFIG_SCHED_BORE
12654	place_entity(cfs_rq, se, ENQUEUE_INITIAL);	12868	place_entity(cfs_rq, se, ENQUEUE_INITIAL);
12655	rq_unlock(rq, &rf);	12869	rq_unlock(rq, &rf);
12656	}	12870	}

Lines 6-12 Link Here

(-)a/kernel/sched/features.h (+4 lines)
6	*/	6	*/
7	SCHED_FEAT(PLACE_LAG, true)	7	SCHED_FEAT(PLACE_LAG, true)
8	SCHED_FEAT(PLACE_DEADLINE_INITIAL, true)	8	SCHED_FEAT(PLACE_DEADLINE_INITIAL, true)
		9	#ifdef CONFIG_SCHED_BORE
		10	SCHED_FEAT(RUN_TO_PARITY, false)
		11	#else // !CONFIG_SCHED_BORE
9	SCHED_FEAT(RUN_TO_PARITY, true)	12	SCHED_FEAT(RUN_TO_PARITY, true)
		13	#endif // CONFIG_SCHED_BORE
10		14
11	/*	15	/*
12	* Prefer to schedule the task we woke last (assuming it failed	16	* Prefer to schedule the task we woke last (assuming it failed

Lines 1965-1971 static inline void dirty_sched_domain_sysctl(int cpu) Link Here

(-)a/kernel/sched/sched.h (-1 / +7 lines)
1965	}	1965	}
1966	#endif	1966	#endif
1967		1967
		1968	#ifdef CONFIG_SCHED_BORE
		1969	extern void sched_update_min_base_slice(void);
		1970	#else // !CONFIG_SCHED_BORE
1968	extern int sched_update_scaling(void);	1971	extern int sched_update_scaling(void);
		1972	#endif // CONFIG_SCHED_BORE
1969		1973
1970	static inline const struct cpumask task_user_cpus(struct task_struct p)	1974	static inline const struct cpumask task_user_cpus(struct task_struct p)
1971	{	1975	{
Lines 2552-2557 extern const_debug unsigned int sysctl_sched_nr_migrate; Link Here
2552	extern const_debug unsigned int sysctl_sched_migration_cost;	2556	extern const_debug unsigned int sysctl_sched_migration_cost;
2553		2557
2554	extern unsigned int sysctl_sched_base_slice;	2558	extern unsigned int sysctl_sched_base_slice;
		2559	#ifdef CONFIG_SCHED_BORE
		2560	extern unsigned int sysctl_sched_min_base_slice;
		2561	#endif // CONFIG_SCHED_BORE
2555		2562
2556	#ifdef CONFIG_SCHED_DEBUG	2563	#ifdef CONFIG_SCHED_DEBUG
2557	extern int sysctl_resched_latency_warn_ms;	2564	extern int sysctl_resched_latency_warn_ms;
2558	-




			Force threading of all interrupt handlers except those
			marked explicitly IRQF_NO_THREAD.

	threadprintk	[KNL]
			Force threaded printing of all legacy consoles. Be
			aware that with this option, the shutdown, reboot, and
			panic messages may not be printed on the legacy
			consoles. Also, earlycon/earlyprintk printing will be
			delayed until a regular console or the kthread is
			available.

			Users can view /proc/consoles to see if their console
			driver is legacy or not. Non-legacy (NBCON) console
			drivers are already threaded and are shown with 'N'.

	topology=	[S390]
			Format: {off | on}
			Specify if the kernel should make use of the cpu




	select ARCH_SUPPORTS_ATOMIC_RMW
	select ARCH_SUPPORTS_HUGETLBFS if ARM_LPAE
	select ARCH_SUPPORTS_PER_VMA_LOCK
	select ARCH_SUPPORTS_RT if HAVE_POSIX_CPU_TIMERS_TASK_WORK
	select ARCH_USE_BUILTIN_BSWAP
	select ARCH_USE_CMPXCHG_LOCKREF
	select ARCH_USE_MEMTEST

	select HAS_IOPORT
	select HAVE_ARCH_AUDITSYSCALL if AEABI && !OABI_COMPAT
	select HAVE_ARCH_BITREVERSE if (CPU_32v7M || CPU_32v7) && !CPU_32v6
	select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU && !PREEMPT_RT
	select HAVE_ARCH_KFENCE if MMU && !XIP_KERNEL
	select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU
	select HAVE_ARCH_KASAN if MMU && !XIP_KERNEL

	select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE
	select HAVE_EFFICIENT_UNALIGNED_ACCESS if (CPU_V6 || CPU_V6K || CPU_V7) && MMU
	select HAVE_EXIT_THREAD
	select HAVE_FAST_GUP if ARM_LPAE && !(PREEMPT_RT && HIGHPTE)
	select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL
	select HAVE_FUNCTION_ERROR_INJECTION
	select HAVE_FUNCTION_GRAPH_TRACER

	select HAVE_PERF_EVENTS
	select HAVE_PERF_REGS
	select HAVE_PERF_USER_STACK_DUMP
	select HAVE_POSIX_CPU_TIMERS_TASK_WORK if !KVM
	select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE
	select HAVE_REGS_AND_STACK_ACCESS_API
	select HAVE_RSEQ

Lines 436-441 do_translation_fault(unsigned long addr, unsigned int fsr, Link Here

(-)a/arch/arm/mm/fault.c (+6 lines)
436	if (addr < TASK_SIZE)	436	if (addr < TASK_SIZE)
437	return do_page_fault(addr, fsr, regs);	437	return do_page_fault(addr, fsr, regs);
438		438
		439	if (interrupts_enabled(regs))
		440	local_irq_enable();
		441
439	if (user_mode(regs))	442	if (user_mode(regs))
440	goto bad_area;	443	goto bad_area;
441		444
Lines 506-511 do_translation_fault(unsigned long addr, unsigned int fsr, Link Here
506	static int	509	static int
507	do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)	510	do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
508	{	511	{
		512	if (interrupts_enabled(regs))
		513	local_irq_enable();
		514
509	do_bad_area(addr, fsr, regs);	515	do_bad_area(addr, fsr, regs);
510	return 0;	516	return 0;
511	}	517	}




 */
union vfp_state *vfp_current_hw_state[NR_CPUS];

/*
 * Claim ownership of the VFP unit.
 *
 * The caller may change VFP registers until vfp_unlock() is called.
 *
 * local_bh_disable() is used to disable preemption and to disable VFP
 * processing in softirq context. On PREEMPT_RT kernels local_bh_disable() is
 * not sufficient because it only serializes soft interrupt related sections
 * via a local lock, but stays preemptible. Disabling preemption is the right
 * choice here as bottom half processing is always in thread context on RT
 * kernels so it implicitly prevents bottom half processing as well.
 */
static void vfp_lock(void)
{
	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		local_bh_disable();
	else
		preempt_disable();
}

static void vfp_unlock(void)
{
	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		local_bh_enable();
	else
		preempt_enable();
}

/*
 * Is 'thread's most up to date state stored in this CPUs hardware?
 * Must be called from non-preemptible context.

/*
 * Process bitmask of exception conditions.
 */
static int vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr)
{
	int si_code = 0;



	if (exceptions == VFP_EXCEPTION_ERROR) {
		vfp_panic("unhandled bounce", inst);
		return FPE_FLTINV;
		return;
	}

	/*

	RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF);
	RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV);

	return si_code;
		vfp_raise_sigfpe(si_code, regs);
}

/*

static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
{
	u32 fpscr, orig_fpscr, fpsid, exceptions;
	int si_code2 = 0;
	int si_code = 0;

	pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);


		 * unallocated VFP instruction but with FPSCR.IXE set and not
		 * on VFP subarch 1.
		 */
		si_code = vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr);
		goto exit;
	}

	/*

	 */
	exceptions = vfp_emulate_instruction(trigger, fpscr, regs);
	if (exceptions)
		si_code2 = vfp_raise_exceptions(exceptions, trigger, orig_fpscr);

	/*
	 * If there isn't a second FP instruction, exit now. Note that
	 * the FPEXC.FP2V bit is valid only if FPEXC.EX is 1.
	 */
	if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V))
		goto exit;

	/*
	 * The barrier() here prevents fpinst2 being read

 emulate:
	exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);
	if (exceptions)
		si_code = vfp_raise_exceptions(exceptions, trigger, orig_fpscr);
exit:
	vfp_unlock();
	if (si_code2)
		vfp_raise_sigfpe(si_code2, regs);
	if (si_code)
		vfp_raise_sigfpe(si_code, regs);
}

static void vfp_enable(void *unused)

 */
void vfp_sync_hwstate(struct thread_info *thread)
{
	vfp_lock();

	if (vfp_state_in_hw(raw_smp_processor_id(), thread)) {

	if (vfp_state_in_hw(cpu, thread)) {
		u32 fpexc = fmrx(FPEXC);

		/*

		fmxr(FPEXC, fpexc);
	}

	vfp_unlock();
	put_cpu();
}

/* Ensure that the thread reloads the hardware VFP state on the next use. */

	if (!user_mode(regs))
		return vfp_kmode_exception(regs, trigger);

	vfp_lock();
	fpexc = fmrx(FPEXC);

	/*

		 * replay the instruction that trapped.
		 */
		fmxr(FPEXC, fpexc);
		vfp_unlock();
	} else {
		/* Check for synchronous or asynchronous exceptions */
		if (!(fpexc & (FPEXC_EX | FPEXC_DEX))) {

			if (!(fpscr & FPSCR_IXE)) {
				if (!(fpscr & FPSCR_LENGTH_MASK)) {
					pr_debug("not VFP\n");
					vfp_unlock();
					return -ENOEXEC;
				}
				fpexc |= FPEXC_DEX;
			}
		}
bounce:		regs->ARM_pc += 4;
		/* VFP_bounce() will invoke vfp_unlock() */
		VFP_bounce(trigger, fpexc, regs);
	}

	local_bh_enable();
	return 0;
}


	unsigned int cpu;
	u32 fpexc;

	vfp_lock();

	/*
	 * Kernel mode NEON is only allowed outside of hardirq context with

{
	/* Disable the NEON/VFP unit. */
	fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
	vfp_unlock();
}
EXPORT_SYMBOL(kernel_neon_end);


Lines 98-103 config ARM64 Link Here

(-)a/arch/arm64/Kconfig (+1 lines)
98	select ARCH_SUPPORTS_NUMA_BALANCING	98	select ARCH_SUPPORTS_NUMA_BALANCING
99	select ARCH_SUPPORTS_PAGE_TABLE_CHECK	99	select ARCH_SUPPORTS_PAGE_TABLE_CHECK
100	select ARCH_SUPPORTS_PER_VMA_LOCK	100	select ARCH_SUPPORTS_PER_VMA_LOCK
		101	select ARCH_SUPPORTS_RT
101	select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH	102	select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
102	select ARCH_WANT_COMPAT_IPC_PARSE_VERSION if COMPAT	103	select ARCH_WANT_COMPAT_IPC_PARSE_VERSION if COMPAT
103	select ARCH_WANT_DEFAULT_BPF_JIT	104	select ARCH_WANT_DEFAULT_BPF_JIT

Lines 166-171 config PPC Link Here

(-)a/arch/powerpc/Kconfig (+2 lines)
166	select ARCH_STACKWALK	166	select ARCH_STACKWALK
167	select ARCH_SUPPORTS_ATOMIC_RMW	167	select ARCH_SUPPORTS_ATOMIC_RMW
168	select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S \|\| PPC_8xx \|\| 40x	168	select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S \|\| PPC_8xx \|\| 40x
		169	select ARCH_SUPPORTS_RT if HAVE_POSIX_CPU_TIMERS_TASK_WORK
169	select ARCH_USE_BUILTIN_BSWAP	170	select ARCH_USE_BUILTIN_BSWAP
170	select ARCH_USE_CMPXCHG_LOCKREF if PPC64	171	select ARCH_USE_CMPXCHG_LOCKREF if PPC64
171	select ARCH_USE_MEMTEST	172	select ARCH_USE_MEMTEST
Lines 270-275 config PPC Link Here
270	select HAVE_PERF_USER_STACK_DUMP	271	select HAVE_PERF_USER_STACK_DUMP
271	select HAVE_REGS_AND_STACK_ACCESS_API	272	select HAVE_REGS_AND_STACK_ACCESS_API
272	select HAVE_RELIABLE_STACKTRACE	273	select HAVE_RELIABLE_STACKTRACE
		274	select HAVE_POSIX_CPU_TIMERS_TASK_WORK if !KVM
273	select HAVE_RSEQ	275	select HAVE_RSEQ
274	select HAVE_SETUP_PER_CPU_AREA if PPC64	276	select HAVE_SETUP_PER_CPU_AREA if PPC64
275	select HAVE_SOFTIRQ_ON_OWN_STACK	277	select HAVE_SOFTIRQ_ON_OWN_STACK

Lines 19-26 Link Here

(-)a/arch/powerpc/include/asm/stackprotector.h (-1 / +6 lines)
19	*/	19	*/
20	static __always_inline void boot_init_stack_canary(void)	20	static __always_inline void boot_init_stack_canary(void)
21	{	21	{
22	unsigned long canary = get_random_canary();	22	unsigned long canary;
23		23
		24	#ifndef CONFIG_PREEMPT_RT
		25	canary = get_random_canary();
		26	#else
		27	canary = ((unsigned long)&canary) & CANARY_MASK;
		28	#endif
24	current->stack_canary = canary;	29	current->stack_canary = canary;
25	#ifdef CONFIG_PPC64	30	#ifdef CONFIG_PPC64
26	get_paca()->canary = canary;	31	get_paca()->canary = canary;

Lines 261-272 static char *get_mmu_str(void) Link Here

(-)a/arch/powerpc/kernel/traps.c (-1 / +6 lines)
261		261
262	static int __die(const char str, struct pt_regs regs, long err)	262	static int __die(const char str, struct pt_regs regs, long err)
263	{	263	{
		264	const char *pr = "";
		265
264	printk("Oops: %s, sig: %ld [#%d]\n", str, err, ++die_counter);	266	printk("Oops: %s, sig: %ld [#%d]\n", str, err, ++die_counter);
265		267
		268	if (IS_ENABLED(CONFIG_PREEMPTION))
		269	pr = IS_ENABLED(CONFIG_PREEMPT_RT) ? " PREEMPT_RT" : " PREEMPT";
		270
266	printk("%s PAGE_SIZE=%luK%s%s%s%s%s%s %s\n",	271	printk("%s PAGE_SIZE=%luK%s%s%s%s%s%s %s\n",
267	IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? "LE" : "BE",	272	IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? "LE" : "BE",
268	PAGE_SIZE / 1024, get_mmu_str(),	273	PAGE_SIZE / 1024, get_mmu_str(),
269	IS_ENABLED(CONFIG_PREEMPT) ? " PREEMPT" : "",	274	pr,
270	IS_ENABLED(CONFIG_SMP) ? " SMP" : "",	275	IS_ENABLED(CONFIG_SMP) ? " SMP" : "",
271	IS_ENABLED(CONFIG_SMP) ? (" NR_CPUS=" __stringify(NR_CPUS)) : "",	276	IS_ENABLED(CONFIG_SMP) ? (" NR_CPUS=" __stringify(NR_CPUS)) : "",
272	debug_pagealloc_enabled() ? " DEBUG_PAGEALLOC" : "",	277	debug_pagealloc_enabled() ? " DEBUG_PAGEALLOC" : "",

Lines 222-227 config KVM_E500MC Link Here

(-)a/arch/powerpc/kvm/Kconfig (+1 lines)
222	config KVM_MPIC	222	config KVM_MPIC
223	bool "KVM in-kernel MPIC emulation"	223	bool "KVM in-kernel MPIC emulation"
224	depends on KVM && PPC_E500	224	depends on KVM && PPC_E500
		225	depends on !PREEMPT_RT
225	select HAVE_KVM_IRQCHIP	226	select HAVE_KVM_IRQCHIP
226	select HAVE_KVM_IRQ_ROUTING	227	select HAVE_KVM_IRQ_ROUTING
227	select HAVE_KVM_MSI	228	select HAVE_KVM_MSI

Lines 2-7 Link Here

(-)a/arch/powerpc/platforms/pseries/Kconfig (+1 lines)
2	config PPC_PSERIES	2	config PPC_PSERIES
3	depends on PPC64 && PPC_BOOK3S	3	depends on PPC64 && PPC_BOOK3S
4	bool "IBM pSeries & new (POWER5-based) iSeries"	4	bool "IBM pSeries & new (POWER5-based) iSeries"
		5	select GENERIC_ALLOCATOR
5	select HAVE_PCSPKR_PLATFORM	6	select HAVE_PCSPKR_PLATFORM
6	select MPIC	7	select MPIC
7	select OF_DYNAMIC	8	select OF_DYNAMIC

Lines 25-30 Link Here

(-)a/arch/powerpc/platforms/pseries/iommu.c (-11 / +20 lines)
25	#include <linux/of_address.h>	25	#include <linux/of_address.h>
26	#include <linux/iommu.h>	26	#include <linux/iommu.h>
27	#include <linux/rculist.h>	27	#include <linux/rculist.h>
		28	#include <linux/local_lock.h>
28	#include <asm/io.h>	29	#include <asm/io.h>
29	#include <asm/prom.h>	30	#include <asm/prom.h>
30	#include <asm/rtas.h>	31	#include <asm/rtas.h>
Lines 206-212 static int tce_build_pSeriesLP(unsigned long liobn, long tcenum, long tceshift, Link Here
206	return ret;	207	return ret;
207	}	208	}
208		209
209	static DEFINE_PER_CPU(__be64 *, tce_page);	210	struct tce_page {
		211	__be64 * page;
		212	local_lock_t lock;
		213	};
		214	static DEFINE_PER_CPU(struct tce_page, tce_page) = {
		215	.lock = INIT_LOCAL_LOCK(lock),
		216	};
210		217
211	static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,	218	static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
212	long npages, unsigned long uaddr,	219	long npages, unsigned long uaddr,
Lines 229-237 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, Link Here
229	direction, attrs);	236	direction, attrs);
230	}	237	}
231		238
232	local_irq_save(flags); /* to protect tcep and the page behind it */	239	/* to protect tcep and the page behind it */
		240	local_lock_irqsave(&tce_page.lock, flags);
233		241
234	tcep = __this_cpu_read(tce_page);	242	tcep = __this_cpu_read(tce_page.page);
235		243
236	/* This is safe to do since interrupts are off when we're called	244	/* This is safe to do since interrupts are off when we're called
237	* from iommu_alloc{,_sg}()	245	* from iommu_alloc{,_sg}()
Lines 240-251 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, Link Here
240	tcep = (__be64 *)__get_free_page(GFP_ATOMIC);	248	tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
241	/* If allocation fails, fall back to the loop implementation */	249	/* If allocation fails, fall back to the loop implementation */
242	if (!tcep) {	250	if (!tcep) {
243	local_irq_restore(flags);	251	local_unlock_irqrestore(&tce_page.lock, flags);
244	return tce_build_pSeriesLP(tbl->it_index, tcenum,	252	return tce_build_pSeriesLP(tbl->it_index, tcenum,
245	tceshift,	253	tceshift,
246	npages, uaddr, direction, attrs);	254	npages, uaddr, direction, attrs);
247	}	255	}
248	__this_cpu_write(tce_page, tcep);	256	__this_cpu_write(tce_page.page, tcep);
249	}	257	}
250		258
251	rpn = __pa(uaddr) >> tceshift;	259	rpn = __pa(uaddr) >> tceshift;
Lines 275-281 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, Link Here
275	tcenum += limit;	283	tcenum += limit;
276	} while (npages > 0 && !rc);	284	} while (npages > 0 && !rc);
277		285
278	local_irq_restore(flags);	286	local_unlock_irqrestore(&tce_page.lock, flags);
279		287
280	if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {	288	if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
281	ret = (int)rc;	289	ret = (int)rc;
Lines 459-474 static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn, Link Here
459	DMA_BIDIRECTIONAL, 0);	467	DMA_BIDIRECTIONAL, 0);
460	}	468	}
461		469
462	local_irq_disable(); /* to protect tcep and the page behind it */	470	/* to protect tcep and the page behind it */
463	tcep = __this_cpu_read(tce_page);	471	local_lock_irq(&tce_page.lock);
		472	tcep = __this_cpu_read(tce_page.page);
464		473
465	if (!tcep) {	474	if (!tcep) {
466	tcep = (__be64 *)__get_free_page(GFP_ATOMIC);	475	tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
467	if (!tcep) {	476	if (!tcep) {
468	local_irq_enable();	477	local_unlock_irq(&tce_page.lock);
469	return -ENOMEM;	478	return -ENOMEM;
470	}	479	}
471	__this_cpu_write(tce_page, tcep);	480	__this_cpu_write(tce_page.page, tcep);
472	}	481	}
473		482
474	proto_tce = TCE_PCI_READ \| TCE_PCI_WRITE;	483	proto_tce = TCE_PCI_READ \| TCE_PCI_WRITE;
Lines 511-517 static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn, Link Here
511		520
512	/* error cleanup: caller will clear whole range */	521	/* error cleanup: caller will clear whole range */
513		522
514	local_irq_enable();	523	local_unlock_irq(&tce_page.lock);
515	return rc;	524	return rc;
516	}	525	}
517		526

Lines 94-99 int arch_dup_task_struct(struct task_struct dst, struct task_struct src); Link Here

(-)a/arch/riscv/include/asm/thread_info.h (+2 lines)
94	* - pending work-to-be-done flags are in lowest half-word	94	* - pending work-to-be-done flags are in lowest half-word
95	* - other flags in upper half-word(s)	95	* - other flags in upper half-word(s)
96	*/	96	*/
		97	#define TIF_ARCH_RESCHED_LAZY 0 /* Lazy rescheduling */
97	#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */	98	#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */
98	#define TIF_SIGPENDING 2 /* signal pending */	99	#define TIF_SIGPENDING 2 /* signal pending */
99	#define TIF_NEED_RESCHED 3 /* rescheduling necessary */	100	#define TIF_NEED_RESCHED 3 /* rescheduling necessary */
Lines 104-109 int arch_dup_task_struct(struct task_struct dst, struct task_struct src); Link Here
104	#define TIF_32BIT 11 /* compat-mode 32bit process */	105	#define TIF_32BIT 11 /* compat-mode 32bit process */
105	#define TIF_RISCV_V_DEFER_RESTORE 12 /* restore Vector before returing to user */	106	#define TIF_RISCV_V_DEFER_RESTORE 12 /* restore Vector before returing to user */
106		107
		108	#define _TIF_ARCH_RESCHED_LAZY (1 << TIF_ARCH_RESCHED_LAZY)
107	#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)	109	#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
108	#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)	110	#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
109	#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)	111	#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)




	select ARCH_HAS_GIGANTIC_PAGE
	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
	select ARCH_SUPPORTS_PER_VMA_LOCK
	select ARCH_SUPPORTS_RT
	select HAVE_ARCH_SOFT_DIRTY
	select MODULES_USE_ELF_RELA
	select NEED_DMA_MAP_STATE

	select ARCH_USES_CFI_TRAPS		if X86_64 && CFI_CLANG
	select ARCH_SUPPORTS_LTO_CLANG
	select ARCH_SUPPORTS_LTO_CLANG_THIN
	select ARCH_SUPPORTS_RT
	select ARCH_USE_BUILTIN_BSWAP
	select ARCH_USE_CMPXCHG_LOCKREF		if X86_CMPXCHG64
	select ARCH_USE_MEMTEST

	select HAVE_STATIC_CALL
	select HAVE_STATIC_CALL_INLINE		if HAVE_OBJTOOL
	select HAVE_PREEMPT_DYNAMIC_CALL
	select HAVE_PREEMPT_AUTO
	select HAVE_RSEQ
	select HAVE_RUST			if X86_64
	select HAVE_SYSCALL_TRACEPOINTS




#define TIF_NOTIFY_RESUME	1	/* callback before returning to user */
#define TIF_SIGPENDING		2	/* signal pending */
#define TIF_NEED_RESCHED	3	/* rescheduling necessary */
#define TIF_ARCH_RESCHED_LAZY	4	/* Lazy rescheduling */
#define TIF_SINGLESTEP		5	/* reenable singlestep on user return*/
#define TIF_SSBD		6	/* Speculative store bypass disable */
#define TIF_SPEC_IB		9	/* Indirect branch speculation mitigation */
#define TIF_SPEC_L1D_FLUSH	10	/* Flush L1D on mm switches (processes) */
#define TIF_USER_RETURN_NOTIFY	11	/* notify kernel of userspace return */

#define _TIF_NOTIFY_RESUME	(1 << TIF_NOTIFY_RESUME)
#define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)
#define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)
#define _TIF_ARCH_RESCHED_LAZY	(1 << TIF_ARCH_RESCHED_LAZY)
#define _TIF_SINGLESTEP		(1 << TIF_SINGLESTEP)
#define _TIF_SSBD		(1 << TIF_SSBD)
#define _TIF_SPEC_IB		(1 << TIF_SPEC_IB)

Lines 108-114 static const struct dmi_system_id processor_power_dmi_table[] = { Link Here

(-)a/drivers/acpi/processor_idle.c (-1 / +1 lines)
108	*/	108	*/
109	static void __cpuidle acpi_safe_halt(void)	109	static void __cpuidle acpi_safe_halt(void)
110	{	110	{
111	if (!tif_need_resched()) {	111	if (!need_resched()) {
112	raw_safe_halt();	112	raw_safe_halt();
113	raw_local_irq_disable();	113	raw_local_irq_disable();
114	}	114	}




static int zram_read_page(struct zram *zram, struct page *page, u32 index,
			  struct bio *parent);

#ifdef CONFIG_PREEMPT_RT
static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages)
{
	size_t index;

	for (index = 0; index < num_pages; index++)
		spin_lock_init(&zram->table[index].lock);
}

static int zram_slot_trylock(struct zram *zram, u32 index)
{
	int ret;

	ret = spin_trylock(&zram->table[index].lock);
	if (ret)
		__set_bit(ZRAM_LOCK, &zram->table[index].flags);
	return ret;
}

static void zram_slot_lock(struct zram *zram, u32 index)
{
	spin_lock(&zram->table[index].lock);
	__set_bit(ZRAM_LOCK, &zram->table[index].flags);
}

static void zram_slot_unlock(struct zram *zram, u32 index)
{
	__clear_bit(ZRAM_LOCK, &zram->table[index].flags);
	spin_unlock(&zram->table[index].lock);
}

#else

static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages) { }

static int zram_slot_trylock(struct zram *zram, u32 index)
{
	return bit_spin_trylock(ZRAM_LOCK, &zram->table[index].flags);

{
	bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags);
}
#endif

static inline bool init_done(struct zram *zram)
{


	if (!huge_class_size)
		huge_class_size = zs_huge_class_size(zram->mem_pool);
	zram_meta_init_table_locks(zram, num_pages);
	return true;
}


Lines 69-74 struct zram_table_entry { Link Here

(-)a/drivers/block/zram/zram_drv.h (+3 lines)
69	unsigned long element;	69	unsigned long element;
70	};	70	};
71	unsigned long flags;	71	unsigned long flags;
		72	#ifdef CONFIG_PREEMPT_RT
		73	spinlock_t lock;
		74	#endif
72	#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME	75	#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
73	ktime_t ac_time;	76	ktime_t ac_time;
74	#endif	77	#endif

Lines 3-9 config DRM_I915 Link Here

(-)a/drivers/gpu/drm/i915/Kconfig (-1 lines)
3	tristate "Intel 8xx/9xx/G3x/G4x/HD Graphics"	3	tristate "Intel 8xx/9xx/G3x/G4x/HD Graphics"
4	depends on DRM	4	depends on DRM
5	depends on X86 && PCI	5	depends on X86 && PCI
6	depends on !PREEMPT_RT
7	select INTEL_GTT if X86	6	select INTEL_GTT if X86
8	select INTERVAL_TREE	7	select INTERVAL_TREE
9	# we need shmfs for the swappable backing store, and in particular	8	# we need shmfs for the swappable backing store, and in particular




	 */
	intel_psr_wait_for_idle_locked(new_crtc_state);

	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		local_irq_disable();

	crtc->debug.min_vbl = min;
	crtc->debug.max_vbl = max;

			break;
		}

		if (!IS_ENABLED(CONFIG_PREEMPT_RT))
			local_irq_enable();

		timeout = schedule_timeout(timeout);

		if (!IS_ENABLED(CONFIG_PREEMPT_RT))
			local_irq_disable();
	}

	finish_wait(wq, &wait);

	return;

irq_disable:
	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		local_irq_disable();
}

#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)

	 */
	intel_vrr_send_push(new_crtc_state);

	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		local_irq_enable();

	if (intel_vgpu_active(dev_priv))
		goto out;




 * all register accesses to the same cacheline to be serialized,
 * otherwise they may hang.
 */
static void intel_vblank_section_enter_irqsave(struct drm_i915_private *i915, unsigned long *flags)
	__acquires(i915->uncore.lock)
{
#ifdef I915
	spin_lock_irqsave(&i915->uncore.lock, *flags);
#else
	*flags = 0;
#endif
}

static void intel_vblank_section_exit_irqrestore(struct drm_i915_private *i915, unsigned long flags)
	__releases(i915->uncore.lock)
{
#ifdef I915
	spin_unlock_irqrestore(&i915->uncore.lock, flags);
#else
	if (flags)
		return;
#endif
}
static void intel_vblank_section_enter(struct drm_i915_private *i915)
	__acquires(i915->uncore.lock)
{

	 * timing critical raw register reads, potentially with
	 * preemption disabled, so the following code must not block.
	 */
	intel_vblank_section_enter_irqsave(dev_priv, &irqflags);
	intel_vblank_section_enter(dev_priv);

	if (IS_ENABLED(CONFIG_PREEMPT_RT))
		preempt_disable();

	/* Get optional system timestamp before query. */
	if (stime)

	if (etime)
		*etime = ktime_get();

	if (IS_ENABLED(CONFIG_PREEMPT_RT))
		preempt_enable();

	intel_vblank_section_exit_irqrestore(dev_priv, irqflags);
	local_irq_restore(irqflags);

	/*
	 * While in vblank, position will be negative

	unsigned long irqflags;
	int position;

	intel_vblank_section_enter_irqsave(dev_priv, &irqflags);
	intel_vblank_section_enter(dev_priv);

	position = __intel_get_crtc_scanline(crtc);

	intel_vblank_section_exit_irqrestore(dev_priv, irqflags);
	local_irq_restore(irqflags);

	return position;
}

Lines 317-326 void __intel_breadcrumbs_park(struct intel_breadcrumbs *b) Link Here

(-)a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c (-3 / +2 lines)
317	/* Kick the work once more to drain the signalers, and disarm the irq */	317	/* Kick the work once more to drain the signalers, and disarm the irq */
318	irq_work_sync(&b->irq_work);	318	irq_work_sync(&b->irq_work);
319	while (READ_ONCE(b->irq_armed) && !atomic_read(&b->active)) {	319	while (READ_ONCE(b->irq_armed) && !atomic_read(&b->active)) {
320	local_irq_disable();	320	irq_work_queue(&b->irq_work);
321	signal_irq_work(&b->irq_work);
322	local_irq_enable();
323	cond_resched();	321	cond_resched();
		322	irq_work_sync(&b->irq_work);
324	}	323	}
325	}	324	}
326		325




	 * and context switches) submission.
	 */

	spin_lock_irq(&sched_engine->lock);

	/*
	 * If the queue is higher priority than the last

				 * Even if ELSP[1] is occupied and not worthy
				 * of timeslices, our queue might be.
				 */
				spin_unlock_irq(&sched_engine->lock);
				return;
			}
		}


		if (last && !can_merge_rq(last, rq)) {
			spin_unlock(&ve->base.sched_engine->lock);
			spin_unlock_irq(&engine->sched_engine->lock);
			return; /* leave this for another sibling */
		}


	 */
	sched_engine->queue_priority_hint = queue_prio(sched_engine);
	i915_sched_engine_reset_on_empty(sched_engine);
	spin_unlock_irq(&sched_engine->lock);

	/*
	 * We can skip poking the HW if we ended up with exactly the same set

	}
}

static void execlists_dequeue_irq(struct intel_engine_cs *engine)
{
	local_irq_disable(); /* Suspend interrupts across request submission */
	execlists_dequeue(engine);
	local_irq_enable(); /* flush irq_work (e.g. breadcrumb enabling) */
}

static void clear_ports(struct i915_request **ports, int count)
{
	memset_p((void **)ports, NULL, count);

	}

	if (!engine->execlists.pending[0]) {
		execlists_dequeue(engine);
		start_timeslice(engine);
	}


Lines 362-368 static inline int intel_guc_send_busy_loop(struct intel_guc *guc, Link Here

(-)a/drivers/gpu/drm/i915/gt/uc/intel_guc.h (-1 / +1 lines)
362	{	362	{
363	int err;	363	int err;
364	unsigned int sleep_period_ms = 1;	364	unsigned int sleep_period_ms = 1;
365	bool not_atomic = !in_atomic() && !irqs_disabled();	365	bool not_atomic = !in_atomic() && !irqs_disabled() && !rcu_preempt_depth();
366		366
367	/*	367	/*
368	* FIXME: Have caller pass in if we are in an atomic context to avoid	368	* FIXME: Have caller pass in if we are in an atomic context to avoid

Lines 609-615 bool __i915_request_submit(struct i915_request *request) Link Here

(-)a/drivers/gpu/drm/i915/i915_request.c (-2 lines)
609		609
610	RQ_TRACE(request, "\n");	610	RQ_TRACE(request, "\n");
611		611
612	GEM_BUG_ON(!irqs_disabled());
613	lockdep_assert_held(&engine->sched_engine->lock);	612	lockdep_assert_held(&engine->sched_engine->lock);
614		613
615	/*	614	/*
Lines 718-724 void __i915_request_unsubmit(struct i915_request *request) Link Here
718	*/	717	*/
719	RQ_TRACE(request, "\n");	718	RQ_TRACE(request, "\n");
720		719
721	GEM_BUG_ON(!irqs_disabled());
722	lockdep_assert_held(&engine->sched_engine->lock);	720	lockdep_assert_held(&engine->sched_engine->lock);
723		721
724	/*	722	/*

Lines 6-11 Link Here

(-)a/drivers/gpu/drm/i915/i915_trace.h (-1 / +5 lines)
6	#if !defined(_I915_TRACE_H_) \|\| defined(TRACE_HEADER_MULTI_READ)	6	#if !defined(_I915_TRACE_H_) \|\| defined(TRACE_HEADER_MULTI_READ)
7	#define _I915_TRACE_H_	7	#define _I915_TRACE_H_
8		8
		9	#ifdef CONFIG_PREEMPT_RT
		10	#define NOTRACE
		11	#endif
		12
9	#include <linux/stringify.h>	13	#include <linux/stringify.h>
10	#include <linux/types.h>	14	#include <linux/types.h>
11	#include <linux/tracepoint.h>	15	#include <linux/tracepoint.h>
Lines 322-328 DEFINE_EVENT(i915_request, i915_request_add, Link Here
322	TP_ARGS(rq)	326	TP_ARGS(rq)
323	);	327	);
324		328
325	#if defined(CONFIG_DRM_I915_LOW_LEVEL_TRACEPOINTS)	329	#if defined(CONFIG_DRM_I915_LOW_LEVEL_TRACEPOINTS) && !defined(NOTRACE)
326	DEFINE_EVENT(i915_request, i915_request_guc_submit,	330	DEFINE_EVENT(i915_request, i915_request_guc_submit,
327	TP_PROTO(struct i915_request *rq),	331	TP_PROTO(struct i915_request *rq),
328	TP_ARGS(rq)	332	TP_ARGS(rq)

Lines 288-294 wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) Link Here

(-)a/drivers/gpu/drm/i915/i915_utils.h (-1 / +1 lines)
288	#define wait_for(COND, MS) _wait_for((COND), (MS) * 1000, 10, 1000)	288	#define wait_for(COND, MS) _wait_for((COND), (MS) * 1000, 10, 1000)
289		289
290	/* If CONFIG_PREEMPT_COUNT is disabled, in_atomic() always reports false. */	290	/* If CONFIG_PREEMPT_COUNT is disabled, in_atomic() always reports false. */
291	#if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT)	291	#if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT) && !defined(CONFIG_PREEMPT_RT)
292	# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) WARN_ON_ONCE((ATOMIC) && !in_atomic())	292	# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) WARN_ON_ONCE((ATOMIC) && !in_atomic())
293	#else	293	#else
294	# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) do { } while (0)	294	# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) do { } while (0)





#ifdef CONFIG_SERIAL_8250_CONSOLE

#ifdef CONFIG_SERIAL_8250_LEGACY_CONSOLE
static void univ8250_console_write(struct console *co, const char *s,
				   unsigned int count)
{


	serial8250_console_write(up, s, count);
}
#else
static void univ8250_console_write_atomic(struct console *co,
					  struct nbcon_write_context *wctxt)
{
	struct uart_8250_port *up = &serial8250_ports[co->index];

	serial8250_console_write_atomic(up, wctxt);
}

static void univ8250_console_write_thread(struct console *co,
					  struct nbcon_write_context *wctxt)
{
	struct uart_8250_port *up = &serial8250_ports[co->index];

	serial8250_console_write_thread(up, wctxt);
}

static void univ8250_console_device_lock(struct console *con, unsigned long *flags)
{
	struct uart_port *up = &serial8250_ports[con->index].port;

	__uart_port_lock_irqsave(up, flags);
}

static void univ8250_console_device_unlock(struct console *con, unsigned long flags)
{
	struct uart_port *up = &serial8250_ports[con->index].port;

	__uart_port_unlock_irqrestore(up, flags);
}

static struct nbcon_drvdata serial8250_nbcon_drvdata;
#endif /* CONFIG_SERIAL_8250_LEGACY_CONSOLE */

static int univ8250_console_setup(struct console *co, char *options)
{


	port = &serial8250_ports[co->index].port;
	/* link port to console */
	uart_port_set_cons(port, co);

	retval = serial8250_console_setup(port, options, false);
	if (retval != 0)
		uart_port_set_cons(port, NULL);
	return retval;
}


			continue;

		co->index = i;
		uart_port_set_cons(port, co);
		return serial8250_console_setup(port, options, true);
	}



static struct console univ8250_console = {
	.name		= "ttyS",
#ifdef CONFIG_SERIAL_8250_LEGACY_CONSOLE
	.write		= univ8250_console_write,
	.flags		= CON_PRINTBUFFER | CON_ANYTIME,
#else
	.write_atomic	= univ8250_console_write_atomic,
	.write_thread	= univ8250_console_write_thread,
	.device_lock	= univ8250_console_device_lock,
	.device_unlock	= univ8250_console_device_unlock,
	.flags		= CON_PRINTBUFFER | CON_ANYTIME | CON_NBCON,
	.nbcon_drvdata	= &serial8250_nbcon_drvdata,
#endif
	.device		= uart_console_device,
	.setup		= univ8250_console_setup,
	.exit		= univ8250_console_exit,
	.match		= univ8250_console_match,
	.flags		= CON_PRINTBUFFER | CON_ANYTIME,
	.index		= -1,
	.data		= &serial8250_reg,
};

Lines 550-555 static int serial8250_em485_init(struct uart_8250_port *p) Link Here

(-)a/drivers/tty/serial/8250/8250_port.c (-2 / +155 lines)
550	if (!p->em485)	550	if (!p->em485)
551	return -ENOMEM;	551	return -ENOMEM;
552		552
		553	#ifndef CONFIG_SERIAL_8250_LEGACY_CONSOLE
		554	if (uart_console(&p->port)) {
		555	dev_warn(p->port.dev, "no atomic printing for rs485 consoles\n");
		556	p->port.cons->write_atomic = NULL;
		557	}
		558	#endif
		559
553	hrtimer_init(&p->em485->stop_tx_timer, CLOCK_MONOTONIC,	560	hrtimer_init(&p->em485->stop_tx_timer, CLOCK_MONOTONIC,
554	HRTIMER_MODE_REL);	561	HRTIMER_MODE_REL);
555	hrtimer_init(&p->em485->start_tx_timer, CLOCK_MONOTONIC,	562	hrtimer_init(&p->em485->start_tx_timer, CLOCK_MONOTONIC,
Lines 702-708 static void serial8250_set_sleep(struct uart_8250_port *p, int sleep) Link Here
702	serial8250_rpm_put(p);	709	serial8250_rpm_put(p);
703	}	710	}
704		711
705	static void serial8250_clear_IER(struct uart_8250_port *up)	712	/*
		713	* Only to be used by write_atomic() and the legacy write(), which do not
		714	* require port lock.
		715	*/
		716	static void __serial8250_clear_IER(struct uart_8250_port *up)
706	{	717	{
707	if (up->capabilities & UART_CAP_UUE)	718	if (up->capabilities & UART_CAP_UUE)
708	serial_out(up, UART_IER, UART_IER_UUE);	719	serial_out(up, UART_IER, UART_IER_UUE);
Lines 710-715 static void serial8250_clear_IER(struct uart_8250_port *up) Link Here
710	serial_out(up, UART_IER, 0);	721	serial_out(up, UART_IER, 0);
711	}	722	}
712		723
		724	static inline void serial8250_clear_IER(struct uart_8250_port *up)
		725	{
		726	/* Port locked to synchronize UART_IER access against the console. */
		727	lockdep_assert_held_once(&up->port.lock);
		728
		729	__serial8250_clear_IER(up);
		730	}
		731
713	#ifdef CONFIG_SERIAL_8250_RSA	732	#ifdef CONFIG_SERIAL_8250_RSA
714	/*	733	/*
715	* Attempts to turn on the RSA FIFO. Returns zero on failure.	734	* Attempts to turn on the RSA FIFO. Returns zero on failure.
Lines 3320-3325 static void serial8250_console_putchar(struct uart_port *port, unsigned char ch) Link Here
3320		3339
3321	wait_for_xmitr(up, UART_LSR_THRE);	3340	wait_for_xmitr(up, UART_LSR_THRE);
3322	serial_port_out(port, UART_TX, ch);	3341	serial_port_out(port, UART_TX, ch);
		3342
		3343	if (ch == '\n')
		3344	up->console_newline_needed = false;
		3345	else
		3346	up->console_newline_needed = true;
3323	}	3347	}
3324		3348
3325	/*	3349	/*
Lines 3348-3353 static void serial8250_console_restore(struct uart_8250_port *up) Link Here
3348	serial8250_out_MCR(up, up->mcr \| UART_MCR_DTR \| UART_MCR_RTS);	3372	serial8250_out_MCR(up, up->mcr \| UART_MCR_DTR \| UART_MCR_RTS);
3349	}	3373	}
3350		3374
		3375	#ifdef CONFIG_SERIAL_8250_LEGACY_CONSOLE
3351	/*	3376	/*
3352	* Print a string to the serial port using the device FIFO	3377	* Print a string to the serial port using the device FIFO
3353	*	3378	*
Lines 3406-3412 void serial8250_console_write(struct uart_8250_port up, const char s, Link Here
3406	* First save the IER then disable the interrupts	3431	* First save the IER then disable the interrupts
3407	*/	3432	*/
3408	ier = serial_port_in(port, UART_IER);	3433	ier = serial_port_in(port, UART_IER);
3409	serial8250_clear_IER(up);	3434	__serial8250_clear_IER(up);
3410		3435
3411	/* check scratch reg to see if port powered off during system sleep */	3436	/* check scratch reg to see if port powered off during system sleep */
3412	if (up->canary && (up->canary != serial_port_in(port, UART_SCR))) {	3437	if (up->canary && (up->canary != serial_port_in(port, UART_SCR))) {
Lines 3472-3477 void serial8250_console_write(struct uart_8250_port up, const char s, Link Here
3472	if (locked)	3497	if (locked)
3473	uart_port_unlock_irqrestore(port, flags);	3498	uart_port_unlock_irqrestore(port, flags);
3474	}	3499	}
		3500	#else
		3501	void serial8250_console_write_thread(struct uart_8250_port *up,
		3502	struct nbcon_write_context *wctxt)
		3503	{
		3504	struct uart_8250_em485 *em485 = up->em485;
		3505	struct uart_port *port = &up->port;
		3506	unsigned int ier;
		3507
		3508	touch_nmi_watchdog();
		3509
		3510	if (!nbcon_enter_unsafe(wctxt))
		3511	return;
		3512
		3513	/* First save IER then disable the interrupts. */
		3514	ier = serial_port_in(port, UART_IER);
		3515	serial8250_clear_IER(up);
		3516
		3517	/* Check scratch reg if port powered off during system sleep. */
		3518	if (up->canary && (up->canary != serial_port_in(port, UART_SCR))) {
		3519	serial8250_console_restore(up);
		3520	up->canary = 0;
		3521	}
		3522
		3523	if (em485) {
		3524	if (em485->tx_stopped)
		3525	up->rs485_start_tx(up);
		3526	mdelay(port->rs485.delay_rts_before_send);
		3527	}
		3528
		3529	if (nbcon_exit_unsafe(wctxt)) {
		3530	int len = READ_ONCE(wctxt->len);
		3531	int i;
		3532
		3533	/*
		3534	* Write out the message. Toggle unsafe for each byte in order
		3535	* to give another (higher priority) context the opportunity
		3536	* for a friendly takeover. If such a takeover occurs, this
		3537	* context must reacquire ownership in order to perform final
		3538	* actions (such as re-enabling the interrupts).
		3539	*
		3540	* IMPORTANT: wctxt->outbuf and wctxt->len are no longer valid
		3541	* after a reacquire so writing the message must be
		3542	* aborted.
		3543	*/
		3544	for (i = 0; i < len; i++) {
		3545	if (!nbcon_enter_unsafe(wctxt)) {
		3546	nbcon_reacquire(wctxt);
		3547	break;
		3548	}
		3549
		3550	uart_console_write(port, wctxt->outbuf + i, 1, serial8250_console_putchar);
		3551
		3552	if (!nbcon_exit_unsafe(wctxt)) {
		3553	nbcon_reacquire(wctxt);
		3554	break;
		3555	}
		3556	}
		3557	} else {
		3558	nbcon_reacquire(wctxt);
		3559	}
		3560
		3561	while (!nbcon_enter_unsafe(wctxt))
		3562	nbcon_reacquire(wctxt);
		3563
3564	/* Finally, wait for transmitter to become empty and restore IER. */
3565	wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
3566	if (em485) {
3567	mdelay(port->rs485.delay_rts_after_send);
3568	if (em485->tx_stopped)
3569	up->rs485_stop_tx(up);
3570	}
3571	serial_port_out(port, UART_IER, ier);
3572
3573	/*
3574	* The receive handling will happen properly because the receive ready
3575	* bit will still be set; it is not cleared on read. However, modem
3576	* control will not, we must call it if we have saved something in the
3577	* saved flags while processing with interrupts off.
3578	*/
3579	if (up->msr_saved_flags)
3580	serial8250_modem_status(up);
3581
3582	nbcon_exit_unsafe(wctxt);
3583	}
3584
3585	void serial8250_console_write_atomic(struct uart_8250_port *up,
3586	struct nbcon_write_context *wctxt)
3587	{
3588	struct uart_port *port = &up->port;
3589	unsigned int ier;
3590
3591	/* Atomic console not supported for rs485 mode. */
3592	if (WARN_ON_ONCE(up->em485))
3593	return;
3594
3595	touch_nmi_watchdog();
3596
3597	if (!nbcon_enter_unsafe(wctxt))
3598	return;
3599
3600	/*
3601	* First save IER then disable the interrupts. The special variant to
3602	* clear IER is used because atomic printing may occur without holding
3603	* the port lock.
3604	*/
3605	ier = serial_port_in(port, UART_IER);
3606	__serial8250_clear_IER(up);
3607
3608	/* Check scratch reg if port powered off during system sleep. */
3609	if (up->canary && (up->canary != serial_port_in(port, UART_SCR))) {
3610	serial8250_console_restore(up);
3611	up->canary = 0;
3612	}
3613
3614	if (up->console_newline_needed)
3615	uart_console_write(port, "\n", 1, serial8250_console_putchar);
3616	uart_console_write(port, wctxt->outbuf, wctxt->len, serial8250_console_putchar);
3617
3618	/* Finally, wait for transmitter to become empty and restore IER. */
3619	wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
3620	serial_port_out(port, UART_IER, ier);
3621
3622	nbcon_exit_unsafe(wctxt);
3623	}
3624	#endif /* CONFIG_SERIAL_8250_LEGACY_CONSOLE */
3475		3625
3476	static unsigned int probe_baud(struct uart_port *port)	3626	static unsigned int probe_baud(struct uart_port *port)
3477	{	3627	{
Lines 3490-3495 static unsigned int probe_baud(struct uart_port *port) Link Here
3490		3640
3491	int serial8250_console_setup(struct uart_port port, char options, bool probe)	3641	int serial8250_console_setup(struct uart_port port, char options, bool probe)
3492	{	3642	{
		3643	struct uart_8250_port *up = up_to_u8250p(port);
3493	int baud = 9600;	3644	int baud = 9600;
3494	int bits = 8;	3645	int bits = 8;
3495	int parity = 'n';	3646	int parity = 'n';
Lines 3499-3504 int serial8250_console_setup(struct uart_port port, char options, bool probe) Link Here
3499	if (!port->iobase && !port->membase)	3650	if (!port->iobase && !port->membase)
3500	return -ENODEV;	3651	return -ENODEV;
3501		3652
		3653	up->console_newline_needed = false;
		3654
3502	if (options)	3655	if (options)
3503	uart_parse_options(options, &baud, &parity, &bits, &flow);	3656	uart_parse_options(options, &baud, &parity, &bits, &flow);
3504	else if (probe)	3657	else if (probe)




				flag = TTY_FRAME;
		}

		sysrq = uart_prepare_sysrq_char(&uap->port, ch & 255);
		sysrq = uart_handle_sysrq_char(&uap->port, ch & 255);
		uart_port_lock(&uap->port);

		if (!sysrq)
			uart_insert_char(&uap->port, ch, UART011_DR_OE, ch, flag);
	}

	ret = pl011_dma_rx_trigger_dma(uap);

	pl011_dma_rx_chars(uap, pending, lastbuf, false);
	uart_unlock_and_check_sysrq(&uap->port);
	/*
	 * Do this check after we picked the DMA chars so we don't
	 * get some IRQ immediately from RX.

static irqreturn_t pl011_int(int irq, void *dev_id)
{
	struct uart_amba_port *uap = dev_id;
	unsigned long flags;
	unsigned int status, pass_counter = AMBA_ISR_PASS_LIMIT;
	int handled = 0;

	uart_port_lock(&uap->port);
	status = pl011_read(uap, REG_RIS) & uap->im;
	if (status) {
		do {

		handled = 1;
	}

	uart_unlock_and_check_sysrq(&uap->port);

	return IRQ_RETVAL(handled);
}


	clk_enable(uap->clk);

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&uap->port, &flags);
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&uap->port);
	else
		uart_port_lock_irqsave(&uap->port, &flags);

	/*
	 *	First save the CR then disable the interrupts

		pl011_write(old_cr, uap, REG_CR);

	if (locked)
		uart_port_unlock_irqrestore(&uap->port, flags);
	local_irq_restore(flags);

	clk_disable(uap->clk);
}

			continue;

		co->index = i;
		uart_port_set_cons(port, co);
		return pl011_console_setup(co, options);
	}





		up->port.icount.rx++;
		ch = rdata & AR933X_UART_DATA_TX_RX_MASK;

		if (uart_prepare_sysrq_char(&up->port, ch))
			continue;

		if ((up->port.ignore_status_mask & AR933X_DUMMY_STATUS_RD) == 0)

		ar933x_uart_tx_chars(up);
	}

	uart_unlock_and_check_sysrq(&up->port);

	return IRQ_HANDLED;
}

	unsigned int int_en;
	int locked = 1;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&up->port, &flags);
	if (up->port.sysrq)
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&up->port);
	else
		uart_port_lock_irqsave(&up->port, &flags);

	/*
	 * First save the IER then disable the interrupts

	ar933x_uart_write(up, AR933X_UART_INT_REG, AR933X_UART_INT_ALLINTS);

	if (locked)
		uart_port_unlock_irqrestore(&up->port, flags);

	local_irq_restore(flags);
}

static int ar933x_uart_console_setup(struct console *co, char *options)




				flag = TTY_PARITY;
		}

		if (uart_prepare_sysrq_char(port, c))
			continue;


		if ((cstat & port->ignore_status_mask) == 0)
			tty_insert_flip_char(tty_port, c, flag);


					       estat & UART_EXTINP_DCD_MASK);
	}

	uart_unlock_and_check_sysrq(port);
	return IRQ_HANDLED;
}


{
	struct uart_port *port;
	unsigned long flags;
	int locked = 1;

	port = &ports[co->index];

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(port, &flags);
	else
		uart_port_lock_irqsave(port, &flags);
	} else if (oops_in_progress) {
		locked = uart_port_trylock(port);
	} else {
		uart_port_lock(port);
		locked = 1;
	}

	/* call helper to deal with \r\n */
	uart_console_write(port, s, count, bcm_console_putchar);

	wait_for_xmitr(port);

	if (locked)
		uart_port_unlock_irqrestore(port, flags);
	local_irq_restore(flags);
}

/*




	int locked = 1;

	touch_nmi_watchdog();
	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&up->port, &flags);
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&up->port);
	else
		uart_port_lock_irqsave(&up->port, &flags);

	uart_console_write(&up->port, s, count, lpc32xx_hsuart_console_putchar);
	wait_for_xmit_empty(&up->port);

	if (locked)
		uart_port_unlock_irqrestore(&up->port, flags);
	local_irq_restore(flags);
}

static int __init lpc32xx_hsuart_console_setup(struct console *co,

			tty_insert_flip_char(tport, 0, TTY_FRAME);
		}

		if (!uart_prepare_sysrq_char(port, tmp & 0xff))
			tty_insert_flip_char(tport, (tmp & 0xFF), flag);

		tmp = readl(LPC32XX_HSUART_FIFO(port->membase));
	}

		__serial_lpc32xx_tx(port);
	}

	uart_unlock_and_check_sysrq(port);

	return IRQ_HANDLED;
}




				continue;
		}

		if (uart_prepare_sysrq_char(port, ch))
			continue;

		if ((status & port->ignore_status_mask) == 0)

			meson_uart_start_tx(port);
	}

	uart_unlock_and_check_sysrq(port);

	return IRQ_HANDLED;
}

				    u_int count)
{
	unsigned long flags;
	int locked = 1;
	u32 val, tmp;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(port, &flags);
	else
		uart_port_lock_irqsave(port, &flags);
		locked = uart_port_trylock(port);
	} else {
		uart_port_lock(port);
		locked = 1;
	}

	val = readl(port->membase + AML_UART_CONTROL);
	tmp = val & ~(AML_UART_TX_INT_EN | AML_UART_RX_INT_EN);

	writel(val, port->membase + AML_UART_CONTROL);

	if (locked)
		uart_port_unlock_irqrestore(port, flags);
	local_irq_restore(flags);
}

static void meson_serial_console_write(struct console *co, const char *s,




		if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))
			flag = TTY_NORMAL;

		sysrq = uart_prepare_sysrq_char(port, dma->virt[i]);
		sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
		uart_port_lock_irqsave(port, &flags);
		if (!sysrq)
			tty_insert_flip_char(tport, dma->virt[i], flag);
	}

	msm_start_rx_dma(msm_port);
done:
	uart_unlock_and_check_sysrq_irqrestore(port, flags);

	if (count)
		tty_flip_buffer_push(tport);

			if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))
				flag = TTY_NORMAL;

			sysrq = uart_prepare_sysrq_char(port, buf[i]);
			sysrq = uart_handle_sysrq_char(port, buf[i]);
			uart_port_lock(port);
			if (!sysrq)
				tty_insert_flip_char(tport, buf[i], flag);
		}

		else if (sr & MSM_UART_SR_PAR_FRAME_ERR)
			flag = TTY_FRAME;

		sysrq = uart_prepare_sysrq_char(port, c);
		sysrq = uart_handle_sysrq_char(port, c);
		uart_port_lock(port);
		if (!sysrq)
			tty_insert_flip_char(tport, c, flag);
	}

	struct uart_port *port = dev_id;
	struct msm_port *msm_port = to_msm_port(port);
	struct msm_dma *dma = &msm_port->rx_dma;
	unsigned long flags;
	unsigned int misr;
	u32 val;

	uart_port_lock(port);
	misr = msm_read(port, MSM_UART_MISR);
	msm_write(port, 0, MSM_UART_IMR); /* disable interrupt */


		msm_handle_delta_cts(port);

	msm_write(port, msm_port->imr, MSM_UART_IMR); /* restore interrupt */
	uart_unlock_and_check_sysrq(port);

	return IRQ_HANDLED;
}

			num_newlines++;
	count += num_newlines;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(port, &flags);
	if (port->sysrq)
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(port);
	else
		uart_port_lock_irqsave(port, &flags);

	if (is_uartdm)
		msm_reset_dm_count(port, count);

	}

	if (locked)
		uart_port_unlock_irqrestore(port, flags);

	local_irq_restore(flags);
}

static void msm_console_write(struct console *co, const char *s,





	up->port.icount.rx++;

	if (uart_prepare_sysrq_char(&up->port, ch))
		return;

	uart_insert_char(&up->port, lsr, UART_LSR_OE, ch, TTY_NORMAL);

		}
	} while (max_count--);

	uart_unlock_and_check_sysrq(&up->port);

	tty_flip_buffer_push(&up->port.state->port);


	unsigned int ier;
	int locked = 1;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&up->port, &flags);
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&up->port);
	else
		uart_port_lock_irqsave(&up->port, &flags);

	/*
	 * First save the IER then disable the interrupts

		check_modem_status(up);

	if (locked)
		uart_port_unlock_irqrestore(&up->port, flags);
	local_irq_restore(flags);
}

static int __init

Lines 199-204 static void owl_uart_receive_chars(struct uart_port *port) Link Here

(-)a/drivers/tty/serial/owl-uart.c (-18 / +12 lines)
199	stat = owl_uart_read(port, OWL_UART_STAT);	199	stat = owl_uart_read(port, OWL_UART_STAT);
200	while (!(stat & OWL_UART_STAT_RFEM)) {	200	while (!(stat & OWL_UART_STAT_RFEM)) {
201	char flag = TTY_NORMAL;	201	char flag = TTY_NORMAL;
		202	bool sysrq;
202		203
203	if (stat & OWL_UART_STAT_RXER)	204	if (stat & OWL_UART_STAT_RXER)
204	port->icount.overrun++;	205	port->icount.overrun++;
Lines 217-223 static void owl_uart_receive_chars(struct uart_port *port) Link Here
217	val = owl_uart_read(port, OWL_UART_RXDAT);	218	val = owl_uart_read(port, OWL_UART_RXDAT);
218	val &= 0xff;	219	val &= 0xff;
219		220
220	if ((stat & port->ignore_status_mask) == 0)	221	sysrq = uart_prepare_sysrq_char(port, val);
		222
		223	if (!sysrq && (stat & port->ignore_status_mask) == 0)
221	tty_insert_flip_char(&port->state->port, val, flag);	224	tty_insert_flip_char(&port->state->port, val, flag);
222		225
223	stat = owl_uart_read(port, OWL_UART_STAT);	226	stat = owl_uart_read(port, OWL_UART_STAT);
Lines 229-238 static void owl_uart_receive_chars(struct uart_port *port) Link Here
229	static irqreturn_t owl_uart_irq(int irq, void *dev_id)	232	static irqreturn_t owl_uart_irq(int irq, void *dev_id)
230	{	233	{
231	struct uart_port *port = dev_id;	234	struct uart_port *port = dev_id;
232	unsigned long flags;
233	u32 stat;	235	u32 stat;
234		236
235	uart_port_lock_irqsave(port, &flags);	237	uart_port_lock(port);
236		238
237	stat = owl_uart_read(port, OWL_UART_STAT);	239	stat = owl_uart_read(port, OWL_UART_STAT);
238		240
Lines 246-252 static irqreturn_t owl_uart_irq(int irq, void *dev_id) Link Here
246	stat \|= OWL_UART_STAT_RIP \| OWL_UART_STAT_TIP;	248	stat \|= OWL_UART_STAT_RIP \| OWL_UART_STAT_TIP;
247	owl_uart_write(port, stat, OWL_UART_STAT);	249	owl_uart_write(port, stat, OWL_UART_STAT);
248		250
249	uart_port_unlock_irqrestore(port, flags);	251	uart_unlock_and_check_sysrq(port);
250		252
251	return IRQ_HANDLED;	253	return IRQ_HANDLED;
252	}	254	}
Lines 508-525 static void owl_uart_port_write(struct uart_port port, const char s, Link Here
508	{	510	{
509	u32 old_ctl, val;	511	u32 old_ctl, val;
510	unsigned long flags;	512	unsigned long flags;
511	int locked;	513	int locked = 1;
512		514
513	local_irq_save(flags);	515	if (oops_in_progress)
514		516	locked = uart_port_trylock_irqsave(port, &flags);
515	if (port->sysrq)	517	else
516	locked = 0;	518	uart_port_lock_irqsave(port, &flags);
517	else if (oops_in_progress)
518	locked = uart_port_trylock(port);
519	else {
520	uart_port_lock(port);
521	locked = 1;
522	}
523		519
524	old_ctl = owl_uart_read(port, OWL_UART_CTL);	520	old_ctl = owl_uart_read(port, OWL_UART_CTL);
525	val = old_ctl \| OWL_UART_CTL_TRFS_TX;	521	val = old_ctl \| OWL_UART_CTL_TRFS_TX;
Lines 541-549 static void owl_uart_port_write(struct uart_port port, const char s, Link Here
541	owl_uart_write(port, old_ctl, OWL_UART_CTL);	537	owl_uart_write(port, old_ctl, OWL_UART_CTL);
542		538
543	if (locked)	539	if (locked)
544	uart_port_unlock(port);	540	uart_port_unlock_irqrestore(port, flags);
545
546	local_irq_restore(flags);
547	}	541	}
548		542
549	static void owl_uart_console_write(struct console co, const char s,	543	static void owl_uart_console_write(struct console co, const char s,





#define IRQ_NAME_SIZE 17
	char				irq_name[IRQ_NAME_SIZE];

	/* protect the eg20t_port private structure and io access to membase */
	spinlock_t lock;
};

/**

			if (uart_handle_break(port))
				continue;
		}
		if (uart_prepare_sysrq_char(port, rbr))
			continue;

		buf[i++] = rbr;

	iowrite8(lcr, priv->membase + UART_LCR);
}

static void push_rx(struct eg20t_port *priv, const unsigned char *buf,
		    int size)
{
	struct uart_port *port = &priv->port;
	struct tty_port *tport = &port->state->port;

	tty_insert_flip_string(tport, buf, size);
	tty_flip_buffer_push(tport);

	return 0;
}

static int dma_push_rx(struct eg20t_port *priv, int size)

{
	struct pch_uart_buffer *buf;
	int rx_size;

	if (!priv->start_rx) {
		pch_uart_hal_disable_interrupt(priv, PCH_UART_HAL_RX_INT |
						     PCH_UART_HAL_RX_ERR_INT);

	buf = &priv->rxbuf;
	do {
		rx_size = pch_uart_hal_read(priv, buf->buf, buf->size);
		push_rx(priv, buf->buf, rx_size);
		if (ret)
			return 0;
	} while (rx_size == buf->size);

	return PCH_UART_HANDLED_RX_INT;
}

static int handle_rx(struct eg20t_port *priv)
{
	return handle_rx_to(priv);
}

static int dma_handle_rx(struct eg20t_port *priv)
{
	struct uart_port *port = &priv->port;

	u8 lsr;
	int ret = 0;
	unsigned char iid;
	unsigned long flags;
	int next = 1;
	u8 msr;

	uart_port_lock(&priv->port);
	handled = 0;
	while (next) {
		iid = pch_uart_hal_get_iid(priv);

						PCH_UART_HAL_RX_INT |
						PCH_UART_HAL_RX_ERR_INT);
			} else {
				ret = handle_rx_to(priv);
			}
			break;
		case PCH_UART_IID_RDR_TO:	/* Received Data Ready

		handled |= (unsigned int)ret;
	}

	uart_unlock_and_check_sysrq(&priv->port);
	return IRQ_RETVAL(handled);
}


	unsigned long flags;

	priv = container_of(port, struct eg20t_port, port);
	uart_port_lock_irqsave(&priv->port, &flags);
	pch_uart_hal_set_break(priv, ctl);
	uart_port_unlock_irqrestore(&priv->port, flags);
}

/* Grab any interrupt resources and initialise any low level driver state. */


	baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);

	uart_port_lock_irqsave(port, &flags);
	uart_port_lock(port);

	uart_update_timeout(port, termios->c_cflag, baud);
	rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);

		tty_termios_encode_baud_rate(termios, baud, baud);

out:
	uart_port_unlock_irqrestore(port, flags);
	spin_unlock_irqrestore(&priv->lock, flags);
}

static const char *pch_uart_type(struct uart_port *port)

{
	struct eg20t_port *priv;
	unsigned long flags;
	int locked = 1;
	int port_locked = 1;
	u8 ier;

	priv = pch_uart_ports[co->index];

	touch_nmi_watchdog();

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&priv->port, &flags);
	else
		uart_port_lock_irqsave(&priv->port, &flags);
		/* serial8250_handle_port() already took the port lock */
		port_locked = 0;
	} else if (oops_in_progress) {
		priv_locked = spin_trylock(&priv->lock);
		port_locked = uart_port_trylock(&priv->port);
	} else {
		spin_lock(&priv->lock);
		uart_port_lock(&priv->port);
	}

	/*
	 *	First save the IER then disable the interrupts

	wait_for_xmitr(priv, UART_LSR_BOTH_EMPTY);
	iowrite8(ier, priv->membase + UART_IER);

	if (locked)
		uart_port_unlock_irqrestore(&priv->port, flags);
	if (priv_locked)
		spin_unlock(&priv->lock);
	local_irq_restore(flags);
}

static int __init pch_console_setup(struct console *co, char *options)

	pci_enable_msi(pdev);
	pci_set_master(pdev);

	spin_lock_init(&priv->lock);

	iobase = pci_resource_start(pdev, 0);
	mapbase = pci_resource_start(pdev, 1);
	priv->mapbase = mapbase;

		 KBUILD_MODNAME ":" PCH_UART_DRIVER_DEVICE "%d",
		 priv->port.line);

	spin_lock_init(&priv->port.lock);

	pci_set_drvdata(pdev, priv);
	priv->trigger_level = 1;
	priv->fcr = 0;




				flag = TTY_FRAME;
		}

		if (uart_prepare_sysrq_char(&up->port, ch))
			goto ignore_char;

		uart_insert_char(&up->port, *status, UART_LSR_OE, ch, flag);

	check_modem_status(up);
	if (lsr & UART_LSR_THRE)
		transmit_chars(up);
	uart_unlock_and_check_sysrq(&up->port);
	return IRQ_HANDLED;
}


	int locked = 1;

	clk_enable(up->clk);
	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&up->port, &flags);
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&up->port);
	else
		uart_port_lock_irqsave(&up->port, &flags);

	/*
	 *	First save the IER then disable the interrupts

	serial_out(up, UART_IER, ier);

	if (locked)
		uart_port_unlock_irqrestore(&up->port, flags);
	local_irq_restore(flags);
	clk_disable(up->clk);

}

#ifdef CONFIG_CONSOLE_POLL




		val &= 0xff;

		port->icount.rx++;
		if (!uart_prepare_sysrq_char(port, val))
			tty_insert_flip_char(&port->state->port, val, flag);

		status = rda_uart_read(port, RDA_UART_STATUS);
	}

static irqreturn_t rda_interrupt(int irq, void *dev_id)
{
	struct uart_port *port = dev_id;
	unsigned long flags;
	u32 val, irq_mask;

	uart_port_lock(port);

	/* Clear IRQ cause */
	val = rda_uart_read(port, RDA_UART_IRQ_CAUSE);

		rda_uart_send_chars(port);
	}

	uart_unlock_and_check_sysrq(port);

	return IRQ_HANDLED;
}

{
	u32 old_irq_mask;
	unsigned long flags;
	int locked = 1;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(port, &flags);
	else
		uart_port_lock_irqsave(port, &flags);
	} else if (oops_in_progress) {
		locked = uart_port_trylock(port);
	} else {
		uart_port_lock(port);
		locked = 1;
	}

	old_irq_mask = rda_uart_read(port, RDA_UART_IRQ_MASK);
	rda_uart_write(port, 0, RDA_UART_IRQ_MASK);

	rda_uart_write(port, old_irq_mask, RDA_UART_IRQ_MASK);

	if (locked)
		uart_port_unlock_irqrestore(port, flags);

	local_irq_restore(flags);
}

static void rda_uart_console_write(struct console *co, const char *s,




	state->uart_port = uport;
	uport->state = state;

	/*
	 * If this port is in use as a console then the spinlock is already
	 * initialised.
	 */
	if (!uart_console_registered(uport))
		uart_port_spin_lock_init(uport);

	state->pm_state = UART_PM_STATE_UNDEFINED;
	uart_port_set_cons(uport, drv->cons);
	uport->minor = drv->tty_driver->minor_start + uport->line;
	uport->name = kasprintf(GFP_KERNEL, "%s%d", drv->dev_name,
				drv->tty_driver->name_base + uport->line);

		goto out;
	}

	/*
	 * If this port is in use as a console then the spinlock is already
	 * initialised.
	 */
	if (!uart_console_registered(uport))
		uart_port_spin_lock_init(uport);

	if (uport->cons && uport->dev)
		of_console_check(uport->dev->of_node, uport->cons->name, uport->line);





			break;

		ssp->port.icount.rx++;
		if (!uart_prepare_sysrq_char(&ssp->port, ch))
			uart_insert_char(&ssp->port, 0, 0, ch, TTY_NORMAL);
	}

	tty_flip_buffer_push(&ssp->port.state->port);

	if (ip & SIFIVE_SERIAL_IP_TXWM_MASK)
		__ssp_transmit_chars(ssp);

	uart_unlock_and_check_sysrq(&ssp->port);

	return IRQ_HANDLED;
}

	if (!ssp)
		return;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&ssp->port, &flags);
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&ssp->port);
	else
		uart_port_lock_irqsave(&ssp->port, &flags);

	ier = __ssp_readl(ssp, SIFIVE_SERIAL_IE_OFFS);
	__ssp_writel(0, SIFIVE_SERIAL_IE_OFFS, ssp);

	__ssp_writel(ier, SIFIVE_SERIAL_IE_OFFS, ssp);

	if (locked)
		uart_port_unlock_irqrestore(&ssp->port, flags);
	local_irq_restore(flags);
}

static int sifive_serial_console_setup(struct console *co, char *options)




		if (port->ignore_status_mask & SUP_DUMMY_READ)
			goto ignore_char;

		if (uart_prepare_sysrq_char(port, ch))
			goto ignore_char;

		uart_insert_char(port, lsr, SUP_UART_LSR_OE, ch, flag);

	if (isc & SUP_UART_ISC_TX)
		transmit_chars(port);

	uart_unlock_and_check_sysrq(port);

	return IRQ_HANDLED;
}

	unsigned long flags;
	int locked = 1;

	if (oops_in_progress)
		locked = uart_port_trylock_irqsave(&sunplus_console_ports[co->index]->port, &flags);
	if (sunplus_console_ports[co->index]->port.sysrq)
		locked = 0;
	else if (oops_in_progress)
		locked = uart_port_trylock(&sunplus_console_ports[co->index]->port);
	else
		uart_port_lock_irqsave(&sunplus_console_ports[co->index]->port, &flags);

	uart_console_write(&sunplus_console_ports[co->index]->port, s, count,
			   sunplus_uart_console_putchar);

	if (locked)
		uart_port_unlock_irqrestore(&sunplus_console_ports[co->index]->port, flags);

	local_irq_restore(flags);
}

static int __init sunplus_console_setup(struct console *co, char *options)

Lines 3567-3574 static ssize_t show_cons_active(struct device *dev, Link Here

(-)a/drivers/tty/tty_io.c (-2 / +7 lines)
3567	for_each_console(c) {	3567	for_each_console(c) {
3568	if (!c->device)	3568	if (!c->device)
3569	continue;	3569	continue;
3570	if (!c->write)	3570	if (c->flags & CON_NBCON) {
3571	continue;	3571	if (!c->write_atomic && !c->write_thread)
		3572	continue;
		3573	} else {
		3574	if (!c->write)
		3575	continue;
		3576	}
3572	if ((c->flags & CON_ENABLED) == 0)	3577	if ((c->flags & CON_ENABLED) == 0)
3573	continue;	3578	continue;
3574	cs[i++] = c;	3579	cs[i++] = c;




		{ CON_ENABLED,		'E' },
		{ CON_CONSDEV,		'C' },
		{ CON_BOOT,		'B' },
		{ CON_NBCON,		'N' },
		{ CON_PRINTBUFFER,	'p' },
		{ CON_BRL,		'b' },
		{ CON_ANYTIME,		'a' },
	};
	char flags[ARRAY_SIZE(con_flags) + 1];
	struct console *con = v;
	char con_write = '-';
	unsigned int a;
	dev_t dev = 0;


	seq_setwidth(m, 21 - 1);
	seq_printf(m, "%s%d", con->name, con->index);
	seq_pad(m, ' ');
	if (con->flags & CON_NBCON) {
		if (con->write_atomic || con->write_thread)
			con_write = 'W';
	} else {
		if (con->write)
			con_write = 'W';
	}
	seq_printf(m, "%c%c%c (%s)", con->read ? 'R' : '-', con_write,
		   con->unblank ? 'U' : '-', flags);
	if (dev)
		seq_printf(m, " %4d:%d", MAJOR(dev), MINOR(dev));


Lines 35-42 static inline void local_bh_enable(void) Link Here

(-)a/include/linux/bottom_half.h (+2 lines)
35		35
36	#ifdef CONFIG_PREEMPT_RT	36	#ifdef CONFIG_PREEMPT_RT
37	extern bool local_bh_blocked(void);	37	extern bool local_bh_blocked(void);
		38	extern void softirq_preempt(void);
38	#else	39	#else
39	static inline bool local_bh_blocked(void) { return false; }	40	static inline bool local_bh_blocked(void) { return false; }
		41	static inline void softirq_preempt(void) { }
40	#endif	42	#endif
41		43
42	#endif /* _LINUX_BH_H */	44	#endif /* _LINUX_BH_H */





#include <linux/atomic.h>
#include <linux/bits.h>
#include <linux/irq_work.h>
#include <linux/rculist.h>
#include <linux/rcuwait.h>
#include <linux/types.h>

struct vc_data;

 */

/**
 * enum cons_flags - General console flags
 * @CON_PRINTBUFFER:	Used by newly registered consoles to avoid duplicate
 *			output of messages that were already shown by boot
 *			consoles or read by userspace via syslog() syscall.

static_assert(sizeof(struct nbcon_state) <= sizeof(int));

/**
 * enum nbcon_prio - console owner priority for nbcon consoles
 * @NBCON_PRIO_NONE:		Unused
 * @NBCON_PRIO_NORMAL:		Normal (non-emergency) usage
 * @NBCON_PRIO_EMERGENCY:	Emergency output (WARN/OOPS...)

	bool			unsafe_takeover;
};

/**
 * struct nbcon_drvdata - Data to allow nbcon acquire in non-print context
 * @ctxt:		The core console context
 * @srcu_cookie:	Storage for a console_srcu_lock cookie, if needed
 * @owner_index:	Storage for the owning console index, if needed
 * @locked:		Storage for the locked state, if needed
 *
 * All fields (except for @ctxt) are available exclusively to the driver to
 * use as needed. They are not used by the printk subsystem.
 */
struct nbcon_drvdata {
	struct nbcon_context	__private ctxt;

	/* reserved for driver use */
	int			srcu_cookie;
	short			owner_index;
	bool			locked;
};

/**
 * struct console - The console descriptor structure
 * @name:		The name of the console driver
 * @write:		Legacy write callback to output messages (Optional)
 * @read:		Read callback for console input (Optional)
 * @device:		The underlying TTY device driver (Optional)
 * @unblank:		Callback to unblank the console (Optional)

 * @data:		Driver private data
 * @node:		hlist node for the console list
 *
 * @write_atomic:	Write callback for atomic context
 * @nbcon_state:	State for nbcon consoles
 * @nbcon_seq:		Sequence number of the next record for nbcon to print
 * @nbcon_prev_seq:	Seq num the previous nbcon owner was assigned to print
 * @pbufs:		Pointer to nbcon private buffer
 * @kthread:		Printer kthread for this console
 * @rcuwait:		RCU-safe wait object for @kthread waking
 * @irq_work:		Defer @kthread waking to IRQ work context
 */
struct console {
	char			name[16];

	struct hlist_node	node;

	/* nbcon console specific members */

	/**
	 * @write_atomic:
	 *
	 * NBCON callback to write out text in any context.
	 *
	 * This callback is called with the console already acquired. The
	 * callback can use nbcon_can_proceed() at any time to verify that
	 * it is still the owner of the console. In the case that it has
	 * lost ownership, it is no longer allowed to go forward. In this
	 * case it must back out immediately and carefully. The buffer
	 * content is also no longer trusted since it no longer belongs to
	 * the context.
	 *
	 * If the callback needs to perform actions where ownership is not
	 * allowed to be taken over, nbcon_enter_unsafe() and
	 * nbcon_exit_unsafe() can be used to mark such sections. These
	 * functions are also points of possible ownership transfer. If
	 * either function returns false, ownership has been lost.
	 *
	 * If the driver must reacquire ownership in order to finalize or
	 * revert hardware changes, nbcon_reacquire() can be used. However,
	 * on reacquire the buffer content is no longer available. A
	 * reacquire cannot be used to resume printing.
	 *
	 * This callback can be called from any context (including NMI).
	 * Therefore it must avoid usage of any locking and instead rely
	 * on the console ownership for synchronization.
	 */
	void (*write_atomic)(struct console *con, struct nbcon_write_context *wctxt);

	/**
	 * @write_thread:
	 *
	 * NBCON callback to write out text in task context. (Optional)
	 *
	 * This callback is called with the console already acquired. Any
	 * additional driver synchronization should have been performed by
	 * device_lock().
	 *
	 * This callback is always called from task context but with migration
	 * disabled.
	 *
	 * The same criteria for console ownership verification and unsafe
	 * sections applies as with write_atomic(). The difference between
	 * this callback and write_atomic() is that this callback is used
	 * during normal operation and is always called from task context.
	 * This provides drivers with a relatively relaxed locking context
	 * for synchronizing output to the hardware.
	 */
	void (*write_thread)(struct console *con, struct nbcon_write_context *wctxt);

	/**
	 * @device_lock:
	 *
	 * NBCON callback to begin synchronization with driver code.
	 *
	 * Console drivers typically must deal with access to the hardware
	 * via user input/output (such as an interactive login shell) and
	 * output of kernel messages via printk() calls. This callback is
	 * called by the printk-subsystem whenever it needs to synchronize
	 * with hardware access by the driver. It should be implemented to
	 * use whatever synchronization mechanism the driver is using for
	 * itself (for example, the port lock for uart serial consoles).
	 *
	 * This callback is always called from task context. It may use any
	 * synchronization method required by the driver. BUT this callback
	 * MUST also disable migration. The console driver may be using a
	 * synchronization mechanism that already takes care of this (such as
	 * spinlocks). Otherwise this function must explicitly call
	 * migrate_disable().
	 *
	 * The flags argument is provided as a convenience to the driver. It
	 * will be passed again to device_unlock(). It can be ignored if the
	 * driver does not need it.
	 */
	void (*device_lock)(struct console *con, unsigned long *flags);

	/**
	 * @device_unlock:
	 *
	 * NBCON callback to finish synchronization with driver code.
	 *
	 * It is the counterpart to device_lock().
	 *
	 * This callback is always called from task context. It must
	 * appropriately re-enable migration (depending on how device_lock()
	 * disabled migration).
	 *
	 * The flags argument is the value of the same variable that was
	 * passed to device_lock().
	 */
	void (*device_unlock)(struct console *con, unsigned long flags);

	atomic_t		__private nbcon_state;
	atomic_long_t		__private nbcon_seq;
	atomic_long_t           __private nbcon_prev_seq;

	/**
	 * @nbcon_drvdata:
	 *
	 * Data for nbcon ownership tracking to allow acquiring nbcon consoles
	 * in non-printing contexts.
	 *
	 * Drivers may need to acquire nbcon consoles in non-printing
	 * contexts. This is achieved by providing a struct nbcon_drvdata.
	 * Then the driver can call nbcon_driver_acquire() and
	 * nbcon_driver_release(). The struct does not require any special
	 * initialization.
	 */
	struct nbcon_drvdata	*nbcon_drvdata;

	struct printk_buffers	*pbufs;
	struct task_struct	*kthread;
	struct rcuwait		rcuwait;
	struct irq_work		irq_work;
};

#ifdef CONFIG_LOCKDEP

extern struct hlist_head console_list;

/**
 * console_srcu_read_flags - Locklessly read flags of a possibly registered
 *				console
 * @con:	struct console pointer of console to read flags from
 *
 * Locklessly reading @con->flags provides a consistent read value because
 * there is at most one CPU modifying @con->flags and that CPU is using only
 * read-modify-write operations to do so.
 * for registered consoles with console_srcu_write_flags().
 *
 * Requires console_srcu_read_lock to be held, which implies that @con might
 * be a registered console. If the caller is holding the console_list_lock or
 * it is certain that the console is not registered, the caller may read
 * @con->flags directly instead.
 *
 * Context: Any context.
 * Return: The current value of the @con->flags field.
 */
static inline short console_srcu_read_flags(const struct console *con)
{
	WARN_ON_ONCE(!console_srcu_read_lock_is_held());

	/*
	 * The READ_ONCE() matches the WRITE_ONCE() when @flags are modified
	 * for registered consoles with console_srcu_write_flags().
	 * console->flags and that CPU is using only read-modify-write
	 * operations to do so.
	 */
	return data_race(READ_ONCE(con->flags));
}

	hlist_for_each_entry(con, &console_list, node)

#ifdef CONFIG_PRINTK
extern void nbcon_cpu_emergency_enter(void);
extern void nbcon_cpu_emergency_exit(void);
extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt);
extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt);
extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt);
extern void nbcon_reacquire(struct nbcon_write_context *wctxt);
#else
static inline void nbcon_cpu_emergency_enter(void) { }
static inline void nbcon_cpu_emergency_exit(void) { }
static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; }
static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; }
static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; }
static inline void nbcon_reacquire(struct nbcon_write_context *wctxt) { }
#endif

extern int console_set_on_cmdline;

Lines 65-71 Link Here

(-)a/include/linux/entry-common.h (-1 / +1 lines)
65	#define EXIT_TO_USER_MODE_WORK \	65	#define EXIT_TO_USER_MODE_WORK \
66	(_TIF_SIGPENDING \| _TIF_NOTIFY_RESUME \| _TIF_UPROBE \| \	66	(_TIF_SIGPENDING \| _TIF_NOTIFY_RESUME \| _TIF_UPROBE \| \
67	_TIF_NEED_RESCHED \| _TIF_PATCH_PENDING \| _TIF_NOTIFY_SIGNAL \| \	67	_TIF_NEED_RESCHED \| _TIF_PATCH_PENDING \| _TIF_NOTIFY_SIGNAL \| \
68	ARCH_EXIT_TO_USER_MODE_WORK)	68	_TIF_NEED_RESCHED_LAZY \| ARCH_EXIT_TO_USER_MODE_WORK)
69		69
70	/**	70	/**
71	* arch_enter_from_user_mode - Architecture specific sanity check for user mode regs	71	* arch_enter_from_user_mode - Architecture specific sanity check for user mode regs

Lines 18-24 Link Here

(-)a/include/linux/entry-kvm.h (-1 / +1 lines)
18		18
19	#define XFER_TO_GUEST_MODE_WORK \	19	#define XFER_TO_GUEST_MODE_WORK \
20	(_TIF_NEED_RESCHED \| _TIF_SIGPENDING \| _TIF_NOTIFY_SIGNAL \| \	20	(_TIF_NEED_RESCHED \| _TIF_SIGPENDING \| _TIF_NOTIFY_SIGNAL \| \
21	_TIF_NOTIFY_RESUME \| ARCH_XFER_TO_GUEST_MODE_WORK)	21	_TIF_NOTIFY_RESUME \| _TIF_NEED_RESCHED_LAZY \| ARCH_XFER_TO_GUEST_MODE_WORK)
22		22
23	struct kvm_vcpu;	23	struct kvm_vcpu;
24		24




extern void raise_softirq_irqoff(unsigned int nr);
extern void raise_softirq(unsigned int nr);

#ifdef CONFIG_PREEMPT_RT
DECLARE_PER_CPU(struct task_struct *, timersd);
DECLARE_PER_CPU(unsigned long, pending_timer_softirq);

extern void raise_timer_softirq(void);
extern void raise_hrtimer_softirq(void);

static inline unsigned int local_pending_timers(void)
{
        return __this_cpu_read(pending_timer_softirq);
}

#else
static inline void raise_timer_softirq(void)
{
	raise_softirq(TIMER_SOFTIRQ);
}

static inline void raise_hrtimer_softirq(void)
{
	raise_softirq_irqoff(HRTIMER_SOFTIRQ);
}

static inline unsigned int local_pending_timers(void)
{
        return local_softirq_pending();
}
#endif

DECLARE_PER_CPU(struct task_struct *, ksoftirqd);

static inline struct task_struct *this_cpu_ksoftirqd(void)

Lines 3365-3370 static inline void dev_xmit_recursion_dec(void) Link Here

(-)a/include/linux/netdevice.h (+1 lines)
3365	__this_cpu_dec(softnet_data.xmit.recursion);	3365	__this_cpu_dec(softnet_data.xmit.recursion);
3366	}	3366	}
3367		3367
		3368	void kick_defer_list_purge(struct softnet_data *sd, unsigned int cpu);
3368	void __netif_schedule(struct Qdisc *q);	3369	void __netif_schedule(struct Qdisc *q);
3369	void netif_schedule_queue(struct netdev_queue *txq);	3370	void netif_schedule_queue(struct netdev_queue *txq);
3370		3371




	unsigned int			pending_wakeup;
	unsigned int			pending_kill;
	unsigned int			pending_disable;
	unsigned int			pending_sigtrap;
	unsigned long			pending_addr;	/* SIGTRAP */
	struct irq_work			pending_irq;
	struct irq_work			pending_disable_irq;
	struct callback_head		pending_task;
	unsigned int			pending_work;


	struct rcu_head			rcu_head;

	/*
	 * Sum (event->pending_work + event->pending_work)
	 *
	 * The SIGTRAP is targeted at ctx->task, as such it won't do changing
	 * that until the signal is delivered.

Lines 9-14 Link Here

(-)a/include/linux/printk.h (-4 / +30 lines)
9	#include <linux/ratelimit_types.h>	9	#include <linux/ratelimit_types.h>
10	#include <linux/once_lite.h>	10	#include <linux/once_lite.h>
11		11
		12	struct console;
		13
12	extern const char linux_banner[];	14	extern const char linux_banner[];
13	extern const char linux_proc_banner[];	15	extern const char linux_proc_banner[];
14		16
Lines 157-171 int _printk(const char *fmt, ...); Link Here
157	*/	159	*/
158	__printf(1, 2) __cold int _printk_deferred(const char *fmt, ...);	160	__printf(1, 2) __cold int _printk_deferred(const char *fmt, ...);
159		161
160	extern void __printk_safe_enter(void);	162	extern void __printk_deferred_enter(void);
161	extern void __printk_safe_exit(void);	163	extern void __printk_deferred_exit(void);
		164
162	/*	165	/*
163	* The printk_deferred_enter/exit macros are available only as a hack for	166	* The printk_deferred_enter/exit macros are available only as a hack for
164	* some code paths that need to defer all printk console printing. Interrupts	167	* some code paths that need to defer all printk console printing. Interrupts
165	* must be disabled for the deferred duration.	168	* must be disabled for the deferred duration.
166	*/	169	*/
167	#define printk_deferred_enter __printk_safe_enter	170	#define printk_deferred_enter() __printk_deferred_enter()
168	#define printk_deferred_exit __printk_safe_exit	171	#define printk_deferred_exit() __printk_deferred_exit()
169		172
170	/*	173	/*
171	* Please don't use printk_ratelimit(), because it shares ratelimiting state	174	* Please don't use printk_ratelimit(), because it shares ratelimiting state
Lines 192-197 void show_regs_print_info(const char *log_lvl); Link Here
192	extern asmlinkage void dump_stack_lvl(const char *log_lvl) __cold;	195	extern asmlinkage void dump_stack_lvl(const char *log_lvl) __cold;
193	extern asmlinkage void dump_stack(void) __cold;	196	extern asmlinkage void dump_stack(void) __cold;
194	void printk_trigger_flush(void);	197	void printk_trigger_flush(void);
		198	void printk_legacy_allow_panic_sync(void);
		199	extern void nbcon_driver_acquire(struct console *con);
		200	extern void nbcon_driver_release(struct console *con);
		201	void nbcon_atomic_flush_unsafe(void);
195	#else	202	#else
196	static inline __printf(1, 0)	203	static inline __printf(1, 0)
197	int vprintk(const char *s, va_list args)	204	int vprintk(const char *s, va_list args)
Lines 271-278 static inline void dump_stack(void) Link Here
271	static inline void printk_trigger_flush(void)	278	static inline void printk_trigger_flush(void)
272	{	279	{
273	}	280	}
		281
		282	static inline void printk_legacy_allow_panic_sync(void)
		283	{
		284	}
		285
		286	static inline void nbcon_driver_acquire(struct console *con)
		287	{
		288	}
		289
		290	static inline void nbcon_driver_release(struct console *con)
		291	{
		292	}
		293
		294	static inline void nbcon_atomic_flush_unsafe(void)
		295	{
		296	}
		297
274	#endif	298	#endif
275		299
		300	bool this_cpu_in_panic(void);
		301
276	#ifdef CONFIG_SMP	302	#ifdef CONFIG_SMP
277	extern int __printk_cpu_sync_try_get(void);	303	extern int __printk_cpu_sync_try_get(void);
278	extern void __printk_cpu_sync_wait(void);	304	extern void __printk_cpu_sync_wait(void);




}
#endif

extern bool task_is_pi_boosted(const struct task_struct *p);
extern int yield_to(struct task_struct *p, bool preempt);
extern void set_user_nice(struct task_struct *p, long nice);
extern int task_prio(const struct task_struct *p);

	update_ti_thread_flag(task_thread_info(tsk), flag, value);
}

static inline bool test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
{
	return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
}

static inline bool test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
{
	return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
}

static inline bool test_tsk_thread_flag(struct task_struct *tsk, int flag)
{
	return test_ti_thread_flag(task_thread_info(tsk), flag);
}

static inline void clear_tsk_need_resched(struct task_struct *tsk)
{
	clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
	if (IS_ENABLED(CONFIG_PREEMPT_BUILD_AUTO))
		clear_tsk_thread_flag(tsk, TIF_NEED_RESCHED_LAZY);
}

static inline bool test_tsk_need_resched(struct task_struct *tsk)
{
	return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
}


static __always_inline bool need_resched(void)
{
	return unlikely(tif_need_resched_lazy() || tif_need_resched());
}

/*




	 */
	smp_mb__after_atomic();

	return unlikely(need_resched());
}

static __always_inline bool __must_check current_clr_polling_and_test(void)

	 */
	smp_mb__after_atomic();

	return unlikely(need_resched());
}

#else


static inline bool __must_check current_set_polling_and_test(void)
{
	return unlikely(need_resched());
}
static inline bool __must_check current_clr_polling_and_test(void)
{
	return unlikely(need_resched());
}
#endif


Lines 153-158 struct uart_8250_port { Link Here

(-)a/include/linux/serial_8250.h (+6 lines)
153	#define MSR_SAVE_FLAGS UART_MSR_ANY_DELTA	153	#define MSR_SAVE_FLAGS UART_MSR_ANY_DELTA
154	unsigned char msr_saved_flags;	154	unsigned char msr_saved_flags;
155		155
		156	bool console_newline_needed;
		157
156	struct uart_8250_dma *dma;	158	struct uart_8250_dma *dma;
157	const struct uart_8250_ops *ops;	159	const struct uart_8250_ops *ops;
158		160
Lines 204-209 void serial8250_init_port(struct uart_8250_port *up); Link Here
204	void serial8250_set_defaults(struct uart_8250_port *up);	206	void serial8250_set_defaults(struct uart_8250_port *up);
205	void serial8250_console_write(struct uart_8250_port up, const char s,	207	void serial8250_console_write(struct uart_8250_port up, const char s,
206	unsigned int count);	208	unsigned int count);
		209	void serial8250_console_write_atomic(struct uart_8250_port *up,
		210	struct nbcon_write_context *wctxt);
		211	void serial8250_console_write_thread(struct uart_8250_port *up,
		212	struct nbcon_write_context *wctxt);
207	int serial8250_console_setup(struct uart_port port, char options, bool probe);	213	int serial8250_console_setup(struct uart_port port, char options, bool probe);
208	int serial8250_console_exit(struct uart_port *port);	214	int serial8250_console_exit(struct uart_port *port);
209		215

Lines 8-17 Link Here

(-)a/include/linux/serial_core.h (-2 / +114 lines)
8	#define LINUX_SERIAL_CORE_H	8	#define LINUX_SERIAL_CORE_H
9		9
10	#include <linux/bitops.h>	10	#include <linux/bitops.h>
		11	#include <linux/bug.h>
11	#include <linux/compiler.h>	12	#include <linux/compiler.h>
12	#include <linux/console.h>	13	#include <linux/console.h>
13	#include <linux/interrupt.h>	14	#include <linux/interrupt.h>
14	#include <linux/circ_buf.h>	15	#include <linux/circ_buf.h>
		16	#include <linux/lockdep.h>
		17	#include <linux/printk.h>
15	#include <linux/spinlock.h>	18	#include <linux/spinlock.h>
16	#include <linux/sched.h>	19	#include <linux/sched.h>
17	#include <linux/tty.h>	20	#include <linux/tty.h>
Lines 588-593 struct uart_port { Link Here
588	void private_data; / generic platform data pointer */	591	void private_data; / generic platform data pointer */
589	};	592	};
590		593
		594	/*
		595	* Only for console->device_lock()/_unlock() callbacks and internal
		596	* port lock wrapper synchronization.
		597	*/
		598	static inline void __uart_port_lock_irqsave(struct uart_port up, unsigned long flags)
		599	{
		600	spin_lock_irqsave(&up->lock, *flags);
		601	}
		602
		603	/*
		604	* Only for console->device_lock()/_unlock() callbacks and internal
		605	* port lock wrapper synchronization.
		606	*/
		607	static inline void __uart_port_unlock_irqrestore(struct uart_port *up, unsigned long flags)
		608	{
		609	spin_unlock_irqrestore(&up->lock, flags);
		610	}
		611
		612	/**
		613	* uart_port_set_cons - Safely set the @cons field for a uart
		614	* @up: The uart port to set
		615	* @con: The new console to set to
		616	*
		617	* This function must be used to set @up->cons. It uses the port lock to
		618	* synchronize with the port lock wrappers in order to ensure that the console
		619	* cannot change or disappear while another context is holding the port lock.
		620	*/
		621	static inline void uart_port_set_cons(struct uart_port up, struct console con)
		622	{
		623	unsigned long flags;
		624
		625	__uart_port_lock_irqsave(up, &flags);
		626	up->cons = con;
		627	__uart_port_unlock_irqrestore(up, flags);
		628	}
		629
		630	/* Only for internal port lock wrapper usage. */
		631	static inline void __uart_port_nbcon_acquire(struct uart_port *up)
		632	{
		633	lockdep_assert_held_once(&up->lock);
		634
		635	if (likely(!uart_console(up)))
		636	return;
		637
		638	if (up->cons->nbcon_drvdata) {
		639	/*
		640	* If @up->cons is registered, prevent it from fully
		641	* unregistering until this context releases the nbcon.
		642	*/
		643	int cookie = console_srcu_read_lock();
		644
		645	/* Ensure console is registered and is an nbcon console. */
		646	if (!hlist_unhashed_lockless(&up->cons->node) &&
		647	(console_srcu_read_flags(up->cons) & CON_NBCON)) {
		648	WARN_ON_ONCE(up->cons->nbcon_drvdata->locked);
		649
		650	nbcon_driver_acquire(up->cons);
		651
		652	/*
		653	* Record @up->line to be used during release because
		654	* @up->cons->index can change while the port and
		655	* nbcon are locked.
		656	*/
		657	up->cons->nbcon_drvdata->owner_index = up->line;
658	up->cons->nbcon_drvdata->srcu_cookie = cookie;
659	up->cons->nbcon_drvdata->locked = true;
660	} else {
661	console_srcu_read_unlock(cookie);
662	}
663	}
664	}
665
666	/* Only for internal port lock wrapper usage. */
667	static inline void __uart_port_nbcon_release(struct uart_port *up)
668	{
669	lockdep_assert_held_once(&up->lock);
670
671	/*
672	* uart_console() cannot be used here because @up->cons->index might
673	* have changed. Check against @up->cons->nbcon_drvdata->owner_index
674	* instead.
675	*/
676
677	if (unlikely(up->cons &&
678	up->cons->nbcon_drvdata &&
679	up->cons->nbcon_drvdata->locked &&
680	up->cons->nbcon_drvdata->owner_index == up->line)) {
681	WARN_ON_ONCE(!up->cons->nbcon_drvdata->locked);
682
683	up->cons->nbcon_drvdata->locked = false;
684	nbcon_driver_release(up->cons);
685	console_srcu_read_unlock(up->cons->nbcon_drvdata->srcu_cookie);
686	}
687	}
688
591	/**	689	/**
592	* uart_port_lock - Lock the UART port	690	* uart_port_lock - Lock the UART port
593	* @up: Pointer to UART port structure	691	* @up: Pointer to UART port structure
Lines 595-600 struct uart_port { Link Here
595	static inline void uart_port_lock(struct uart_port *up)	693	static inline void uart_port_lock(struct uart_port *up)
596	{	694	{
597	spin_lock(&up->lock);	695	spin_lock(&up->lock);
		696	__uart_port_nbcon_acquire(up);
598	}	697	}
599		698
600	/**	699	/**
Lines 604-609 static inline void uart_port_lock(struct uart_port *up) Link Here
604	static inline void uart_port_lock_irq(struct uart_port *up)	703	static inline void uart_port_lock_irq(struct uart_port *up)
605	{	704	{
606	spin_lock_irq(&up->lock);	705	spin_lock_irq(&up->lock);
		706	__uart_port_nbcon_acquire(up);
607	}	707	}
608		708
609	/**	709	/**
Lines 614-619 static inline void uart_port_lock_irq(struct uart_port *up) Link Here
614	static inline void uart_port_lock_irqsave(struct uart_port up, unsigned long flags)	714	static inline void uart_port_lock_irqsave(struct uart_port up, unsigned long flags)
615	{	715	{
616	spin_lock_irqsave(&up->lock, *flags);	716	spin_lock_irqsave(&up->lock, *flags);
		717	__uart_port_nbcon_acquire(up);
617	}	718	}
618		719
619	/**	720	/**
Lines 624-630 static inline void uart_port_lock_irqsave(struct uart_port up, unsigned long f Link Here
624	*/	725	*/
625	static inline bool uart_port_trylock(struct uart_port *up)	726	static inline bool uart_port_trylock(struct uart_port *up)
626	{	727	{
627	return spin_trylock(&up->lock);	728	if (!spin_trylock(&up->lock))
		729	return false;
		730
		731	__uart_port_nbcon_acquire(up);
		732	return true;
628	}	733	}
629		734
630	/**	735	/**
Lines 636-642 static inline bool uart_port_trylock(struct uart_port *up) Link Here
636	*/	741	*/
637	static inline bool uart_port_trylock_irqsave(struct uart_port up, unsigned long flags)	742	static inline bool uart_port_trylock_irqsave(struct uart_port up, unsigned long flags)
638	{	743	{
639	return spin_trylock_irqsave(&up->lock, *flags);	744	if (!spin_trylock_irqsave(&up->lock, *flags))
		745	return false;
		746
		747	__uart_port_nbcon_acquire(up);
		748	return true;
640	}	749	}
641		750
642	/**	751	/**
Lines 645-650 static inline bool uart_port_trylock_irqsave(struct uart_port *up, unsigned long Link Here
645	*/	754	*/
646	static inline void uart_port_unlock(struct uart_port *up)	755	static inline void uart_port_unlock(struct uart_port *up)
647	{	756	{
		757	__uart_port_nbcon_release(up);
648	spin_unlock(&up->lock);	758	spin_unlock(&up->lock);
649	}	759	}
650		760
Lines 654-659 static inline void uart_port_unlock(struct uart_port *up) Link Here
654	*/	764	*/
655	static inline void uart_port_unlock_irq(struct uart_port *up)	765	static inline void uart_port_unlock_irq(struct uart_port *up)
656	{	766	{
		767	__uart_port_nbcon_release(up);
657	spin_unlock_irq(&up->lock);	768	spin_unlock_irq(&up->lock);
658	}	769	}
659		770
Lines 664-669 static inline void uart_port_unlock_irq(struct uart_port *up) Link Here
664	*/	775	*/
665	static inline void uart_port_unlock_irqrestore(struct uart_port *up, unsigned long flags)	776	static inline void uart_port_unlock_irqrestore(struct uart_port *up, unsigned long flags)
666	{	777	{
		778	__uart_port_nbcon_release(up);
667	spin_unlock_irqrestore(&up->lock, flags);	779	spin_unlock_irqrestore(&up->lock, flags);
668	}	780	}
669		781

Lines 59-64 enum syscall_work_bit { Link Here

(-)a/include/linux/thread_info.h (+24 lines)
59		59
60	#include <asm/thread_info.h>	60	#include <asm/thread_info.h>
61		61
		62	#ifdef CONFIG_PREEMPT_BUILD_AUTO
		63	# define TIF_NEED_RESCHED_LAZY TIF_ARCH_RESCHED_LAZY
		64	# define _TIF_NEED_RESCHED_LAZY _TIF_ARCH_RESCHED_LAZY
		65	# define TIF_NEED_RESCHED_LAZY_OFFSET (TIF_NEED_RESCHED_LAZY - TIF_NEED_RESCHED)
		66	#else
		67	# define TIF_NEED_RESCHED_LAZY TIF_NEED_RESCHED
		68	# define _TIF_NEED_RESCHED_LAZY _TIF_NEED_RESCHED
		69	# define TIF_NEED_RESCHED_LAZY_OFFSET 0
		70	#endif
		71
62	#ifdef __KERNEL__	72	#ifdef __KERNEL__
63		73
64	#ifndef arch_set_restart_data	74	#ifndef arch_set_restart_data
Lines 185-190 static __always_inline bool tif_need_resched(void) Link Here
185	(unsigned long *)(&current_thread_info()->flags));	195	(unsigned long *)(&current_thread_info()->flags));
186	}	196	}
187		197
		198	static __always_inline bool tif_need_resched_lazy(void)
		199	{
		200	return IS_ENABLED(CONFIG_PREEMPT_BUILD_AUTO) &&
		201	arch_test_bit(TIF_NEED_RESCHED_LAZY,
		202	(unsigned long *)(&current_thread_info()->flags));
		203	}
		204
188	#else	205	#else
189		206
190	static __always_inline bool tif_need_resched(void)	207	static __always_inline bool tif_need_resched(void)
Lines 193-198 static __always_inline bool tif_need_resched(void) Link Here
193	(unsigned long *)(&current_thread_info()->flags));	210	(unsigned long *)(&current_thread_info()->flags));
194	}	211	}
195		212
		213	static __always_inline bool tif_need_resched_lazy(void)
		214	{
		215	return IS_ENABLED(CONFIG_PREEMPT_BUILD_AUTO) &&
		216	test_bit(TIF_NEED_RESCHED_LAZY,
		217	(unsigned long *)(&current_thread_info()->flags));
		218	}
		219
196	#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */	220	#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */
197		221
198	#ifndef CONFIG_HAVE_ARCH_WITHIN_STACK_FRAMES	222	#ifndef CONFIG_HAVE_ARCH_WITHIN_STACK_FRAMES





enum trace_flag_type {
	TRACE_FLAG_IRQS_OFF		= 0x01,
	TRACE_FLAG_NEED_RESCHED		= 0x02,
	TRACE_FLAG_NEED_RESCHED_LAZY	= 0x04,
	TRACE_FLAG_HARDIRQ		= 0x08,
	TRACE_FLAG_SOFTIRQ		= 0x10,
	TRACE_FLAG_PREEMPT_RESCHED	= 0x20,


static inline unsigned int tracing_gen_ctx_flags(unsigned long irqflags)
{
	return tracing_gen_ctx_irq_test(0);
}
static inline unsigned int tracing_gen_ctx(void)
{
	return tracing_gen_ctx_irq_test(0);
}
#endif


Lines 11-16 config PREEMPT_BUILD Link Here

(-)a/kernel/Kconfig.preempt (-1 / +16 lines)
11	select PREEMPTION	11	select PREEMPTION
12	select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK	12	select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK
13		13
		14	config PREEMPT_BUILD_AUTO
		15	bool
		16	select PREEMPT_BUILD
		17
		18	config HAVE_PREEMPT_AUTO
		19	bool
		20
14	choice	21	choice
15	prompt "Preemption Model"	22	prompt "Preemption Model"
16	default PREEMPT_NONE	23	default PREEMPT_NONE
Lines 67-75 config PREEMPT Link Here
67	embedded system with latency requirements in the milliseconds	74	embedded system with latency requirements in the milliseconds
68	range.	75	range.
69		76
		77	config PREEMPT_AUTO
		78	bool "Automagic preemption mode with runtime tweaking support"
		79	depends on HAVE_PREEMPT_AUTO
		80	select PREEMPT_BUILD_AUTO
		81	help
		82	Add some sensible blurb here
		83
70	config PREEMPT_RT	84	config PREEMPT_RT
71	bool "Fully Preemptible Kernel (Real-Time)"	85	bool "Fully Preemptible Kernel (Real-Time)"
72	depends on EXPERT && ARCH_SUPPORTS_RT	86	depends on EXPERT && ARCH_SUPPORTS_RT
		87	select PREEMPT_BUILD_AUTO if HAVE_PREEMPT_AUTO
73	select PREEMPTION	88	select PREEMPTION
74	help	89	help
75	This option turns the kernel into a real-time kernel by replacing	90	This option turns the kernel into a real-time kernel by replacing
Lines 95-101 config PREEMPTION Link Here
95		110
96	config PREEMPT_DYNAMIC	111	config PREEMPT_DYNAMIC
97	bool "Preemption behaviour defined on boot"	112	bool "Preemption behaviour defined on boot"
98	depends on HAVE_PREEMPT_DYNAMIC && !PREEMPT_RT	113	depends on HAVE_PREEMPT_DYNAMIC && !PREEMPT_RT && !PREEMPT_AUTO
99	select JUMP_LABEL if HAVE_PREEMPT_DYNAMIC_KEY	114	select JUMP_LABEL if HAVE_PREEMPT_DYNAMIC_KEY
100	select PREEMPT_BUILD	115	select PREEMPT_BUILD
101	default y if HAVE_PREEMPT_DYNAMIC_CALL	116	default y if HAVE_PREEMPT_DYNAMIC_CALL

Lines 92-98 __always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs, Link Here

(-)a/kernel/entry/common.c (-2 / +2 lines)
92		92
93	local_irq_enable_exit_to_user(ti_work);	93	local_irq_enable_exit_to_user(ti_work);
94		94
95	if (ti_work & _TIF_NEED_RESCHED)	95	if (ti_work & (_TIF_NEED_RESCHED \| _TIF_NEED_RESCHED_LAZY))
96	schedule();	96	schedule();
97		97
98	if (ti_work & _TIF_UPROBE)	98	if (ti_work & _TIF_UPROBE)
Lines 301-307 void raw_irqentry_exit_cond_resched(void) Link Here
301	rcu_irq_exit_check_preempt();	301	rcu_irq_exit_check_preempt();
302	if (IS_ENABLED(CONFIG_DEBUG_ENTRY))	302	if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
303	WARN_ON_ONCE(!on_thread_stack());	303	WARN_ON_ONCE(!on_thread_stack());
304	if (need_resched())	304	if (test_tsk_need_resched(current))
305	preempt_schedule_irq();	305	preempt_schedule_irq();
306	}	306	}
307	}	307	}

Lines 13-19 static int xfer_to_guest_mode_work(struct kvm_vcpu *vcpu, unsigned long ti_work) Link Here

(-)a/kernel/entry/kvm.c (-1 / +1 lines)
13	return -EINTR;	13	return -EINTR;
14	}	14	}
15		15
16	if (ti_work & _TIF_NEED_RESCHED)	16	if (ti_work & (_TIF_NEED_RESCHED \| TIF_NEED_RESCHED_LAZY))
17	schedule();	17	schedule();
18		18
19	if (ti_work & _TIF_NOTIFY_RESUME)	19	if (ti_work & _TIF_NOTIFY_RESUME)




		state = PERF_EVENT_STATE_OFF;
	}

	if (event->pending_sigtrap) {
		bool dec = true;

		event->pending_sigtrap = 0;
		if (state != PERF_EVENT_STATE_OFF &&
		    !event->pending_work) {
			event->pending_work = 1;
			dec = false;
			WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));
			task_work_add(current, &event->pending_task, TWA_RESUME);
		}
		if (dec)
			local_dec(&event->ctx->nr_pending);
	}

	perf_event_set_state(event, state);

	if (!is_software_event(event))

 * hold the top-level event's child_mutex, so any descendant that
 * goes to exit will block in perf_event_exit_event().
 *
 * When called from perf_pending_disable it's OK because event->ctx
 * is the current context on this CPU and preemption is disabled,
 * hence we can't get into perf_event_task_sched_out for this context.
 */

void perf_event_disable_inatomic(struct perf_event *event)
{
	event->pending_disable = 1;
	irq_work_queue(&event->pending_disable_irq);
}

#define MAX_INTERRUPTS (~0ULL)

static void _free_event(struct perf_event *event)
{
	irq_work_sync(&event->pending_irq);
	irq_work_sync(&event->pending_disable_irq);

	unaccount_event(event);


/*
 * Deliver the pending work in-event-context or follow the context.
 */
static void __perf_pending_disable(struct perf_event *event)
{
	int cpu = READ_ONCE(event->oncpu);


	 * Yay, we hit home and are in the context of the event.
	 */
	if (cpu == smp_processor_id()) {
		if (event->pending_sigtrap) {
			event->pending_sigtrap = 0;
			perf_sigtrap(event);
			local_dec(&event->ctx->nr_pending);
		}
		if (event->pending_disable) {
			event->pending_disable = 0;
			perf_event_disable_local(event);

	 *				  irq_work_queue(); // FAILS
	 *
	 *  irq_work_run()
	 *    perf_pending_disable()
	 *
	 * But the event runs on CPU-B and wants disabling there.
	 */
	irq_work_queue_on(&event->pending_disable_irq, cpu);
}

static void perf_pending_disable(struct irq_work *entry)
{
	struct perf_event *event = container_of(entry, struct perf_event, pending_disable_irq);
	int rctx;

	/*
	 * If we 'fail' here, that's OK, it means recursion is already disabled
	 * and we won't recurse 'further'.
	 */
	rctx = perf_swevent_get_recursion_context();
	__perf_pending_disable(event);
	if (rctx >= 0)
		perf_swevent_put_recursion_context(rctx);
}

static void perf_pending_irq(struct irq_work *entry)

		perf_event_wakeup(event);
	}

	__perf_pending_irq(event);

	if (rctx >= 0)
		perf_swevent_put_recursion_context(rctx);
}

static void perf_pending_task(struct callback_head *head)
{
	struct perf_event *event = container_of(head, struct perf_event, pending_task);
	int rctx;

	/*
	 * If we 'fail' here, that's OK, it means recursion is already disabled
	 * and we won't recurse 'further'.
	 */
	preempt_disable_notrace();
	rctx = perf_swevent_get_recursion_context();

	if (event->pending_work) {
		event->pending_work = 0;

		local_dec(&event->ctx->nr_pending);
	}

	if (rctx >= 0)
		perf_swevent_put_recursion_context(rctx);
	preempt_enable_notrace();

	put_event(event);
}



		if (regs)
			pending_id = hash32_ptr((void *)instruction_pointer(regs)) ?: 1;
		if (!event->pending_work) {
			event->pending_work = pending_id;
			local_inc(&event->ctx->nr_pending);
			WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));
			task_work_add(current, &event->pending_task, TWA_RESUME);
			/*
			 * The NMI path returns directly to userland. The
			 * irq_work is raised as a dummy interrupt to ensure
			 * regular return path to user is taken and task_work
			 * is processed.
			 */
			if (in_nmi())
				irq_work_queue(&event->pending_disable_irq);
		} else if (event->attr.exclude_kernel && valid_sample) {
			/*
			 * Should not be able to return to user space without
			 * consuming pending_work; with exceptions:
			 *
			 *  1. Where !exclude_kernel, events can overflow again
			 *     in the kernel without returning to user space.

			 *     To approximate progress (with false negatives),
			 *     check 32-bit hash of the current IP.
			 */
			WARN_ON_ONCE(event->pending_work != pending_id);
		}

		event->pending_addr = 0;
		if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR))
			event->pending_addr = data->addr;
		irq_work_queue(&event->pending_irq);
	}

	READ_ONCE(event->overflow_handler)(event, data, regs);


	init_waitqueue_head(&event->waitq);
	init_irq_work(&event->pending_irq, perf_pending_irq);
	event->pending_disable_irq = IRQ_WORK_INIT_HARD(perf_pending_disable);
	init_task_work(&event->pending_task, perf_pending_task);

	mutex_init(&event->mmap_mutex);

		     &parent_event->child_total_time_running);
}

static bool task_work_cb_match(struct callback_head *cb, void *data)
{
	struct perf_event *event = container_of(cb, struct perf_event, pending_task);

	return event == data;
}

static void
perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx)
{

		 * Kick perf_poll() for is_event_hup();
		 */
		perf_event_wakeup(parent_event);
		/*
		 * Cancel pending task_work and update counters if it has not
		 * yet been delivered to userland. free_event() expects the
		 * reference counter at 1 and keeping the event around until the
		 * task return to userland will be a unexpected.
		 */
		if (event->pending_work &&
		    task_work_cancel_match(current, task_work_cb_match, event)) {
			put_event(event);
			local_dec(&event->ctx->nr_pending);
		}
		free_event(event);
		put_event(parent_event);
		return;

Lines 179-184 KERNEL_ATTR_RO(crash_elfcorehdr_size); Link Here

(-)a/kernel/ksysfs.c (+12 lines)
179		179
180	#endif /* CONFIG_CRASH_CORE */	180	#endif /* CONFIG_CRASH_CORE */
181		181
		182	#if defined(CONFIG_PREEMPT_RT)
		183	static ssize_t realtime_show(struct kobject *kobj,
		184	struct kobj_attribute attr, char buf)
		185	{
		186	return sprintf(buf, "%d\n", 1);
		187	}
		188	KERNEL_ATTR_RO(realtime);
		189	#endif
		190
182	/* whether file capabilities are enabled */	191	/* whether file capabilities are enabled */
183	static ssize_t fscaps_show(struct kobject *kobj,	192	static ssize_t fscaps_show(struct kobject *kobj,
184	struct kobj_attribute attr, char buf)	193	struct kobj_attribute attr, char buf)
Lines 274-279 static struct attribute * kernel_attrs[] = { Link Here
274	#ifndef CONFIG_TINY_RCU	283	#ifndef CONFIG_TINY_RCU
275	&rcu_expedited_attr.attr,	284	&rcu_expedited_attr.attr,
276	&rcu_normal_attr.attr,	285	&rcu_normal_attr.attr,
		286	#endif
		287	#ifdef CONFIG_PREEMPT_RT
		288	&realtime_attr.attr,
277	#endif	289	#endif
278	NULL	290	NULL
279	};	291	};

Lines 56-61 Link Here

(-)a/kernel/locking/lockdep.c (-3 / +88 lines)
56	#include <linux/kprobes.h>	56	#include <linux/kprobes.h>
57	#include <linux/lockdep.h>	57	#include <linux/lockdep.h>
58	#include <linux/context_tracking.h>	58	#include <linux/context_tracking.h>
		59	#include <linux/console.h>
59		60
60	#include <asm/sections.h>	61	#include <asm/sections.h>
61		62
Lines 574-581 static struct lock_trace *save_trace(void) Link Here
574	if (!debug_locks_off_graph_unlock())	575	if (!debug_locks_off_graph_unlock())
575	return NULL;	576	return NULL;
576		577
		578	nbcon_cpu_emergency_enter();
577	print_lockdep_off("BUG: MAX_STACK_TRACE_ENTRIES too low!");	579	print_lockdep_off("BUG: MAX_STACK_TRACE_ENTRIES too low!");
578	dump_stack();	580	dump_stack();
		581	nbcon_cpu_emergency_exit();
579		582
580	return NULL;	583	return NULL;
581	}	584	}
Lines 782-787 static void lockdep_print_held_locks(struct task_struct *p) Link Here
782	{	785	{
783	int i, depth = READ_ONCE(p->lockdep_depth);	786	int i, depth = READ_ONCE(p->lockdep_depth);
784		787
		788	nbcon_cpu_emergency_enter();
		789
785	if (!depth)	790	if (!depth)
786	printk("no locks held by %s/%d.\n", p->comm, task_pid_nr(p));	791	printk("no locks held by %s/%d.\n", p->comm, task_pid_nr(p));
787	else	792	else
Lines 792-802 static void lockdep_print_held_locks(struct task_struct *p) Link Here
792	* and it's not the current task.	797	* and it's not the current task.
793	*/	798	*/
794	if (p != current && task_is_running(p))	799	if (p != current && task_is_running(p))
795	return;	800	goto out;
796	for (i = 0; i < depth; i++) {	801	for (i = 0; i < depth; i++) {
797	printk(" #%d: ", i);	802	printk(" #%d: ", i);
798	print_lock(p->held_locks + i);	803	print_lock(p->held_locks + i);
799	}	804	}
		805	out:
		806	nbcon_cpu_emergency_exit();
800	}	807	}
801		808
802	static void print_kernel_ident(void)	809	static void print_kernel_ident(void)
Lines 888-898 look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass) Link Here
888	if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) {	895	if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) {
889	instrumentation_begin();	896	instrumentation_begin();
890	debug_locks_off();	897	debug_locks_off();
		898	nbcon_cpu_emergency_enter();
891	printk(KERN_ERR	899	printk(KERN_ERR
892	"BUG: looking up invalid subclass: %u\n", subclass);	900	"BUG: looking up invalid subclass: %u\n", subclass);
893	printk(KERN_ERR	901	printk(KERN_ERR
894	"turning off the locking correctness validator.\n");	902	"turning off the locking correctness validator.\n");
895	dump_stack();	903	dump_stack();
		904	nbcon_cpu_emergency_exit();
896	instrumentation_end();	905	instrumentation_end();
897	return NULL;	906	return NULL;
898	}	907	}
Lines 969-979 static bool assign_lock_key(struct lockdep_map *lock) Link Here
969	else {	978	else {
970	/* Debug-check: all keys must be persistent! */	979	/* Debug-check: all keys must be persistent! */
971	debug_locks_off();	980	debug_locks_off();
		981	nbcon_cpu_emergency_enter();
972	pr_err("INFO: trying to register non-static key.\n");	982	pr_err("INFO: trying to register non-static key.\n");
973	pr_err("The code is fine but needs lockdep annotation, or maybe\n");	983	pr_err("The code is fine but needs lockdep annotation, or maybe\n");
974	pr_err("you didn't initialize this object before use?\n");	984	pr_err("you didn't initialize this object before use?\n");
975	pr_err("turning off the locking correctness validator.\n");	985	pr_err("turning off the locking correctness validator.\n");
976	dump_stack();	986	dump_stack();
		987	nbcon_cpu_emergency_exit();
977	return false;	988	return false;
978	}	989	}
979		990
Lines 1317-1324 register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force) Link Here
1317	return NULL;	1328	return NULL;
1318	}	1329	}
1319		1330
		1331	nbcon_cpu_emergency_enter();
1320	print_lockdep_off("BUG: MAX_LOCKDEP_KEYS too low!");	1332	print_lockdep_off("BUG: MAX_LOCKDEP_KEYS too low!");
1321	dump_stack();	1333	dump_stack();
		1334	nbcon_cpu_emergency_exit();
1322	return NULL;	1335	return NULL;
1323	}	1336	}
1324	nr_lock_classes++;	1337	nr_lock_classes++;
Lines 1350-1360 register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force) Link Here
1350	if (verbose(class)) {	1363	if (verbose(class)) {
1351	graph_unlock();	1364	graph_unlock();
1352		1365
		1366	nbcon_cpu_emergency_enter();
1353	printk("\nnew class %px: %s", class->key, class->name);	1367	printk("\nnew class %px: %s", class->key, class->name);
1354	if (class->name_version > 1)	1368	if (class->name_version > 1)
1355	printk(KERN_CONT "#%d", class->name_version);	1369	printk(KERN_CONT "#%d", class->name_version);
1356	printk(KERN_CONT "\n");	1370	printk(KERN_CONT "\n");
1357	dump_stack();	1371	dump_stack();
		1372	nbcon_cpu_emergency_exit();
1358		1373
1359	if (!graph_lock()) {	1374	if (!graph_lock()) {
1360	return NULL;	1375	return NULL;
Lines 1393-1400 static struct lock_list *alloc_list_entry(void) Link Here
1393	if (!debug_locks_off_graph_unlock())	1408	if (!debug_locks_off_graph_unlock())
1394	return NULL;	1409	return NULL;
1395		1410
		1411	nbcon_cpu_emergency_enter();
1396	print_lockdep_off("BUG: MAX_LOCKDEP_ENTRIES too low!");	1412	print_lockdep_off("BUG: MAX_LOCKDEP_ENTRIES too low!");
1397	dump_stack();	1413	dump_stack();
		1414	nbcon_cpu_emergency_exit();
1398	return NULL;	1415	return NULL;
1399	}	1416	}
1400	nr_list_entries++;	1417	nr_list_entries++;
Lines 2040-2045 static noinline void print_circular_bug(struct lock_list *this, Link Here
2040		2057
2041	depth = get_lock_depth(target);	2058	depth = get_lock_depth(target);
2042		2059
		2060	nbcon_cpu_emergency_enter();
		2061
2043	print_circular_bug_header(target, depth, check_src, check_tgt);	2062	print_circular_bug_header(target, depth, check_src, check_tgt);
2044		2063
2045	parent = get_lock_parent(target);	2064	parent = get_lock_parent(target);
Lines 2058-2063 static noinline void print_circular_bug(struct lock_list *this, Link Here
2058		2077
2059	printk("\nstack backtrace:\n");	2078	printk("\nstack backtrace:\n");
2060	dump_stack();	2079	dump_stack();
		2080
		2081	nbcon_cpu_emergency_exit();
2061	}	2082	}
2062		2083
2063	static noinline void print_bfs_bug(int ret)	2084	static noinline void print_bfs_bug(int ret)
Lines 2570-2575 print_bad_irq_dependency(struct task_struct *curr, Link Here
2570	if (!debug_locks_off_graph_unlock() \|\| debug_locks_silent)	2591	if (!debug_locks_off_graph_unlock() \|\| debug_locks_silent)
2571	return;	2592	return;
2572		2593
		2594	nbcon_cpu_emergency_enter();
		2595
2573	pr_warn("\n");	2596	pr_warn("\n");
2574	pr_warn("=====================================================\n");	2597	pr_warn("=====================================================\n");
2575	pr_warn("WARNING: %s-safe -> %s-unsafe lock order detected\n",	2598	pr_warn("WARNING: %s-safe -> %s-unsafe lock order detected\n",
Lines 2619-2629 print_bad_irq_dependency(struct task_struct *curr, Link Here
2619	pr_warn(" and %s-irq-unsafe lock:\n", irqclass);	2642	pr_warn(" and %s-irq-unsafe lock:\n", irqclass);
2620	next_root->trace = save_trace();	2643	next_root->trace = save_trace();
2621	if (!next_root->trace)	2644	if (!next_root->trace)
2622	return;	2645	goto out;
2623	print_shortest_lock_dependencies(forwards_entry, next_root);	2646	print_shortest_lock_dependencies(forwards_entry, next_root);
2624		2647
2625	pr_warn("\nstack backtrace:\n");	2648	pr_warn("\nstack backtrace:\n");
2626	dump_stack();	2649	dump_stack();
		2650	out:
		2651	nbcon_cpu_emergency_exit();
2627	}	2652	}
2628		2653
2629	static const char *state_names[] = {	2654	static const char *state_names[] = {
Lines 2988-2993 print_deadlock_bug(struct task_struct curr, struct held_lock prev, Link Here
2988	if (!debug_locks_off_graph_unlock() \|\| debug_locks_silent)	3013	if (!debug_locks_off_graph_unlock() \|\| debug_locks_silent)
2989	return;	3014	return;
2990		3015
		3016	nbcon_cpu_emergency_enter();
		3017
2991	pr_warn("\n");	3018	pr_warn("\n");
2992	pr_warn("============================================\n");	3019	pr_warn("============================================\n");
2993	pr_warn("WARNING: possible recursive locking detected\n");	3020	pr_warn("WARNING: possible recursive locking detected\n");
Lines 3010-3015 print_deadlock_bug(struct task_struct curr, struct held_lock prev, Link Here
3010		3037
3011	pr_warn("\nstack backtrace:\n");	3038	pr_warn("\nstack backtrace:\n");
3012	dump_stack();	3039	dump_stack();
		3040
		3041	nbcon_cpu_emergency_exit();
3013	}	3042	}
3014		3043
3015	/*	3044	/*
Lines 3607-3612 static void print_collision(struct task_struct *curr, Link Here
3607	struct held_lock *hlock_next,	3636	struct held_lock *hlock_next,
3608	struct lock_chain *chain)	3637	struct lock_chain *chain)
3609	{	3638	{
		3639	nbcon_cpu_emergency_enter();
		3640
3610	pr_warn("\n");	3641	pr_warn("\n");
3611	pr_warn("============================\n");	3642	pr_warn("============================\n");
3612	pr_warn("WARNING: chain_key collision\n");	3643	pr_warn("WARNING: chain_key collision\n");
Lines 3623-3628 static void print_collision(struct task_struct *curr, Link Here
3623		3654
3624	pr_warn("\nstack backtrace:\n");	3655	pr_warn("\nstack backtrace:\n");
3625	dump_stack();	3656	dump_stack();
		3657
		3658	nbcon_cpu_emergency_exit();
3626	}	3659	}
3627	#endif	3660	#endif
3628		3661
Lines 3713-3720 static inline int add_chain_cache(struct task_struct *curr, Link Here
3713	if (!debug_locks_off_graph_unlock())	3746	if (!debug_locks_off_graph_unlock())
3714	return 0;	3747	return 0;
3715		3748
		3749	nbcon_cpu_emergency_enter();
3716	print_lockdep_off("BUG: MAX_LOCKDEP_CHAINS too low!");	3750	print_lockdep_off("BUG: MAX_LOCKDEP_CHAINS too low!");
3717	dump_stack();	3751	dump_stack();
		3752	nbcon_cpu_emergency_exit();
3718	return 0;	3753	return 0;
3719	}	3754	}
3720	chain->chain_key = chain_key;	3755	chain->chain_key = chain_key;
Lines 3731-3738 static inline int add_chain_cache(struct task_struct *curr, Link Here
3731	if (!debug_locks_off_graph_unlock())	3766	if (!debug_locks_off_graph_unlock())
3732	return 0;	3767	return 0;
3733		3768
		3769	nbcon_cpu_emergency_enter();
3734	print_lockdep_off("BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!");	3770	print_lockdep_off("BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!");
3735	dump_stack();	3771	dump_stack();
		3772	nbcon_cpu_emergency_exit();
3736	return 0;	3773	return 0;
3737	}	3774	}
3738		3775
Lines 3971-3976 print_usage_bug(struct task_struct curr, struct held_lock this, Link Here
3971	if (!debug_locks_off() \|\| debug_locks_silent)	4008	if (!debug_locks_off() \|\| debug_locks_silent)
3972	return;	4009	return;
3973		4010
		4011	nbcon_cpu_emergency_enter();
		4012
3974	pr_warn("\n");	4013	pr_warn("\n");
3975	pr_warn("================================\n");	4014	pr_warn("================================\n");
3976	pr_warn("WARNING: inconsistent lock state\n");	4015	pr_warn("WARNING: inconsistent lock state\n");
Lines 3999-4004 print_usage_bug(struct task_struct curr, struct held_lock this, Link Here
3999		4038
4000	pr_warn("\nstack backtrace:\n");	4039	pr_warn("\nstack backtrace:\n");
4001	dump_stack();	4040	dump_stack();
		4041
		4042	nbcon_cpu_emergency_exit();
4002	}	4043	}
4003		4044
4004	/*	4045	/*
Lines 4033-4038 print_irq_inversion_bug(struct task_struct *curr, Link Here
4033	if (!debug_locks_off_graph_unlock() \|\| debug_locks_silent)	4074	if (!debug_locks_off_graph_unlock() \|\| debug_locks_silent)
4034	return;	4075	return;
4035		4076
		4077	nbcon_cpu_emergency_enter();
		4078
4036	pr_warn("\n");	4079	pr_warn("\n");
4037	pr_warn("========================================================\n");	4080	pr_warn("========================================================\n");
4038	pr_warn("WARNING: possible irq lock inversion dependency detected\n");	4081	pr_warn("WARNING: possible irq lock inversion dependency detected\n");
Lines 4073-4083 print_irq_inversion_bug(struct task_struct *curr, Link Here
4073	pr_warn("\nthe shortest dependencies between 2nd lock and 1st lock:\n");	4116	pr_warn("\nthe shortest dependencies between 2nd lock and 1st lock:\n");
4074	root->trace = save_trace();	4117	root->trace = save_trace();
4075	if (!root->trace)	4118	if (!root->trace)
4076	return;	4119	goto out;
4077	print_shortest_lock_dependencies(other, root);	4120	print_shortest_lock_dependencies(other, root);
4078		4121
4079	pr_warn("\nstack backtrace:\n");	4122	pr_warn("\nstack backtrace:\n");
4080	dump_stack();	4123	dump_stack();
		4124	out:
		4125	nbcon_cpu_emergency_exit();
4081	}	4126	}
4082		4127
4083	/*	4128	/*
Lines 4154-4159 void print_irqtrace_events(struct task_struct *curr) Link Here
4154	{	4199	{
4155	const struct irqtrace_events *trace = &curr->irqtrace;	4200	const struct irqtrace_events *trace = &curr->irqtrace;
4156		4201
		4202	nbcon_cpu_emergency_enter();
		4203
4157	printk("irq event stamp: %u\n", trace->irq_events);	4204	printk("irq event stamp: %u\n", trace->irq_events);
4158	printk("hardirqs last enabled at (%u): [<%px>] %pS\n",	4205	printk("hardirqs last enabled at (%u): [<%px>] %pS\n",
4159	trace->hardirq_enable_event, (void *)trace->hardirq_enable_ip,	4206	trace->hardirq_enable_event, (void *)trace->hardirq_enable_ip,
Lines 4167-4172 void print_irqtrace_events(struct task_struct *curr) Link Here
4167	printk("softirqs last disabled at (%u): [<%px>] %pS\n",	4214	printk("softirqs last disabled at (%u): [<%px>] %pS\n",
4168	trace->softirq_disable_event, (void *)trace->softirq_disable_ip,	4215	trace->softirq_disable_event, (void *)trace->softirq_disable_ip,
4169	(void *)trace->softirq_disable_ip);	4216	(void *)trace->softirq_disable_ip);
		4217
		4218	nbcon_cpu_emergency_exit();
4170	}	4219	}
4171		4220
4172	static int HARDIRQ_verbose(struct lock_class *class)	4221	static int HARDIRQ_verbose(struct lock_class *class)
Lines 4687-4696 static int mark_lock(struct task_struct curr, struct held_lock this, Link Here
4687	* We must printk outside of the graph_lock:	4736	* We must printk outside of the graph_lock:
4688	*/	4737	*/
4689	if (ret == 2) {	4738	if (ret == 2) {
		4739	nbcon_cpu_emergency_enter();
4690	printk("\nmarked lock as {%s}:\n", usage_str[new_bit]);	4740	printk("\nmarked lock as {%s}:\n", usage_str[new_bit]);
4691	print_lock(this);	4741	print_lock(this);
4692	print_irqtrace_events(curr);	4742	print_irqtrace_events(curr);
4693	dump_stack();	4743	dump_stack();
		4744	nbcon_cpu_emergency_exit();
4694	}	4745	}
4695		4746
4696	return ret;	4747	return ret;
Lines 4731-4736 print_lock_invalid_wait_context(struct task_struct *curr, Link Here
4731	if (debug_locks_silent)	4782	if (debug_locks_silent)
4732	return 0;	4783	return 0;
4733		4784
		4785	nbcon_cpu_emergency_enter();
		4786
4734	pr_warn("\n");	4787	pr_warn("\n");
4735	pr_warn("=============================\n");	4788	pr_warn("=============================\n");
4736	pr_warn("[ BUG: Invalid wait context ]\n");	4789	pr_warn("[ BUG: Invalid wait context ]\n");
Lines 4750-4755 print_lock_invalid_wait_context(struct task_struct *curr, Link Here
4750	pr_warn("stack backtrace:\n");	4803	pr_warn("stack backtrace:\n");
4751	dump_stack();	4804	dump_stack();
4752		4805
		4806	nbcon_cpu_emergency_exit();
		4807
4753	return 0;	4808	return 0;
4754	}	4809	}
4755		4810
Lines 4954-4959 print_lock_nested_lock_not_held(struct task_struct *curr, Link Here
4954	if (debug_locks_silent)	5009	if (debug_locks_silent)
4955	return;	5010	return;
4956		5011
		5012	nbcon_cpu_emergency_enter();
		5013
4957	pr_warn("\n");	5014	pr_warn("\n");
4958	pr_warn("==================================\n");	5015	pr_warn("==================================\n");
4959	pr_warn("WARNING: Nested lock was not taken\n");	5016	pr_warn("WARNING: Nested lock was not taken\n");
Lines 4974-4979 print_lock_nested_lock_not_held(struct task_struct *curr, Link Here
4974		5031
4975	pr_warn("\nstack backtrace:\n");	5032	pr_warn("\nstack backtrace:\n");
4976	dump_stack();	5033	dump_stack();
		5034
		5035	nbcon_cpu_emergency_exit();
4977	}	5036	}
4978		5037
4979	static int __lock_is_held(const struct lockdep_map *lock, int read);	5038	static int __lock_is_held(const struct lockdep_map *lock, int read);
Lines 5019-5029 static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, Link Here
5019	debug_class_ops_inc(class);	5078	debug_class_ops_inc(class);
5020		5079
5021	if (very_verbose(class)) {	5080	if (very_verbose(class)) {
		5081	nbcon_cpu_emergency_enter();
5022	printk("\nacquire class [%px] %s", class->key, class->name);	5082	printk("\nacquire class [%px] %s", class->key, class->name);
5023	if (class->name_version > 1)	5083	if (class->name_version > 1)
5024	printk(KERN_CONT "#%d", class->name_version);	5084	printk(KERN_CONT "#%d", class->name_version);
5025	printk(KERN_CONT "\n");	5085	printk(KERN_CONT "\n");
5026	dump_stack();	5086	dump_stack();
		5087	nbcon_cpu_emergency_exit();
5027	}	5088	}
5028		5089
5029	/*	5090	/*
Lines 5150-5155 static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, Link Here
5150	#endif	5211	#endif
5151	if (unlikely(curr->lockdep_depth >= MAX_LOCK_DEPTH)) {	5212	if (unlikely(curr->lockdep_depth >= MAX_LOCK_DEPTH)) {
5152	debug_locks_off();	5213	debug_locks_off();
		5214	nbcon_cpu_emergency_enter();
5153	print_lockdep_off("BUG: MAX_LOCK_DEPTH too low!");	5215	print_lockdep_off("BUG: MAX_LOCK_DEPTH too low!");
5154	printk(KERN_DEBUG "depth: %i max: %lu!\n",	5216	printk(KERN_DEBUG "depth: %i max: %lu!\n",
5155	curr->lockdep_depth, MAX_LOCK_DEPTH);	5217	curr->lockdep_depth, MAX_LOCK_DEPTH);
Lines 5157-5162 static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, Link Here
5157	lockdep_print_held_locks(current);	5219	lockdep_print_held_locks(current);
5158	debug_show_all_locks();	5220	debug_show_all_locks();
5159	dump_stack();	5221	dump_stack();
		5222	nbcon_cpu_emergency_exit();
5160		5223
5161	return 0;	5224	return 0;
5162	}	5225	}
Lines 5176-5181 static void print_unlock_imbalance_bug(struct task_struct *curr, Link Here
5176	if (debug_locks_silent)	5239	if (debug_locks_silent)
5177	return;	5240	return;
5178		5241
		5242	nbcon_cpu_emergency_enter();
		5243
5179	pr_warn("\n");	5244	pr_warn("\n");
5180	pr_warn("=====================================\n");	5245	pr_warn("=====================================\n");
5181	pr_warn("WARNING: bad unlock balance detected!\n");	5246	pr_warn("WARNING: bad unlock balance detected!\n");
Lines 5192-5197 static void print_unlock_imbalance_bug(struct task_struct *curr, Link Here
5192		5257
5193	pr_warn("\nstack backtrace:\n");	5258	pr_warn("\nstack backtrace:\n");
5194	dump_stack();	5259	dump_stack();
		5260
		5261	nbcon_cpu_emergency_exit();
5195	}	5262	}
5196		5263
5197	static noinstr int match_held_lock(const struct held_lock *hlock,	5264	static noinstr int match_held_lock(const struct held_lock *hlock,
Lines 5895-5900 static void print_lock_contention_bug(struct task_struct *curr, Link Here
5895	if (debug_locks_silent)	5962	if (debug_locks_silent)
5896	return;	5963	return;
5897		5964
		5965	nbcon_cpu_emergency_enter();
		5966
5898	pr_warn("\n");	5967	pr_warn("\n");
5899	pr_warn("=================================\n");	5968	pr_warn("=================================\n");
5900	pr_warn("WARNING: bad contention detected!\n");	5969	pr_warn("WARNING: bad contention detected!\n");
Lines 5911-5916 static void print_lock_contention_bug(struct task_struct *curr, Link Here
5911		5980
5912	pr_warn("\nstack backtrace:\n");	5981	pr_warn("\nstack backtrace:\n");
5913	dump_stack();	5982	dump_stack();
		5983
		5984	nbcon_cpu_emergency_exit();
5914	}	5985	}
5915		5986
5916	static void	5987	static void
Lines 6524-6529 print_freed_lock_bug(struct task_struct curr, const void mem_from, Link Here
6524	if (debug_locks_silent)	6595	if (debug_locks_silent)
6525	return;	6596	return;
6526		6597
		6598	nbcon_cpu_emergency_enter();
		6599
6527	pr_warn("\n");	6600	pr_warn("\n");
6528	pr_warn("=========================\n");	6601	pr_warn("=========================\n");
6529	pr_warn("WARNING: held lock freed!\n");	6602	pr_warn("WARNING: held lock freed!\n");
Lines 6536-6541 print_freed_lock_bug(struct task_struct curr, const void mem_from, Link Here
6536		6609
6537	pr_warn("\nstack backtrace:\n");	6610	pr_warn("\nstack backtrace:\n");
6538	dump_stack();	6611	dump_stack();
		6612
		6613	nbcon_cpu_emergency_exit();
6539	}	6614	}
6540		6615
6541	static inline int not_in_range(const void* mem_from, unsigned long mem_len,	6616	static inline int not_in_range(const void* mem_from, unsigned long mem_len,
Lines 6582-6587 static void print_held_locks_bug(void) Link Here
6582	if (debug_locks_silent)	6657	if (debug_locks_silent)
6583	return;	6658	return;
6584		6659
		6660	nbcon_cpu_emergency_enter();
		6661
6585	pr_warn("\n");	6662	pr_warn("\n");
6586	pr_warn("====================================\n");	6663	pr_warn("====================================\n");
6587	pr_warn("WARNING: %s/%d still has locks held!\n",	6664	pr_warn("WARNING: %s/%d still has locks held!\n",
Lines 6591-6596 static void print_held_locks_bug(void) Link Here
6591	lockdep_print_held_locks(current);	6668	lockdep_print_held_locks(current);
6592	pr_warn("\nstack backtrace:\n");	6669	pr_warn("\nstack backtrace:\n");
6593	dump_stack();	6670	dump_stack();
		6671
		6672	nbcon_cpu_emergency_exit();
6594	}	6673	}
6595		6674
6596	void debug_check_no_locks_held(void)	6675	void debug_check_no_locks_held(void)
Lines 6609-6614 void debug_show_all_locks(void) Link Here
6609	pr_warn("INFO: lockdep is turned off.\n");	6688	pr_warn("INFO: lockdep is turned off.\n");
6610	return;	6689	return;
6611	}	6690	}
		6691	nbcon_cpu_emergency_enter();
6612	pr_warn("\nShowing all locks held in the system:\n");	6692	pr_warn("\nShowing all locks held in the system:\n");
6613		6693
6614	rcu_read_lock();	6694	rcu_read_lock();
Lines 6623-6628 void debug_show_all_locks(void) Link Here
6623		6703
6624	pr_warn("\n");	6704	pr_warn("\n");
6625	pr_warn("=============================================\n\n");	6705	pr_warn("=============================================\n\n");
		6706	nbcon_cpu_emergency_exit();
6626	}	6707	}
6627	EXPORT_SYMBOL_GPL(debug_show_all_locks);	6708	EXPORT_SYMBOL_GPL(debug_show_all_locks);
6628	#endif	6709	#endif
Lines 6648-6653 asmlinkage __visible void lockdep_sys_exit(void) Link Here
6648	if (unlikely(curr->lockdep_depth)) {	6729	if (unlikely(curr->lockdep_depth)) {
6649	if (!debug_locks_off())	6730	if (!debug_locks_off())
6650	return;	6731	return;
		6732	nbcon_cpu_emergency_enter();
6651	pr_warn("\n");	6733	pr_warn("\n");
6652	pr_warn("================================================\n");	6734	pr_warn("================================================\n");
6653	pr_warn("WARNING: lock held when returning to user space!\n");	6735	pr_warn("WARNING: lock held when returning to user space!\n");
Lines 6656-6661 asmlinkage __visible void lockdep_sys_exit(void) Link Here
6656	pr_warn("%s/%d is leaving the kernel with locks still held!\n",	6738	pr_warn("%s/%d is leaving the kernel with locks still held!\n",
6657	curr->comm, curr->pid);	6739	curr->comm, curr->pid);
6658	lockdep_print_held_locks(curr);	6740	lockdep_print_held_locks(curr);
		6741	nbcon_cpu_emergency_exit();
6659	}	6742	}
6660		6743
6661	/*	6744	/*
Lines 6672-6677 void lockdep_rcu_suspicious(const char file, const int line, const char s) Link Here
6672	bool rcu = warn_rcu_enter();	6755	bool rcu = warn_rcu_enter();
6673		6756
6674	/* Note: the following can be executed concurrently, so be careful. */	6757	/* Note: the following can be executed concurrently, so be careful. */
		6758	nbcon_cpu_emergency_enter();
6675	pr_warn("\n");	6759	pr_warn("\n");
6676	pr_warn("=============================\n");	6760	pr_warn("=============================\n");
6677	pr_warn("WARNING: suspicious RCU usage\n");	6761	pr_warn("WARNING: suspicious RCU usage\n");
Lines 6710-6715 void lockdep_rcu_suspicious(const char file, const int line, const char s) Link Here
6710	lockdep_print_held_locks(curr);	6794	lockdep_print_held_locks(curr);
6711	pr_warn("\nstack backtrace:\n");	6795	pr_warn("\nstack backtrace:\n");
6712	dump_stack();	6796	dump_stack();
		6797	nbcon_cpu_emergency_exit();
6713	warn_rcu_exit(rcu);	6798	warn_rcu_exit(rcu);
6714	}	6799	}
6715	EXPORT_SYMBOL_GPL(lockdep_rcu_suspicious);	6800	EXPORT_SYMBOL_GPL(lockdep_rcu_suspicious);

Lines 364-369 void panic(const char *fmt, ...) Link Here

(-)a/kernel/panic.c (+17 lines)
364		364
365	panic_other_cpus_shutdown(_crash_kexec_post_notifiers);	365	panic_other_cpus_shutdown(_crash_kexec_post_notifiers);
366		366
		367	printk_legacy_allow_panic_sync();
		368
367	/*	369	/*
368	* Run any panic handlers, including those that might need to	370	* Run any panic handlers, including those that might need to
369	* add information to the kmsg dump output.	371	* add information to the kmsg dump output.
Lines 446-451 void panic(const char *fmt, ...) Link Here
446		448
447	/* Do not scroll important messages printed above */	449	/* Do not scroll important messages printed above */
448	suppress_printk = 1;	450	suppress_printk = 1;
		451
		452	/*
		453	* The final messages may not have been printed if in a context that
		454	* defers printing (such as NMI) and irq_work is not available.
		455	* Explicitly flush the kernel log buffer one last time.
		456	*/
		457	console_flush_on_panic(CONSOLE_FLUSH_PENDING);
		458	nbcon_atomic_flush_unsafe();
		459
449	local_irq_enable();	460	local_irq_enable();
450	for (i = 0; ; i += PANIC_TIMER_STEP) {	461	for (i = 0; ; i += PANIC_TIMER_STEP) {
451	touch_softlockup_watchdog();	462	touch_softlockup_watchdog();
Lines 623-628 bool oops_may_print(void) Link Here
623	*/	634	*/
624	void oops_enter(void)	635	void oops_enter(void)
625	{	636	{
		637	nbcon_cpu_emergency_enter();
626	tracing_off();	638	tracing_off();
627	/* can't trust the integrity of the kernel anymore: */	639	/* can't trust the integrity of the kernel anymore: */
628	debug_locks_off();	640	debug_locks_off();
Lines 645-650 void oops_exit(void) Link Here
645	{	657	{
646	do_oops_enter_exit();	658	do_oops_enter_exit();
647	print_oops_end_marker();	659	print_oops_end_marker();
		660	nbcon_cpu_emergency_exit();
648	kmsg_dump(KMSG_DUMP_OOPS);	661	kmsg_dump(KMSG_DUMP_OOPS);
649	}	662	}
650		663
Lines 656-661 struct warn_args { Link Here
656	void __warn(const char file, int line, void caller, unsigned taint,	669	void __warn(const char file, int line, void caller, unsigned taint,
657	struct pt_regs regs, struct warn_args args)	670	struct pt_regs regs, struct warn_args args)
658	{	671	{
		672	nbcon_cpu_emergency_enter();
		673
659	disable_trace_on_warning();	674	disable_trace_on_warning();
660		675
661	if (file)	676	if (file)
Lines 686-691 void __warn(const char file, int line, void caller, unsigned taint, Link Here
686		701
687	/* Just a warning, don't kill lockdep. */	702	/* Just a warning, don't kill lockdep. */
688	add_taint(taint, LOCKDEP_STILL_OK);	703	add_taint(taint, LOCKDEP_STILL_OK);
		704
		705	nbcon_cpu_emergency_exit();
689	}	706	}
690		707
691	#ifdef CONFIG_BUG	708	#ifdef CONFIG_BUG

Lines 2-12 Link Here

(-)a/kernel/printk/internal.h (-3 / +108 lines)
2	/*	2	/*
3	* internal.h - printk internal definitions	3	* internal.h - printk internal definitions
4	*/	4	*/
5	#include <linux/percpu.h>
6	#include <linux/console.h>	5	#include <linux/console.h>
7	#include "printk_ringbuffer.h"	6	#include <linux/jump_label.h>
		7	#include <linux/percpu.h>
		8	#include <linux/types.h>
8		9
9	#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)	10	#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)
		11	struct ctl_table;
10	void __init printk_sysctl_init(void);	12	void __init printk_sysctl_init(void);
11	int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write,	13	int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write,
12	void buffer, size_t lenp, loff_t *ppos);	14	void buffer, size_t lenp, loff_t *ppos);
Lines 20-25 int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write, Link Here
20	(con->flags & CON_BOOT) ? "boot" : "", \	22	(con->flags & CON_BOOT) ? "boot" : "", \
21	con->name, con->index, ##__VA_ARGS__)	23	con->name, con->index, ##__VA_ARGS__)
22		24
		25	#ifdef CONFIG_PREEMPT_RT
		26	# define force_printkthreads() (true)
		27	#else
		28	DECLARE_STATIC_KEY_FALSE(force_printkthreads_key);
		29	# define force_printkthreads() (static_branch_unlikely(&force_printkthreads_key))
		30	#endif
		31
23	#ifdef CONFIG_PRINTK	32	#ifdef CONFIG_PRINTK
24		33
25	#ifdef CONFIG_PRINTK_CALLER	34	#ifdef CONFIG_PRINTK_CALLER
Lines 43-49 enum printk_info_flags { Link Here
43	LOG_CONT = 8, /* text is a fragment of a continuation line */	52	LOG_CONT = 8, /* text is a fragment of a continuation line */
44	};	53	};
45		54
		55	struct printk_ringbuffer;
		56	struct dev_printk_info;
		57
46	extern struct printk_ringbuffer *prb;	58	extern struct printk_ringbuffer *prb;
		59	extern bool printk_threads_enabled;
47		60
48	__printf(4, 0)	61	__printf(4, 0)
49	int vprintk_store(int facility, int level,	62	int vprintk_store(int facility, int level,
Lines 53-58 int vprintk_store(int facility, int level, Link Here
53	__printf(1, 0) int vprintk_default(const char *fmt, va_list args);	66	__printf(1, 0) int vprintk_default(const char *fmt, va_list args);
54	__printf(1, 0) int vprintk_deferred(const char *fmt, va_list args);	67	__printf(1, 0) int vprintk_deferred(const char *fmt, va_list args);
55		68
		69	void __printk_safe_enter(void);
		70	void __printk_safe_exit(void);
		71
56	bool printk_percpu_data_ready(void);	72	bool printk_percpu_data_ready(void);
57		73
58	#define printk_safe_enter_irqsave(flags) \	74	#define printk_safe_enter_irqsave(flags) \
Lines 71-82 void defer_console_output(void); Link Here
71		87
72	u16 printk_parse_prefix(const char text, int level,	88	u16 printk_parse_prefix(const char text, int level,
73	enum printk_info_flags *flags);	89	enum printk_info_flags *flags);
		90	void console_lock_spinning_enable(void);
		91	int console_lock_spinning_disable_and_check(int cookie);
74		92
75	u64 nbcon_seq_read(struct console *con);	93	u64 nbcon_seq_read(struct console *con);
76	void nbcon_seq_force(struct console *con, u64 seq);	94	void nbcon_seq_force(struct console *con, u64 seq);
77	bool nbcon_alloc(struct console *con);	95	bool nbcon_alloc(struct console *con);
78	void nbcon_init(struct console *con);	96	void nbcon_init(struct console *con);
79	void nbcon_free(struct console *con);	97	void nbcon_free(struct console *con);
		98	enum nbcon_prio nbcon_get_default_prio(void);
		99	void nbcon_atomic_flush_pending(void);
		100	bool nbcon_legacy_emit_next_record(struct console con, bool handover,
		101	int cookie, bool use_atomic);
		102	void nbcon_kthread_create(struct console *con);
		103	void nbcon_wake_threads(void);
		104	void nbcon_legacy_kthread_create(void);
		105
		106	/*
		107	* Check if the given console is currently capable and allowed to print
		108	* records. Note that this function does not consider the current context,
		109	* which can also play a role in deciding if @con can be used to print
		110	* records.
		111	*/
		112	static inline bool console_is_usable(struct console *con, short flags, bool use_atomic)
		113	{
		114	if (!(flags & CON_ENABLED))
		115	return false;
		116
		117	if ((flags & CON_SUSPENDED))
		118	return false;
		119
		120	if (flags & CON_NBCON) {
		121	if (use_atomic) {
		122	if (!con->write_atomic)
		123	return false;
		124	} else {
		125	if (!con->write_thread)
		126	return false;
		127	}
		128	} else {
		129	if (!con->write)
		130	return false;
		131	}
		132
		133	/*
		134	* Console drivers may assume that per-cpu resources have been
		135	* allocated. So unless they're explicitly marked as being able to
		136	* cope (CON_ANYTIME) don't call them until this CPU is officially up.
		137	*/
		138	if (!cpu_online(raw_smp_processor_id()) && !(flags & CON_ANYTIME))
		139	return false;
		140
		141	return true;
		142	}
		143
		144	/**
		145	* nbcon_kthread_wake - Wake up a printk thread
		146	* @con: Console to operate on
		147	*/
		148	static inline void nbcon_kthread_wake(struct console *con)
		149	{
		150	/*
		151	* Guarantee any new records can be seen by tasks preparing to wait
		152	* before this context checks if the rcuwait is empty.
		153	*
		154	* The full memory barrier in rcuwait_wake_up() pairs with the full
		155	* memory barrier within set_current_state() of
		156	* ___rcuwait_wait_event(), which is called after prepare_to_rcuwait()
		157	* adds the waiter but before it has checked the wait condition.
		158	*
		159	* This pairs with nbcon_kthread_func:A.
		160	*/
		161	rcuwait_wake_up(&con->rcuwait); /* LMM(nbcon_kthread_wake:A) */
162	}
80		163
81	#else	164	#else
82		165
Lines 84-89 void nbcon_free(struct console *con); Link Here
84	#define PRINTK_MESSAGE_MAX 0	167	#define PRINTK_MESSAGE_MAX 0
85	#define PRINTKRB_RECORD_MAX 0	168	#define PRINTKRB_RECORD_MAX 0
86		169
		170	static inline void nbcon_kthread_wake(struct console *con) { }
		171	static inline void nbcon_kthread_create(struct console *con) { }
		172	#define printk_threads_enabled (false)
		173
87	/*	174	/*
88	* In !PRINTK builds we still export console_sem	175	* In !PRINTK builds we still export console_sem
89	* semaphore and some of console functions (console_unlock()/etc.), so	176	* semaphore and some of console functions (console_unlock()/etc.), so
Lines 98-106 static inline void nbcon_seq_force(struct console *con, u64 seq) { } Link Here
98	static inline bool nbcon_alloc(struct console *con) { return false; }	185	static inline bool nbcon_alloc(struct console *con) { return false; }
99	static inline void nbcon_init(struct console *con) { }	186	static inline void nbcon_init(struct console *con) { }
100	static inline void nbcon_free(struct console *con) { }	187	static inline void nbcon_free(struct console *con) { }
		188	static inline enum nbcon_prio nbcon_get_default_prio(void) { return NBCON_PRIO_NONE; }
		189	static inline void nbcon_atomic_flush_pending(void) { }
		190	static inline bool nbcon_legacy_emit_next_record(struct console con, bool handover,
		191	int cookie, bool use_atomic) { return false; }
		192
		193	static inline bool console_is_usable(struct console *con, short flags,
		194	bool use_atomic) { return false; }
101		195
102	#endif /* CONFIG_PRINTK */	196	#endif /* CONFIG_PRINTK */
103		197
		198	extern bool have_boot_console;
		199	extern bool have_legacy_console;
		200
		201	/*
		202	* Specifies if the console lock/unlock dance is needed for console
		203	* printing. If @have_boot_console is true, the nbcon consoles will
		204	* be printed serially along with the legacy consoles because nbcon
		205	* consoles cannot print simultaneously with boot consoles.
		206	*/
		207	#define printing_via_unlock (have_legacy_console \|\| have_boot_console)
		208
104	extern struct printk_buffers printk_shared_pbufs;	209	extern struct printk_buffers printk_shared_pbufs;
105		210
106	/**	211	/**
Lines 130-139 struct printk_message { Link Here
130	};	235	};
131		236
132	bool other_cpu_in_panic(void);	237	bool other_cpu_in_panic(void);
133	bool this_cpu_in_panic(void);
134	bool printk_get_next_message(struct printk_message *pmsg, u64 seq,	238	bool printk_get_next_message(struct printk_message *pmsg, u64 seq,
135	bool is_extended, bool may_supress);	239	bool is_extended, bool may_supress);
136		240
137	#ifdef CONFIG_PRINTK	241	#ifdef CONFIG_PRINTK
138	void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped);	242	void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped);
		243	void console_prepend_replay(struct printk_message *pmsg);
139	#endif	244	#endif

Lines 2-12 Link Here

(-)a/kernel/printk/nbcon.c (-17 / +726 lines)
2	// Copyright (C) 2022 Linutronix GmbH, John Ogness	2	// Copyright (C) 2022 Linutronix GmbH, John Ogness
3	// Copyright (C) 2022 Intel, Thomas Gleixner	3	// Copyright (C) 2022 Intel, Thomas Gleixner
4		4
5	#include <linux/kernel.h>	5	#include <linux/atomic.h>
		6	#include <linux/bug.h>
6	#include <linux/console.h>	7	#include <linux/console.h>
7	#include <linux/delay.h>	8	#include <linux/delay.h>
		9	#include <linux/errno.h>
		10	#include <linux/export.h>
		11	#include <linux/init.h>
		12	#include <linux/irqflags.h>
		13	#include <linux/kthread.h>
		14	#include <linux/minmax.h>
		15	#include <linux/percpu.h>
		16	#include <linux/preempt.h>
8	#include <linux/slab.h>	17	#include <linux/slab.h>
		18	#include <linux/smp.h>
		19	#include <linux/stddef.h>
		20	#include <linux/string.h>
		21	#include <linux/syscore_ops.h>
		22	#include <linux/types.h>
9	#include "internal.h"	23	#include "internal.h"
		24	#include "printk_ringbuffer.h"
10	/*	25	/*
11	* Printk console printing implementation for consoles which does not depend	26	* Printk console printing implementation for consoles which does not depend
12	* on the legacy style console_lock mechanism.	27	* on the legacy style console_lock mechanism.
Lines 172-180 void nbcon_seq_force(struct console *con, u64 seq) Link Here
172	u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb));	187	u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb));
173		188
174	atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __u64seq_to_ulseq(valid_seq));	189	atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __u64seq_to_ulseq(valid_seq));
175
176	/* Clear con->seq since nbcon consoles use con->nbcon_seq instead. */
177	con->seq = 0;
178	}	190	}
179		191
180	/**	192	/**
Lines 201-206 static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq) Link Here
201	}	213	}
202	}	214	}
203		215
		216	bool printk_threads_enabled __ro_after_init;
		217
204	/**	218	/**
205	* nbcon_context_try_acquire_direct - Try to acquire directly	219	* nbcon_context_try_acquire_direct - Try to acquire directly
206	* @ctxt: The context of the caller	220	* @ctxt: The context of the caller
Lines 531-536 static struct printk_buffers panic_nbcon_pbufs; Link Here
531	* nbcon_context_try_acquire - Try to acquire nbcon console	545	* nbcon_context_try_acquire - Try to acquire nbcon console
532	* @ctxt: The context of the caller	546	* @ctxt: The context of the caller
533	*	547	*
		548	* Context: Any context which could not be migrated to another CPU.
534	* Return: True if the console was acquired. False otherwise.	549	* Return: True if the console was acquired. False otherwise.
535	*	550	*
536	* If the caller allowed an unsafe hostile takeover, on success the	551	* If the caller allowed an unsafe hostile takeover, on success the
Lines 538-544 static struct printk_buffers panic_nbcon_pbufs; Link Here
538	* in an unsafe state. Otherwise, on success the caller may assume	553	* in an unsafe state. Otherwise, on success the caller may assume
539	* the console is not in an unsafe state.	554	* the console is not in an unsafe state.
540	*/	555	*/
541	__maybe_unused
542	static bool nbcon_context_try_acquire(struct nbcon_context *ctxt)	556	static bool nbcon_context_try_acquire(struct nbcon_context *ctxt)
543	{	557	{
544	unsigned int cpu = smp_processor_id();	558	unsigned int cpu = smp_processor_id();
Lines 824-832 bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) Link Here
824	}	838	}
825	EXPORT_SYMBOL_GPL(nbcon_exit_unsafe);	839	EXPORT_SYMBOL_GPL(nbcon_exit_unsafe);
826		840
		841	/**
		842	* nbcon_reacquire - Reacquire a console after losing ownership
		843	* @wctxt: The write context that was handed to the write function
		844	*
		845	* Since ownership can be lost at any time due to handover or takeover, a
		846	* printing context _should_ be prepared to back out immediately and
		847	* carefully. However, there are many scenarios where the context _must_
		848	* reacquire ownership in order to finalize or revert hardware changes.
		849	*
		850	* This function allows a context to reacquire ownership using the same
		851	* priority as its previous ownership.
		852	*
		853	* Note that for printing contexts, after a successful reacquire the
		854	* context will have no output buffer because that has been lost. This
		855	* function cannot be used to resume printing.
		856	*/
		857	void nbcon_reacquire(struct nbcon_write_context *wctxt)
		858	{
		859	struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
		860	struct console *con = ctxt->console;
		861	struct nbcon_state cur;
		862
		863	while (!nbcon_context_try_acquire(ctxt))
		864	cpu_relax();
		865
		866	wctxt->outbuf = NULL;
		867	wctxt->len = 0;
		868	nbcon_state_read(con, &cur);
		869	wctxt->unsafe_takeover = cur.unsafe_takeover;
		870	}
		871	EXPORT_SYMBOL_GPL(nbcon_reacquire);
		872
827	/**	873	/**
828	* nbcon_emit_next_record - Emit a record in the acquired context	874	* nbcon_emit_next_record - Emit a record in the acquired context
829	* @wctxt: The write context that will be handed to the write function	875	* @wctxt: The write context that will be handed to the write function
		876	* @use_atomic: True if the write_atomic callback is to be used
830	*	877	*
831	* Return: True if this context still owns the console. False if	878	* Return: True if this context still owns the console. False if
832	* ownership was handed over or taken.	879	* ownership was handed over or taken.
Lines 840-847 EXPORT_SYMBOL_GPL(nbcon_exit_unsafe); Link Here
840	* When true is returned, @wctxt->ctxt.backlog indicates whether there are	887	* When true is returned, @wctxt->ctxt.backlog indicates whether there are
841	* still records pending in the ringbuffer,	888	* still records pending in the ringbuffer,
842	*/	889	*/
843	__maybe_unused	890	static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt, bool use_atomic)
844	static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt)
845	{	891	{
846	struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);	892	struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
847	struct console *con = ctxt->console;	893	struct console *con = ctxt->console;
Lines 852-858 static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) Link Here
852	unsigned long con_dropped;	898	unsigned long con_dropped;
853	struct nbcon_state cur;	899	struct nbcon_state cur;
854	unsigned long dropped;	900	unsigned long dropped;
855	bool done;	901	unsigned long ulseq;
856		902
857	/*	903	/*
858	* The printk buffers are filled within an unsafe section. This	904	* The printk buffers are filled within an unsafe section. This
Lines 878-883 static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) Link Here
878	if (dropped && !is_extended)	924	if (dropped && !is_extended)
879	console_prepend_dropped(&pmsg, dropped);	925	console_prepend_dropped(&pmsg, dropped);
880		926
		927	/*
		928	* If the previous owner was assigned the same record, this context
		929	* has taken over ownership and is replaying the record. Prepend a
		930	* message to let the user know the record is replayed.
		931	*/
		932	ulseq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_prev_seq));
		933	if (__ulseq_to_u64seq(prb, ulseq) == pmsg.seq) {
		934	console_prepend_replay(&pmsg);
		935	} else {
		936	/*
		937	* Ensure this context is still the owner before trying to
		938	* update @nbcon_prev_seq. Otherwise the value in @ulseq may
		939	* not be from the previous owner.
		940	*/
		941	nbcon_state_read(con, &cur);
		942	if (!nbcon_context_can_proceed(ctxt, &cur))
		943	return false;
		944
		945	atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_prev_seq), &ulseq,
		946	__u64seq_to_ulseq(pmsg.seq));
		947	}
		948
881	if (!nbcon_context_exit_unsafe(ctxt))	949	if (!nbcon_context_exit_unsafe(ctxt))
882	return false;	950	return false;
883		951
Lines 891-907 static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) Link Here
891	nbcon_state_read(con, &cur);	959	nbcon_state_read(con, &cur);
892	wctxt->unsafe_takeover = cur.unsafe_takeover;	960	wctxt->unsafe_takeover = cur.unsafe_takeover;
893		961
894	if (con->write_atomic) {	962	if (use_atomic &&
895	done = con->write_atomic(con, wctxt);	963	con->write_atomic) {
		964	con->write_atomic(con, wctxt);
		965
		966	} else if (!use_atomic &&
		967	con->write_thread) {
		968	con->write_thread(con, wctxt);
		969
896	} else {	970	} else {
897	nbcon_context_release(ctxt);	971	/*
		972	* This function should never be called for legacy consoles.
		973	* Handle it as if ownership was lost and try to continue.
		974	*/
898	WARN_ON_ONCE(1);	975	WARN_ON_ONCE(1);
899	done = false;	976	nbcon_context_release(ctxt);
		977	return false;
900	}	978	}
901		979
902	/* If not done, the emit was aborted. */	980	if (!wctxt->outbuf) {
903	if (!done)	981	/*
		982	* Ownership was lost and reacquired by the driver.
		983	* Handle it as if ownership was lost and try to continue.
		984	*/
		985	nbcon_context_release(ctxt);
904	return false;	986	return false;
		987	}
905		988
906	/*	989	/*
907	* Since any dropped message was successfully output, reset the	990	* Since any dropped message was successfully output, reset the
Lines 928-933 static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) Link Here
928	return nbcon_context_exit_unsafe(ctxt);	1011	return nbcon_context_exit_unsafe(ctxt);
929	}	1012	}
930		1013
		1014	/**
		1015	* nbcon_kthread_should_wakeup - Check whether a printer thread should wakeup
		1016	* @con: Console to operate on
		1017	* @ctxt: The acquire context that contains the state
		1018	* at console_acquire()
		1019	*
		1020	* Return: True if the thread should shutdown or if the console is
		1021	* allowed to print and a record is available. False otherwise.
		1022	*
		1023	* After the thread wakes up, it must first check if it should shutdown before
		1024	* attempting any printing.
		1025	*/
		1026	static bool nbcon_kthread_should_wakeup(struct console con, struct nbcon_context ctxt)
		1027	{
		1028	bool ret = false;
		1029	short flags;
		1030	int cookie;
		1031
		1032	if (kthread_should_stop())
		1033	return true;
		1034
		1035	cookie = console_srcu_read_lock();
		1036
		1037	flags = console_srcu_read_flags(con);
		1038	if (console_is_usable(con, flags, false)) {
		1039	/* Bring the sequence in @ctxt up to date */
		1040	ctxt->seq = nbcon_seq_read(con);
		1041
		1042	ret = prb_read_valid(prb, ctxt->seq, NULL);
		1043	}
		1044
		1045	console_srcu_read_unlock(cookie);
		1046	return ret;
		1047	}
		1048
		1049	/**
		1050	* nbcon_kthread_func - The printer thread function
		1051	* @__console: Console to operate on
		1052	*/
		1053	static int nbcon_kthread_func(void *__console)
		1054	{
		1055	struct console *con = __console;
		1056	struct nbcon_write_context wctxt = {
		1057	.ctxt.console = con,
		1058	.ctxt.prio = NBCON_PRIO_NORMAL,
		1059	};
		1060	struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
		1061	unsigned long flags;
		1062	short con_flags;
		1063	bool backlog;
		1064	int cookie;
		1065	int ret;
		1066
		1067	wait_for_event:
		1068	/*
		1069	* Guarantee this task is visible on the rcuwait before
		1070	* checking the wake condition.
		1071	*
		1072	* The full memory barrier within set_current_state() of
		1073	* ___rcuwait_wait_event() pairs with the full memory
		1074	* barrier within rcuwait_has_sleeper().
		1075	*
		1076	* This pairs with rcuwait_has_sleeper:A and nbcon_kthread_wake:A.
		1077	*/
1078	ret = rcuwait_wait_event(&con->rcuwait,
1079	nbcon_kthread_should_wakeup(con, ctxt),
1080	TASK_INTERRUPTIBLE); /* LMM(nbcon_kthread_func:A) */
1081
1082	if (kthread_should_stop())
1083	return 0;
1084
1085	/* Wait was interrupted by a spurious signal, go back to sleep. */
1086	if (ret)
1087	goto wait_for_event;
1088
1089	do {
1090	backlog = false;
1091
1092	cookie = console_srcu_read_lock();
1093
1094	con_flags = console_srcu_read_flags(con);
1095
1096	if (console_is_usable(con, con_flags, false)) {
1097	con->device_lock(con, &flags);
1098
1099	/*
1100	* Ensure this stays on the CPU to make handover and
1101	* takeover possible.
1102	*/
1103	cant_migrate();
1104
1105	if (nbcon_context_try_acquire(ctxt)) {
1106	/*
1107	* If the emit fails, this context is no
1108	* longer the owner.
1109	*/
1110	if (nbcon_emit_next_record(&wctxt, false)) {
1111	nbcon_context_release(ctxt);
1112	backlog = ctxt->backlog;
1113	}
1114	}
1115
1116	con->device_unlock(con, flags);
1117	}
1118
1119	console_srcu_read_unlock(cookie);
1120
1121	} while (backlog);
1122
1123	goto wait_for_event;
1124	}
1125
1126	/**
1127	* nbcon_irq_work - irq work to wake printk thread
1128	* @irq_work: The irq work to operate on
1129	*/
1130	static void nbcon_irq_work(struct irq_work *irq_work)
1131	{
1132	struct console *con = container_of(irq_work, struct console, irq_work);
1133
1134	nbcon_kthread_wake(con);
1135	}
1136
1137	static inline bool rcuwait_has_sleeper(struct rcuwait *w)
1138	{
1139	bool has_sleeper;
1140
1141	rcu_read_lock();
1142	/*
1143	* Guarantee any new records can be seen by tasks preparing to wait
1144	* before this context checks if the rcuwait is empty.
1145	*
1146	* This full memory barrier pairs with the full memory barrier within
1147	* set_current_state() of ___rcuwait_wait_event(), which is called
1148	* after prepare_to_rcuwait() adds the waiter but before it has
1149	* checked the wait condition.
1150	*
1151	* This pairs with nbcon_kthread_func:A.
1152	*/
1153	smp_mb(); /* LMM(rcuwait_has_sleeper:A) */
1154	has_sleeper = !!rcu_dereference(w->task);
1155	rcu_read_unlock();
1156
1157	return has_sleeper;
1158	}
1159
1160	/**
1161	* nbcon_wake_threads - Wake up printing threads using irq_work
1162	*/
1163	void nbcon_wake_threads(void)
1164	{
1165	struct console *con;
1166	int cookie;
1167
1168	cookie = console_srcu_read_lock();
1169	for_each_console_srcu(con) {
1170	/*
1171	* Only schedule irq_work if the printing thread is
1172	* actively waiting. If not waiting, the thread will
1173	* notice by itself that it has work to do.
1174	*/
1175	if (con->kthread && rcuwait_has_sleeper(&con->rcuwait))
1176	irq_work_queue(&con->irq_work);
1177	}
1178	console_srcu_read_unlock(cookie);
1179	}
1180
1181	/* Track the nbcon emergency nesting per CPU. */
1182	static DEFINE_PER_CPU(unsigned int, nbcon_pcpu_emergency_nesting);
1183	static unsigned int early_nbcon_pcpu_emergency_nesting __initdata;
1184
1185	/**
1186	* nbcon_get_cpu_emergency_nesting - Get the per CPU emergency nesting pointer
1187	*
1188	* Return: Either a pointer to the per CPU emergency nesting counter of
1189	* the current CPU or to the init data during early boot.
1190	*/
1191	static __ref unsigned int *nbcon_get_cpu_emergency_nesting(void)
1192	{
1193	/*
1194	* The value of __printk_percpu_data_ready gets set in normal
1195	* context and before SMP initialization. As a result it could
1196	* never change while inside an nbcon emergency section.
1197	*/
1198	if (!printk_percpu_data_ready())
1199	return &early_nbcon_pcpu_emergency_nesting;
1200
1201	return this_cpu_ptr(&nbcon_pcpu_emergency_nesting);
1202	}
1203
1204	/**
1205	* nbcon_emit_one - Print one record for an nbcon console using the
1206	* specified callback
1207	* @wctxt: An initialized write context struct to use for this context
1208	* @use_atomic: True if the write_atomic callback is to be used
1209	*
1210	* Return: False if it is known there are no more records to print,
1211	* otherwise true.
1212	*
1213	* This is an internal helper to handle the locking of the console before
1214	* calling nbcon_emit_next_record().
1215	*/
1216	static bool nbcon_emit_one(struct nbcon_write_context *wctxt, bool use_atomic)
1217	{
1218	struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
1219
1220	if (!nbcon_context_try_acquire(ctxt))
1221	return true;
1222
1223	/*
1224	* nbcon_emit_next_record() returns false when the console was
1225	* handed over or taken over. In both cases the context is no
1226	* longer valid.
1227	*/
1228	if (!nbcon_emit_next_record(wctxt, use_atomic))
1229	return true;
1230
1231	nbcon_context_release(ctxt);
1232
1233	return ctxt->backlog;
1234	}
1235
1236	/**
1237	* nbcon_get_default_prio - The appropriate nbcon priority to use for nbcon
1238	* printing on the current CPU
1239	*
1240	* Context: Any context which could not be migrated to another CPU.
1241	* Return: The nbcon_prio to use for acquiring an nbcon console in this
1242	* context for printing.
1243	*/
1244	enum nbcon_prio nbcon_get_default_prio(void)
1245	{
1246	unsigned int *cpu_emergency_nesting;
1247
1248	if (this_cpu_in_panic())
1249	return NBCON_PRIO_PANIC;
1250
1251	cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
1252	if (*cpu_emergency_nesting)
1253	return NBCON_PRIO_EMERGENCY;
1254
1255	return NBCON_PRIO_NORMAL;
1256	}
1257
1258	/**
1259	* nbcon_legacy_emit_next_record - Print one record for an nbcon console
1260	* in legacy contexts
1261	* @con: The console to print on
1262	* @handover: Will be set to true if a printk waiter has taken over the
1263	* console_lock, in which case the caller is no longer holding
1264	* both the console_lock and the SRCU read lock. Otherwise it
1265	* is set to false.
1266	* @cookie: The cookie from the SRCU read lock.
1267	* @use_atomic: True if the write_atomic callback is to be used
1268	*
1269	* Context: Any context except NMI.
1270	* Return: False if the given console has no next record to print,
1271	* otherwise true.
1272	*
1273	* This function is meant to be called by console_flush_all() to print records
1274	* on nbcon consoles from legacy context (printing via console unlocking).
1275	* Essentially it is the nbcon version of console_emit_next_record().
1276	*/
1277	bool nbcon_legacy_emit_next_record(struct console con, bool handover,
1278	int cookie, bool use_atomic)
1279	{
1280	struct nbcon_write_context wctxt = { };
1281	struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
1282	unsigned long flags;
1283	bool progress;
1284
1285	*handover = false;
1286
1287	ctxt->console = con;
1288
1289	if (use_atomic) {
1290	/* Use the same procedure as console_emit_next_record(). */
1291	printk_safe_enter_irqsave(flags);
1292	console_lock_spinning_enable();
1293	stop_critical_timings();
1294
1295	ctxt->prio = nbcon_get_default_prio();
1296	progress = nbcon_emit_one(&wctxt, use_atomic);
1297
1298	start_critical_timings();
1299	*handover = console_lock_spinning_disable_and_check(cookie);
1300	printk_safe_exit_irqrestore(flags);
1301	} else {
1302	con->device_lock(con, &flags);
1303	cant_migrate();
1304
1305	ctxt->prio = nbcon_get_default_prio();
1306	progress = nbcon_emit_one(&wctxt, use_atomic);
1307
1308	con->device_unlock(con, flags);
1309	}
1310
1311	return progress;
1312	}
1313
1314	/**
1315	* __nbcon_atomic_flush_pending_con - Flush specified nbcon console using its
1316	* write_atomic() callback
1317	* @con: The nbcon console to flush
1318	* @stop_seq: Flush up until this record
1319	* @allow_unsafe_takeover: True, to allow unsafe hostile takeovers
1320	*
1321	* Return: True if taken over while printing. Otherwise false.
1322	*
1323	* If flushing up to @stop_seq was not successful, it only makes sense for the
1324	* caller to try again when true was returned. When false is returned, either
1325	* there are no more records available to read or this context is not allowed
1326	* to acquire the console.
1327	*/
1328	static bool __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq,
1329	bool allow_unsafe_takeover)
1330	{
1331	struct nbcon_write_context wctxt = { };
1332	struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
1333
1334	ctxt->console = con;
1335	ctxt->spinwait_max_us = 2000;
1336	ctxt->prio = nbcon_get_default_prio();
1337	ctxt->allow_unsafe_takeover = allow_unsafe_takeover;
1338
1339	if (!nbcon_context_try_acquire(ctxt))
1340	return false;
1341
1342	while (nbcon_seq_read(con) < stop_seq) {
1343	/*
1344	* nbcon_emit_next_record() returns false when the console was
1345	* handed over or taken over. In both cases the context is no
1346	* longer valid.
1347	*/
1348	if (!nbcon_emit_next_record(&wctxt, true))
1349	return true;
1350
1351	if (!ctxt->backlog)
1352	break;
1353	}
1354
1355	nbcon_context_release(ctxt);
1356
1357	return false;
1358	}
1359
1360	/**
1361	* __nbcon_atomic_flush_pending - Flush all nbcon consoles using their
1362	* write_atomic() callback
1363	* @stop_seq: Flush up until this record
1364	* @allow_unsafe_takeover: True, to allow unsafe hostile takeovers
1365	*/
1366	static void __nbcon_atomic_flush_pending(u64 stop_seq, bool allow_unsafe_takeover)
1367	{
1368	struct console *con;
1369	bool should_retry;
1370	int cookie;
1371
1372	do {
1373	should_retry = false;
1374
1375	cookie = console_srcu_read_lock();
1376	for_each_console_srcu(con) {
1377	short flags = console_srcu_read_flags(con);
1378	unsigned long irq_flags;
1379
1380	if (!(flags & CON_NBCON))
1381	continue;
1382
1383	if (!console_is_usable(con, flags, true))
1384	continue;
1385
1386	if (nbcon_seq_read(con) >= stop_seq)
1387	continue;
1388
1389	/*
1390	* Atomic flushing does not use console driver
1391	* synchronization (i.e. it does not hold the port
1392	* lock for uart consoles). Therefore IRQs must be
1393	* disabled to avoid being interrupted and then
1394	* calling into a driver that will deadlock trying
1395	* to acquire console ownership.
1396	*/
1397	local_irq_save(irq_flags);
1398
1399	should_retry \|= __nbcon_atomic_flush_pending_con(con, stop_seq,
1400	allow_unsafe_takeover);
1401	local_irq_restore(irq_flags);
1402	}
1403	console_srcu_read_unlock(cookie);
1404	} while (should_retry);
1405	}
1406
1407	/**
1408	* nbcon_atomic_flush_pending - Flush all nbcon consoles using their
1409	* write_atomic() callback
1410	*
1411	* Flush the backlog up through the currently newest record. Any new
1412	* records added while flushing will not be flushed. This is to avoid
1413	* one CPU printing unbounded because other CPUs continue to add records.
1414	*/
1415	void nbcon_atomic_flush_pending(void)
1416	{
1417	__nbcon_atomic_flush_pending(prb_next_reserve_seq(prb), false);
1418	}
1419
1420	/**
1421	* nbcon_atomic_flush_unsafe - Flush all nbcon consoles using their
1422	* write_atomic() callback and allowing unsafe hostile takeovers
1423	*
1424	* Flush the backlog up through the currently newest record. Unsafe hostile
1425	* takeovers will be performed, if necessary.
1426	*/
1427	void nbcon_atomic_flush_unsafe(void)
1428	{
1429	__nbcon_atomic_flush_pending(prb_next_reserve_seq(prb), true);
1430	}
1431
1432	/**
1433	* nbcon_cpu_emergency_enter - Enter an emergency section where printk()
1434	* messages for that CPU are only stored
1435	*
1436	* Upon exiting the emergency section, all stored messages are flushed.
1437	*
1438	* Context: Any context. Disables preemption.
1439	*
1440	* When within an emergency section, no printing occurs on that CPU. This
1441	* is to allow all emergency messages to be dumped into the ringbuffer before
1442	* flushing the ringbuffer. The actual printing occurs when exiting the
1443	* outermost emergency section.
1444	*/
1445	void nbcon_cpu_emergency_enter(void)
1446	{
1447	unsigned int *cpu_emergency_nesting;
1448
1449	preempt_disable();
1450
1451	cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
1452	(*cpu_emergency_nesting)++;
1453	}
1454
1455	/**
1456	* nbcon_cpu_emergency_exit - Exit an emergency section and flush the
1457	* stored messages
1458	*
1459	* Flushing only occurs when exiting all nesting for the CPU.
1460	*
1461	* Context: Any context. Enables preemption.
1462	*/
1463	void nbcon_cpu_emergency_exit(void)
1464	{
1465	unsigned int *cpu_emergency_nesting;
1466	bool do_trigger_flush = false;
1467
1468	cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
1469
1470	WARN_ON_ONCE(*cpu_emergency_nesting == 0);
1471
1472	if (*cpu_emergency_nesting == 1) {
1473	nbcon_atomic_flush_pending();
1474	do_trigger_flush = true;
1475	}
1476
1477	/* Undo the nesting count of nbcon_cpu_emergency_enter(). */
1478	(*cpu_emergency_nesting)--;
1479
1480	preempt_enable();
1481
1482	if (do_trigger_flush)
1483	printk_trigger_flush();
1484	}
1485
1486	/**
1487	* nbcon_kthread_stop - Stop a printer thread
1488	* @con: Console to operate on
1489	*/
1490	static void nbcon_kthread_stop(struct console *con)
1491	{
1492	lockdep_assert_console_list_lock_held();
1493
1494	if (!con->kthread)
1495	return;
1496
1497	kthread_stop(con->kthread);
1498	con->kthread = NULL;
1499	}
1500
1501	/**
1502	* nbcon_kthread_create - Create a printer thread
1503	* @con: Console to operate on
1504	*
1505	* If it fails, let the console proceed. The atomic part might
1506	* be usable and useful.
1507	*/
1508	void nbcon_kthread_create(struct console *con)
1509	{
1510	struct task_struct *kt;
1511
1512	lockdep_assert_console_list_lock_held();
1513
1514	if (!(con->flags & CON_NBCON) \|\| !con->write_thread)
1515	return;
1516
1517	if (!printk_threads_enabled \|\| con->kthread)
1518	return;
1519
1520	/*
1521	* Printer threads cannot be started as long as any boot console is
1522	* registered because there is no way to synchronize the hardware
1523	* registers between boot console code and regular console code.
1524	*/
1525	if (have_boot_console)
1526	return;
1527
1528	kt = kthread_run(nbcon_kthread_func, con, "pr/%s%d", con->name, con->index);
1529	if (IS_ERR(kt)) {
1530	con_printk(KERN_ERR, con, "failed to start printing thread\n");
1531	return;
1532	}
1533
1534	con->kthread = kt;
1535
1536	/*
1537	* It is important that console printing threads are scheduled
1538	* shortly after a printk call and with generous runtime budgets.
1539	*/
1540	sched_set_normal(con->kthread, -20);
1541	}
1542
1543	static int __init printk_setup_threads(void)
1544	{
1545	struct console *con;
1546
1547	console_list_lock();
1548	printk_threads_enabled = true;
1549	for_each_console(con)
1550	nbcon_kthread_create(con);
1551	if (force_printkthreads() && printing_via_unlock)
1552	nbcon_legacy_kthread_create();
1553	console_list_unlock();
1554	return 0;
1555	}
1556	early_initcall(printk_setup_threads);
1557
931	/**	1558	/**
932	* nbcon_alloc - Allocate buffers needed by the nbcon console	1559	* nbcon_alloc - Allocate buffers needed by the nbcon console
933	* @con: Console to allocate buffers for	1560	* @con: Console to allocate buffers for
Lines 964-971 bool nbcon_alloc(struct console *con) Link Here
964	*	1591	*
965	* nbcon_alloc() must be called and succeed before this function	1592	* nbcon_alloc() must be called and succeed before this function
966	* is called.	1593	* is called.
967	*
968	* This function expects that the legacy @con->seq has been set.
969	*/	1594	*/
970	void nbcon_init(struct console *con)	1595	void nbcon_init(struct console *con)
971	{	1596	{
Lines 974-981 void nbcon_init(struct console *con) Link Here
974	/* nbcon_alloc() must have been called and successful! */	1599	/* nbcon_alloc() must have been called and successful! */
975	BUG_ON(!con->pbufs);	1600	BUG_ON(!con->pbufs);
976		1601
977	nbcon_seq_force(con, con->seq);	1602	rcuwait_init(&con->rcuwait);
		1603	init_irq_work(&con->irq_work, nbcon_irq_work);
		1604	nbcon_seq_force(con, 0);
		1605	atomic_long_set(&ACCESS_PRIVATE(con, nbcon_prev_seq), -1UL);
978	nbcon_state_set(con, &state);	1606	nbcon_state_set(con, &state);
		1607	nbcon_kthread_create(con);
979	}	1608	}
980		1609
981	/**	1610	/**
Lines 986-991 void nbcon_free(struct console *con) Link Here
986	{	1615	{
987	struct nbcon_state state = { };	1616	struct nbcon_state state = { };
988		1617
		1618	nbcon_kthread_stop(con);
989	nbcon_state_set(con, &state);	1619	nbcon_state_set(con, &state);
990		1620
991	/* Boot consoles share global printk buffers. */	1621	/* Boot consoles share global printk buffers. */
Lines 994-996 void nbcon_free(struct console *con) Link Here
994		1624
995	con->pbufs = NULL;	1625	con->pbufs = NULL;
996	}	1626	}
		1627
		1628	/**
		1629	* nbcon_driver_acquire - Acquire nbcon console and enter unsafe section
		1630	* @con: The nbcon console to acquire
		1631	*
		1632	* Context: Any context which could not be migrated to another CPU.
		1633	*
		1634	* Console drivers will usually use their own internal synchronization
		1635	* mechasism to synchronize between console printing and non-printing
		1636	* activities (such as setting baud rates). However, nbcon console drivers
		1637	* supporting atomic consoles may also want to mark unsafe sections when
		1638	* performing non-printing activities.
		1639	*
		1640	* This function acquires the nbcon console using priority NBCON_PRIO_NORMAL
		1641	* and marks it unsafe for handover/takeover.
		1642	*
		1643	* Console drivers using this function must have provided @nbcon_drvdata in
		1644	* their struct console, which is used to track ownership and state
		1645	* information.
		1646	*/
		1647	void nbcon_driver_acquire(struct console *con)
		1648	{
		1649	struct nbcon_context *ctxt = &ACCESS_PRIVATE(con->nbcon_drvdata, ctxt);
		1650
		1651	cant_migrate();
		1652
		1653	do {
		1654	do {
		1655	memset(ctxt, 0, sizeof(*ctxt));
		1656	ctxt->console = con;
		1657	ctxt->prio = NBCON_PRIO_NORMAL;
		1658	} while (!nbcon_context_try_acquire(ctxt));
		1659
		1660	} while (!nbcon_context_enter_unsafe(ctxt));
		1661	}
		1662	EXPORT_SYMBOL_GPL(nbcon_driver_acquire);
		1663
		1664	/**
		1665	* nbcon_driver_release - Exit unsafe section and release the nbcon console
		1666	* @con: The nbcon console acquired in nbcon_driver_acquire()
		1667	*/
		1668	void nbcon_driver_release(struct console *con)
		1669	{
		1670	struct nbcon_context *ctxt = &ACCESS_PRIVATE(con->nbcon_drvdata, ctxt);
		1671
		1672	if (nbcon_context_exit_unsafe(ctxt))
		1673	nbcon_context_release(ctxt);
		1674	}
		1675	EXPORT_SYMBOL_GPL(nbcon_driver_release);
		1676
		1677	/**
		1678	* printk_kthread_shutdown - shutdown all threaded printers
		1679	*
		1680	* On system shutdown all threaded printers are stopped. This allows printk
		1681	* to transition back to atomic printing, thus providing a robust mechanism
		1682	* for the final shutdown/reboot messages to be output.
		1683	*/
		1684	static void printk_kthread_shutdown(void)
		1685	{
		1686	struct console *con;
		1687
		1688	console_list_lock();
		1689	for_each_console(con) {
		1690	if (con->flags & CON_NBCON)
1691	nbcon_kthread_stop(con);
1692	}
1693	console_list_unlock();
1694	}
1695
1696	static struct syscore_ops printk_syscore_ops = {
1697	.shutdown = printk_kthread_shutdown,
1698	};
1699
1700	static int __init printk_init_ops(void)
1701	{
1702	register_syscore_ops(&printk_syscore_ops);
1703	return 0;
1704	}
1705	device_initcall(printk_init_ops);

Lines 195-200 static int __init control_devkmsg(char *str) Link Here

(-)a/kernel/printk/printk.c (-133 / +561 lines)
195	}	195	}
196	__setup("printk.devkmsg=", control_devkmsg);	196	__setup("printk.devkmsg=", control_devkmsg);
197		197
		198	#if !defined(CONFIG_PREEMPT_RT)
		199	DEFINE_STATIC_KEY_FALSE(force_printkthreads_key);
		200
		201	static int __init setup_forced_printkthreads(char *arg)
		202	{
		203	static_branch_enable(&force_printkthreads_key);
		204	return 0;
		205	}
		206	early_param("threadprintk", setup_forced_printkthreads);
		207	#endif
		208
198	char devkmsg_log_str[DEVKMSG_STR_MAX_SIZE] = "ratelimit";	209	char devkmsg_log_str[DEVKMSG_STR_MAX_SIZE] = "ratelimit";
199	#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)	210	#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)
200	int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write,	211	int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write,
Lines 282-287 EXPORT_SYMBOL(console_list_unlock); Link Here
282	* Return: A cookie to pass to console_srcu_read_unlock().	293	* Return: A cookie to pass to console_srcu_read_unlock().
283	*/	294	*/
284	int console_srcu_read_lock(void)	295	int console_srcu_read_lock(void)
		296	__acquires(&console_srcu)
285	{	297	{
286	return srcu_read_lock_nmisafe(&console_srcu);	298	return srcu_read_lock_nmisafe(&console_srcu);
287	}	299	}
Lines 295-300 EXPORT_SYMBOL(console_srcu_read_lock); Link Here
295	* Counterpart to console_srcu_read_lock()	307	* Counterpart to console_srcu_read_lock()
296	*/	308	*/
297	void console_srcu_read_unlock(int cookie)	309	void console_srcu_read_unlock(int cookie)
		310	__releases(&console_srcu)
298	{	311	{
299	srcu_read_unlock_nmisafe(&console_srcu, cookie);	312	srcu_read_unlock_nmisafe(&console_srcu, cookie);
300	}	313	}
Lines 461-474 static int console_msg_format = MSG_FORMAT_DEFAULT; Link Here
461	/* syslog_lock protects syslog_* variables and write access to clear_seq. */	474	/* syslog_lock protects syslog_* variables and write access to clear_seq. */
462	static DEFINE_MUTEX(syslog_lock);	475	static DEFINE_MUTEX(syslog_lock);
463		476
464	#ifdef CONFIG_PRINTK
465	/*	477	/*
466	* During panic, heavy printk by other CPUs can delay the	478	* Specifies if a legacy console is registered. If legacy consoles are
467	* panic and risk deadlock on console resources.	479	* present, it is necessary to perform the console lock/unlock dance
		480	* whenever console flushing should occur.
468	*/	481	*/
469	static int __read_mostly suppress_panic_printk;	482	bool have_legacy_console;
470		483
		484	/*
		485	* Specifies if an nbcon console is registered. If nbcon consoles are present,
		486	* synchronous printing of legacy consoles will not occur during panic until
		487	* the backtrace has been stored to the ringbuffer.
		488	*/
		489	static bool have_nbcon_console;
		490
		491	/*
		492	* Specifies if a boot console is registered. If boot consoles are present,
		493	* nbcon consoles cannot print simultaneously and must be synchronized by
		494	* the console lock. This is because boot consoles and nbcon consoles may
		495	* have mapped the same hardware.
		496	*/
		497	bool have_boot_console;
		498
		499	#ifdef CONFIG_PRINTK
471	DECLARE_WAIT_QUEUE_HEAD(log_wait);	500	DECLARE_WAIT_QUEUE_HEAD(log_wait);
		501
		502	static DECLARE_WAIT_QUEUE_HEAD(legacy_wait);
		503
472	/* All 3 protected by @syslog_lock. */	504	/* All 3 protected by @syslog_lock. */
473	/* the next printk record to read by syslog(READ) or /proc/kmsg */	505	/* the next printk record to read by syslog(READ) or /proc/kmsg */
474	static u64 syslog_seq;	506	static u64 syslog_seq;
Lines 1867-1873 static bool console_waiter; Link Here
1867	* there may be a waiter spinning (like a spinlock). Also it must be	1899	* there may be a waiter spinning (like a spinlock). Also it must be
1868	* ready to hand over the lock at the end of the section.	1900	* ready to hand over the lock at the end of the section.
1869	*/	1901	*/
1870	static void console_lock_spinning_enable(void)	1902	void console_lock_spinning_enable(void)
1871	{	1903	{
1872	/*	1904	/*
1873	* Do not use spinning in panic(). The panic CPU wants to keep the lock.	1905	* Do not use spinning in panic(). The panic CPU wants to keep the lock.
Lines 1906-1912 static void console_lock_spinning_enable(void) Link Here
1906	*	1938	*
1907	* Return: 1 if the lock rights were passed, 0 otherwise.	1939	* Return: 1 if the lock rights were passed, 0 otherwise.
1908	*/	1940	*/
1909	static int console_lock_spinning_disable_and_check(int cookie)	1941	int console_lock_spinning_disable_and_check(int cookie)
1910	{	1942	{
1911	int waiter;	1943	int waiter;
1912		1944
Lines 2311-2365 int vprintk_store(int facility, int level, Link Here
2311	return ret;	2343	return ret;
2312	}	2344	}
2313		2345
		2346	static bool legacy_allow_panic_sync;
		2347
		2348	/*
		2349	* This acts as a one-way switch to allow legacy consoles to print from
		2350	* the printk() caller context on a panic CPU. It also attempts to flush
		2351	* the legacy consoles in this context.
		2352	*/
		2353	void printk_legacy_allow_panic_sync(void)
		2354	{
		2355	legacy_allow_panic_sync = true;
		2356
		2357	if (printing_via_unlock && !in_nmi()) {
		2358	if (console_trylock())
		2359	console_unlock();
		2360	}
		2361	}
		2362
2314	asmlinkage int vprintk_emit(int facility, int level,	2363	asmlinkage int vprintk_emit(int facility, int level,
2315	const struct dev_printk_info *dev_info,	2364	const struct dev_printk_info *dev_info,
2316	const char *fmt, va_list args)	2365	const char *fmt, va_list args)
2317	{	2366	{
		2367	bool do_trylock_unlock = printing_via_unlock &&
		2368	!force_printkthreads();
2318	int printed_len;	2369	int printed_len;
2319	bool in_sched = false;
2320		2370
2321	/* Suppress unimportant messages after panic happens */	2371	/* Suppress unimportant messages after panic happens */
2322	if (unlikely(suppress_printk))	2372	if (unlikely(suppress_printk))
2323	return 0;	2373	return 0;
2324		2374
2325	if (unlikely(suppress_panic_printk) &&	2375	/*
2326	atomic_read(&panic_cpu) != raw_smp_processor_id())	2376	* The messages on the panic CPU are the most important. If
		2377	* non-panic CPUs are generating any messages, they will be
		2378	* silently dropped.
		2379	*/
		2380	if (other_cpu_in_panic())
2327	return 0;	2381	return 0;
2328		2382
2329	if (level == LOGLEVEL_SCHED) {	2383	if (level == LOGLEVEL_SCHED) {
2330	level = LOGLEVEL_DEFAULT;	2384	level = LOGLEVEL_DEFAULT;
2331	in_sched = true;	2385	/* If called from the scheduler, we can not call up(). */
		2386	do_trylock_unlock = false;
2332	}	2387	}
2333		2388
2334	printk_delay(level);	2389	printk_delay(level);
2335		2390
2336	printed_len = vprintk_store(facility, level, dev_info, fmt, args);	2391	printed_len = vprintk_store(facility, level, dev_info, fmt, args);
2337		2392
2338	/* If called from the scheduler, we can not call up(). */	2393	if (have_nbcon_console && !have_boot_console) {
2339	if (!in_sched) {	2394	bool is_panic_context = this_cpu_in_panic();
		2395
		2396	/*
		2397	* In panic, the legacy consoles are not allowed to print from
		2398	* the printk calling context unless explicitly allowed. This
		2399	* gives the safe nbcon consoles a chance to print out all the
		2400	* panic messages first. This restriction only applies if
		2401	* there are nbcon consoles registered.
		2402	*/
		2403	if (is_panic_context)
		2404	do_trylock_unlock &= legacy_allow_panic_sync;
		2405
		2406	/*
		2407	* There are situations where nbcon atomic printing should
		2408	* happen in the printk() caller context:
		2409	*
		2410	* - When this CPU is in panic.
		2411	*
		2412	* - When booting, before the printing threads have been
		2413	* started.
		2414	*
		2415	* - During shutdown, since the printing threads may not get
		2416	* a chance to print the final messages.
		2417	*
		2418	* Note that if boot consoles are registered, the console
		2419	* lock/unlock dance must be relied upon instead because nbcon
		2420	* consoles cannot print simultaneously with boot consoles.
		2421	*/
		2422	if (is_panic_context \|\|
		2423	!printk_threads_enabled \|\|
		2424	(system_state > SYSTEM_RUNNING)) {
		2425	nbcon_atomic_flush_pending();
		2426	}
		2427	}
		2428
		2429	nbcon_wake_threads();
		2430
		2431	if (do_trylock_unlock) {
2340	/*	2432	/*
2341	* The caller may be holding system-critical or	2433	* The caller may be holding system-critical or
2342	* timing-sensitive locks. Disable preemption during	2434	* timing-sensitive locks. Disable preemption during
2343	* printing of all remaining records to all consoles so that	2435	* printing of all remaining records to all consoles so that
2344	* this context can return as soon as possible. Hopefully	2436	* this context can return as soon as possible. Hopefully
2345	* another printk() caller will take over the printing.	2437	* another printk() caller will take over the printing.
		2438	*
		2439	* Also, nbcon_get_default_prio() requires migration disabled.
2346	*/	2440	*/
2347	preempt_disable();	2441	preempt_disable();
		2442
2348	/*	2443	/*
2349	* Try to acquire and then immediately release the console	2444	* Try to acquire and then immediately release the console
2350	* semaphore. The release will print out buffers. With the	2445	* semaphore. The release will print out buffers. With the
2351	* spinning variant, this context tries to take over the	2446	* spinning variant, this context tries to take over the
2352	* printing from another printing context.	2447	* printing from another printing context.
		2448	*
		2449	* Skip it in EMERGENCY priority. The console will be
		2450	* explicitly flushed when exiting the emergency section.
2353	*/	2451	*/
2354	if (console_trylock_spinning())	2452	if (nbcon_get_default_prio() != NBCON_PRIO_EMERGENCY) {
2355	console_unlock();	2453	if (console_trylock_spinning())
		2454	console_unlock();
		2455	}
		2456
2356	preempt_enable();	2457	preempt_enable();
2357	}	2458	}
2358		2459
2359	if (in_sched)	2460	if (do_trylock_unlock)
2360	defer_console_output();
2361	else
2362	wake_up_klogd();	2461	wake_up_klogd();
		2462	else
		2463	defer_console_output();
2363		2464
2364	return printed_len;	2465	return printed_len;
2365	}	2466	}
Lines 2387-2392 EXPORT_SYMBOL(_printk); Link Here
2387	static bool pr_flush(int timeout_ms, bool reset_on_progress);	2488	static bool pr_flush(int timeout_ms, bool reset_on_progress);
2388	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress);	2489	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress);
2389		2490
		2491	static struct task_struct *nbcon_legacy_kthread;
		2492
		2493	static inline void wake_up_legacy_kthread(void)
		2494	{
		2495	if (nbcon_legacy_kthread)
		2496	wake_up_interruptible(&legacy_wait);
		2497	}
		2498
2390	#else /* CONFIG_PRINTK */	2499	#else /* CONFIG_PRINTK */
2391		2500
2392	#define printk_time false	2501	#define printk_time false
Lines 2400-2405 static u64 syslog_seq; Link Here
2400	static bool pr_flush(int timeout_ms, bool reset_on_progress) { return true; }	2509	static bool pr_flush(int timeout_ms, bool reset_on_progress) { return true; }
2401	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; }	2510	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; }
2402		2511
		2512	static inline void nbcon_legacy_kthread_create(void) { }
		2513	static inline void wake_up_legacy_kthread(void) { }
2403	#endif /* CONFIG_PRINTK */	2514	#endif /* CONFIG_PRINTK */
2404		2515
2405	#ifdef CONFIG_EARLY_PRINTK	2516	#ifdef CONFIG_EARLY_PRINTK
Lines 2615-2620 void suspend_console(void) Link Here
2615	void resume_console(void)	2726	void resume_console(void)
2616	{	2727	{
2617	struct console *con;	2728	struct console *con;
		2729	short flags;
		2730	int cookie;
2618		2731
2619	if (!console_suspend_enabled)	2732	if (!console_suspend_enabled)
2620	return;	2733	return;
Lines 2631-2636 void resume_console(void) Link Here
2631	*/	2744	*/
2632	synchronize_srcu(&console_srcu);	2745	synchronize_srcu(&console_srcu);
2633		2746
		2747	/*
		2748	* Since this runs in task context, wake the threaded printers
		2749	* directly rather than scheduling irq_work to do it.
		2750	*/
		2751	cookie = console_srcu_read_lock();
		2752	for_each_console_srcu(con) {
		2753	flags = console_srcu_read_flags(con);
		2754	if (flags & CON_NBCON)
		2755	nbcon_kthread_wake(con);
		2756	}
		2757	console_srcu_read_unlock(cookie);
		2758
		2759	wake_up_legacy_kthread();
		2760
2634	pr_flush(1000, true);	2761	pr_flush(1000, true);
2635	}	2762	}
2636		2763
Lines 2645-2651 void resume_console(void) Link Here
2645	*/	2772	*/
2646	static int console_cpu_notify(unsigned int cpu)	2773	static int console_cpu_notify(unsigned int cpu)
2647	{	2774	{
2648	if (!cpuhp_tasks_frozen) {	2775	if (!cpuhp_tasks_frozen && printing_via_unlock &&
		2776	!force_printkthreads()) {
2649	/* If trylock fails, someone else is doing the printing */	2777	/* If trylock fails, someone else is doing the printing */
2650	if (console_trylock())	2778	if (console_trylock())
2651	console_unlock();	2779	console_unlock();
Lines 2702-2737 int is_console_locked(void) Link Here
2702	}	2830	}
2703	EXPORT_SYMBOL(is_console_locked);	2831	EXPORT_SYMBOL(is_console_locked);
2704		2832
2705	/*
2706	* Check if the given console is currently capable and allowed to print
2707	* records.
2708	*
2709	* Requires the console_srcu_read_lock.
2710	*/
2711	static inline bool console_is_usable(struct console *con)
2712	{
2713	short flags = console_srcu_read_flags(con);
2714
2715	if (!(flags & CON_ENABLED))
2716	return false;
2717
2718	if ((flags & CON_SUSPENDED))
2719	return false;
2720
2721	if (!con->write)
2722	return false;
2723
2724	/*
2725	* Console drivers may assume that per-cpu resources have been
2726	* allocated. So unless they're explicitly marked as being able to
2727	* cope (CON_ANYTIME) don't call them until this CPU is officially up.
2728	*/
2729	if (!cpu_online(raw_smp_processor_id()) && !(flags & CON_ANYTIME))
2730	return false;
2731
2732	return true;
2733	}
2734
2735	static void __console_unlock(void)	2833	static void __console_unlock(void)
2736	{	2834	{
2737	console_locked = 0;	2835	console_locked = 0;
Lines 2741-2770 static void __console_unlock(void) Link Here
2741	#ifdef CONFIG_PRINTK	2839	#ifdef CONFIG_PRINTK
2742		2840
2743	/*	2841	/*
2744	* Prepend the message in @pmsg->pbufs->outbuf with a "dropped message". This	2842	* Prepend the message in @pmsg->pbufs->outbuf with the message in
2745	* is achieved by shifting the existing message over and inserting the dropped	2843	* @pmsg->pbufs->scratchbuf. This is achieved by shifting the existing message
2746	* message.	2844	* over and inserting the scratchbuf message.
2747	*	2845	*
2748	* @pmsg is the printk message to prepend.	2846	* @pmsg is the printk message to prepend.
2749	*	2847	*
2750	* @dropped is the dropped count to report in the dropped message.	2848	* @len is the length of the message in @pmsg->pbufs->scratchbuf.
2751	*	2849	*
2752	* If the message text in @pmsg->pbufs->outbuf does not have enough space for	2850	* If the message text in @pmsg->pbufs->outbuf does not have enough space for
2753	* the dropped message, the message text will be sufficiently truncated.	2851	* the scratchbuf message, the message text will be sufficiently truncated.
2754	*	2852	*
2755	* If @pmsg->pbufs->outbuf is modified, @pmsg->outbuf_len is updated.	2853	* If @pmsg->pbufs->outbuf is modified, @pmsg->outbuf_len is updated.
2756	*/	2854	*/
2757	void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped)	2855	static void __console_prepend_scratch(struct printk_message *pmsg, size_t len)
2758	{	2856	{
2759	struct printk_buffers *pbufs = pmsg->pbufs;	2857	struct printk_buffers *pbufs = pmsg->pbufs;
2760	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
2761	const size_t outbuf_sz = sizeof(pbufs->outbuf);	2858	const size_t outbuf_sz = sizeof(pbufs->outbuf);
2762	char *scratchbuf = &pbufs->scratchbuf[0];	2859	char *scratchbuf = &pbufs->scratchbuf[0];
2763	char *outbuf = &pbufs->outbuf[0];	2860	char *outbuf = &pbufs->outbuf[0];
2764	size_t len;
2765
2766	len = scnprintf(scratchbuf, scratchbuf_sz,
2767	" %lu printk messages dropped \n", dropped);
2768		2861
2769	/*	2862	/*
2770	* Make sure outbuf is sufficiently large before prepending.	2863	* Make sure outbuf is sufficiently large before prepending.
Lines 2786-2791 void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) Link Here
2786	pmsg->outbuf_len += len;	2879	pmsg->outbuf_len += len;
2787	}	2880	}
2788		2881
		2882	/*
		2883	* Prepend the message in @pmsg->pbufs->outbuf with a "dropped message".
		2884	* @pmsg->outbuf_len is updated appropriately.
		2885	*
		2886	* @pmsg is the printk message to prepend.
		2887	*
		2888	* @dropped is the dropped count to report in the dropped message.
		2889	*/
		2890	void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped)
		2891	{
		2892	struct printk_buffers *pbufs = pmsg->pbufs;
		2893	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
		2894	char *scratchbuf = &pbufs->scratchbuf[0];
		2895	size_t len;
		2896
		2897	len = scnprintf(scratchbuf, scratchbuf_sz,
		2898	" %lu printk messages dropped \n", dropped);
		2899
		2900	__console_prepend_scratch(pmsg, len);
		2901	}
		2902
		2903	/*
		2904	* Prepend the message in @pmsg->pbufs->outbuf with a "replay message".
		2905	* @pmsg->outbuf_len is updated appropriately.
		2906	*
		2907	* @pmsg is the printk message to prepend.
		2908	*/
		2909	void console_prepend_replay(struct printk_message *pmsg)
		2910	{
		2911	struct printk_buffers *pbufs = pmsg->pbufs;
		2912	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
		2913	char *scratchbuf = &pbufs->scratchbuf[0];
		2914	size_t len;
		2915
		2916	len = scnprintf(scratchbuf, scratchbuf_sz,
		2917	" replaying previous printk message \n");
		2918
		2919	__console_prepend_scratch(pmsg, len);
		2920	}
		2921
2789	/*	2922	/*
2790	* Read and format the specified record (or a later record if the specified	2923	* Read and format the specified record (or a later record if the specified
2791	* record is not available).	2924	* record is not available).
Lines 2808-2815 void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) Link Here
2808	bool printk_get_next_message(struct printk_message *pmsg, u64 seq,	2941	bool printk_get_next_message(struct printk_message *pmsg, u64 seq,
2809	bool is_extended, bool may_suppress)	2942	bool is_extended, bool may_suppress)
2810	{	2943	{
2811	static int panic_console_dropped;
2812
2813	struct printk_buffers *pbufs = pmsg->pbufs;	2944	struct printk_buffers *pbufs = pmsg->pbufs;
2814	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);	2945	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
2815	const size_t outbuf_sz = sizeof(pbufs->outbuf);	2946	const size_t outbuf_sz = sizeof(pbufs->outbuf);
Lines 2837-2853 bool printk_get_next_message(struct printk_message *pmsg, u64 seq, Link Here
2837	pmsg->seq = r.info->seq;	2968	pmsg->seq = r.info->seq;
2838	pmsg->dropped = r.info->seq - seq;	2969	pmsg->dropped = r.info->seq - seq;
2839		2970
2840	/*
2841	* Check for dropped messages in panic here so that printk
2842	* suppression can occur as early as possible if necessary.
2843	*/
2844	if (pmsg->dropped &&
2845	panic_in_progress() &&
2846	panic_console_dropped++ > 10) {
2847	suppress_panic_printk = 1;
2848	pr_warn_once("Too many dropped messages. Suppress messages on non-panic CPUs to prevent livelock.\n");
2849	}
2850
2851	/* Skip record that has level above the console loglevel. */	2971	/* Skip record that has level above the console loglevel. */
2852	if (may_suppress && suppress_message_printing(r.info->level))	2972	if (may_suppress && suppress_message_printing(r.info->level))
2853	goto out;	2973	goto out;
Lines 2864-2869 bool printk_get_next_message(struct printk_message *pmsg, u64 seq, Link Here
2864	return true;	2984	return true;
2865	}	2985	}
2866		2986
		2987	/*
		2988	* Legacy console printing from printk() caller context does not respect
		2989	* raw_spinlock/spinlock nesting. For !PREEMPT_RT the lockdep warning is a
		2990	* false positive. For PREEMPT_RT the false positive condition does not
		2991	* occur.
		2992	*
		2993	* This map is used to establish LD_WAIT_SLEEP context for the console write
		2994	* callbacks when legacy printing to avoid false positive lockdep complaints,
		2995	* thus allowing lockdep to continue to function for real issues.
		2996	*/
		2997	#ifdef CONFIG_PREEMPT_RT
		2998	static inline void printk_legacy_lock_map_acquire_try(void) { }
		2999	static inline void printk_legacy_lock_map_release(void) { }
		3000	#else
		3001	static DEFINE_WAIT_OVERRIDE_MAP(printk_legacy_map, LD_WAIT_SLEEP);
		3002
		3003	static inline void printk_legacy_lock_map_acquire_try(void)
		3004	{
		3005	lock_map_acquire_try(&printk_legacy_map);
		3006	}
		3007
		3008	static inline void printk_legacy_lock_map_release(void)
		3009	{
		3010	lock_map_release(&printk_legacy_map);
		3011	}
		3012	#endif /* CONFIG_PREEMPT_RT */
		3013
2867	/*	3014	/*
2868	* Used as the printk buffers for non-panic, serialized console printing.	3015	* Used as the printk buffers for non-panic, serialized console printing.
2869	* This is for legacy (!CON_NBCON) as well as all boot (CON_BOOT) consoles.	3016	* This is for legacy (!CON_NBCON) as well as all boot (CON_BOOT) consoles.
Lines 2913-2943 static bool console_emit_next_record(struct console con, bool handover, int co Link Here
2913	con->dropped = 0;	3060	con->dropped = 0;
2914	}	3061	}
2915		3062
2916	/*
2917	* While actively printing out messages, if another printk()
2918	* were to occur on another CPU, it may wait for this one to
2919	* finish. This task can not be preempted if there is a
2920	* waiter waiting to take over.
2921	*
2922	* Interrupts are disabled because the hand over to a waiter
2923	* must not be interrupted until the hand over is completed
2924	* (@console_waiter is cleared).
2925	*/
2926	printk_safe_enter_irqsave(flags);
2927	console_lock_spinning_enable();
2928
2929	/* Do not trace print latency. */
2930	stop_critical_timings();
2931
2932	/* Write everything out to the hardware. */	3063	/* Write everything out to the hardware. */
2933	con->write(con, outbuf, pmsg.outbuf_len);
2934		3064
2935	start_critical_timings();	3065	if (force_printkthreads()) {
		3066	/*
		3067	* With forced threading this function is either in a thread
		3068	* or panic context. So there is no need for concern about
		3069	* printk reentrance, handovers, or lockdep complaints.
		3070	*/
2936		3071
2937	con->seq = pmsg.seq + 1;	3072	con->write(con, outbuf, pmsg.outbuf_len);
		3073	con->seq = pmsg.seq + 1;
		3074	} else {
		3075	/*
		3076	* While actively printing out messages, if another printk()
		3077	* were to occur on another CPU, it may wait for this one to
		3078	* finish. This task can not be preempted if there is a
		3079	* waiter waiting to take over.
		3080	*
		3081	* Interrupts are disabled because the hand over to a waiter
		3082	* must not be interrupted until the hand over is completed
		3083	* (@console_waiter is cleared).
		3084	*/
		3085	printk_safe_enter_irqsave(flags);
		3086	console_lock_spinning_enable();
2938		3087
2939	*handover = console_lock_spinning_disable_and_check(cookie);	3088	/* Do not trace print latency. */
2940	printk_safe_exit_irqrestore(flags);	3089	stop_critical_timings();
		3090
		3091	printk_legacy_lock_map_acquire_try();
		3092	con->write(con, outbuf, pmsg.outbuf_len);
		3093	printk_legacy_lock_map_release();
		3094
		3095	start_critical_timings();
		3096
		3097	con->seq = pmsg.seq + 1;
		3098
		3099	*handover = console_lock_spinning_disable_and_check(cookie);
		3100	printk_safe_exit_irqrestore(flags);
		3101	}
2941	skip:	3102	skip:
2942	return true;	3103	return true;
2943	}	3104	}
Lines 2990-3002 static bool console_flush_all(bool do_cond_resched, u64 next_seq, bool handove Link Here
2990		3151
2991	cookie = console_srcu_read_lock();	3152	cookie = console_srcu_read_lock();
2992	for_each_console_srcu(con) {	3153	for_each_console_srcu(con) {
		3154	short flags = console_srcu_read_flags(con);
		3155	u64 printk_seq;
2993	bool progress;	3156	bool progress;
2994		3157
2995	if (!console_is_usable(con))	3158	/*
		3159	* console_flush_all() is only for legacy consoles,
		3160	* unless the nbcon console has no kthread printer.
		3161	*/
		3162	if ((flags & CON_NBCON) && con->kthread)
		3163	continue;
		3164
		3165	if (!console_is_usable(con, flags, !do_cond_resched))
2996	continue;	3166	continue;
2997	any_usable = true;	3167	any_usable = true;
2998		3168
2999	progress = console_emit_next_record(con, handover, cookie);	3169	if (flags & CON_NBCON) {
		3170	progress = nbcon_legacy_emit_next_record(con, handover, cookie,
		3171	!do_cond_resched);
		3172	printk_seq = nbcon_seq_read(con);
		3173	} else {
		3174	progress = console_emit_next_record(con, handover, cookie);
		3175	printk_seq = con->seq;
		3176	}
3000		3177
3001	/*	3178	/*
3002	* If a handover has occurred, the SRCU read lock	3179	* If a handover has occurred, the SRCU read lock
Lines 3006-3013 static bool console_flush_all(bool do_cond_resched, u64 next_seq, bool handove Link Here
3006	return false;	3183	return false;
3007		3184
3008	/* Track the next of the highest seq flushed. */	3185	/* Track the next of the highest seq flushed. */
3009	if (con->seq > *next_seq)	3186	if (printk_seq > *next_seq)
3010	*next_seq = con->seq;	3187	*next_seq = printk_seq;
3011		3188
3012	if (!progress)	3189	if (!progress)
3013	continue;	3190	continue;
Lines 3030-3048 static bool console_flush_all(bool do_cond_resched, u64 next_seq, bool handove Link Here
3030	return false;	3207	return false;
3031	}	3208	}
3032		3209
3033	/**	3210	static void console_flush_and_unlock(void)
3034	* console_unlock - unblock the console subsystem from printing
3035	*
3036	* Releases the console_lock which the caller holds to block printing of
3037	* the console subsystem.
3038	*
3039	* While the console_lock was held, console output may have been buffered
3040	* by printk(). If this is the case, console_unlock(); emits
3041	* the output prior to releasing the lock.
3042	*
3043	* console_unlock(); may be called from any context.
3044	*/
3045	void console_unlock(void)
3046	{	3211	{
3047	bool do_cond_resched;	3212	bool do_cond_resched;
3048	bool handover;	3213	bool handover;
Lines 3086-3091 void console_unlock(void) Link Here
3086	*/	3251	*/
3087	} while (prb_read_valid(prb, next_seq, NULL) && console_trylock());	3252	} while (prb_read_valid(prb, next_seq, NULL) && console_trylock());
3088	}	3253	}
		3254
		3255	/**
		3256	* console_unlock - unblock the console subsystem from printing
		3257	*
		3258	* Releases the console_lock which the caller holds to block printing of
		3259	* the console subsystem.
		3260	*
		3261	* While the console_lock was held, console output may have been buffered
		3262	* by printk(). If this is the case, console_unlock(); emits
		3263	* the output prior to releasing the lock.
		3264	*
		3265	* console_unlock(); may be called from any context.
		3266	*/
		3267	void console_unlock(void)
		3268	{
		3269	/*
		3270	* Forced threading relies on kthread and atomic consoles for
		3271	* printing. It never attempts to print from console_unlock().
		3272	*/
		3273	if (force_printkthreads()) {
		3274	__console_unlock();
		3275	return;
		3276	}
		3277
		3278	console_flush_and_unlock();
		3279	}
3089	EXPORT_SYMBOL(console_unlock);	3280	EXPORT_SYMBOL(console_unlock);
3090		3281
3091	/**	3282	/**
Lines 3219-3225 void console_flush_on_panic(enum con_flush_mode mode) Link Here
3219	console_srcu_read_unlock(cookie);	3410	console_srcu_read_unlock(cookie);
3220	}	3411	}
3221		3412
3222	console_flush_all(false, &next_seq, &handover);	3413	nbcon_atomic_flush_pending();
		3414
		3415	if (printing_via_unlock)
		3416	console_flush_all(false, &next_seq, &handover);
3223	}	3417	}
3224		3418
3225	/*	3419	/*
Lines 3276-3288 EXPORT_SYMBOL(console_stop); Link Here
3276		3470
3277	void console_start(struct console *console)	3471	void console_start(struct console *console)
3278	{	3472	{
		3473	short flags;
		3474
3279	console_list_lock();	3475	console_list_lock();
3280	console_srcu_write_flags(console, console->flags \| CON_ENABLED);	3476	console_srcu_write_flags(console, console->flags \| CON_ENABLED);
		3477	flags = console->flags;
3281	console_list_unlock();	3478	console_list_unlock();
		3479
		3480	/*
		3481	* Ensure that all SRCU list walks have completed. The related
		3482	* printing context must be able to see it is enabled so that
		3483	* it is guaranteed to wake up and resume printing.
		3484	*/
		3485	synchronize_srcu(&console_srcu);
		3486
		3487	if (flags & CON_NBCON)
		3488	nbcon_kthread_wake(console);
		3489	else
		3490	wake_up_legacy_kthread();
		3491
3282	__pr_flush(console, 1000, true);	3492	__pr_flush(console, 1000, true);
3283	}	3493	}
3284	EXPORT_SYMBOL(console_start);	3494	EXPORT_SYMBOL(console_start);
3285		3495
		3496	#ifdef CONFIG_PRINTK
		3497	static bool printer_should_wake(void)
		3498	{
		3499	bool available = false;
		3500	struct console *con;
		3501	int cookie;
		3502
		3503	if (kthread_should_stop())
		3504	return true;
		3505
		3506	cookie = console_srcu_read_lock();
		3507	for_each_console_srcu(con) {
		3508	short flags = console_srcu_read_flags(con);
		3509	u64 printk_seq;
		3510
		3511	/*
		3512	* The legacy printer thread is only for legacy consoles,
		3513	* unless the nbcon console has no kthread printer.
		3514	*/
		3515	if ((flags & CON_NBCON) && con->kthread)
		3516	continue;
		3517
		3518	if (!console_is_usable(con, flags, true))
		3519	continue;
		3520
		3521	if (flags & CON_NBCON) {
		3522	printk_seq = nbcon_seq_read(con);
		3523	} else {
		3524	/*
		3525	* It is safe to read @seq because only this
		3526	* thread context updates @seq.
		3527	*/
		3528	printk_seq = con->seq;
		3529	}
		3530
		3531	if (prb_read_valid(prb, printk_seq, NULL)) {
		3532	available = true;
		3533	break;
		3534	}
		3535	}
		3536	console_srcu_read_unlock(cookie);
		3537
		3538	return available;
		3539	}
		3540
		3541	static int nbcon_legacy_kthread_func(void *unused)
		3542	{
		3543	int error;
		3544
		3545	for (;;) {
		3546	error = wait_event_interruptible(legacy_wait, printer_should_wake());
		3547
		3548	if (kthread_should_stop())
		3549	break;
		3550
		3551	if (error)
		3552	continue;
		3553
		3554	console_lock();
		3555	console_flush_and_unlock();
		3556	}
		3557
		3558	return 0;
		3559	}
3560
3561	void nbcon_legacy_kthread_create(void)
3562	{
3563	struct task_struct *kt;
3564
3565	lockdep_assert_held(&console_mutex);
3566
3567	if (!force_printkthreads())
3568	return;
3569
3570	if (!printk_threads_enabled \|\| nbcon_legacy_kthread)
3571	return;
3572
3573	kt = kthread_run(nbcon_legacy_kthread_func, NULL, "pr/legacy");
3574	if (IS_ERR(kt)) {
3575	pr_err("unable to start legacy printing thread\n");
3576	return;
3577	}
3578
3579	nbcon_legacy_kthread = kt;
3580
3581	/*
3582	* It is important that console printing threads are scheduled
3583	* shortly after a printk call and with generous runtime budgets.
3584	*/
3585	sched_set_normal(nbcon_legacy_kthread, -20);
3586	}
3587	#endif /* CONFIG_PRINTK */
3588
3286	static int __read_mostly keep_bootcon;	3589	static int __read_mostly keep_bootcon;
3287		3590
3288	static int __init keep_bootcon_setup(char *str)	3591	static int __init keep_bootcon_setup(char *str)
Lines 3366-3371 static void try_enable_default_console(struct console *newcon) Link Here
3366	newcon->flags \|= CON_CONSDEV;	3669	newcon->flags \|= CON_CONSDEV;
3367	}	3670	}
3368		3671
		3672	/* Set @newcon->seq to the first record this console should print. */
3369	static void console_init_seq(struct console *newcon, bool bootcon_registered)	3673	static void console_init_seq(struct console *newcon, bool bootcon_registered)
3370	{	3674	{
3371	struct console *con;	3675	struct console *con;
Lines 3414-3424 static void console_init_seq(struct console *newcon, bool bootcon_registered) Link Here
3414		3718
3415	newcon->seq = prb_next_seq(prb);	3719	newcon->seq = prb_next_seq(prb);
3416	for_each_console(con) {	3720	for_each_console(con) {
3417	if ((con->flags & CON_BOOT) &&	3721	u64 seq;
3418	(con->flags & CON_ENABLED) &&	3722
3419	con->seq < newcon->seq) {	3723	if (!((con->flags & CON_BOOT) &&
3420	newcon->seq = con->seq;	3724	(con->flags & CON_ENABLED))) {
		3725	continue;
3421	}	3726	}
		3727
		3728	if (con->flags & CON_NBCON)
		3729	seq = nbcon_seq_read(con);
		3730	else
		3731	seq = con->seq;
		3732
		3733	if (seq < newcon->seq)
		3734	newcon->seq = seq;
3422	}	3735	}
3423	}	3736	}
3424		3737
Lines 3456-3461 void register_console(struct console *newcon) Link Here
3456	struct console *con;	3769	struct console *con;
3457	bool bootcon_registered = false;	3770	bool bootcon_registered = false;
3458	bool realcon_registered = false;	3771	bool realcon_registered = false;
		3772	unsigned long flags;
3459	int err;	3773	int err;
3460		3774
3461	console_list_lock();	3775	console_list_lock();
Lines 3535-3543 void register_console(struct console *newcon) Link Here
3535	newcon->dropped = 0;	3849	newcon->dropped = 0;
3536	console_init_seq(newcon, bootcon_registered);	3850	console_init_seq(newcon, bootcon_registered);
3537		3851
3538	if (newcon->flags & CON_NBCON)	3852	if (newcon->flags & CON_NBCON) {
		3853	have_nbcon_console = true;
3539	nbcon_init(newcon);	3854	nbcon_init(newcon);
3540		3855
		3856	/*
		3857	* nbcon consoles have their own sequence counter. The legacy
		3858	* sequence counter is reset so that it is clear it is not
		3859	* being used.
		3860	*/
		3861	nbcon_seq_force(newcon, newcon->seq);
		3862	newcon->seq = 0;
		3863	} else {
		3864	have_legacy_console = true;
		3865	nbcon_legacy_kthread_create();
		3866	}
		3867
		3868	if (newcon->flags & CON_BOOT)
		3869	have_boot_console = true;
		3870
		3871	/*
		3872	* If another context is actively using the hardware of this new
		3873	* console, it will not be aware of the nbcon synchronization. This
		3874	* is a risk that two contexts could access the hardware
		3875	* simultaneously if this new console is used for atomic printing
		3876	* and the other context is still using the hardware.
		3877	*
		3878	* Use the driver synchronization to ensure that the hardware is not
		3879	* in use while this new console transitions to being registered.
		3880	*/
		3881	if ((newcon->flags & CON_NBCON) && newcon->write_atomic)
		3882	newcon->device_lock(newcon, &flags);
		3883
3541	/*	3884	/*
3542	* Put this console in the list - keep the	3885	* Put this console in the list - keep the
3543	* preferred driver at the head of the list.	3886	* preferred driver at the head of the list.
Lines 3562-3567 void register_console(struct console *newcon) Link Here
3562	* register_console() completes.	3905	* register_console() completes.
3563	*/	3906	*/
3564		3907
		3908	/* This new console is now registered. */
		3909	if ((newcon->flags & CON_NBCON) && newcon->write_atomic)
		3910	newcon->device_unlock(newcon, flags);
		3911
3565	console_sysfs_notify();	3912	console_sysfs_notify();
3566		3913
3567	/*	3914	/*
Lines 3590-3595 EXPORT_SYMBOL(register_console); Link Here
3590	/* Must be called under console_list_lock(). */	3937	/* Must be called under console_list_lock(). */
3591	static int unregister_console_locked(struct console *console)	3938	static int unregister_console_locked(struct console *console)
3592	{	3939	{
		3940	bool is_boot_con = (console->flags & CON_BOOT);
		3941	bool found_legacy_con = false;
		3942	bool found_nbcon_con = false;
		3943	bool found_boot_con = false;
		3944	struct console *c;
3593	int res;	3945	int res;
3594		3946
3595	lockdep_assert_console_list_lock_held();	3947	lockdep_assert_console_list_lock_held();
Lines 3637-3642 static int unregister_console_locked(struct console *console) Link Here
3637	if (console->exit)	3989	if (console->exit)
3638	res = console->exit(console);	3990	res = console->exit(console);
3639		3991
		3992	/*
		3993	* With this console gone, the global flags tracking registered
		3994	* console types may have changed. Update them.
		3995	*/
		3996	for_each_console(c) {
		3997	if (c->flags & CON_BOOT)
		3998	found_boot_con = true;
		3999
		4000	if (c->flags & CON_NBCON)
		4001	found_nbcon_con = true;
		4002	else
		4003	found_legacy_con = true;
		4004	}
		4005	if (!found_boot_con)
		4006	have_boot_console = found_boot_con;
		4007	if (!found_legacy_con)
		4008	have_legacy_console = found_legacy_con;
		4009	if (!found_nbcon_con)
		4010	have_nbcon_console = found_nbcon_con;
		4011
		4012	/*
		4013	* When the last boot console unregisters, start up the
		4014	* printing threads.
		4015	*/
		4016	if (is_boot_con && !have_boot_console) {
		4017	for_each_console(c)
		4018	nbcon_kthread_create(c);
		4019	}
		4020
		4021	#ifdef CONFIG_PRINTK
		4022	if (!printing_via_unlock && nbcon_legacy_kthread) {
		4023	kthread_stop(nbcon_legacy_kthread);
		4024	nbcon_legacy_kthread = NULL;
		4025	}
		4026	#endif
		4027
3640	return res;	4028	return res;
3641	}	4029	}
3642		4030
Lines 3795-3817 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre Link Here
3795		4183
3796	seq = prb_next_reserve_seq(prb);	4184	seq = prb_next_reserve_seq(prb);
3797		4185
3798	/* Flush the consoles so that records up to @seq are printed. */	4186	/*
3799	console_lock();	4187	* Flush the consoles so that records up to @seq are printed.
3800	console_unlock();	4188	* Otherwise this function will just wait for the threaded printers
		4189	* to print up to @seq.
		4190	*/
		4191	if (printing_via_unlock && !force_printkthreads()) {
		4192	console_lock();
		4193	console_unlock();
		4194	}
3801		4195
3802	for (;;) {	4196	for (;;) {
3803	unsigned long begin_jiffies;	4197	unsigned long begin_jiffies;
3804	unsigned long slept_jiffies;	4198	unsigned long slept_jiffies;
		4199	bool use_console_lock = printing_via_unlock;
		4200
		4201	/*
		4202	* Ensure the compiler does not optimize @use_console_lock to
		4203	* be @printing_via_unlock since the latter can change at any
		4204	* time.
		4205	*/
		4206	barrier();
3805		4207
3806	diff = 0;	4208	diff = 0;
3807		4209
3808	/*	4210	if (use_console_lock) {
3809	* Hold the console_lock to guarantee safe access to	4211	/*
3810	* console->seq. Releasing console_lock flushes more	4212	* Hold the console_lock to guarantee safe access to
3811	* records in case @seq is still not printed on all	4213	* console->seq. Releasing console_lock flushes more
3812	* usable consoles.	4214	* records in case @seq is still not printed on all
3813	*/	4215	* usable consoles.
3814	console_lock();	4216	*/
		4217	console_lock();
		4218	}
3815		4219
3816	cookie = console_srcu_read_lock();	4220	cookie = console_srcu_read_lock();
3817	for_each_console_srcu(c) {	4221	for_each_console_srcu(c) {
Lines 3825-3836 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre Link Here
3825	* that they make forward progress, so only increment	4229	* that they make forward progress, so only increment
3826	* @diff for usable consoles.	4230	* @diff for usable consoles.
3827	*/	4231	*/
3828	if (!console_is_usable(c))	4232	if (!console_is_usable(c, flags, true) &&
		4233	!console_is_usable(c, flags, false)) {
3829	continue;	4234	continue;
		4235	}
3830		4236
3831	if (flags & CON_NBCON) {	4237	if (flags & CON_NBCON) {
3832	printk_seq = nbcon_seq_read(c);	4238	printk_seq = nbcon_seq_read(c);
3833	} else {	4239	} else {
		4240	WARN_ON_ONCE(!use_console_lock);
3834	printk_seq = c->seq;	4241	printk_seq = c->seq;
3835	}	4242	}
3836		4243
Lines 3842-3848 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre Link Here
3842	if (diff != last_diff && reset_on_progress)	4249	if (diff != last_diff && reset_on_progress)
3843	remaining_jiffies = timeout_jiffies;	4250	remaining_jiffies = timeout_jiffies;
3844		4251
3845	console_unlock();	4252	if (use_console_lock)
		4253	console_unlock();
3846		4254
3847	/* Note: @diff is 0 if there are no usable consoles. */	4255	/* Note: @diff is 0 if there are no usable consoles. */
3848	if (diff == 0 \|\| remaining_jiffies == 0)	4256	if (diff == 0 \|\| remaining_jiffies == 0)
Lines 3894-3902 static void wake_up_klogd_work_func(struct irq_work *irq_work) Link Here
3894	int pending = this_cpu_xchg(printk_pending, 0);	4302	int pending = this_cpu_xchg(printk_pending, 0);
3895		4303
3896	if (pending & PRINTK_PENDING_OUTPUT) {	4304	if (pending & PRINTK_PENDING_OUTPUT) {
3897	/* If trylock fails, someone else is doing the printing */	4305	if (force_printkthreads()) {
3898	if (console_trylock())	4306	wake_up_legacy_kthread();
3899	console_unlock();	4307	} else {
		4308	/*
		4309	* If trylock fails, some other context
		4310	* will do the printing.
		4311	*/
		4312	if (console_trylock())
		4313	console_unlock();
		4314	}
3900	}	4315	}
3901		4316
3902	if (pending & PRINTK_PENDING_WAKEUP)	4317	if (pending & PRINTK_PENDING_WAKEUP)
Lines 3912-3917 static void __wake_up_klogd(int val) Link Here
3912	return;	4327	return;
3913		4328
3914	preempt_disable();	4329	preempt_disable();
		4330
3915	/*	4331	/*
3916	* Guarantee any new records can be seen by tasks preparing to wait	4332	* Guarantee any new records can be seen by tasks preparing to wait
3917	* before this context checks if the wait queue is empty.	4333	* before this context checks if the wait queue is empty.
Lines 3923-3933 static void __wake_up_klogd(int val) Link Here
3923	*	4339	*
3924	* This pairs with devkmsg_read:A and syslog_print:A.	4340	* This pairs with devkmsg_read:A and syslog_print:A.
3925	*/	4341	*/
3926	if (wq_has_sleeper(&log_wait) \|\| /* LMM(__wake_up_klogd:A) */	4342	if (!wq_has_sleeper(&log_wait)) /* LMM(__wake_up_klogd:A) */
3927	(val & PRINTK_PENDING_OUTPUT)) {	4343	val &= ~PRINTK_PENDING_WAKEUP;
		4344
		4345	/*
		4346	* Simple read is safe. register_console() would flush a newly
		4347	* registered legacy console when writing the message about it
		4348	* being enabled.
		4349	*/
		4350	if (!printing_via_unlock)
		4351	val &= ~PRINTK_PENDING_OUTPUT;
		4352
		4353	if (val) {
3928	this_cpu_or(printk_pending, val);	4354	this_cpu_or(printk_pending, val);
3929	irq_work_queue(this_cpu_ptr(&wake_up_klogd_work));	4355	irq_work_queue(this_cpu_ptr(&wake_up_klogd_work));
3930	}	4356	}
		4357
3931	preempt_enable();	4358	preempt_enable();
3932	}	4359	}
3933		4360
Lines 3969-3974 void defer_console_output(void) Link Here
3969		4396
3970	void printk_trigger_flush(void)	4397	void printk_trigger_flush(void)
3971	{	4398	{
		4399	nbcon_wake_threads();
3972	defer_console_output();	4400	defer_console_output();
3973	}	4401	}
3974		4402

Lines 1034-1042 static char data_alloc(struct printk_ringbuffer rb, unsigned int size, Link Here

(-)a/kernel/printk/printk_ringbuffer.c (-4 / +16 lines)
1034	unsigned long next_lpos;	1034	unsigned long next_lpos;
1035		1035
1036	if (size == 0) {	1036	if (size == 0) {
1037	/* Specify a data-less block. */	1037	/*
1038	blk_lpos->begin = NO_LPOS;	1038	* Data blocks are not created for empty lines. Instead, the
1039	blk_lpos->next = NO_LPOS;	1039	* reader will recognize these special lpos values and handle
		1040	* it appropriately.
		1041	*/
		1042	blk_lpos->begin = EMPTY_LINE_LPOS;
		1043	blk_lpos->next = EMPTY_LINE_LPOS;
1040	return NULL;	1044	return NULL;
1041	}	1045	}
1042		1046
Lines 1214-1223 static const char get_data(struct prb_data_ring data_ring, Link Here
1214		1218
1215	/* Data-less data block description. */	1219	/* Data-less data block description. */
1216	if (BLK_DATALESS(blk_lpos)) {	1220	if (BLK_DATALESS(blk_lpos)) {
1217	if (blk_lpos->begin == NO_LPOS && blk_lpos->next == NO_LPOS) {	1221	/*
		1222	* Records that are just empty lines are also valid, even
		1223	* though they do not have a data block. For such records
		1224	* explicitly return empty string data to signify success.
		1225	*/
		1226	if (blk_lpos->begin == EMPTY_LINE_LPOS &&
		1227	blk_lpos->next == EMPTY_LINE_LPOS) {
1218	*data_size = 0;	1228	*data_size = 0;
1219	return "";	1229	return "";
1220	}	1230	}
		1231
		1232	/* Data lost, invalid, or otherwise unavailable. */
1221	return NULL;	1233	return NULL;
1222	}	1234	}
1223		1235

Lines 5-10 Link Here

(-)a/kernel/printk/printk_ringbuffer.h (-1 / +17 lines)
5		5
6	#include <linux/atomic.h>	6	#include <linux/atomic.h>
7	#include <linux/dev_printk.h>	7	#include <linux/dev_printk.h>
		8	#include <linux/stddef.h>
		9	#include <linux/types.h>
8		10
9	/*	11	/*
10	* Meta information about each stored message.	12	* Meta information about each stored message.
Lines 127-134 enum desc_state { Link Here
127	#define DESC_SV(id, state) (((unsigned long)state << DESC_FLAGS_SHIFT) \| id)	129	#define DESC_SV(id, state) (((unsigned long)state << DESC_FLAGS_SHIFT) \| id)
128	#define DESC_ID_MASK (~DESC_FLAGS_MASK)	130	#define DESC_ID_MASK (~DESC_FLAGS_MASK)
129	#define DESC_ID(sv) ((sv) & DESC_ID_MASK)	131	#define DESC_ID(sv) ((sv) & DESC_ID_MASK)
		132
		133	/*
		134	* Special data block logical position values (for fields of
		135	* @prb_desc.text_blk_lpos).
		136	*
		137	* - Bit0 is used to identify if the record has no data block. (Implemented in
		138	* the LPOS_DATALESS() macro.)
		139	*
		140	* - Bit1 specifies the reason for not having a data block.
		141	*
		142	* These special values could never be real lpos values because of the
		143	* meta data and alignment padding of data blocks. (See to_blk_size() for
		144	* details.)
		145	*/
130	#define FAILED_LPOS 0x1	146	#define FAILED_LPOS 0x1
131	#define NO_LPOS 0x3	147	#define EMPTY_LINE_LPOS 0x3
132		148
133	#define FAILED_BLK_LPOS \	149	#define FAILED_BLK_LPOS \
134	{ \	150	{ \




	this_cpu_dec(printk_context);
}

void __printk_deferred_enter(void)
{
	cant_migrate();
	__printk_safe_enter();
}

void __printk_deferred_exit(void)
{
	cant_migrate();
	__printk_safe_exit();
}

asmlinkage int vprintk(const char *fmt, va_list args)
{
#ifdef CONFIG_KGDB_KDB

Lines 2409-2414 static int rcutorture_booster_init(unsigned int cpu) Link Here

(-)a/kernel/rcu/rcutorture.c (+6 lines)
2409	WARN_ON_ONCE(!t);	2409	WARN_ON_ONCE(!t);
2410	sp.sched_priority = 2;	2410	sp.sched_priority = 2;
2411	sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);	2411	sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
		2412	#ifdef CONFIG_PREEMPT_RT
		2413	t = per_cpu(timersd, cpu);
		2414	WARN_ON_ONCE(!t);
		2415	sp.sched_priority = 2;
		2416	sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
		2417	#endif
2412	}	2418	}
2413		2419
2414	/* Don't allow time recalculation while creating a new task. */	2420	/* Don't allow time recalculation while creating a new task. */

Lines 7-12 Link Here

(-)a/kernel/rcu/tree_exp.h (+7 lines)
7	* Authors: Paul E. McKenney <paulmck@linux.ibm.com>	7	* Authors: Paul E. McKenney <paulmck@linux.ibm.com>
8	*/	8	*/
9		9
		10	#include <linux/console.h>
10	#include <linux/lockdep.h>	11	#include <linux/lockdep.h>
11		12
12	static void rcu_exp_handler(void *unused);	13	static void rcu_exp_handler(void *unused);
Lines 636-641 static void synchronize_rcu_expedited_wait(void) Link Here
636	return;	637	return;
637	if (rcu_stall_is_suppressed())	638	if (rcu_stall_is_suppressed())
638	continue;	639	continue;
		640
		641	nbcon_cpu_emergency_enter();
		642
639	j = jiffies;	643	j = jiffies;
640	rcu_stall_notifier_call_chain(RCU_STALL_NOTIFY_EXP, (void *)(j - jiffies_start));	644	rcu_stall_notifier_call_chain(RCU_STALL_NOTIFY_EXP, (void *)(j - jiffies_start));
641	trace_rcu_stall_warning(rcu_state.name, TPS("ExpeditedStall"));	645	trace_rcu_stall_warning(rcu_state.name, TPS("ExpeditedStall"));
Lines 689-694 static void synchronize_rcu_expedited_wait(void) Link Here
689	rcu_exp_print_detail_task_stall_rnp(rnp);	693	rcu_exp_print_detail_task_stall_rnp(rnp);
690	}	694	}
691	jiffies_stall = 3 * rcu_exp_jiffies_till_stall_check() + 3;	695	jiffies_stall = 3 * rcu_exp_jiffies_till_stall_check() + 3;
		696
		697	nbcon_cpu_emergency_exit();
		698
692	panic_on_rcu_stall();	699	panic_on_rcu_stall();
693	}	700	}
694	}	701	}

Lines 7-12 Link Here

(-)a/kernel/rcu/tree_stall.h (+9 lines)
7	* Author: Paul E. McKenney <paulmck@linux.ibm.com>	7	* Author: Paul E. McKenney <paulmck@linux.ibm.com>
8	*/	8	*/
9		9
		10	#include <linux/console.h>
10	#include <linux/kvm_para.h>	11	#include <linux/kvm_para.h>
11	#include <linux/rcu_notifier.h>	12	#include <linux/rcu_notifier.h>
12		13
Lines 604-609 static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) Link Here
604	if (rcu_stall_is_suppressed())	605	if (rcu_stall_is_suppressed())
605	return;	606	return;
606		607
		608	nbcon_cpu_emergency_enter();
		609
607	/*	610	/*
608	* OK, time to rat on our buddy...	611	* OK, time to rat on our buddy...
609	* See Documentation/RCU/stallwarn.rst for info on how to debug	612	* See Documentation/RCU/stallwarn.rst for info on how to debug
Lines 655-660 static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) Link Here
655	rcu_check_gp_kthread_expired_fqs_timer();	658	rcu_check_gp_kthread_expired_fqs_timer();
656	rcu_check_gp_kthread_starvation();	659	rcu_check_gp_kthread_starvation();
657		660
		661	nbcon_cpu_emergency_exit();
		662
658	panic_on_rcu_stall();	663	panic_on_rcu_stall();
659		664
660	rcu_force_quiescent_state(); /* Kick them all. */	665	rcu_force_quiescent_state(); /* Kick them all. */
Lines 675-680 static void print_cpu_stall(unsigned long gps) Link Here
675	if (rcu_stall_is_suppressed())	680	if (rcu_stall_is_suppressed())
676	return;	681	return;
677		682
		683	nbcon_cpu_emergency_enter();
		684
678	/*	685	/*
679	* OK, time to rat on ourselves...	686	* OK, time to rat on ourselves...
680	* See Documentation/RCU/stallwarn.rst for info on how to debug	687	* See Documentation/RCU/stallwarn.rst for info on how to debug
Lines 703-708 static void print_cpu_stall(unsigned long gps) Link Here
703	jiffies + 3 * rcu_jiffies_till_stall_check() + 3);	710	jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
704	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);	711	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
705		712
		713	nbcon_cpu_emergency_exit();
		714
706	panic_on_rcu_stall();	715	panic_on_rcu_stall();
707		716
708	/*	717	/*





#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
/*
 * Atomically set TIF_NEED_RESCHED[_LAZY] and test for TIF_POLLING_NRFLAG,
 * this avoids any races wrt polling state changes and thereby avoids
 * spurious IPIs.
 */
static inline bool set_nr_and_not_polling(struct task_struct *p, int tif_bit)
{
	struct thread_info *ti = task_thread_info(p);

	return !(fetch_or(&ti->flags, 1 << tif_bit) & _TIF_POLLING_NRFLAG);
}

/*

	do {
		if (!(val & _TIF_POLLING_NRFLAG))
			return false;
		if (val & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY))
			return true;
	} while (!try_cmpxchg(&ti->flags, &val, val | _TIF_NEED_RESCHED));


}

#else
static inline bool set_nr_and_not_polling(struct task_struct *p, int tif_bit)
{
	set_tsk_thread_flag(p, tif_bit);
	return true;
}


 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
static void __resched_curr(struct rq *rq, int lazy)
{
	int cpu, tif_bit = TIF_NEED_RESCHED + lazy;
	struct task_struct *curr = rq->curr;
	int cpu;

	lockdep_assert_rq_held(rq);

	if (unlikely(test_tsk_thread_flag(curr, tif_bit)))
		return;

	cpu = cpu_of(rq);

	if (cpu == smp_processor_id()) {
		set_tsk_thread_flag(curr, tif_bit);
		if (!lazy)
			set_preempt_need_resched();
		return;
	}

	if (set_nr_and_not_polling(curr, tif_bit)) {
		if (!lazy)
			smp_send_reschedule(cpu);
	} else {
		trace_sched_wake_idle_without_ipi(cpu);
	}
}

void resched_curr(struct rq *rq)
{
	__resched_curr(rq, 0);
}

void resched_curr_lazy(struct rq *rq)
{
	int lazy = IS_ENABLED(CONFIG_PREEMPT_BUILD_AUTO) && !sched_feat(FORCE_NEED_RESCHED) ?
		TIF_NEED_RESCHED_LAZY_OFFSET : 0;

	if (lazy && unlikely(test_tsk_thread_flag(rq->curr, TIF_NEED_RESCHED)))
		return;

	__resched_curr(rq, lazy);
}

void resched_cpu(int cpu)

	 * and testing of the above solutions didn't appear to report
	 * much benefits.
	 */
	if (set_nr_and_not_polling(rq->idle, TIF_NEED_RESCHED))
		smp_send_reschedule(cpu);
	else
		trace_sched_wake_idle_without_ipi(cpu);


#endif /* #ifdef CONFIG_PREEMPT_DYNAMIC */

/*
 * task_is_pi_boosted - Check if task has been PI boosted.
 * @p:	Task to check.
 *
 * Return true if task is subject to priority inheritance.
 */
bool task_is_pi_boosted(const struct task_struct *p)
{
	int prio = p->prio;

	if (!rt_prio(prio))
		return false;
	return prio != p->normal_prio;
}

/**
 * yield - yield the current processor to other threads.
 *




	.release	= seq_release,
};

static ssize_t sched_hog_write(struct file *filp, const char __user *ubuf,
			       size_t cnt, loff_t *ppos)
{
	unsigned long end = jiffies + 60 * HZ;

	for (; time_before(jiffies, end) && !signal_pending(current);)
		cpu_relax();

	return cnt;
}

static const struct file_operations sched_hog_fops = {
	.write		= sched_hog_write,
	.open		= simple_open,
	.llseek		= default_llseek,
};

static struct dentry *debugfs_sched;

static __init int sched_init_debug(void)


	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);

	debugfs_create_file("hog", 0200, debugfs_sched, NULL, &sched_hog_fops);

	return 0;
}
late_initcall(sched_init_debug);

Lines 975-982 static void clear_buddies(struct cfs_rq cfs_rq, struct sched_entity se); Link Here

(-)a/kernel/sched/fair.c (-15 / +31 lines)
975	* XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i	975	* XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i
976	* this is probably good enough.	976	* this is probably good enough.
977	*/	977	*/
978	static void update_deadline(struct cfs_rq cfs_rq, struct sched_entity se)	978	static void update_deadline(struct cfs_rq cfs_rq, struct sched_entity se, bool tick)
979	{	979	{
		980	struct rq *rq = rq_of(cfs_rq);
		981
980	if ((s64)(se->vruntime - se->deadline) < 0)	982	if ((s64)(se->vruntime - se->deadline) < 0)
981	return;	983	return;
982		984
Lines 995-1004 static void update_deadline(struct cfs_rq cfs_rq, struct sched_entity se) Link Here
995	/*	997	/*
996	* The task has consumed its request, reschedule.	998	* The task has consumed its request, reschedule.
997	*/	999	*/
998	if (cfs_rq->nr_running > 1) {	1000	if (cfs_rq->nr_running < 2)
999	resched_curr(rq_of(cfs_rq));	1001	return;
1000	clear_buddies(cfs_rq, se);	1002
		1003	if (!IS_ENABLED(CONFIG_PREEMPT_BUILD_AUTO) \|\| sched_feat(FORCE_NEED_RESCHED)) {
		1004	resched_curr(rq);
		1005	} else {
		1006	/* Did the task ignore the lazy reschedule request? */
		1007	if (tick && test_tsk_thread_flag(rq->curr, TIF_NEED_RESCHED_LAZY))
		1008	resched_curr(rq);
		1009	else
		1010	resched_curr_lazy(rq);
1001	}	1011	}
		1012	clear_buddies(cfs_rq, se);
1002	}	1013	}
1003		1014
1004	#include "pelt.h"	1015	#include "pelt.h"
Lines 1153-1159 s64 update_curr_common(struct rq *rq) Link Here
1153	/*	1164	/*
1154	* Update the current task's runtime statistics.	1165	* Update the current task's runtime statistics.
1155	*/	1166	*/
1156	static void update_curr(struct cfs_rq *cfs_rq)	1167	static void __update_curr(struct cfs_rq *cfs_rq, bool tick)
1157	{	1168	{
1158	struct sched_entity *curr = cfs_rq->curr;	1169	struct sched_entity *curr = cfs_rq->curr;
1159	s64 delta_exec;	1170	s64 delta_exec;
Lines 1357-1363 Link Here
1357	#else // !CONFIG_SCHED_BORE	1357	#else // !CONFIG_SCHED_BORE
1358	curr->vruntime += calc_delta_fair(delta_exec, curr);	1358	curr->vruntime += calc_delta_fair(delta_exec, curr);
1359	#endif // CONFIG_SCHED_BORE	1359	#endif // CONFIG_SCHED_BORE
1360	update_deadline(cfs_rq, curr);	1360	update_deadline(cfs_rq, curr, tick);
1361	update_min_vruntime(cfs_rq);	1361	update_min_vruntime(cfs_rq);
1362		1362
1363	if (entity_is_task(curr))	1363	if (entity_is_task(curr))
Lines 1175-1180 static void update_curr(struct cfs_rq *cfs_rq) Link Here
1175	account_cfs_rq_runtime(cfs_rq, delta_exec);	1186	account_cfs_rq_runtime(cfs_rq, delta_exec);
1176	}	1187	}
1177		1188
		1189	static inline void update_curr(struct cfs_rq *cfs_rq)
		1190	{
		1191	__update_curr(cfs_rq, false);
		1192	}
		1193
1178	static void update_curr_fair(struct rq *rq)	1194	static void update_curr_fair(struct rq *rq)
1179	{	1195	{
1180	update_curr(cfs_rq_of(&rq->curr->se));	1196	update_curr(cfs_rq_of(&rq->curr->se));
Lines 5493-5499 entity_tick(struct cfs_rq cfs_rq, struct sched_entity curr, int queued) Link Here
5493	/*	5509	/*
5494	* Update run-time statistics of the 'current'.	5510	* Update run-time statistics of the 'current'.
5495	*/	5511	*/
5496	update_curr(cfs_rq);	5512	__update_curr(cfs_rq, true);
5497		5513
5498	/*	5514	/*
5499	* Ensure that runnable average is periodically updated.	5515	* Ensure that runnable average is periodically updated.
Lines 5507-5513 entity_tick(struct cfs_rq cfs_rq, struct sched_entity curr, int queued) Link Here
5507	* validating it and just reschedule.	5523	* validating it and just reschedule.
5508	*/	5524	*/
5509	if (queued) {	5525	if (queued) {
5510	resched_curr(rq_of(cfs_rq));	5526	resched_curr_lazy(rq_of(cfs_rq));
5511	return;	5527	return;
5512	}	5528	}
5513	/*	5529	/*
Lines 5653-5659 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) Link Here
5653	* hierarchy can be throttled	5669	* hierarchy can be throttled
5654	*/	5670	*/
5655	if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))	5671	if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
5656	resched_curr(rq_of(cfs_rq));	5672	resched_curr_lazy(rq_of(cfs_rq));
5657	}	5673	}
5658		5674
5659	static __always_inline	5675	static __always_inline
Lines 5913-5919 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) Link Here
5913		5929
5914	/* Determine whether we need to wake up potentially idle CPU: */	5930	/* Determine whether we need to wake up potentially idle CPU: */
5915	if (rq->curr == rq->idle && rq->cfs.nr_running)	5931	if (rq->curr == rq->idle && rq->cfs.nr_running)
5916	resched_curr(rq);	5932	resched_curr_lazy(rq);
5917	}	5933	}
5918		5934
5919	#ifdef CONFIG_SMP	5935	#ifdef CONFIG_SMP
Lines 6628-6634 static void hrtick_start_fair(struct rq rq, struct task_struct p) Link Here
6628		6644
6629	if (delta < 0) {	6645	if (delta < 0) {
6630	if (task_current(rq, p))	6646	if (task_current(rq, p))
6631	resched_curr(rq);	6647	resched_curr_lazy(rq);
6632	return;	6648	return;
6633	}	6649	}
6634	hrtick_start(rq, delta);	6650	hrtick_start(rq, delta);
Lines 8304-8310 static void check_preempt_wakeup_fair(struct rq rq, struct task_struct p, int Link Here
8304	* prevents us from potentially nominating it as a false LAST_BUDDY	8320	* prevents us from potentially nominating it as a false LAST_BUDDY
8305	* below.	8321	* below.
8306	*/	8322	*/
8307	if (test_tsk_need_resched(curr))	8323	if (need_resched())
8308	return;	8324	return;
8309		8325
8310	/* Idle tasks are by definition preempted by non-idle tasks. */	8326	/* Idle tasks are by definition preempted by non-idle tasks. */
Lines 8346-8352 static void check_preempt_wakeup_fair(struct rq rq, struct task_struct p, int Link Here
8346	return;	8362	return;
8347		8363
8348	preempt:	8364	preempt:
8349	resched_curr(rq);	8365	resched_curr_lazy(rq);
8350	}	8366	}
8351		8367
8352	#ifdef CONFIG_SMP	8368	#ifdef CONFIG_SMP
Lines 12516-12522 static inline void task_tick_core(struct rq rq, struct task_struct curr) Link Here
12516	*/	12532	*/
12517	if (rq->core->core_forceidle_count && rq->cfs.nr_running == 1 &&	12533	if (rq->core->core_forceidle_count && rq->cfs.nr_running == 1 &&
12518	__entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE))	12534	__entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE))
12519	resched_curr(rq);	12535	resched_curr_lazy(rq);
12520	}	12536	}
12521		12537
12522	/*	12538	/*
Lines 12681-12687 prio_changed_fair(struct rq rq, struct task_struct p, int oldprio) Link Here
12681	*/	12697	*/
12682	if (task_current(rq, p)) {	12698	if (task_current(rq, p)) {
12683	if (p->prio > oldprio)	12699	if (p->prio > oldprio)
12684	resched_curr(rq);	12700	resched_curr_lazy(rq);
12685	} else	12701	} else
12686	wakeup_preempt(rq, p, 0);	12702	wakeup_preempt(rq, p, 0);
12687	}	12703	}

Lines 87-89 SCHED_FEAT(UTIL_EST, true) Link Here

(-)a/kernel/sched/features.h (+2 lines)
87	SCHED_FEAT(LATENCY_WARN, false)	87	SCHED_FEAT(LATENCY_WARN, false)
88		88
89	SCHED_FEAT(HZ_BW, true)	89	SCHED_FEAT(HZ_BW, true)
		90
		91	SCHED_FEAT(FORCE_NEED_RESCHED, false)

Lines 57-64 static noinline int __cpuidle cpu_idle_poll(void) Link Here

(-)a/kernel/sched/idle.c (-2 / +1 lines)
57	ct_cpuidle_enter();	57	ct_cpuidle_enter();
58		58
59	raw_local_irq_enable();	59	raw_local_irq_enable();
60	while (!tif_need_resched() &&	60	while (!need_resched() && (cpu_idle_force_poll \|\| tick_check_broadcast_expired()))
61	(cpu_idle_force_poll \|\| tick_check_broadcast_expired()))
62	cpu_relax();	61	cpu_relax();
63	raw_local_irq_disable();	62	raw_local_irq_disable();
64		63

Lines 2194-2201 static int rto_next_cpu(struct root_domain *rd) Link Here

(-)a/kernel/sched/rt.c (-1 / +4 lines)
2194		2194
2195	rd->rto_cpu = cpu;	2195	rd->rto_cpu = cpu;
2196		2196
2197	if (cpu < nr_cpu_ids)	2197	if (cpu < nr_cpu_ids) {
		2198	if (!has_pushable_tasks(cpu_rq(cpu)))
		2199	continue;
2198	return cpu;	2200	return cpu;
		2201	}
2199		2202
2200	rd->rto_cpu = -1;	2203	rd->rto_cpu = -1;
2201		2204

Lines 2463-2468 extern void init_sched_fair_class(void); Link Here

(-)a/kernel/sched/sched.h (+1 lines)
2463	extern void reweight_task(struct task_struct *p, int prio);	2463	extern void reweight_task(struct task_struct *p, int prio);
2464		2464
2465	extern void resched_curr(struct rq *rq);	2465	extern void resched_curr(struct rq *rq);
		2466	extern void resched_curr_lazy(struct rq *rq);
2466	extern void resched_cpu(int cpu);	2467	extern void resched_cpu(int cpu);
2467		2468
2468	extern struct rt_bandwidth def_rt_bandwidth;	2469	extern struct rt_bandwidth def_rt_bandwidth;

Lines 247-252 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) Link Here

(-)a/kernel/softirq.c (-1 / +94 lines)
247	}	247	}
248	EXPORT_SYMBOL(__local_bh_enable_ip);	248	EXPORT_SYMBOL(__local_bh_enable_ip);
249		249
		250	void softirq_preempt(void)
		251	{
		252	if (WARN_ON_ONCE(!preemptible()))
		253	return;
		254
		255	if (WARN_ON_ONCE(__this_cpu_read(softirq_ctrl.cnt) != SOFTIRQ_OFFSET))
		256	return;
		257
		258	__local_bh_enable(SOFTIRQ_OFFSET, true);
		259	/* preemption point */
		260	__local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
		261	}
		262
250	/*	263	/*
251	* Invoked from ksoftirqd_run() outside of the interrupt disabled section	264	* Invoked from ksoftirqd_run() outside of the interrupt disabled section
252	* to acquire the per CPU local lock for reentrancy protection.	265	* to acquire the per CPU local lock for reentrancy protection.
Lines 619-624 static inline void tick_irq_exit(void) Link Here
619	#endif	632	#endif
620	}	633	}
621		634
		635	#ifdef CONFIG_PREEMPT_RT
		636	DEFINE_PER_CPU(struct task_struct *, timersd);
		637	DEFINE_PER_CPU(unsigned long, pending_timer_softirq);
		638
		639	static void wake_timersd(void)
		640	{
		641	struct task_struct *tsk = __this_cpu_read(timersd);
		642
		643	if (tsk)
		644	wake_up_process(tsk);
		645	}
		646
		647	#else
		648
		649	static inline void wake_timersd(void) { }
		650
		651	#endif
		652
622	static inline void __irq_exit_rcu(void)	653	static inline void __irq_exit_rcu(void)
623	{	654	{
624	#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED	655	#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
Lines 631-636 static inline void __irq_exit_rcu(void) Link Here
631	if (!in_interrupt() && local_softirq_pending())	662	if (!in_interrupt() && local_softirq_pending())
632	invoke_softirq();	663	invoke_softirq();
633		664
		665	if (IS_ENABLED(CONFIG_PREEMPT_RT) && local_pending_timers() &&
		666	!(in_nmi() \| in_hardirq()))
		667	wake_timersd();
		668
634	tick_irq_exit();	669	tick_irq_exit();
635	}	670	}
636		671
Lines 963-974 static struct smp_hotplug_thread softirq_threads = { Link Here
963	.thread_comm = "ksoftirqd/%u",	998	.thread_comm = "ksoftirqd/%u",
964	};	999	};
965		1000
		1001	#ifdef CONFIG_PREEMPT_RT
		1002	static void timersd_setup(unsigned int cpu)
		1003	{
		1004	sched_set_fifo_low(current);
		1005	}
		1006
		1007	static int timersd_should_run(unsigned int cpu)
		1008	{
		1009	return local_pending_timers();
		1010	}
		1011
		1012	static void run_timersd(unsigned int cpu)
		1013	{
		1014	unsigned int timer_si;
		1015
		1016	ksoftirqd_run_begin();
		1017
		1018	timer_si = local_pending_timers();
		1019	__this_cpu_write(pending_timer_softirq, 0);
		1020	or_softirq_pending(timer_si);
		1021
		1022	__do_softirq();
		1023
		1024	ksoftirqd_run_end();
		1025	}
		1026
		1027	static void raise_ktimers_thread(unsigned int nr)
		1028	{
		1029	trace_softirq_raise(nr);
		1030	__this_cpu_or(pending_timer_softirq, 1 << nr);
		1031	}
		1032
		1033	void raise_hrtimer_softirq(void)
		1034	{
		1035	raise_ktimers_thread(HRTIMER_SOFTIRQ);
		1036	}
		1037
		1038	void raise_timer_softirq(void)
		1039	{
		1040	unsigned long flags;
		1041
		1042	local_irq_save(flags);
		1043	raise_ktimers_thread(TIMER_SOFTIRQ);
		1044	wake_timersd();
		1045	local_irq_restore(flags);
		1046	}
		1047
		1048	static struct smp_hotplug_thread timer_threads = {
		1049	.store = &timersd,
		1050	.setup = timersd_setup,
		1051	.thread_should_run = timersd_should_run,
		1052	.thread_fn = run_timersd,
		1053	.thread_comm = "ktimers/%u",
		1054	};
		1055	#endif
		1056
966	static __init int spawn_ksoftirqd(void)	1057	static __init int spawn_ksoftirqd(void)
967	{	1058	{
968	cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,	1059	cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
969	takeover_tasklets);	1060	takeover_tasklets);
970	BUG_ON(smpboot_register_percpu_thread(&softirq_threads));	1061	BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
971		1062	#ifdef CONFIG_PREEMPT_RT
		1063	BUG_ON(smpboot_register_percpu_thread(&timer_threads));
		1064	#endif
972	return 0;	1065	return 0;
973	}	1066	}
974	early_initcall(spawn_ksoftirqd);	1067	early_initcall(spawn_ksoftirqd);

Lines 1809-1815 void hrtimer_interrupt(struct clock_event_device *dev) Link Here

(-)a/kernel/time/hrtimer.c (-2 / +2 lines)
1809	if (!ktime_before(now, cpu_base->softirq_expires_next)) {	1809	if (!ktime_before(now, cpu_base->softirq_expires_next)) {
1810	cpu_base->softirq_expires_next = KTIME_MAX;	1810	cpu_base->softirq_expires_next = KTIME_MAX;
1811	cpu_base->softirq_activated = 1;	1811	cpu_base->softirq_activated = 1;
1812	raise_softirq_irqoff(HRTIMER_SOFTIRQ);	1812	raise_hrtimer_softirq();
1813	}	1813	}
1814		1814
1815	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);	1815	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
Lines 1922-1928 void hrtimer_run_queues(void) Link Here
1922	if (!ktime_before(now, cpu_base->softirq_expires_next)) {	1922	if (!ktime_before(now, cpu_base->softirq_expires_next)) {
1923	cpu_base->softirq_expires_next = KTIME_MAX;	1923	cpu_base->softirq_expires_next = KTIME_MAX;
1924	cpu_base->softirq_activated = 1;	1924	cpu_base->softirq_activated = 1;
1925	raise_softirq_irqoff(HRTIMER_SOFTIRQ);	1925	raise_hrtimer_softirq();
1926	}	1926	}
1927		1927
1928	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);	1928	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);

Lines 796-802 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) Link Here

(-)a/kernel/time/tick-sched.c (-1 / +1 lines)
796		796
797	static inline bool local_timer_softirq_pending(void)	797	static inline bool local_timer_softirq_pending(void)
798	{	798	{
799	return local_softirq_pending() & BIT(TIMER_SOFTIRQ);	799	return local_pending_timers() & BIT(TIMER_SOFTIRQ);
800	}	800	}
801		801
802	static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)	802	static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)

Lines 1470-1478 static inline void timer_base_unlock_expiry(struct timer_base *base) Link Here

(-)a/kernel/time/timer.c (-2 / +9 lines)
1470	*/	1470	*/
1471	static void timer_sync_wait_running(struct timer_base *base)	1471	static void timer_sync_wait_running(struct timer_base *base)
1472	{	1472	{
1473	if (atomic_read(&base->timer_waiters)) {	1473	bool need_preempt;
		1474
		1475	need_preempt = task_is_pi_boosted(current);
		1476	if (need_preempt \|\| atomic_read(&base->timer_waiters)) {
1474	raw_spin_unlock_irq(&base->lock);	1477	raw_spin_unlock_irq(&base->lock);
1475	spin_unlock(&base->expiry_lock);	1478	spin_unlock(&base->expiry_lock);
		1479
		1480	if (need_preempt)
		1481	softirq_preempt();
		1482
1476	spin_lock(&base->expiry_lock);	1483	spin_lock(&base->expiry_lock);
1477	raw_spin_lock_irq(&base->lock);	1484	raw_spin_lock_irq(&base->lock);
1478	}	1485	}
Lines 2070-2076 static void run_local_timers(void) Link Here
2070	if (time_before(jiffies, base->next_expiry))	2077	if (time_before(jiffies, base->next_expiry))
2071	return;	2078	return;
2072	}	2079	}
2073	raise_softirq(TIMER_SOFTIRQ);	2080	raise_timer_softirq();
2074	}	2081	}
2075		2082
2076	/*	2083	/*

Lines 2717-2722 unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status) Link Here

(-)a/kernel/trace/trace.c (+2 lines)
2717		2717
2718	if (tif_need_resched())	2718	if (tif_need_resched())
2719	trace_flags \|= TRACE_FLAG_NEED_RESCHED;	2719	trace_flags \|= TRACE_FLAG_NEED_RESCHED;
		2720	if (tif_need_resched_lazy())
		2721	trace_flags \|= TRACE_FLAG_NEED_RESCHED_LAZY;
2720	if (test_preempt_need_resched())	2722	if (test_preempt_need_resched())
2721	trace_flags \|= TRACE_FLAG_PREEMPT_RESCHED;	2723	trace_flags \|= TRACE_FLAG_PREEMPT_RESCHED;
2722	return (trace_flags << 16) \| (min_t(unsigned int, pc & 0xff, 0xf)) \|	2724	return (trace_flags << 16) \| (min_t(unsigned int, pc & 0xff, 0xf)) \|

Lines 460-476 int trace_print_lat_fmt(struct trace_seq s, struct trace_entry entry) Link Here

(-)a/kernel/trace/trace_output.c (-2 / +14 lines)
460	(entry->flags & TRACE_FLAG_IRQS_OFF && bh_off) ? 'D' :	460	(entry->flags & TRACE_FLAG_IRQS_OFF && bh_off) ? 'D' :
461	(entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :	461	(entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
462	bh_off ? 'b' :	462	bh_off ? 'b' :
463	(entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' :	463	!IS_ENABLED(CONFIG_TRACE_IRQFLAGS_SUPPORT) ? 'X' :
464	'.';	464	'.';
465		465
466	switch (entry->flags & (TRACE_FLAG_NEED_RESCHED \|	466	switch (entry->flags & (TRACE_FLAG_NEED_RESCHED \| TRACE_FLAG_NEED_RESCHED_LAZY \|
467	TRACE_FLAG_PREEMPT_RESCHED)) {	467	TRACE_FLAG_PREEMPT_RESCHED)) {
		468	case TRACE_FLAG_NEED_RESCHED \| TRACE_FLAG_NEED_RESCHED_LAZY \| TRACE_FLAG_PREEMPT_RESCHED:
		469	need_resched = 'B';
		470	break;
468	case TRACE_FLAG_NEED_RESCHED \| TRACE_FLAG_PREEMPT_RESCHED:	471	case TRACE_FLAG_NEED_RESCHED \| TRACE_FLAG_PREEMPT_RESCHED:
469	need_resched = 'N';	472	need_resched = 'N';
470	break;	473	break;
		474	case TRACE_FLAG_NEED_RESCHED_LAZY \| TRACE_FLAG_PREEMPT_RESCHED:
		475	need_resched = 'L';
		476	break;
		477	case TRACE_FLAG_NEED_RESCHED \| TRACE_FLAG_NEED_RESCHED_LAZY:
		478	need_resched = 'b';
		479	break;
471	case TRACE_FLAG_NEED_RESCHED:	480	case TRACE_FLAG_NEED_RESCHED:
472	need_resched = 'n';	481	need_resched = 'n';
473	break;	482	break;
		483	case TRACE_FLAG_NEED_RESCHED_LAZY:
		484	need_resched = 'l';
		485	break;
474	case TRACE_FLAG_PREEMPT_RESCHED:	486	case TRACE_FLAG_PREEMPT_RESCHED:
475	need_resched = 'p';	487	need_resched = 'p';
476	break;	488	break;

Lines 96-110 static void __dump_stack(const char *log_lvl) Link Here

(-)a/lib/dump_stack.c (-3 / +13 lines)
96	*/	96	*/
97	asmlinkage __visible void dump_stack_lvl(const char *log_lvl)	97	asmlinkage __visible void dump_stack_lvl(const char *log_lvl)
98	{	98	{
		99	bool in_panic = this_cpu_in_panic();
99	unsigned long flags;	100	unsigned long flags;
100		101
101	/*	102	/*
102	* Permit this cpu to perform nested stack dumps while serialising	103	* Permit this cpu to perform nested stack dumps while serialising
103	* against other CPUs	104	* against other CPUs, unless this CPU is in panic.
		105	*
		106	* When in panic, non-panic CPUs are not permitted to store new
		107	* printk messages so there is no need to synchronize the output.
		108	* This avoids potential deadlock in panic() if another CPU is
		109	* holding and unable to release the printk_cpu_sync.
104	*/	110	*/
105	printk_cpu_sync_get_irqsave(flags);	111	if (!in_panic)
		112	printk_cpu_sync_get_irqsave(flags);
		113
106	__dump_stack(log_lvl);	114	__dump_stack(log_lvl);
107	printk_cpu_sync_put_irqrestore(flags);	115
		116	if (!in_panic)
		117	printk_cpu_sync_put_irqrestore(flags);
108	}	118	}
109	EXPORT_SYMBOL(dump_stack_lvl);	119	EXPORT_SYMBOL(dump_stack_lvl);
110		120

Line 0 Link Here

(-)a/localversion-rt (+1 lines)
	1	-rt11

Lines 78-83 Link Here

(-)a/net/core/dev.c (-71 / +156 lines)
78	#include <linux/slab.h>	78	#include <linux/slab.h>
79	#include <linux/sched.h>	79	#include <linux/sched.h>
80	#include <linux/sched/mm.h>	80	#include <linux/sched/mm.h>
		81	#include <linux/smpboot.h>
81	#include <linux/mutex.h>	82	#include <linux/mutex.h>
82	#include <linux/rwsem.h>	83	#include <linux/rwsem.h>
83	#include <linux/string.h>	84	#include <linux/string.h>
Lines 216-250 static inline struct hlist_head dev_index_hash(struct net net, int ifindex) Link Here
216	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];	217	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
217	}	218	}
218		219
219	static inline void rps_lock_irqsave(struct softnet_data *sd,	220	#ifndef CONFIG_PREEMPT_RT
220	unsigned long *flags)	221
		222	static DEFINE_STATIC_KEY_FALSE(use_backlog_threads_key);
		223
		224	static int __init setup_backlog_napi_threads(char *arg)
221	{	225	{
222	if (IS_ENABLED(CONFIG_RPS))	226	static_branch_enable(&use_backlog_threads_key);
		227	return 0;
		228	}
		229	early_param("thread_backlog_napi", setup_backlog_napi_threads);
		230
		231	static bool use_backlog_threads(void)
		232	{
		233	return static_branch_unlikely(&use_backlog_threads_key);
		234	}
		235
		236	#else
		237
		238	static bool use_backlog_threads(void)
		239	{
		240	return true;
		241	}
		242
		243	#endif
		244
		245	static inline void backlog_lock_irq_save(struct softnet_data *sd,
		246	unsigned long *flags)
		247	{
		248	if (IS_ENABLED(CONFIG_RPS) \|\| use_backlog_threads())
223	spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);	249	spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
224	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))	250	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
225	local_irq_save(*flags);	251	local_irq_save(*flags);
226	}	252	}
227		253
228	static inline void rps_lock_irq_disable(struct softnet_data *sd)	254	static inline void backlog_lock_irq_disable(struct softnet_data *sd)
229	{	255	{
230	if (IS_ENABLED(CONFIG_RPS))	256	if (IS_ENABLED(CONFIG_RPS) \|\| use_backlog_threads())
231	spin_lock_irq(&sd->input_pkt_queue.lock);	257	spin_lock_irq(&sd->input_pkt_queue.lock);
232	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))	258	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
233	local_irq_disable();	259	local_irq_disable();
234	}	260	}
235		261
236	static inline void rps_unlock_irq_restore(struct softnet_data *sd,	262	static inline void backlog_unlock_irq_restore(struct softnet_data *sd,
237	unsigned long *flags)	263	unsigned long *flags)
238	{	264	{
239	if (IS_ENABLED(CONFIG_RPS))	265	if (IS_ENABLED(CONFIG_RPS) \|\| use_backlog_threads())
240	spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);	266	spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);
241	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))	267	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
242	local_irq_restore(*flags);	268	local_irq_restore(*flags);
243	}	269	}
244		270
245	static inline void rps_unlock_irq_enable(struct softnet_data *sd)	271	static inline void backlog_unlock_irq_enable(struct softnet_data *sd)
246	{	272	{
247	if (IS_ENABLED(CONFIG_RPS))	273	if (IS_ENABLED(CONFIG_RPS) \|\| use_backlog_threads())
248	spin_unlock_irq(&sd->input_pkt_queue.lock);	274	spin_unlock_irq(&sd->input_pkt_queue.lock);
249	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))	275	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
250	local_irq_enable();	276	local_irq_enable();
Lines 4420-4425 EXPORT_SYMBOL(__dev_direct_xmit); Link Here
4420	/*************************************************************************	4446	/*************************************************************************
4421	* Receiver routines	4447	* Receiver routines
4422	*************************************************************************/	4448	*************************************************************************/
		4449	static DEFINE_PER_CPU(struct task_struct *, backlog_napi);
4423		4450
4424	int netdev_max_backlog __read_mostly = 1000;	4451	int netdev_max_backlog __read_mostly = 1000;
4425	EXPORT_SYMBOL(netdev_max_backlog);	4452	EXPORT_SYMBOL(netdev_max_backlog);
Lines 4452-4469 static inline void ____napi_schedule(struct softnet_data *sd, Link Here
4452	*/	4479	*/
4453	thread = READ_ONCE(napi->thread);	4480	thread = READ_ONCE(napi->thread);
4454	if (thread) {	4481	if (thread) {
4455	/* Avoid doing set_bit() if the thread is in	4482	if (use_backlog_threads() && thread == raw_cpu_read(backlog_napi))
4456	* INTERRUPTIBLE state, cause napi_thread_wait()	4483	goto use_local_napi;
4457	* makes sure to proceed with napi polling	4484
4458	* if the thread is explicitly woken from here.	4485	set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4459	*/
4460	if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE)
4461	set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4462	wake_up_process(thread);	4486	wake_up_process(thread);
4463	return;	4487	return;
4464	}	4488	}
4465	}	4489	}
4466		4490
		4491	use_local_napi:
4467	list_add_tail(&napi->poll_list, &sd->poll_list);	4492	list_add_tail(&napi->poll_list, &sd->poll_list);
4468	WRITE_ONCE(napi->list_owner, smp_processor_id());	4493	WRITE_ONCE(napi->list_owner, smp_processor_id());
4469	/* If not called from net_rx_action()	4494	/* If not called from net_rx_action()
Lines 4709-4714 static void napi_schedule_rps(struct softnet_data *sd) Link Here
4709		4734
4710	#ifdef CONFIG_RPS	4735	#ifdef CONFIG_RPS
4711	if (sd != mysd) {	4736	if (sd != mysd) {
		4737	if (use_backlog_threads()) {
		4738	__napi_schedule_irqoff(&sd->backlog);
		4739	return;
		4740	}
		4741
4712	sd->rps_ipi_next = mysd->rps_ipi_list;	4742	sd->rps_ipi_next = mysd->rps_ipi_list;
4713	mysd->rps_ipi_list = sd;	4743	mysd->rps_ipi_list = sd;
4714		4744
Lines 4723-4728 static void napi_schedule_rps(struct softnet_data *sd) Link Here
4723	__napi_schedule_irqoff(&mysd->backlog);	4753	__napi_schedule_irqoff(&mysd->backlog);
4724	}	4754	}
4725		4755
		4756	void kick_defer_list_purge(struct softnet_data *sd, unsigned int cpu)
		4757	{
		4758	unsigned long flags;
		4759
		4760	if (use_backlog_threads()) {
		4761	backlog_lock_irq_save(sd, &flags);
		4762
		4763	if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
		4764	__napi_schedule_irqoff(&sd->backlog);
		4765
		4766	backlog_unlock_irq_restore(sd, &flags);
		4767
		4768	} else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) {
		4769	smp_call_function_single_async(cpu, &sd->defer_csd);
		4770	}
		4771	}
		4772
4726	#ifdef CONFIG_NET_FLOW_LIMIT	4773	#ifdef CONFIG_NET_FLOW_LIMIT
4727	int netdev_flow_limit_table_len __read_mostly = (1 << 12);	4774	int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4728	#endif	4775	#endif
Lines 4778-4784 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, Link Here
4778	reason = SKB_DROP_REASON_NOT_SPECIFIED;	4825	reason = SKB_DROP_REASON_NOT_SPECIFIED;
4779	sd = &per_cpu(softnet_data, cpu);	4826	sd = &per_cpu(softnet_data, cpu);
4780		4827
4781	rps_lock_irqsave(sd, &flags);	4828	backlog_lock_irq_save(sd, &flags);
4782	if (!netif_running(skb->dev))	4829	if (!netif_running(skb->dev))
4783	goto drop;	4830	goto drop;
4784	qlen = skb_queue_len(&sd->input_pkt_queue);	4831	qlen = skb_queue_len(&sd->input_pkt_queue);
Lines 4787-4793 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, Link Here
4787	enqueue:	4834	enqueue:
4788	__skb_queue_tail(&sd->input_pkt_queue, skb);	4835	__skb_queue_tail(&sd->input_pkt_queue, skb);
4789	input_queue_tail_incr_save(sd, qtail);	4836	input_queue_tail_incr_save(sd, qtail);
4790	rps_unlock_irq_restore(sd, &flags);	4837	backlog_unlock_irq_restore(sd, &flags);
4791	return NET_RX_SUCCESS;	4838	return NET_RX_SUCCESS;
4792	}	4839	}
4793		4840
Lines 4802-4808 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, Link Here
4802		4849
4803	drop:	4850	drop:
4804	sd->dropped++;	4851	sd->dropped++;
4805	rps_unlock_irq_restore(sd, &flags);	4852	backlog_unlock_irq_restore(sd, &flags);
4806		4853
4807	dev_core_stats_rx_dropped_inc(skb->dev);	4854	dev_core_stats_rx_dropped_inc(skb->dev);
4808	kfree_skb_reason(skb, reason);	4855	kfree_skb_reason(skb, reason);
Lines 5833-5839 static void flush_backlog(struct work_struct *work) Link Here
5833	local_bh_disable();	5880	local_bh_disable();
5834	sd = this_cpu_ptr(&softnet_data);	5881	sd = this_cpu_ptr(&softnet_data);
5835		5882
5836	rps_lock_irq_disable(sd);	5883	backlog_lock_irq_disable(sd);
5837	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {	5884	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
5838	if (skb->dev->reg_state == NETREG_UNREGISTERING) {	5885	if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5839	__skb_unlink(skb, &sd->input_pkt_queue);	5886	__skb_unlink(skb, &sd->input_pkt_queue);
Lines 5841-5847 static void flush_backlog(struct work_struct *work) Link Here
5841	input_queue_head_incr(sd);	5888	input_queue_head_incr(sd);
5842	}	5889	}
5843	}	5890	}
5844	rps_unlock_irq_enable(sd);	5891	backlog_unlock_irq_enable(sd);
5845		5892
5846	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {	5893	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
5847	if (skb->dev->reg_state == NETREG_UNREGISTERING) {	5894	if (skb->dev->reg_state == NETREG_UNREGISTERING) {
Lines 5859-5872 static bool flush_required(int cpu) Link Here
5859	struct softnet_data *sd = &per_cpu(softnet_data, cpu);	5906	struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5860	bool do_flush;	5907	bool do_flush;
5861		5908
5862	rps_lock_irq_disable(sd);	5909	backlog_lock_irq_disable(sd);
5863		5910
5864	/* as insertion into process_queue happens with the rps lock held,	5911	/* as insertion into process_queue happens with the rps lock held,
5865	* process_queue access may race only with dequeue	5912	* process_queue access may race only with dequeue
5866	*/	5913	*/
5867	do_flush = !skb_queue_empty(&sd->input_pkt_queue) \|\|	5914	do_flush = !skb_queue_empty(&sd->input_pkt_queue) \|\|
5868	!skb_queue_empty_lockless(&sd->process_queue);	5915	!skb_queue_empty_lockless(&sd->process_queue);
5869	rps_unlock_irq_enable(sd);	5916	backlog_unlock_irq_enable(sd);
5870		5917
5871	return do_flush;	5918	return do_flush;
5872	#endif	5919	#endif
Lines 5932-5938 static void net_rps_action_and_irq_enable(struct softnet_data *sd) Link Here
5932	#ifdef CONFIG_RPS	5979	#ifdef CONFIG_RPS
5933	struct softnet_data *remsd = sd->rps_ipi_list;	5980	struct softnet_data *remsd = sd->rps_ipi_list;
5934		5981
5935	if (remsd) {	5982	if (!use_backlog_threads() && remsd) {
5936	sd->rps_ipi_list = NULL;	5983	sd->rps_ipi_list = NULL;
5937		5984
5938	local_irq_enable();	5985	local_irq_enable();
Lines 5947-5953 static void net_rps_action_and_irq_enable(struct softnet_data *sd) Link Here
5947	static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)	5994	static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5948	{	5995	{
5949	#ifdef CONFIG_RPS	5996	#ifdef CONFIG_RPS
5950	return sd->rps_ipi_list != NULL;	5997	return !use_backlog_threads() && sd->rps_ipi_list;
5951	#else	5998	#else
5952	return false;	5999	return false;
5953	#endif	6000	#endif
Lines 5981-5987 static int process_backlog(struct napi_struct *napi, int quota) Link Here
5981		6028
5982	}	6029	}
5983		6030
5984	rps_lock_irq_disable(sd);	6031	backlog_lock_irq_disable(sd);
5985	if (skb_queue_empty(&sd->input_pkt_queue)) {	6032	if (skb_queue_empty(&sd->input_pkt_queue)) {
5986	/*	6033	/*
5987	* Inline a custom version of __napi_complete().	6034	* Inline a custom version of __napi_complete().
Lines 5991-6003 static int process_backlog(struct napi_struct *napi, int quota) Link Here
5991	* We can use a plain write instead of clear_bit(),	6038	* We can use a plain write instead of clear_bit(),
5992	* and we dont need an smp_mb() memory barrier.	6039	* and we dont need an smp_mb() memory barrier.
5993	*/	6040	*/
5994	napi->state = 0;	6041	napi->state &= NAPIF_STATE_THREADED;
5995	again = false;	6042	again = false;
5996	} else {	6043	} else {
5997	skb_queue_splice_tail_init(&sd->input_pkt_queue,	6044	skb_queue_splice_tail_init(&sd->input_pkt_queue,
5998	&sd->process_queue);	6045	&sd->process_queue);
5999	}	6046	}
6000	rps_unlock_irq_enable(sd);	6047	backlog_unlock_irq_enable(sd);
6001	}	6048	}
6002		6049
6003	return work;	6050	return work;
Lines 6654-6661 static int napi_poll(struct napi_struct n, struct list_head repoll) Link Here
6654		6701
6655	static int napi_thread_wait(struct napi_struct *napi)	6702	static int napi_thread_wait(struct napi_struct *napi)
6656	{	6703	{
6657	bool woken = false;
6658
6659	set_current_state(TASK_INTERRUPTIBLE);	6704	set_current_state(TASK_INTERRUPTIBLE);
6660		6705
6661	while (!kthread_should_stop()) {	6706	while (!kthread_should_stop()) {
Lines 6664-6678 static int napi_thread_wait(struct napi_struct *napi) Link Here
6664	* Testing SCHED bit is not enough because SCHED bit might be	6709	* Testing SCHED bit is not enough because SCHED bit might be
6665	* set by some other busy poll thread or by napi_disable().	6710	* set by some other busy poll thread or by napi_disable().
6666	*/	6711	*/
6667	if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state) \|\| woken) {	6712	if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) {
6668	WARN_ON(!list_empty(&napi->poll_list));	6713	WARN_ON(!list_empty(&napi->poll_list));
6669	__set_current_state(TASK_RUNNING);	6714	__set_current_state(TASK_RUNNING);
6670	return 0;	6715	return 0;
6671	}	6716	}
6672		6717
6673	schedule();	6718	schedule();
6674	/* woken being true indicates this thread owns this napi. */
6675	woken = true;
6676	set_current_state(TASK_INTERRUPTIBLE);	6719	set_current_state(TASK_INTERRUPTIBLE);
6677	}	6720	}
6678	__set_current_state(TASK_RUNNING);	6721	__set_current_state(TASK_RUNNING);
Lines 6701-6743 static void skb_defer_free_flush(struct softnet_data *sd) Link Here
6701	}	6744	}
6702	}	6745	}
6703		6746
		6747	static void napi_threaded_poll_loop(struct napi_struct *napi)
		6748	{
		6749	struct softnet_data *sd;
		6750	unsigned long last_qs = jiffies;
		6751
		6752	for (;;) {
		6753	bool repoll = false;
		6754	void *have;
		6755
		6756	local_bh_disable();
		6757	sd = this_cpu_ptr(&softnet_data);
		6758	sd->in_napi_threaded_poll = true;
		6759
		6760	have = netpoll_poll_lock(napi);
		6761	__napi_poll(napi, &repoll);
		6762	netpoll_poll_unlock(have);
		6763
		6764	sd->in_napi_threaded_poll = false;
		6765	barrier();
		6766
		6767	if (sd_has_rps_ipi_waiting(sd)) {
		6768	local_irq_disable();
		6769	net_rps_action_and_irq_enable(sd);
		6770	}
		6771	skb_defer_free_flush(sd);
		6772	local_bh_enable();
		6773
		6774	if (!repoll)
		6775	break;
		6776
		6777	rcu_softirq_qs_periodic(last_qs);
		6778	cond_resched();
		6779	}
		6780	}
		6781
6704	static int napi_threaded_poll(void *data)	6782	static int napi_threaded_poll(void *data)
6705	{	6783	{
6706	struct napi_struct *napi = data;	6784	struct napi_struct *napi = data;
6707	struct softnet_data *sd;
6708	void *have;
6709		6785
6710	while (!napi_thread_wait(napi)) {	6786	while (!napi_thread_wait(napi))
6711	unsigned long last_qs = jiffies;	6787	napi_threaded_poll_loop(napi);
6712		6788
6713	for (;;) {
6714	bool repoll = false;
6715
6716	local_bh_disable();
6717	sd = this_cpu_ptr(&softnet_data);
6718	sd->in_napi_threaded_poll = true;
6719
6720	have = netpoll_poll_lock(napi);
6721	__napi_poll(napi, &repoll);
6722	netpoll_poll_unlock(have);
6723
6724	sd->in_napi_threaded_poll = false;
6725	barrier();
6726
6727	if (sd_has_rps_ipi_waiting(sd)) {
6728	local_irq_disable();
6729	net_rps_action_and_irq_enable(sd);
6730	}
6731	skb_defer_free_flush(sd);
6732	local_bh_enable();
6733
6734	if (!repoll)
6735	break;
6736
6737	rcu_softirq_qs_periodic(last_qs);
6738	cond_resched();
6739	}
6740	}
6741	return 0;	6789	return 0;
6742	}	6790	}
6743		6791
Lines 11336-11342 static int dev_cpu_dead(unsigned int oldcpu) Link Here
11336		11384
11337	list_del_init(&napi->poll_list);	11385	list_del_init(&napi->poll_list);
11338	if (napi->poll == process_backlog)	11386	if (napi->poll == process_backlog)
11339	napi->state = 0;	11387	napi->state &= NAPIF_STATE_THREADED;
11340	else	11388	else
11341	____napi_schedule(sd, napi);	11389	____napi_schedule(sd, napi);
11342	}	11390	}
Lines 11344-11355 static int dev_cpu_dead(unsigned int oldcpu) Link Here
11344	raise_softirq_irqoff(NET_TX_SOFTIRQ);	11392	raise_softirq_irqoff(NET_TX_SOFTIRQ);
11345	local_irq_enable();	11393	local_irq_enable();
11346		11394
		11395	if (!use_backlog_threads()) {
11347	#ifdef CONFIG_RPS	11396	#ifdef CONFIG_RPS
11348	remsd = oldsd->rps_ipi_list;	11397	remsd = oldsd->rps_ipi_list;
11349	oldsd->rps_ipi_list = NULL;	11398	oldsd->rps_ipi_list = NULL;
11350	#endif	11399	#endif
11351	/* send out pending IPI's on offline CPU */	11400	/* send out pending IPI's on offline CPU */
11352	net_rps_send_ipi(remsd);	11401	net_rps_send_ipi(remsd);
		11402	}
11353		11403
11354	/* Process offline CPU's input_pkt_queue */	11404	/* Process offline CPU's input_pkt_queue */
11355	while ((skb = __skb_dequeue(&oldsd->process_queue))) {	11405	while ((skb = __skb_dequeue(&oldsd->process_queue))) {
Lines 11669-11674 static void __init net_dev_struct_check(void) Link Here
11669	*	11719	*
11670	*/	11720	*/
11671		11721
		11722	static int backlog_napi_should_run(unsigned int cpu)
		11723	{
		11724	struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
		11725	struct napi_struct *napi = &sd->backlog;
		11726
		11727	return test_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
		11728	}
		11729
		11730	static void run_backlog_napi(unsigned int cpu)
		11731	{
		11732	struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
		11733
		11734	napi_threaded_poll_loop(&sd->backlog);
		11735	}
		11736
		11737	static void backlog_napi_setup(unsigned int cpu)
		11738	{
		11739	struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
		11740	struct napi_struct *napi = &sd->backlog;
		11741
		11742	napi->thread = this_cpu_read(backlog_napi);
		11743	set_bit(NAPI_STATE_THREADED, &napi->state);
		11744	}
		11745
		11746	static struct smp_hotplug_thread backlog_threads = {
		11747	.store = &backlog_napi,
		11748	.thread_should_run = backlog_napi_should_run,
		11749	.thread_fn = run_backlog_napi,
		11750	.thread_comm = "backlog_napi/%u",
		11751	.setup = backlog_napi_setup,
		11752	};
		11753
11672	/*	11754	/*
11673	* This is called single threaded during boot, so no need	11755	* This is called single threaded during boot, so no need
11674	* to take the rtnl semaphore.	11756	* to take the rtnl semaphore.
Lines 11721-11727 static int __init net_dev_init(void) Link Here
11721	init_gro_hash(&sd->backlog);	11803	init_gro_hash(&sd->backlog);
11722	sd->backlog.poll = process_backlog;	11804	sd->backlog.poll = process_backlog;
11723	sd->backlog.weight = weight_p;	11805	sd->backlog.weight = weight_p;
		11806	INIT_LIST_HEAD(&sd->backlog.poll_list);
11724	}	11807	}
		11808	if (use_backlog_threads())
		11809	smpboot_register_percpu_thread(&backlog_threads);
11725		11810
11726	dev_boot_phase = 0;	11811	dev_boot_phase = 0;
11727		11812

Lines 6929-6936 nodefer: __kfree_skb(skb); Link Here

(-)a/net/core/skbuff.c (-2 / +2 lines)
6929	/* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU	6929	/* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU
6930	* if we are unlucky enough (this seems very unlikely).	6930	* if we are unlucky enough (this seems very unlikely).
6931	*/	6931	*/
6932	if (unlikely(kick) && !cmpxchg(&sd->defer_ipi_scheduled, 0, 1))	6932	if (unlikely(kick))
6933	smp_call_function_single_async(cpu, &sd->defer_csd);	6933	kick_defer_list_purge(sd, cpu);
6934	}	6934	}
6935		6935
6936	static void skb_splice_csum_page(struct sk_buff skb, struct page page,	6936	static void skb_splice_csum_page(struct sk_buff skb, struct page page,

Return to bug 916954

Lines 4507-4512 int wake_up_state(struct task_struct *p, unsigned int state) Link Here

(-)a/kernel/sched/core.c (+144 lines)
4507	return try_to_wake_up(p, state, 0);	4507	return try_to_wake_up(p, state, 0);
4508	}	4508	}
4509		4509
		4510	#ifdef CONFIG_SCHED_BORE
		4511	extern bool sched_bore;
		4512	extern u8 sched_burst_fork_atavistic;
		4513	extern uint sched_burst_cache_lifetime;
		4514
		4515	static void __init sched_init_bore(void) {
		4516	init_task.se.burst_time = 0;
		4517	init_task.se.prev_burst_penalty = 0;
		4518	init_task.se.curr_burst_penalty = 0;
		4519	init_task.se.burst_penalty = 0;
		4520	init_task.se.burst_score = 0;
		4521	init_task.se.child_burst_last_cached = 0;
		4522	}
		4523
		4524	void inline sched_fork_bore(struct task_struct *p) {
		4525	p->se.burst_time = 0;
		4526	p->se.curr_burst_penalty = 0;
		4527	p->se.burst_score = 0;
		4528	p->se.child_burst_last_cached = 0;
		4529	}
		4530
		4531	static u32 count_child_tasks(struct task_struct *p) {
		4532	struct task_struct *child;
		4533	u32 cnt = 0;
		4534	list_for_each_entry(child, &p->children, sibling) {cnt++;}
		4535	return cnt;
		4536	}
		4537
		4538	static inline bool task_is_inheritable(struct task_struct *p) {
		4539	return (p->sched_class == &fair_sched_class);
		4540	}
		4541
		4542	static inline bool child_burst_cache_expired(struct task_struct *p, u64 now) {
		4543	u64 expiration_time =
		4544	p->se.child_burst_last_cached + sched_burst_cache_lifetime;
		4545	return ((s64)(expiration_time - now) < 0);
		4546	}
		4547
		4548	static void __update_child_burst_cache(
		4549	struct task_struct *p, u32 cnt, u32 sum, u64 now) {
		4550	u8 avg = 0;
		4551	if (cnt) avg = sum / cnt;
		4552	p->se.child_burst = max(avg, p->se.burst_penalty);
		4553	p->se.child_burst_cnt = cnt;
		4554	p->se.child_burst_last_cached = now;
		4555	}
		4556
		4557	static inline void update_child_burst_direct(struct task_struct *p, u64 now) {
		4558	struct task_struct *child;
		4559	u32 cnt = 0;
		4560	u32 sum = 0;
		4561
		4562	list_for_each_entry(child, &p->children, sibling) {
		4563	if (!task_is_inheritable(child)) continue;
		4564	cnt++;
		4565	sum += child->se.burst_penalty;
		4566	}
		4567
		4568	__update_child_burst_cache(p, cnt, sum, now);
		4569	}
		4570
		4571	static inline u8 __inherit_burst_direct(struct task_struct *p, u64 now) {
		4572	struct task_struct *parent = p->real_parent;
		4573	if (child_burst_cache_expired(parent, now))
4574	update_child_burst_direct(parent, now);
4575
4576	return parent->se.child_burst;
4577	}
4578
4579	static void update_child_burst_topological(
4580	struct task_struct p, u64 now, u32 depth, u32 acnt, u32 *asum) {
4581	struct task_struct child, dec;
4582	u32 cnt = 0, dcnt = 0;
4583	u32 sum = 0;
4584
4585	list_for_each_entry(child, &p->children, sibling) {
4586	dec = child;
4587	while ((dcnt = count_child_tasks(dec)) == 1)
4588	dec = list_first_entry(&dec->children, struct task_struct, sibling);
4589
4590	if (!dcnt \|\| !depth) {
4591	if (!task_is_inheritable(dec)) continue;
4592	cnt++;
4593	sum += dec->se.burst_penalty;
4594	continue;
4595	}
4596	if (!child_burst_cache_expired(dec, now)) {
4597	cnt += dec->se.child_burst_cnt;
4598	sum += (u32)dec->se.child_burst * dec->se.child_burst_cnt;
4599	continue;
4600	}
4601	update_child_burst_topological(dec, now, depth - 1, &cnt, &sum);
4602	}
4603
4604	__update_child_burst_cache(p, cnt, sum, now);
4605	*acnt += cnt;
4606	*asum += sum;
4607	}
4608
4609	static inline u8 __inherit_burst_topological(struct task_struct *p, u64 now) {
4610	struct task_struct *anc = p->real_parent;
4611	u32 cnt = 0, sum = 0;
4612
4613	while (anc->real_parent != anc && count_child_tasks(anc) == 1)
4614	anc = anc->real_parent;
4615
4616	if (child_burst_cache_expired(anc, now))
4617	update_child_burst_topological(
4618	anc, now, sched_burst_fork_atavistic - 1, &cnt, &sum);
4619
4620	return anc->se.child_burst;
4621	}
4622
4623	static inline void inherit_burst(struct task_struct *p) {
4624	u8 burst_cache;
4625	u64 now = ktime_get_ns();
4626
4627	read_lock(&tasklist_lock);
4628	burst_cache = likely(sched_burst_fork_atavistic)?
4629	__inherit_burst_topological(p, now):
4630	__inherit_burst_direct(p, now);
4631	read_unlock(&tasklist_lock);
4632
4633	p->se.prev_burst_penalty = max(p->se.prev_burst_penalty, burst_cache);
4634	}
4635
4636	static void sched_post_fork_bore(struct task_struct *p) {
4637	if (p->sched_class == &fair_sched_class)
4638	inherit_burst(p);
4639	p->se.burst_penalty = p->se.prev_burst_penalty;
4640	}
4641	#endif // CONFIG_SCHED_BORE
4642
4510	/*	4643	/*
4511	* Perform scheduler related setup for a newly forked process p.	4644	* Perform scheduler related setup for a newly forked process p.
4512	* p is forked by current.	4645	* p is forked by current.
Lines 4523-4528 static void __sched_fork(unsigned long clone_flags, struct task_struct *p) Link Here
4523	p->se.prev_sum_exec_runtime = 0;	4656	p->se.prev_sum_exec_runtime = 0;
4524	p->se.nr_migrations = 0;	4657	p->se.nr_migrations = 0;
4525	p->se.vruntime = 0;	4658	p->se.vruntime = 0;
		4659	#ifdef CONFIG_SCHED_BORE
		4660	sched_fork_bore(p);
		4661	#endif // CONFIG_SCHED_BORE
4526	p->se.vlag = 0;	4662	p->se.vlag = 0;
4527	p->se.slice = sysctl_sched_base_slice;	4663	p->se.slice = sysctl_sched_base_slice;
4528	INIT_LIST_HEAD(&p->se.group_node);	4664	INIT_LIST_HEAD(&p->se.group_node);
Lines 4839-4844 void sched_cgroup_fork(struct task_struct p, struct kernel_clone_args kargs) Link Here
4839		4975
4840	void sched_post_fork(struct task_struct *p)	4976	void sched_post_fork(struct task_struct *p)
4841	{	4977	{
		4978	#ifdef CONFIG_SCHED_BORE
		4979	sched_post_fork_bore(p);
		4980	#endif // CONFIG_SCHED_BORE
4842	uclamp_post_fork(p);	4981	uclamp_post_fork(p);
4843	}	4982	}
4844		4983
Lines 9910-9915 void __init sched_init(void) Link Here
9910	BUG_ON(&dl_sched_class != &stop_sched_class + 1);	10049	BUG_ON(&dl_sched_class != &stop_sched_class + 1);
9911	#endif	10050	#endif
9912		10051
		10052	#ifdef CONFIG_SCHED_BORE
		10053	sched_init_bore();
		10054	printk(KERN_INFO "BORE (Burst-Oriented Response Enhancer) CPU Scheduler modification 5.0.3 by Masahito Suzuki");
		10055	#endif // CONFIG_SCHED_BORE
		10056
9913	wait_bit_init();	10057	wait_bit_init();
9914		10058
9915	#ifdef CONFIG_FAIR_GROUP_SCHED	10059	#ifdef CONFIG_FAIR_GROUP_SCHED

Lines 6508-6513 Link Here

(-)a/Documentation/admin-guide/kernel-parameters.txt (+12 lines)
6508	Force threading of all interrupt handlers except those	6508	Force threading of all interrupt handlers except those
6509	marked explicitly IRQF_NO_THREAD.	6509	marked explicitly IRQF_NO_THREAD.
6510		6510
		6511	threadprintk [KNL]
		6512	Force threaded printing of all legacy consoles. Be
		6513	aware that with this option, the shutdown, reboot, and
		6514	panic messages may not be printed on the legacy
		6515	consoles. Also, earlycon/earlyprintk printing will be
		6516	delayed until a regular console or the kthread is
		6517	available.
		6518
		6519	Users can view /proc/consoles to see if their console
		6520	driver is legacy or not. Non-legacy (NBCON) console
		6521	drivers are already threaded and are shown with 'N'.
		6522
6511	topology= [S390]	6523	topology= [S390]
6512	Format: {off \| on}	6524	Format: {off \| on}
6513	Specify if the kernel should make use of the cpu	6525	Specify if the kernel should make use of the cpu

Lines 1357-1363 Link Here

(-)a/kernel/sched/fair.c (-2 / +1 lines)
1357	#else // !CONFIG_SCHED_BORE	1357	#else // !CONFIG_SCHED_BORE
1358	curr->vruntime += calc_delta_fair(delta_exec, curr);	1358	curr->vruntime += calc_delta_fair(delta_exec, curr);
1359	#endif // CONFIG_SCHED_BORE	1359	#endif // CONFIG_SCHED_BORE
1360	update_deadline(cfs_rq, curr);	1360	update_deadline(cfs_rq, curr, tick);
1361	update_min_vruntime(cfs_rq);	1361	update_min_vruntime(cfs_rq);
1362		1362
1363	if (entity_is_task(curr))	1363	if (entity_is_task(curr))
1364	--
1365	include/linux/sched.h \| 10 ++	1364	include/linux/sched.h \| 10 ++
1366	init/Kconfig \| 17 ++++	1365	init/Kconfig \| 17 ++++
1367	kernel/sched/core.c \| 144 ++++++++++++++++++++++++++	1366	kernel/sched/core.c \| 144 ++++++++++++++++++++++++++
1368	kernel/sched/debug.c \| 60 ++++++++++-	1367	kernel/sched/debug.c \| 60 ++++++++++-
1369	kernel/sched/fair.c \| 218 +++++++++++++++++++++++++++++++++++++++-	1368	kernel/sched/fair.c \| 218 +++++++++++++++++++++++++++++++++++++++-
1370	kernel/sched/features.h \| 4 +	1369	kernel/sched/features.h \| 4 +
1371	kernel/sched/sched.h \| 7 ++	1370	kernel/sched/sched.h \| 7 ++
1372	7 files changed, 457 insertions(+), 3 deletions(-)	1371	7 files changed, 457 insertions(+), 3 deletions(-)

Lines 547-552 struct sched_entity { Link Here

(-)a/include/linux/sched.h (+10 lines)
547	u64 sum_exec_runtime;	547	u64 sum_exec_runtime;
548	u64 prev_sum_exec_runtime;	548	u64 prev_sum_exec_runtime;
549	u64 vruntime;	549	u64 vruntime;
		550	#ifdef CONFIG_SCHED_BORE
		551	u64 burst_time;
		552	u8 prev_burst_penalty;
		553	u8 curr_burst_penalty;
		554	u8 burst_penalty;
		555	u8 burst_score;
		556	u8 child_burst;
		557	u32 child_burst_cnt;
		558	u64 child_burst_last_cached;
		559	#endif // CONFIG_SCHED_BORE
550	s64 vlag;	560	s64 vlag;
551	u64 slice;	561	u64 slice;
552		562

Lines 1279-1284 config CHECKPOINT_RESTORE Link Here

(-)a/init/Kconfig (+17 lines)
1279		1279
1280	If unsure, say N here.	1280	If unsure, say N here.
1281		1281
		1282	config SCHED_BORE
		1283	bool "Burst-Oriented Response Enhancer"
		1284	default y
		1285	help
		1286	In Desktop and Mobile computing, one might prefer interactive
		1287	tasks to keep responsive no matter what they run in the background.
		1288
		1289	Enabling this kernel feature modifies the scheduler to discriminate
		1290	tasks by their burst time (runtime since it last went sleeping or
		1291	yielding state) and prioritize those that run less bursty.
		1292	Such tasks usually include window compositor, widgets backend,
		1293	terminal emulator, video playback, games and so on.
		1294	With a little impact to scheduling fairness, it may improve
		1295	responsiveness especially under heavy background workload.
		1296
		1297	If unsure, say Y here.
		1298
1282	config SCHED_AUTOGROUP	1299	config SCHED_AUTOGROUP
1283	bool "Automatic process group scheduling"	1300	bool "Automatic process group scheduling"
1284	select CGROUPS	1301	select CGROUPS

Lines 167-173 static const struct file_operations sched_feat_fops = { Link Here

(-)a/kernel/sched/debug.c (-2 / +60 lines)
167	};	167	};
168		168
169	#ifdef CONFIG_SMP	169	#ifdef CONFIG_SMP
		170	#ifdef CONFIG_SCHED_BORE
		171	static ssize_t sched_min_base_slice_write(struct file filp, const char __user ubuf,
		172	size_t cnt, loff_t *ppos)
		173	{
		174	char buf[16];
		175	unsigned int value;
		176
		177	if (cnt > 15)
		178	cnt = 15;
		179
		180	if (copy_from_user(&buf, ubuf, cnt))
		181	return -EFAULT;
		182	buf[cnt] = '\0';
		183
		184	if (kstrtouint(buf, 10, &value))
		185	return -EINVAL;
170		186
		187	if (!value)
		188	return -EINVAL;
		189
		190	sysctl_sched_min_base_slice = value;
		191	sched_update_min_base_slice();
		192
		193	*ppos += cnt;
		194	return cnt;
		195	}
		196
		197	static int sched_min_base_slice_show(struct seq_file m, void v)
		198	{
		199	seq_printf(m, "%d\n", sysctl_sched_min_base_slice);
		200	return 0;
		201	}
		202
		203	static int sched_min_base_slice_open(struct inode inode, struct file filp)
		204	{
		205	return single_open(filp, sched_min_base_slice_show, NULL);
		206	}
		207
		208	static const struct file_operations sched_min_base_slice_fops = {
		209	.open = sched_min_base_slice_open,
		210	.write = sched_min_base_slice_write,
		211	.read = seq_read,
		212	.llseek = seq_lseek,
		213	.release = single_release,
		214	};
		215	#else // !CONFIG_SCHED_BORE
171	static ssize_t sched_scaling_write(struct file filp, const char __user ubuf,	216	static ssize_t sched_scaling_write(struct file filp, const char __user ubuf,
172	size_t cnt, loff_t *ppos)	217	size_t cnt, loff_t *ppos)
173	{	218	{
Lines 213-219 static const struct file_operations sched_scaling_fops = { Link Here
213	.llseek = seq_lseek,	258	.llseek = seq_lseek,
214	.release = single_release,	259	.release = single_release,
215	};	260	};
216		261	#endif // CONFIG_SCHED_BORE
217	#endif /* SMP */	262	#endif /* SMP */
218		263
219	#ifdef CONFIG_PREEMPT_DYNAMIC	264	#ifdef CONFIG_PREEMPT_DYNAMIC
Lines 398-411 Link Here
398	debugfs_create_file("preempt", 0644, debugfs_sched, NULL, &sched_dynamic_fops);	398	debugfs_create_file("preempt", 0644, debugfs_sched, NULL, &sched_dynamic_fops);
399	#endif	399	#endif
400		400
401	debugfs_create_u32("base_slice_ns", 0644, debugfs_sched, &sysctl_sched_base_slice);	401	#ifdef CONFIG_SCHED_BORE
		402	debugfs_create_file("min_base_slice_ns", 0644, debugfs_sched, NULL, &sched_min_base_slice_fops);
		403	debugfs_create_u32("base_slice_ns", 0400, debugfs_sched, &sysctl_sched_base_slice);
		404	#else // !CONFIG_SCHED_BORE
		405	debugfs_create_u32("base_slice_ns", 0644, debugfs_sched, &sysctl_sched_base_slice);
		406	#endif // CONFIG_SCHED_BORE
402		407
403	#ifndef CONFIG_SCHED_ALT	408	#ifndef CONFIG_SCHED_ALT
404	debugfs_create_u32("latency_warn_ms", 0644, debugfs_sched, &sysctl_resched_latency_warn_ms);	409	debugfs_create_u32("latency_warn_ms", 0644, debugfs_sched, &sysctl_resched_latency_warn_ms);
405	debugfs_create_u32("latency_warn_once", 0644, debugfs_sched, &sysctl_resched_latency_warn_once);	410	debugfs_create_u32("latency_warn_once", 0644, debugfs_sched, &sysctl_resched_latency_warn_once);
406		411
407	#ifdef CONFIG_SMP	412	#ifdef CONFIG_SMP
		413	#if !defined(CONFIG_SCHED_BORE)
408	debugfs_create_file("tunable_scaling", 0644, debugfs_sched, NULL, &sched_scaling_fops);	414	debugfs_create_file("tunable_scaling", 0644, debugfs_sched, NULL, &sched_scaling_fops);
		415	#endif // CONFIG_SCHED_BORE
409	debugfs_create_u32("migration_cost_ns", 0644, debugfs_sched, &sysctl_sched_migration_cost);	416	debugfs_create_u32("migration_cost_ns", 0644, debugfs_sched, &sysctl_sched_migration_cost);
410	debugfs_create_u32("nr_migrate", 0644, debugfs_sched, &sysctl_sched_nr_migrate);	417	debugfs_create_u32("nr_migrate", 0644, debugfs_sched, &sysctl_sched_nr_migrate);
411		418
Lines 595-600 print_task(struct seq_file m, struct rq rq, struct task_struct *p) Link Here
595	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)),	647	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)),
596	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime)));	648	SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime)));
597		649
		650	#ifdef CONFIG_SCHED_BORE
		651	SEQ_printf(m, " %2d", p->se.burst_score);
		652	#endif // CONFIG_SCHED_BORE
598	#ifdef CONFIG_NUMA_BALANCING	653	#ifdef CONFIG_NUMA_BALANCING
599	SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));	654	SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
600	#endif	655	#endif
Lines 1068-1073 void proc_sched_show_task(struct task_struct p, struct pid_namespace ns, Link Here
1068		1123
1069	P(se.load.weight);	1124	P(se.load.weight);
1070	#ifdef CONFIG_SMP	1125	#ifdef CONFIG_SMP
		1126	#ifdef CONFIG_SCHED_BORE
		1127	P(se.burst_score);
		1128	#endif // CONFIG_SCHED_BORE
1071	P(se.avg.load_sum);	1129	P(se.avg.load_sum);
1072	P(se.avg.runnable_sum);	1130	P(se.avg.runnable_sum);
1073	P(se.avg.util_sum);	1131	P(se.avg.util_sum);

Lines 36-41 config ARM Link Here

(-)a/arch/arm/Kconfig (-2 / +4 lines)
36	select ARCH_SUPPORTS_ATOMIC_RMW	36	select ARCH_SUPPORTS_ATOMIC_RMW
37	select ARCH_SUPPORTS_HUGETLBFS if ARM_LPAE	37	select ARCH_SUPPORTS_HUGETLBFS if ARM_LPAE
38	select ARCH_SUPPORTS_PER_VMA_LOCK	38	select ARCH_SUPPORTS_PER_VMA_LOCK
		39	select ARCH_SUPPORTS_RT if HAVE_POSIX_CPU_TIMERS_TASK_WORK
39	select ARCH_USE_BUILTIN_BSWAP	40	select ARCH_USE_BUILTIN_BSWAP
40	select ARCH_USE_CMPXCHG_LOCKREF	41	select ARCH_USE_CMPXCHG_LOCKREF
41	select ARCH_USE_MEMTEST	42	select ARCH_USE_MEMTEST
Lines 75-81 config ARM Link Here
75	select HAS_IOPORT	76	select HAS_IOPORT
76	select HAVE_ARCH_AUDITSYSCALL if AEABI && !OABI_COMPAT	77	select HAVE_ARCH_AUDITSYSCALL if AEABI && !OABI_COMPAT
77	select HAVE_ARCH_BITREVERSE if (CPU_32v7M \|\| CPU_32v7) && !CPU_32v6	78	select HAVE_ARCH_BITREVERSE if (CPU_32v7M \|\| CPU_32v7) && !CPU_32v6
78	select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU	79	select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU && !PREEMPT_RT
79	select HAVE_ARCH_KFENCE if MMU && !XIP_KERNEL	80	select HAVE_ARCH_KFENCE if MMU && !XIP_KERNEL
80	select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU	81	select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU
81	select HAVE_ARCH_KASAN if MMU && !XIP_KERNEL	82	select HAVE_ARCH_KASAN if MMU && !XIP_KERNEL
Lines 98-104 config ARM Link Here
98	select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE	99	select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE
99	select HAVE_EFFICIENT_UNALIGNED_ACCESS if (CPU_V6 \|\| CPU_V6K \|\| CPU_V7) && MMU	100	select HAVE_EFFICIENT_UNALIGNED_ACCESS if (CPU_V6 \|\| CPU_V6K \|\| CPU_V7) && MMU
100	select HAVE_EXIT_THREAD	101	select HAVE_EXIT_THREAD
101	select HAVE_FAST_GUP if ARM_LPAE	102	select HAVE_FAST_GUP if ARM_LPAE && !(PREEMPT_RT && HIGHPTE)
102	select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL	103	select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL
103	select HAVE_FUNCTION_ERROR_INJECTION	104	select HAVE_FUNCTION_ERROR_INJECTION
104	select HAVE_FUNCTION_GRAPH_TRACER	105	select HAVE_FUNCTION_GRAPH_TRACER
Lines 120-125 config ARM Link Here
120	select HAVE_PERF_EVENTS	121	select HAVE_PERF_EVENTS
121	select HAVE_PERF_REGS	122	select HAVE_PERF_REGS
122	select HAVE_PERF_USER_STACK_DUMP	123	select HAVE_PERF_USER_STACK_DUMP
		124	select HAVE_POSIX_CPU_TIMERS_TASK_WORK if !KVM
123	select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE	125	select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE
124	select HAVE_REGS_AND_STACK_ACCESS_API	126	select HAVE_REGS_AND_STACK_ACCESS_API
125	select HAVE_RSEQ	127	select HAVE_RSEQ

Lines 55-60 extern unsigned int VFP_arch_feroceon __alias(VFP_arch); Link Here

(-)a/arch/arm/vfp/vfpmodule.c (-21 / +53 lines)
55	*/	55	*/
56	union vfp_state *vfp_current_hw_state[NR_CPUS];	56	union vfp_state *vfp_current_hw_state[NR_CPUS];
57		57
		58	/*
		59	* Claim ownership of the VFP unit.
		60	*
		61	* The caller may change VFP registers until vfp_unlock() is called.
		62	*
		63	* local_bh_disable() is used to disable preemption and to disable VFP
		64	* processing in softirq context. On PREEMPT_RT kernels local_bh_disable() is
		65	* not sufficient because it only serializes soft interrupt related sections
		66	* via a local lock, but stays preemptible. Disabling preemption is the right
		67	* choice here as bottom half processing is always in thread context on RT
		68	* kernels so it implicitly prevents bottom half processing as well.
		69	*/
		70	static void vfp_lock(void)
		71	{
		72	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		73	local_bh_disable();
		74	else
		75	preempt_disable();
		76	}
		77
		78	static void vfp_unlock(void)
		79	{
		80	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		81	local_bh_enable();
		82	else
		83	preempt_enable();
		84	}
		85
58	/*	86	/*
59	* Is 'thread's most up to date state stored in this CPUs hardware?	87	* Is 'thread's most up to date state stored in this CPUs hardware?
60	* Must be called from non-preemptible context.	88	* Must be called from non-preemptible context.
Lines 240-246 static void vfp_panic(char *reason, u32 inst) Link Here
240	/*	268	/*
241	* Process bitmask of exception conditions.	269	* Process bitmask of exception conditions.
242	*/	270	*/
243	static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_regs *regs)	271	static int vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr)
244	{	272	{
245	int si_code = 0;	273	int si_code = 0;
246		274
Lines 248-255 static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_ Link Here
248		276
249	if (exceptions == VFP_EXCEPTION_ERROR) {	277	if (exceptions == VFP_EXCEPTION_ERROR) {
250	vfp_panic("unhandled bounce", inst);	278	vfp_panic("unhandled bounce", inst);
251	vfp_raise_sigfpe(FPE_FLTINV, regs);	279	return FPE_FLTINV;
252	return;
253	}	280	}
254		281
255	/*	282	/*
Lines 277-284 static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_ Link Here
277	RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF);	304	RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF);
278	RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV);	305	RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV);
279		306
280	if (si_code)	307	return si_code;
281	vfp_raise_sigfpe(si_code, regs);
282	}	308	}
283		309
284	/*	310	/*
Lines 324-329 static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs) Link Here
324	static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)	350	static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
325	{	351	{
326	u32 fpscr, orig_fpscr, fpsid, exceptions;	352	u32 fpscr, orig_fpscr, fpsid, exceptions;
		353	int si_code2 = 0;
		354	int si_code = 0;
327		355
328	pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);	356	pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);
329		357
Lines 369-376 static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) Link Here
369	* unallocated VFP instruction but with FPSCR.IXE set and not	397	* unallocated VFP instruction but with FPSCR.IXE set and not
370	* on VFP subarch 1.	398	* on VFP subarch 1.
371	*/	399	*/
372	vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr, regs);	400	si_code = vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr);
373	return;	401	goto exit;
374	}	402	}
375		403
376	/*	404	/*
Lines 394-407 static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) Link Here
394	*/	422	*/
395	exceptions = vfp_emulate_instruction(trigger, fpscr, regs);	423	exceptions = vfp_emulate_instruction(trigger, fpscr, regs);
396	if (exceptions)	424	if (exceptions)
397	vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);	425	si_code2 = vfp_raise_exceptions(exceptions, trigger, orig_fpscr);
398		426
399	/*	427	/*
400	* If there isn't a second FP instruction, exit now. Note that	428	* If there isn't a second FP instruction, exit now. Note that
401	* the FPEXC.FP2V bit is valid only if FPEXC.EX is 1.	429	* the FPEXC.FP2V bit is valid only if FPEXC.EX is 1.
402	*/	430	*/
403	if ((fpexc & (FPEXC_EX \| FPEXC_FP2V)) != (FPEXC_EX \| FPEXC_FP2V))	431	if ((fpexc & (FPEXC_EX \| FPEXC_FP2V)) != (FPEXC_EX \| FPEXC_FP2V))
404	return;	432	goto exit;
405		433
406	/*	434	/*
407	* The barrier() here prevents fpinst2 being read	435	* The barrier() here prevents fpinst2 being read
Lines 413-419 static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) Link Here
413	emulate:	441	emulate:
414	exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);	442	exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);
415	if (exceptions)	443	if (exceptions)
416	vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);	444	si_code = vfp_raise_exceptions(exceptions, trigger, orig_fpscr);
		445	exit:
		446	vfp_unlock();
		447	if (si_code2)
		448	vfp_raise_sigfpe(si_code2, regs);
		449	if (si_code)
		450	vfp_raise_sigfpe(si_code, regs);
417	}	451	}
418		452
419	static void vfp_enable(void *unused)	453	static void vfp_enable(void *unused)
Lines 512-522 static inline void vfp_pm_init(void) { } Link Here
512	*/	546	*/
513	void vfp_sync_hwstate(struct thread_info *thread)	547	void vfp_sync_hwstate(struct thread_info *thread)
514	{	548	{
515	unsigned int cpu = get_cpu();	549	vfp_lock();
516		550
517	local_bh_disable();	551	if (vfp_state_in_hw(raw_smp_processor_id(), thread)) {
518
519	if (vfp_state_in_hw(cpu, thread)) {
520	u32 fpexc = fmrx(FPEXC);	552	u32 fpexc = fmrx(FPEXC);
521		553
522	/*	554	/*
Lines 527-534 void vfp_sync_hwstate(struct thread_info *thread) Link Here
527	fmxr(FPEXC, fpexc);	559	fmxr(FPEXC, fpexc);
528	}	560	}
529		561
530	local_bh_enable();	562	vfp_unlock();
531	put_cpu();
532	}	563	}
533		564
534	/* Ensure that the thread reloads the hardware VFP state on the next use. */	565	/* Ensure that the thread reloads the hardware VFP state on the next use. */
Lines 683-689 static int vfp_support_entry(struct pt_regs *regs, u32 trigger) Link Here
683	if (!user_mode(regs))	714	if (!user_mode(regs))
684	return vfp_kmode_exception(regs, trigger);	715	return vfp_kmode_exception(regs, trigger);
685		716
686	local_bh_disable();	717	vfp_lock();
687	fpexc = fmrx(FPEXC);	718	fpexc = fmrx(FPEXC);
688		719
689	/*	720	/*
Lines 748-753 static int vfp_support_entry(struct pt_regs *regs, u32 trigger) Link Here
748	* replay the instruction that trapped.	779	* replay the instruction that trapped.
749	*/	780	*/
750	fmxr(FPEXC, fpexc);	781	fmxr(FPEXC, fpexc);
		782	vfp_unlock();
751	} else {	783	} else {
752	/* Check for synchronous or asynchronous exceptions */	784	/* Check for synchronous or asynchronous exceptions */
753	if (!(fpexc & (FPEXC_EX \| FPEXC_DEX))) {	785	if (!(fpexc & (FPEXC_EX \| FPEXC_DEX))) {
Lines 762-778 static int vfp_support_entry(struct pt_regs *regs, u32 trigger) Link Here
762	if (!(fpscr & FPSCR_IXE)) {	794	if (!(fpscr & FPSCR_IXE)) {
763	if (!(fpscr & FPSCR_LENGTH_MASK)) {	795	if (!(fpscr & FPSCR_LENGTH_MASK)) {
764	pr_debug("not VFP\n");	796	pr_debug("not VFP\n");
765	local_bh_enable();	797	vfp_unlock();
766	return -ENOEXEC;	798	return -ENOEXEC;
767	}	799	}
768	fpexc \|= FPEXC_DEX;	800	fpexc \|= FPEXC_DEX;
769	}	801	}
770	}	802	}
771	bounce: regs->ARM_pc += 4;	803	bounce: regs->ARM_pc += 4;
		804	/* VFP_bounce() will invoke vfp_unlock() */
772	VFP_bounce(trigger, fpexc, regs);	805	VFP_bounce(trigger, fpexc, regs);
773	}	806	}
774		807
775	local_bh_enable();
776	return 0;	808	return 0;
777	}	809	}
778		810
Lines 837-843 void kernel_neon_begin(void) Link Here
837	unsigned int cpu;	869	unsigned int cpu;
838	u32 fpexc;	870	u32 fpexc;
839		871
840	local_bh_disable();	872	vfp_lock();
841		873
842	/*	874	/*
843	* Kernel mode NEON is only allowed outside of hardirq context with	875	* Kernel mode NEON is only allowed outside of hardirq context with
Lines 868-874 void kernel_neon_end(void) Link Here
868	{	900	{
869	/* Disable the NEON/VFP unit. */	901	/* Disable the NEON/VFP unit. */
870	fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);	902	fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
871	local_bh_enable();	903	vfp_unlock();
872	}	904	}
873	EXPORT_SYMBOL(kernel_neon_end);	905	EXPORT_SYMBOL(kernel_neon_end);
874		906

Lines 49-54 config RISCV Link Here

(-)a/arch/riscv/Kconfig (+2 lines)
49	select ARCH_SUPPORTS_HUGETLBFS if MMU	49	select ARCH_SUPPORTS_HUGETLBFS if MMU
50	select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU	50	select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU
51	select ARCH_SUPPORTS_PER_VMA_LOCK if MMU	51	select ARCH_SUPPORTS_PER_VMA_LOCK if MMU
		52	select ARCH_SUPPORTS_RT
52	select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK	53	select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK
53	select ARCH_USE_MEMTEST	54	select ARCH_USE_MEMTEST
54	select ARCH_USE_QUEUED_RWLOCKS	55	select ARCH_USE_QUEUED_RWLOCKS
Lines 142-147 config RISCV Link Here
142	select HAVE_PERF_USER_STACK_DUMP	143	select HAVE_PERF_USER_STACK_DUMP
143	select HAVE_POSIX_CPU_TIMERS_TASK_WORK	144	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
144	select HAVE_PREEMPT_DYNAMIC_KEY if !XIP_KERNEL	145	select HAVE_PREEMPT_DYNAMIC_KEY if !XIP_KERNEL
		146	select HAVE_PREEMPT_AUTO
145	select HAVE_REGS_AND_STACK_ACCESS_API	147	select HAVE_REGS_AND_STACK_ACCESS_API
146	select HAVE_RETHOOK if !XIP_KERNEL	148	select HAVE_RETHOOK if !XIP_KERNEL
147	select HAVE_RSEQ	149	select HAVE_RSEQ

Lines 28-33 config X86_64 Link Here

(-)a/arch/x86/Kconfig (+3 lines)
28	select ARCH_HAS_GIGANTIC_PAGE	28	select ARCH_HAS_GIGANTIC_PAGE
29	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128	29	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
30	select ARCH_SUPPORTS_PER_VMA_LOCK	30	select ARCH_SUPPORTS_PER_VMA_LOCK
		31	select ARCH_SUPPORTS_RT
31	select HAVE_ARCH_SOFT_DIRTY	32	select HAVE_ARCH_SOFT_DIRTY
32	select MODULES_USE_ELF_RELA	33	select MODULES_USE_ELF_RELA
33	select NEED_DMA_MAP_STATE	34	select NEED_DMA_MAP_STATE
Lines 119-124 config X86 Link Here
119	select ARCH_USES_CFI_TRAPS if X86_64 && CFI_CLANG	120	select ARCH_USES_CFI_TRAPS if X86_64 && CFI_CLANG
120	select ARCH_SUPPORTS_LTO_CLANG	121	select ARCH_SUPPORTS_LTO_CLANG
121	select ARCH_SUPPORTS_LTO_CLANG_THIN	122	select ARCH_SUPPORTS_LTO_CLANG_THIN
		123	select ARCH_SUPPORTS_RT
122	select ARCH_USE_BUILTIN_BSWAP	124	select ARCH_USE_BUILTIN_BSWAP
123	select ARCH_USE_CMPXCHG_LOCKREF if X86_CMPXCHG64	125	select ARCH_USE_CMPXCHG_LOCKREF if X86_CMPXCHG64
124	select ARCH_USE_MEMTEST	126	select ARCH_USE_MEMTEST
Lines 275-280 config X86 Link Here
275	select HAVE_STATIC_CALL	277	select HAVE_STATIC_CALL
276	select HAVE_STATIC_CALL_INLINE if HAVE_OBJTOOL	278	select HAVE_STATIC_CALL_INLINE if HAVE_OBJTOOL
277	select HAVE_PREEMPT_DYNAMIC_CALL	279	select HAVE_PREEMPT_DYNAMIC_CALL
		280	select HAVE_PREEMPT_AUTO
278	select HAVE_RSEQ	281	select HAVE_RSEQ
279	select HAVE_RUST if X86_64	282	select HAVE_RUST if X86_64
280	select HAVE_SYSCALL_TRACEPOINTS	283	select HAVE_SYSCALL_TRACEPOINTS

Lines 81-88 struct thread_info { Link Here

(-)a/arch/x86/include/asm/thread_info.h (-2 / +4 lines)
81	#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */	81	#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */
82	#define TIF_SIGPENDING 2 /* signal pending */	82	#define TIF_SIGPENDING 2 /* signal pending */
83	#define TIF_NEED_RESCHED 3 /* rescheduling necessary */	83	#define TIF_NEED_RESCHED 3 /* rescheduling necessary */
84	#define TIF_SINGLESTEP 4 /* reenable singlestep on user return*/	84	#define TIF_ARCH_RESCHED_LAZY 4 /* Lazy rescheduling */
85	#define TIF_SSBD 5 /* Speculative store bypass disable */	85	#define TIF_SINGLESTEP 5 /* reenable singlestep on user return*/
		86	#define TIF_SSBD 6 /* Speculative store bypass disable */
86	#define TIF_SPEC_IB 9 /* Indirect branch speculation mitigation */	87	#define TIF_SPEC_IB 9 /* Indirect branch speculation mitigation */
87	#define TIF_SPEC_L1D_FLUSH 10 /* Flush L1D on mm switches (processes) */	88	#define TIF_SPEC_L1D_FLUSH 10 /* Flush L1D on mm switches (processes) */
88	#define TIF_USER_RETURN_NOTIFY 11 /* notify kernel of userspace return */	89	#define TIF_USER_RETURN_NOTIFY 11 /* notify kernel of userspace return */
Lines 104-109 struct thread_info { Link Here
104	#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)	105	#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
105	#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)	106	#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
106	#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)	107	#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
		108	#define _TIF_ARCH_RESCHED_LAZY (1 << TIF_ARCH_RESCHED_LAZY)
107	#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP)	109	#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP)
108	#define _TIF_SSBD (1 << TIF_SSBD)	110	#define _TIF_SSBD (1 << TIF_SSBD)
109	#define _TIF_SPEC_IB (1 << TIF_SPEC_IB)	111	#define _TIF_SPEC_IB (1 << TIF_SPEC_IB)

Lines 57-62 static void zram_free_page(struct zram *zram, size_t index); Link Here

(-)a/drivers/block/zram/zram_drv.c (+37 lines)
57	static int zram_read_page(struct zram zram, struct page page, u32 index,	57	static int zram_read_page(struct zram zram, struct page page, u32 index,
58	struct bio *parent);	58	struct bio *parent);
59		59
		60	#ifdef CONFIG_PREEMPT_RT
		61	static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages)
		62	{
		63	size_t index;
		64
		65	for (index = 0; index < num_pages; index++)
		66	spin_lock_init(&zram->table[index].lock);
		67	}
		68
		69	static int zram_slot_trylock(struct zram *zram, u32 index)
		70	{
		71	int ret;
		72
		73	ret = spin_trylock(&zram->table[index].lock);
		74	if (ret)
		75	__set_bit(ZRAM_LOCK, &zram->table[index].flags);
		76	return ret;
		77	}
		78
		79	static void zram_slot_lock(struct zram *zram, u32 index)
		80	{
		81	spin_lock(&zram->table[index].lock);
		82	__set_bit(ZRAM_LOCK, &zram->table[index].flags);
		83	}
		84
		85	static void zram_slot_unlock(struct zram *zram, u32 index)
		86	{
		87	__clear_bit(ZRAM_LOCK, &zram->table[index].flags);
		88	spin_unlock(&zram->table[index].lock);
		89	}
		90
		91	#else
		92
		93	static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages) { }
		94
60	static int zram_slot_trylock(struct zram *zram, u32 index)	95	static int zram_slot_trylock(struct zram *zram, u32 index)
61	{	96	{
62	return bit_spin_trylock(ZRAM_LOCK, &zram->table[index].flags);	97	return bit_spin_trylock(ZRAM_LOCK, &zram->table[index].flags);
Lines 71-76 static void zram_slot_unlock(struct zram *zram, u32 index) Link Here
71	{	106	{
72	bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags);	107	bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags);
73	}	108	}
		109	#endif
74		110
75	static inline bool init_done(struct zram *zram)	111	static inline bool init_done(struct zram *zram)
76	{	112	{
Lines 1241-1246 static bool zram_meta_alloc(struct zram *zram, u64 disksize) Link Here
1241		1277
1242	if (!huge_class_size)	1278	if (!huge_class_size)
1243	huge_class_size = zs_huge_class_size(zram->mem_pool);	1279	huge_class_size = zs_huge_class_size(zram->mem_pool);
		1280	zram_meta_init_table_locks(zram, num_pages);
1244	return true;	1281	return true;
1245	}	1282	}
1246		1283

Lines 580-586 void intel_pipe_update_start(struct intel_atomic_state *state, Link Here

(-)a/drivers/gpu/drm/i915/display/intel_crtc.c (-5 / +10 lines)
580	*/	580	*/
581	intel_psr_wait_for_idle_locked(new_crtc_state);	581	intel_psr_wait_for_idle_locked(new_crtc_state);
582		582
583	local_irq_disable();	583	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		584	local_irq_disable();
584		585
585	crtc->debug.min_vbl = min;	586	crtc->debug.min_vbl = min;
586	crtc->debug.max_vbl = max;	587	crtc->debug.max_vbl = max;
Lines 605-615 void intel_pipe_update_start(struct intel_atomic_state *state, Link Here
605	break;	606	break;
606	}	607	}
607		608
608	local_irq_enable();	609	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		610	local_irq_enable();
609		611
610	timeout = schedule_timeout(timeout);	612	timeout = schedule_timeout(timeout);
611		613
612	local_irq_disable();	614	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		615	local_irq_disable();
613	}	616	}
614		617
615	finish_wait(wq, &wait);	618	finish_wait(wq, &wait);
Lines 642-648 void intel_pipe_update_start(struct intel_atomic_state *state, Link Here
642	return;	645	return;
643		646
644	irq_disable:	647	irq_disable:
645	local_irq_disable();	648	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		649	local_irq_disable();
646	}	650	}
647		651
648	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)	652	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
Lines 744-750 void intel_pipe_update_end(struct intel_atomic_state *state, Link Here
744	*/	748	*/
745	intel_vrr_send_push(new_crtc_state);	749	intel_vrr_send_push(new_crtc_state);
746		750
747	local_irq_enable();	751	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		752	local_irq_enable();
748		753
749	if (intel_vgpu_active(dev_priv))	754	if (intel_vgpu_active(dev_priv))
750	goto out;	755	goto out;

Lines 275-280 int intel_crtc_scanline_to_hw(struct intel_crtc *crtc, int scanline) Link Here

(-)a/drivers/gpu/drm/i915/display/intel_vblank.c (-10 / +28 lines)
275	* all register accesses to the same cacheline to be serialized,	275	* all register accesses to the same cacheline to be serialized,
276	* otherwise they may hang.	276	* otherwise they may hang.
277	*/	277	*/
		278	static void intel_vblank_section_enter_irqsave(struct drm_i915_private i915, unsigned long flags)
		279	__acquires(i915->uncore.lock)
		280	{
		281	#ifdef I915
		282	spin_lock_irqsave(&i915->uncore.lock, *flags);
		283	#else
		284	*flags = 0;
		285	#endif
		286	}
		287
		288	static void intel_vblank_section_exit_irqrestore(struct drm_i915_private *i915, unsigned long flags)
		289	__releases(i915->uncore.lock)
		290	{
		291	#ifdef I915
		292	spin_unlock_irqrestore(&i915->uncore.lock, flags);
		293	#else
		294	if (flags)
		295	return;
		296	#endif
		297	}
278	static void intel_vblank_section_enter(struct drm_i915_private *i915)	298	static void intel_vblank_section_enter(struct drm_i915_private *i915)
279	__acquires(i915->uncore.lock)	299	__acquires(i915->uncore.lock)
280	{	300	{
Lines 332-341 static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc, Link Here
332	* timing critical raw register reads, potentially with	352	* timing critical raw register reads, potentially with
333	* preemption disabled, so the following code must not block.	353	* preemption disabled, so the following code must not block.
334	*/	354	*/
335	local_irq_save(irqflags);	355	intel_vblank_section_enter_irqsave(dev_priv, &irqflags);
336	intel_vblank_section_enter(dev_priv);
337		356
338	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */	357	if (IS_ENABLED(CONFIG_PREEMPT_RT))
		358	preempt_disable();
339		359
340	/* Get optional system timestamp before query. */	360	/* Get optional system timestamp before query. */
341	if (stime)	361	if (stime)
Lines 399-408 static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc, Link Here
399	if (etime)	419	if (etime)
400	*etime = ktime_get();	420	*etime = ktime_get();
401		421
402	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */	422	if (IS_ENABLED(CONFIG_PREEMPT_RT))
		423	preempt_enable();
403		424
404	intel_vblank_section_exit(dev_priv);	425	intel_vblank_section_exit_irqrestore(dev_priv, irqflags);
405	local_irq_restore(irqflags);
406		426
407	/*	427	/*
408	* While in vblank, position will be negative	428	* While in vblank, position will be negative
Lines 440-452 int intel_get_crtc_scanline(struct intel_crtc *crtc) Link Here
440	unsigned long irqflags;	460	unsigned long irqflags;
441	int position;	461	int position;
442		462
443	local_irq_save(irqflags);	463	intel_vblank_section_enter_irqsave(dev_priv, &irqflags);
444	intel_vblank_section_enter(dev_priv);
445		464
446	position = __intel_get_crtc_scanline(crtc);	465	position = __intel_get_crtc_scanline(crtc);
447		466
448	intel_vblank_section_exit(dev_priv);	467	intel_vblank_section_exit_irqrestore(dev_priv, irqflags);
449	local_irq_restore(irqflags);
450		468
451	return position;	469	return position;
452	}	470	}

Lines 1303-1309 static void execlists_dequeue(struct intel_engine_cs *engine) Link Here

(-)a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c (-12 / +5 lines)
1303	* and context switches) submission.	1303	* and context switches) submission.
1304	*/	1304	*/
1305		1305
1306	spin_lock(&sched_engine->lock);	1306	spin_lock_irq(&sched_engine->lock);
1307		1307
1308	/*	1308	/*
1309	* If the queue is higher priority than the last	1309	* If the queue is higher priority than the last
Lines 1403-1409 static void execlists_dequeue(struct intel_engine_cs *engine) Link Here
1403	* Even if ELSP[1] is occupied and not worthy	1403	* Even if ELSP[1] is occupied and not worthy
1404	* of timeslices, our queue might be.	1404	* of timeslices, our queue might be.
1405	*/	1405	*/
1406	spin_unlock(&sched_engine->lock);	1406	spin_unlock_irq(&sched_engine->lock);
1407	return;	1407	return;
1408	}	1408	}
1409	}	1409	}
Lines 1429-1435 static void execlists_dequeue(struct intel_engine_cs *engine) Link Here
1429		1429
1430	if (last && !can_merge_rq(last, rq)) {	1430	if (last && !can_merge_rq(last, rq)) {
1431	spin_unlock(&ve->base.sched_engine->lock);	1431	spin_unlock(&ve->base.sched_engine->lock);
1432	spin_unlock(&engine->sched_engine->lock);	1432	spin_unlock_irq(&engine->sched_engine->lock);
1433	return; /* leave this for another sibling */	1433	return; /* leave this for another sibling */
1434	}	1434	}
1435		1435
Lines 1591-1597 static void execlists_dequeue(struct intel_engine_cs *engine) Link Here
1591	*/	1591	*/
1592	sched_engine->queue_priority_hint = queue_prio(sched_engine);	1592	sched_engine->queue_priority_hint = queue_prio(sched_engine);
1593	i915_sched_engine_reset_on_empty(sched_engine);	1593	i915_sched_engine_reset_on_empty(sched_engine);
1594	spin_unlock(&sched_engine->lock);	1594	spin_unlock_irq(&sched_engine->lock);
1595		1595
1596	/*	1596	/*
1597	* We can skip poking the HW if we ended up with exactly the same set	1597	* We can skip poking the HW if we ended up with exactly the same set
Lines 1617-1629 static void execlists_dequeue(struct intel_engine_cs *engine) Link Here
1617	}	1617	}
1618	}	1618	}
1619		1619
1620	static void execlists_dequeue_irq(struct intel_engine_cs *engine)
1621	{
1622	local_irq_disable(); /* Suspend interrupts across request submission */
1623	execlists_dequeue(engine);
1624	local_irq_enable(); /* flush irq_work (e.g. breadcrumb enabling) */
1625	}
1626
1627	static void clear_ports(struct i915_request **ports, int count)	1620	static void clear_ports(struct i915_request **ports, int count)
1628	{	1621	{
1629	memset_p((void **)ports, NULL, count);	1622	memset_p((void **)ports, NULL, count);
Lines 2478-2484 static void execlists_submission_tasklet(struct tasklet_struct *t) Link Here
2478	}	2471	}
2479		2472
2480	if (!engine->execlists.pending[0]) {	2473	if (!engine->execlists.pending[0]) {
2481	execlists_dequeue_irq(engine);	2474	execlists_dequeue(engine);
2482	start_timeslice(engine);	2475	start_timeslice(engine);
2483	}	2476	}
2484		2477

Lines 592-597 serial8250_register_ports(struct uart_driver drv, struct device dev) Link Here

(-)a/drivers/tty/serial/8250/8250_core.c (-4 / +47 lines)
592		592
593	#ifdef CONFIG_SERIAL_8250_CONSOLE	593	#ifdef CONFIG_SERIAL_8250_CONSOLE
594		594
		595	#ifdef CONFIG_SERIAL_8250_LEGACY_CONSOLE
595	static void univ8250_console_write(struct console co, const char s,	596	static void univ8250_console_write(struct console co, const char s,
596	unsigned int count)	597	unsigned int count)
597	{	598	{
Lines 599-604 static void univ8250_console_write(struct console co, const char s, Link Here
599		600
600	serial8250_console_write(up, s, count);	601	serial8250_console_write(up, s, count);
601	}	602	}
		603	#else
		604	static void univ8250_console_write_atomic(struct console *co,
		605	struct nbcon_write_context *wctxt)
		606	{
		607	struct uart_8250_port *up = &serial8250_ports[co->index];
		608
		609	serial8250_console_write_atomic(up, wctxt);
		610	}
		611
		612	static void univ8250_console_write_thread(struct console *co,
		613	struct nbcon_write_context *wctxt)
		614	{
		615	struct uart_8250_port *up = &serial8250_ports[co->index];
		616
		617	serial8250_console_write_thread(up, wctxt);
		618	}
		619
		620	static void univ8250_console_device_lock(struct console con, unsigned long flags)
		621	{
		622	struct uart_port *up = &serial8250_ports[con->index].port;
		623
		624	__uart_port_lock_irqsave(up, flags);
		625	}
		626
		627	static void univ8250_console_device_unlock(struct console *con, unsigned long flags)
		628	{
		629	struct uart_port *up = &serial8250_ports[con->index].port;
		630
		631	__uart_port_unlock_irqrestore(up, flags);
		632	}
		633
		634	static struct nbcon_drvdata serial8250_nbcon_drvdata;
		635	#endif /* CONFIG_SERIAL_8250_LEGACY_CONSOLE */
602		636
603	static int univ8250_console_setup(struct console co, char options)	637	static int univ8250_console_setup(struct console co, char options)
604	{	638	{
Lines 627-637 static int univ8250_console_setup(struct console co, char options) Link Here
627		661
628	port = &serial8250_ports[co->index].port;	662	port = &serial8250_ports[co->index].port;
629	/* link port to console */	663	/* link port to console */
630	port->cons = co;	664	uart_port_set_cons(port, co);
631		665
632	retval = serial8250_console_setup(port, options, false);	666	retval = serial8250_console_setup(port, options, false);
633	if (retval != 0)	667	if (retval != 0)
634	port->cons = NULL;	668	uart_port_set_cons(port, NULL);
635	return retval;	669	return retval;
636	}	670	}
637		671
Lines 689-695 static int univ8250_console_match(struct console co, char name, int idx, Link Here
689	continue;	723	continue;
690		724
691	co->index = i;	725	co->index = i;
692	port->cons = co;	726	uart_port_set_cons(port, co);
693	return serial8250_console_setup(port, options, true);	727	return serial8250_console_setup(port, options, true);
694	}	728	}
695		729
Lines 698-709 static int univ8250_console_match(struct console co, char name, int idx, Link Here
698		732
699	static struct console univ8250_console = {	733	static struct console univ8250_console = {
700	.name = "ttyS",	734	.name = "ttyS",
		735	#ifdef CONFIG_SERIAL_8250_LEGACY_CONSOLE
701	.write = univ8250_console_write,	736	.write = univ8250_console_write,
		737	.flags = CON_PRINTBUFFER \| CON_ANYTIME,
		738	#else
		739	.write_atomic = univ8250_console_write_atomic,
		740	.write_thread = univ8250_console_write_thread,
		741	.device_lock = univ8250_console_device_lock,
		742	.device_unlock = univ8250_console_device_unlock,
		743	.flags = CON_PRINTBUFFER \| CON_ANYTIME \| CON_NBCON,
		744	.nbcon_drvdata = &serial8250_nbcon_drvdata,
		745	#endif
702	.device = uart_console_device,	746	.device = uart_console_device,
703	.setup = univ8250_console_setup,	747	.setup = univ8250_console_setup,
704	.exit = univ8250_console_exit,	748	.exit = univ8250_console_exit,
705	.match = univ8250_console_match,	749	.match = univ8250_console_match,
706	.flags = CON_PRINTBUFFER \| CON_ANYTIME,
707	.index = -1,	750	.index = -1,
708	.data = &serial8250_reg,	751	.data = &serial8250_reg,
709	};	752	};

Lines 348-357 static int pl011_fifo_to_tty(struct uart_amba_port *uap) Link Here

(-)a/drivers/tty/serial/amba-pl011.c (-17 / +9 lines)
348	flag = TTY_FRAME;	348	flag = TTY_FRAME;
349	}	349	}
350		350
351	uart_port_unlock(&uap->port);	351	sysrq = uart_prepare_sysrq_char(&uap->port, ch & 255);
352	sysrq = uart_handle_sysrq_char(&uap->port, ch & 255);
353	uart_port_lock(&uap->port);
354
355	if (!sysrq)	352	if (!sysrq)
356	uart_insert_char(&uap->port, ch, UART011_DR_OE, ch, flag);	353	uart_insert_char(&uap->port, ch, UART011_DR_OE, ch, flag);
357	}	354	}
Lines 1017-1023 static void pl011_dma_rx_callback(void *data) Link Here
1017	ret = pl011_dma_rx_trigger_dma(uap);	1014	ret = pl011_dma_rx_trigger_dma(uap);
1018		1015
1019	pl011_dma_rx_chars(uap, pending, lastbuf, false);	1016	pl011_dma_rx_chars(uap, pending, lastbuf, false);
1020	uart_port_unlock_irq(&uap->port);	1017	uart_unlock_and_check_sysrq(&uap->port);
1021	/*	1018	/*
1022	* Do this check after we picked the DMA chars so we don't	1019	* Do this check after we picked the DMA chars so we don't
1023	* get some IRQ immediately from RX.	1020	* get some IRQ immediately from RX.
Lines 1540-1550 static void check_apply_cts_event_workaround(struct uart_amba_port *uap) Link Here
1540	static irqreturn_t pl011_int(int irq, void *dev_id)	1537	static irqreturn_t pl011_int(int irq, void *dev_id)
1541	{	1538	{
1542	struct uart_amba_port *uap = dev_id;	1539	struct uart_amba_port *uap = dev_id;
1543	unsigned long flags;
1544	unsigned int status, pass_counter = AMBA_ISR_PASS_LIMIT;	1540	unsigned int status, pass_counter = AMBA_ISR_PASS_LIMIT;
1545	int handled = 0;	1541	int handled = 0;
1546		1542
1547	uart_port_lock_irqsave(&uap->port, &flags);	1543	uart_port_lock(&uap->port);
1548	status = pl011_read(uap, REG_RIS) & uap->im;	1544	status = pl011_read(uap, REG_RIS) & uap->im;
1549	if (status) {	1545	if (status) {
1550	do {	1546	do {
Lines 1573-1579 static irqreturn_t pl011_int(int irq, void *dev_id) Link Here
1573	handled = 1;	1569	handled = 1;
1574	}	1570	}
1575		1571
1576	uart_port_unlock_irqrestore(&uap->port, flags);	1572	uart_unlock_and_check_sysrq(&uap->port);
1577		1573
1578	return IRQ_RETVAL(handled);	1574	return IRQ_RETVAL(handled);
1579	}	1575	}
Lines 2322-2334 pl011_console_write(struct console co, const char s, unsigned int count) Link Here
2322		2318
2323	clk_enable(uap->clk);	2319	clk_enable(uap->clk);
2324		2320
2325	local_irq_save(flags);	2321	if (oops_in_progress)
2326	if (uap->port.sysrq)	2322	locked = uart_port_trylock_irqsave(&uap->port, &flags);
2327	locked = 0;
2328	else if (oops_in_progress)
2329	locked = uart_port_trylock(&uap->port);
2330	else	2323	else
2331	uart_port_lock(&uap->port);	2324	uart_port_lock_irqsave(&uap->port, &flags);
2332		2325
2333	/*	2326	/*
2334	* First save the CR then disable the interrupts	2327	* First save the CR then disable the interrupts
Lines 2354-2361 pl011_console_write(struct console co, const char s, unsigned int count) Link Here
2354	pl011_write(old_cr, uap, REG_CR);	2347	pl011_write(old_cr, uap, REG_CR);
2355		2348
2356	if (locked)	2349	if (locked)
2357	uart_port_unlock(&uap->port);	2350	uart_port_unlock_irqrestore(&uap->port, flags);
2358	local_irq_restore(flags);
2359		2351
2360	clk_disable(uap->clk);	2352	clk_disable(uap->clk);
2361	}	2353	}
Lines 2496-2502 static int pl011_console_match(struct console co, char name, int idx, Link Here
2496	continue;	2488	continue;
2497		2489
2498	co->index = i;	2490	co->index = i;
2499	port->cons = co;	2491	uart_port_set_cons(port, co);
2500	return pl011_console_setup(co, options);	2492	return pl011_console_setup(co, options);
2501	}	2493	}
2502		2494

Lines 378-384 static void ar933x_uart_rx_chars(struct ar933x_uart_port *up) Link Here

(-)a/drivers/tty/serial/ar933x_uart.c (-12 / +6 lines)
378	up->port.icount.rx++;	378	up->port.icount.rx++;
379	ch = rdata & AR933X_UART_DATA_TX_RX_MASK;	379	ch = rdata & AR933X_UART_DATA_TX_RX_MASK;
380		380
381	if (uart_handle_sysrq_char(&up->port, ch))	381	if (uart_prepare_sysrq_char(&up->port, ch))
382	continue;	382	continue;
383		383
384	if ((up->port.ignore_status_mask & AR933X_DUMMY_STATUS_RD) == 0)	384	if ((up->port.ignore_status_mask & AR933X_DUMMY_STATUS_RD) == 0)
Lines 468-474 static irqreturn_t ar933x_uart_interrupt(int irq, void *dev_id) Link Here
468	ar933x_uart_tx_chars(up);	468	ar933x_uart_tx_chars(up);
469	}	469	}
470		470
471	uart_port_unlock(&up->port);	471	uart_unlock_and_check_sysrq(&up->port);
472		472
473	return IRQ_HANDLED;	473	return IRQ_HANDLED;
474	}	474	}
Lines 627-640 static void ar933x_uart_console_write(struct console co, const char s, Link Here
627	unsigned int int_en;	627	unsigned int int_en;
628	int locked = 1;	628	int locked = 1;
629		629
630	local_irq_save(flags);	630	if (oops_in_progress)
631		631	locked = uart_port_trylock_irqsave(&up->port, &flags);
632	if (up->port.sysrq)
633	locked = 0;
634	else if (oops_in_progress)
635	locked = uart_port_trylock(&up->port);
636	else	632	else
637	uart_port_lock(&up->port);	633	uart_port_lock_irqsave(&up->port, &flags);
638		634
639	/*	635	/*
640	* First save the IER then disable the interrupts	636	* First save the IER then disable the interrupts
Lines 654-662 static void ar933x_uart_console_write(struct console co, const char s, Link Here
654	ar933x_uart_write(up, AR933X_UART_INT_REG, AR933X_UART_INT_ALLINTS);	650	ar933x_uart_write(up, AR933X_UART_INT_REG, AR933X_UART_INT_ALLINTS);
655		651
656	if (locked)	652	if (locked)
657	uart_port_unlock(&up->port);	653	uart_port_unlock_irqrestore(&up->port, flags);
658
659	local_irq_restore(flags);
660	}	654	}
661		655
662	static int ar933x_uart_console_setup(struct console co, char options)	656	static int ar933x_uart_console_setup(struct console co, char options)

Lines 285-294 static void bcm_uart_do_rx(struct uart_port *port) Link Here

(-)a/drivers/tty/serial/bcm63xx_uart.c (-16 / +8 lines)
285	flag = TTY_PARITY;	285	flag = TTY_PARITY;
286	}	286	}
287		287
288	if (uart_handle_sysrq_char(port, c))	288	if (uart_prepare_sysrq_char(port, c))
289	continue;	289	continue;
290		290
291
292	if ((cstat & port->ignore_status_mask) == 0)	291	if ((cstat & port->ignore_status_mask) == 0)
293	tty_insert_flip_char(tty_port, c, flag);	292	tty_insert_flip_char(tty_port, c, flag);
294		293
Lines 353-359 static irqreturn_t bcm_uart_interrupt(int irq, void *dev_id) Link Here
353	estat & UART_EXTINP_DCD_MASK);	352	estat & UART_EXTINP_DCD_MASK);
354	}	353	}
355		354
356	uart_port_unlock(port);	355	uart_unlock_and_check_sysrq(port);
357	return IRQ_HANDLED;	356	return IRQ_HANDLED;
358	}	357	}
359		358
Lines 703-722 static void bcm_console_write(struct console co, const char s, Link Here
703	{	702	{
704	struct uart_port *port;	703	struct uart_port *port;
705	unsigned long flags;	704	unsigned long flags;
706	int locked;	705	int locked = 1;
707		706
708	port = &ports[co->index];	707	port = &ports[co->index];
709		708
710	local_irq_save(flags);	709	if (oops_in_progress)
711	if (port->sysrq) {	710	locked = uart_port_trylock_irqsave(port, &flags);
712	/* bcm_uart_interrupt() already took the lock */	711	else
713	locked = 0;	712	uart_port_lock_irqsave(port, &flags);
714	} else if (oops_in_progress) {
715	locked = uart_port_trylock(port);
716	} else {
717	uart_port_lock(port);
718	locked = 1;
719	}
720		713
721	/* call helper to deal with \r\n */	714	/* call helper to deal with \r\n */
722	uart_console_write(port, s, count, bcm_console_putchar);	715	uart_console_write(port, s, count, bcm_console_putchar);
Lines 725-732 static void bcm_console_write(struct console co, const char s, Link Here
725	wait_for_xmitr(port);	718	wait_for_xmitr(port);
726		719
727	if (locked)	720	if (locked)
728	uart_port_unlock(port);	721	uart_port_unlock_irqrestore(port, flags);
729	local_irq_restore(flags);
730	}	722	}
731		723
732	/*	724	/*

Lines 136-155 static void lpc32xx_hsuart_console_write(struct console co, const char s, Link Here

(-)a/drivers/tty/serial/lpc32xx_hs.c (-10 / +7 lines)
136	int locked = 1;	136	int locked = 1;
137		137
138	touch_nmi_watchdog();	138	touch_nmi_watchdog();
139	local_irq_save(flags);	139	if (oops_in_progress)
140	if (up->port.sysrq)	140	locked = uart_port_trylock_irqsave(&up->port, &flags);
141	locked = 0;
142	else if (oops_in_progress)
143	locked = uart_port_trylock(&up->port);
144	else	141	else
145	uart_port_lock(&up->port);	142	uart_port_lock_irqsave(&up->port, &flags);
146		143
147	uart_console_write(&up->port, s, count, lpc32xx_hsuart_console_putchar);	144	uart_console_write(&up->port, s, count, lpc32xx_hsuart_console_putchar);
148	wait_for_xmit_empty(&up->port);	145	wait_for_xmit_empty(&up->port);
149		146
150	if (locked)	147	if (locked)
151	uart_port_unlock(&up->port);	148	uart_port_unlock_irqrestore(&up->port, flags);
152	local_irq_restore(flags);
153	}	149	}
154		150
155	static int __init lpc32xx_hsuart_console_setup(struct console *co,	151	static int __init lpc32xx_hsuart_console_setup(struct console *co,
Lines 268-274 static void __serial_lpc32xx_rx(struct uart_port *port) Link Here
268	tty_insert_flip_char(tport, 0, TTY_FRAME);	264	tty_insert_flip_char(tport, 0, TTY_FRAME);
269	}	265	}
270		266
271	tty_insert_flip_char(tport, (tmp & 0xFF), flag);	267	if (!uart_prepare_sysrq_char(port, tmp & 0xff))
		268	tty_insert_flip_char(tport, (tmp & 0xFF), flag);
272		269
273	tmp = readl(LPC32XX_HSUART_FIFO(port->membase));	270	tmp = readl(LPC32XX_HSUART_FIFO(port->membase));
274	}	271	}
Lines 333-339 static irqreturn_t serial_lpc32xx_interrupt(int irq, void *dev_id) Link Here
333	__serial_lpc32xx_tx(port);	330	__serial_lpc32xx_tx(port);
334	}	331	}
335		332
336	uart_port_unlock(port);	333	uart_unlock_and_check_sysrq(port);
337		334
338	return IRQ_HANDLED;	335	return IRQ_HANDLED;
339	}	336	}

Lines 220-226 static void meson_receive_chars(struct uart_port *port) Link Here

(-)a/drivers/tty/serial/meson_uart.c (-14 / +8 lines)
220	continue;	220	continue;
221	}	221	}
222		222
223	if (uart_handle_sysrq_char(port, ch))	223	if (uart_prepare_sysrq_char(port, ch))
224	continue;	224	continue;
225		225
226	if ((status & port->ignore_status_mask) == 0)	226	if ((status & port->ignore_status_mask) == 0)
Lines 248-254 static irqreturn_t meson_uart_interrupt(int irq, void *dev_id) Link Here
248	meson_uart_start_tx(port);	248	meson_uart_start_tx(port);
249	}	249	}
250		250
251	uart_port_unlock(port);	251	uart_unlock_and_check_sysrq(port);
252		252
253	return IRQ_HANDLED;	253	return IRQ_HANDLED;
254	}	254	}
Lines 556-573 static void meson_serial_port_write(struct uart_port port, const char s, Link Here
556	u_int count)	556	u_int count)
557	{	557	{
558	unsigned long flags;	558	unsigned long flags;
559	int locked;	559	int locked = 1;
560	u32 val, tmp;	560	u32 val, tmp;
561		561
562	local_irq_save(flags);	562	if (oops_in_progress)
563	if (port->sysrq) {	563	locked = uart_port_trylock_irqsave(port, &flags);
564	locked = 0;	564	else
565	} else if (oops_in_progress) {	565	uart_port_lock_irqsave(port, &flags);
566	locked = uart_port_trylock(port);
567	} else {
568	uart_port_lock(port);
569	locked = 1;
570	}
571		566
572	val = readl(port->membase + AML_UART_CONTROL);	567	val = readl(port->membase + AML_UART_CONTROL);
573	tmp = val & ~(AML_UART_TX_INT_EN \| AML_UART_RX_INT_EN);	568	tmp = val & ~(AML_UART_TX_INT_EN \| AML_UART_RX_INT_EN);
Lines 577-584 static void meson_serial_port_write(struct uart_port port, const char s, Link Here
577	writel(val, port->membase + AML_UART_CONTROL);	572	writel(val, port->membase + AML_UART_CONTROL);
578		573
579	if (locked)	574	if (locked)
580	uart_port_unlock(port);	575	uart_port_unlock_irqrestore(port, flags);
581	local_irq_restore(flags);
582	}	576	}
583		577
584	static void meson_serial_console_write(struct console co, const char s,	578	static void meson_serial_console_write(struct console co, const char s,

Lines 588-603 static void msm_complete_rx_dma(void *args) Link Here

(-)a/drivers/tty/serial/msm_serial.c (-23 / +10 lines)
588	if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))	588	if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))
589	flag = TTY_NORMAL;	589	flag = TTY_NORMAL;
590		590
591	uart_port_unlock_irqrestore(port, flags);	591	sysrq = uart_prepare_sysrq_char(port, dma->virt[i]);
592	sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
593	uart_port_lock_irqsave(port, &flags);
594	if (!sysrq)	592	if (!sysrq)
595	tty_insert_flip_char(tport, dma->virt[i], flag);	593	tty_insert_flip_char(tport, dma->virt[i], flag);
596	}	594	}
597		595
598	msm_start_rx_dma(msm_port);	596	msm_start_rx_dma(msm_port);
599	done:	597	done:
600	uart_port_unlock_irqrestore(port, flags);	598	uart_unlock_and_check_sysrq_irqrestore(port, flags);
601		599
602	if (count)	600	if (count)
603	tty_flip_buffer_push(tport);	601	tty_flip_buffer_push(tport);
Lines 763-771 static void msm_handle_rx_dm(struct uart_port *port, unsigned int misr) Link Here
763	if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))	761	if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))
764	flag = TTY_NORMAL;	762	flag = TTY_NORMAL;
765		763
766	uart_port_unlock(port);	764	sysrq = uart_prepare_sysrq_char(port, buf[i]);
767	sysrq = uart_handle_sysrq_char(port, buf[i]);
768	uart_port_lock(port);
769	if (!sysrq)	765	if (!sysrq)
770	tty_insert_flip_char(tport, buf[i], flag);	766	tty_insert_flip_char(tport, buf[i], flag);
771	}	767	}
Lines 825-833 static void msm_handle_rx(struct uart_port *port) Link Here
825	else if (sr & MSM_UART_SR_PAR_FRAME_ERR)	821	else if (sr & MSM_UART_SR_PAR_FRAME_ERR)
826	flag = TTY_FRAME;	822	flag = TTY_FRAME;
827		823
828	uart_port_unlock(port);	824	sysrq = uart_prepare_sysrq_char(port, c);
829	sysrq = uart_handle_sysrq_char(port, c);
830	uart_port_lock(port);
831	if (!sysrq)	825	if (!sysrq)
832	tty_insert_flip_char(tport, c, flag);	826	tty_insert_flip_char(tport, c, flag);
833	}	827	}
Lines 948-958 static irqreturn_t msm_uart_irq(int irq, void *dev_id) Link Here
948	struct uart_port *port = dev_id;	942	struct uart_port *port = dev_id;
949	struct msm_port *msm_port = to_msm_port(port);	943	struct msm_port *msm_port = to_msm_port(port);
950	struct msm_dma *dma = &msm_port->rx_dma;	944	struct msm_dma *dma = &msm_port->rx_dma;
951	unsigned long flags;
952	unsigned int misr;	945	unsigned int misr;
953	u32 val;	946	u32 val;
954		947
955	uart_port_lock_irqsave(port, &flags);	948	uart_port_lock(port);
956	misr = msm_read(port, MSM_UART_MISR);	949	misr = msm_read(port, MSM_UART_MISR);
957	msm_write(port, 0, MSM_UART_IMR); /* disable interrupt */	950	msm_write(port, 0, MSM_UART_IMR); /* disable interrupt */
958		951
Lines 984-990 static irqreturn_t msm_uart_irq(int irq, void *dev_id) Link Here
984	msm_handle_delta_cts(port);	977	msm_handle_delta_cts(port);
985		978
986	msm_write(port, msm_port->imr, MSM_UART_IMR); /* restore interrupt */	979	msm_write(port, msm_port->imr, MSM_UART_IMR); /* restore interrupt */
987	uart_port_unlock_irqrestore(port, flags);	980	uart_unlock_and_check_sysrq(port);
988		981
989	return IRQ_HANDLED;	982	return IRQ_HANDLED;
990	}	983	}
Lines 1621-1634 static void __msm_console_write(struct uart_port port, const char s, Link Here
1621	num_newlines++;	1614	num_newlines++;
1622	count += num_newlines;	1615	count += num_newlines;
1623		1616
1624	local_irq_save(flags);	1617	if (oops_in_progress)
1625		1618	locked = uart_port_trylock_irqsave(port, &flags);
1626	if (port->sysrq)
1627	locked = 0;
1628	else if (oops_in_progress)
1629	locked = uart_port_trylock(port);
1630	else	1619	else
1631	uart_port_lock(port);	1620	uart_port_lock_irqsave(port, &flags);
1632		1621
1633	if (is_uartdm)	1622	if (is_uartdm)
1634	msm_reset_dm_count(port, count);	1623	msm_reset_dm_count(port, count);
Lines 1667-1675 static void __msm_console_write(struct uart_port port, const char s, Link Here
1667	}	1656	}
1668		1657
1669	if (locked)	1658	if (locked)
1670	uart_port_unlock(port);	1659	uart_port_unlock_irqrestore(port, flags);
1671
1672	local_irq_restore(flags);
1673	}	1660	}
1674		1661
1675	static void msm_console_write(struct console co, const char s,	1662	static void msm_console_write(struct console co, const char s,

Lines 508-514 static void serial_omap_rdi(struct uart_omap_port *up, unsigned int lsr) Link Here

(-)a/drivers/tty/serial/omap-serial.c (-10 / +6 lines)
508		508
509	up->port.icount.rx++;	509	up->port.icount.rx++;
510		510
511	if (uart_handle_sysrq_char(&up->port, ch))	511	if (uart_prepare_sysrq_char(&up->port, ch))
512	return;	512	return;
513		513
514	uart_insert_char(&up->port, lsr, UART_LSR_OE, ch, TTY_NORMAL);	514	uart_insert_char(&up->port, lsr, UART_LSR_OE, ch, TTY_NORMAL);
Lines 563-569 static irqreturn_t serial_omap_irq(int irq, void *dev_id) Link Here
563	}	563	}
564	} while (max_count--);	564	} while (max_count--);
565		565
566	uart_port_unlock(&up->port);	566	uart_unlock_and_check_sysrq(&up->port);
567		567
568	tty_flip_buffer_push(&up->port.state->port);	568	tty_flip_buffer_push(&up->port.state->port);
569		569
Lines 1212-1224 serial_omap_console_write(struct console co, const char s, Link Here
1212	unsigned int ier;	1212	unsigned int ier;
1213	int locked = 1;	1213	int locked = 1;
1214		1214
1215	local_irq_save(flags);	1215	if (oops_in_progress)
1216	if (up->port.sysrq)	1216	locked = uart_port_trylock_irqsave(&up->port, &flags);
1217	locked = 0;
1218	else if (oops_in_progress)
1219	locked = uart_port_trylock(&up->port);
1220	else	1217	else
1221	uart_port_lock(&up->port);	1218	uart_port_lock_irqsave(&up->port, &flags);
1222		1219
1223	/*	1220	/*
1224	* First save the IER then disable the interrupts	1221	* First save the IER then disable the interrupts
Lines 1245-1252 serial_omap_console_write(struct console co, const char s, Link Here
1245	check_modem_status(up);	1242	check_modem_status(up);
1246		1243
1247	if (locked)	1244	if (locked)
1248	uart_port_unlock(&up->port);	1245	uart_port_unlock_irqrestore(&up->port, flags);
1249	local_irq_restore(flags);
1250	}	1246	}
1251		1247
1252	static int __init	1248	static int __init

Lines 237-245 struct eg20t_port { Link Here

(-)a/drivers/tty/serial/pch_uart.c (-51 / +19 lines)
237		237
238	#define IRQ_NAME_SIZE 17	238	#define IRQ_NAME_SIZE 17
239	char irq_name[IRQ_NAME_SIZE];	239	char irq_name[IRQ_NAME_SIZE];
240
241	/* protect the eg20t_port private structure and io access to membase */
242	spinlock_t lock;
243	};	240	};
244		241
245	/**	242	/**
Lines 567-573 static int pch_uart_hal_read(struct eg20t_port priv, unsigned char buf, Link Here
567	if (uart_handle_break(port))	564	if (uart_handle_break(port))
568	continue;	565	continue;
569	}	566	}
570	if (uart_handle_sysrq_char(port, rbr))	567	if (uart_prepare_sysrq_char(port, rbr))
571	continue;	568	continue;
572		569
573	buf[i++] = rbr;	570	buf[i++] = rbr;
Lines 599-614 static void pch_uart_hal_set_break(struct eg20t_port *priv, int on) Link Here
599	iowrite8(lcr, priv->membase + UART_LCR);	596	iowrite8(lcr, priv->membase + UART_LCR);
600	}	597	}
601		598
602	static int push_rx(struct eg20t_port priv, const unsigned char buf,	599	static void push_rx(struct eg20t_port priv, const unsigned char buf,
603	int size)	600	int size)
604	{	601	{
605	struct uart_port *port = &priv->port;	602	struct uart_port *port = &priv->port;
606	struct tty_port *tport = &port->state->port;	603	struct tty_port *tport = &port->state->port;
607		604
608	tty_insert_flip_string(tport, buf, size);	605	tty_insert_flip_string(tport, buf, size);
609	tty_flip_buffer_push(tport);	606	tty_flip_buffer_push(tport);
610
611	return 0;
612	}	607	}
613		608
614	static int dma_push_rx(struct eg20t_port *priv, int size)	609	static int dma_push_rx(struct eg20t_port *priv, int size)
Lines 761-767 static int handle_rx_to(struct eg20t_port *priv) Link Here
761	{	756	{
762	struct pch_uart_buffer *buf;	757	struct pch_uart_buffer *buf;
763	int rx_size;	758	int rx_size;
764	int ret;	759
765	if (!priv->start_rx) {	760	if (!priv->start_rx) {
766	pch_uart_hal_disable_interrupt(priv, PCH_UART_HAL_RX_INT \|	761	pch_uart_hal_disable_interrupt(priv, PCH_UART_HAL_RX_INT \|
767	PCH_UART_HAL_RX_ERR_INT);	762	PCH_UART_HAL_RX_ERR_INT);
Lines 770-788 static int handle_rx_to(struct eg20t_port *priv) Link Here
770	buf = &priv->rxbuf;	765	buf = &priv->rxbuf;
771	do {	766	do {
772	rx_size = pch_uart_hal_read(priv, buf->buf, buf->size);	767	rx_size = pch_uart_hal_read(priv, buf->buf, buf->size);
773	ret = push_rx(priv, buf->buf, rx_size);	768	push_rx(priv, buf->buf, rx_size);
774	if (ret)
775	return 0;
776	} while (rx_size == buf->size);	769	} while (rx_size == buf->size);
777		770
778	return PCH_UART_HANDLED_RX_INT;	771	return PCH_UART_HANDLED_RX_INT;
779	}	772	}
780		773
781	static int handle_rx(struct eg20t_port *priv)
782	{
783	return handle_rx_to(priv);
784	}
785
786	static int dma_handle_rx(struct eg20t_port *priv)	774	static int dma_handle_rx(struct eg20t_port *priv)
787	{	775	{
788	struct uart_port *port = &priv->port;	776	struct uart_port *port = &priv->port;
Lines 1019-1029 static irqreturn_t pch_uart_interrupt(int irq, void *dev_id) Link Here
1019	u8 lsr;	1007	u8 lsr;
1020	int ret = 0;	1008	int ret = 0;
1021	unsigned char iid;	1009	unsigned char iid;
1022	unsigned long flags;
1023	int next = 1;	1010	int next = 1;
1024	u8 msr;	1011	u8 msr;
1025		1012
1026	spin_lock_irqsave(&priv->lock, flags);	1013	uart_port_lock(&priv->port);
1027	handled = 0;	1014	handled = 0;
1028	while (next) {	1015	while (next) {
1029	iid = pch_uart_hal_get_iid(priv);	1016	iid = pch_uart_hal_get_iid(priv);
Lines 1051-1057 static irqreturn_t pch_uart_interrupt(int irq, void *dev_id) Link Here
1051	PCH_UART_HAL_RX_INT \|	1038	PCH_UART_HAL_RX_INT \|
1052	PCH_UART_HAL_RX_ERR_INT);	1039	PCH_UART_HAL_RX_ERR_INT);
1053	} else {	1040	} else {
1054	ret = handle_rx(priv);	1041	ret = handle_rx_to(priv);
1055	}	1042	}
1056	break;	1043	break;
1057	case PCH_UART_IID_RDR_TO: /* Received Data Ready	1044	case PCH_UART_IID_RDR_TO: /* Received Data Ready
Lines 1083-1089 static irqreturn_t pch_uart_interrupt(int irq, void *dev_id) Link Here
1083	handled \|= (unsigned int)ret;	1070	handled \|= (unsigned int)ret;
1084	}	1071	}
1085		1072
1086	spin_unlock_irqrestore(&priv->lock, flags);	1073	uart_unlock_and_check_sysrq(&priv->port);
1087	return IRQ_RETVAL(handled);	1074	return IRQ_RETVAL(handled);
1088	}	1075	}
1089		1076
Lines 1194-1202 static void pch_uart_break_ctl(struct uart_port *port, int ctl) Link Here
1194	unsigned long flags;	1181	unsigned long flags;
1195		1182
1196	priv = container_of(port, struct eg20t_port, port);	1183	priv = container_of(port, struct eg20t_port, port);
1197	spin_lock_irqsave(&priv->lock, flags);	1184	uart_port_lock_irqsave(&priv->port, &flags);
1198	pch_uart_hal_set_break(priv, ctl);	1185	pch_uart_hal_set_break(priv, ctl);
1199	spin_unlock_irqrestore(&priv->lock, flags);	1186	uart_port_unlock_irqrestore(&priv->port, flags);
1200	}	1187	}
1201		1188
1202	/* Grab any interrupt resources and initialise any low level driver state. */	1189	/* Grab any interrupt resources and initialise any low level driver state. */
Lines 1346-1353 static void pch_uart_set_termios(struct uart_port *port, Link Here
1346		1333
1347	baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);	1334	baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);
1348		1335
1349	spin_lock_irqsave(&priv->lock, flags);	1336	uart_port_lock_irqsave(port, &flags);
1350	uart_port_lock(port);
1351		1337
1352	uart_update_timeout(port, termios->c_cflag, baud);	1338	uart_update_timeout(port, termios->c_cflag, baud);
1353	rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);	1339	rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);
Lines 1360-1367 static void pch_uart_set_termios(struct uart_port *port, Link Here
1360	tty_termios_encode_baud_rate(termios, baud, baud);	1346	tty_termios_encode_baud_rate(termios, baud, baud);
1361		1347
1362	out:	1348	out:
1363	uart_port_unlock(port);	1349	uart_port_unlock_irqrestore(port, flags);
1364	spin_unlock_irqrestore(&priv->lock, flags);
1365	}	1350	}
1366		1351
1367	static const char pch_uart_type(struct uart_port port)	1352	static const char pch_uart_type(struct uart_port port)
Lines 1565-1591 pch_console_write(struct console co, const char s, unsigned int count) Link Here
1565	{	1550	{
1566	struct eg20t_port *priv;	1551	struct eg20t_port *priv;
1567	unsigned long flags;	1552	unsigned long flags;
1568	int priv_locked = 1;	1553	int locked = 1;
1569	int port_locked = 1;
1570	u8 ier;	1554	u8 ier;
1571		1555
1572	priv = pch_uart_ports[co->index];	1556	priv = pch_uart_ports[co->index];
1573		1557
1574	touch_nmi_watchdog();	1558	touch_nmi_watchdog();
1575		1559
1576	local_irq_save(flags);	1560	if (oops_in_progress)
1577	if (priv->port.sysrq) {	1561	locked = uart_port_trylock_irqsave(&priv->port, &flags);
1578	/* call to uart_handle_sysrq_char already took the priv lock */	1562	else
1579	priv_locked = 0;	1563	uart_port_lock_irqsave(&priv->port, &flags);
1580	/* serial8250_handle_port() already took the port lock */
1581	port_locked = 0;
1582	} else if (oops_in_progress) {
1583	priv_locked = spin_trylock(&priv->lock);
1584	port_locked = uart_port_trylock(&priv->port);
1585	} else {
1586	spin_lock(&priv->lock);
1587	uart_port_lock(&priv->port);
1588	}
1589		1564
1590	/*	1565	/*
1591	* First save the IER then disable the interrupts	1566	* First save the IER then disable the interrupts
Lines 1603-1613 pch_console_write(struct console co, const char s, unsigned int count) Link Here
1603	wait_for_xmitr(priv, UART_LSR_BOTH_EMPTY);	1578	wait_for_xmitr(priv, UART_LSR_BOTH_EMPTY);
1604	iowrite8(ier, priv->membase + UART_IER);	1579	iowrite8(ier, priv->membase + UART_IER);
1605		1580
1606	if (port_locked)	1581	if (locked)
1607	uart_port_unlock(&priv->port);	1582	uart_port_unlock_irqrestore(&priv->port, flags);
1608	if (priv_locked)
1609	spin_unlock(&priv->lock);
1610	local_irq_restore(flags);
1611	}	1583	}
1612		1584
1613	static int __init pch_console_setup(struct console co, char options)	1585	static int __init pch_console_setup(struct console co, char options)
Lines 1704-1711 static struct eg20t_port pch_uart_init_port(struct pci_dev pdev, Link Here
1704	pci_enable_msi(pdev);	1676	pci_enable_msi(pdev);
1705	pci_set_master(pdev);	1677	pci_set_master(pdev);
1706		1678
1707	spin_lock_init(&priv->lock);
1708
1709	iobase = pci_resource_start(pdev, 0);	1679	iobase = pci_resource_start(pdev, 0);
1710	mapbase = pci_resource_start(pdev, 1);	1680	mapbase = pci_resource_start(pdev, 1);
1711	priv->mapbase = mapbase;	1681	priv->mapbase = mapbase;
Lines 1735-1742 static struct eg20t_port pch_uart_init_port(struct pci_dev pdev, Link Here
1735	KBUILD_MODNAME ":" PCH_UART_DRIVER_DEVICE "%d",	1705	KBUILD_MODNAME ":" PCH_UART_DRIVER_DEVICE "%d",
1736	priv->port.line);	1706	priv->port.line);
1737		1707
1738	spin_lock_init(&priv->port.lock);
1739
1740	pci_set_drvdata(pdev, priv);	1708	pci_set_drvdata(pdev, priv);
1741	priv->trigger_level = 1;	1709	priv->trigger_level = 1;
1742	priv->fcr = 0;	1710	priv->fcr = 0;

Lines 151-157 static inline void receive_chars(struct uart_pxa_port up, int status) Link Here

(-)a/drivers/tty/serial/pxa.c (-11 / +6 lines)
151	flag = TTY_FRAME;	151	flag = TTY_FRAME;
152	}	152	}
153		153
154	if (uart_handle_sysrq_char(&up->port, ch))	154	if (uart_prepare_sysrq_char(&up->port, ch))
155	goto ignore_char;	155	goto ignore_char;
156		156
157	uart_insert_char(&up->port, *status, UART_LSR_OE, ch, flag);	157	uart_insert_char(&up->port, *status, UART_LSR_OE, ch, flag);
Lines 232-238 static inline irqreturn_t serial_pxa_irq(int irq, void *dev_id) Link Here
232	check_modem_status(up);	232	check_modem_status(up);
233	if (lsr & UART_LSR_THRE)	233	if (lsr & UART_LSR_THRE)
234	transmit_chars(up);	234	transmit_chars(up);
235	uart_port_unlock(&up->port);	235	uart_unlock_and_check_sysrq(&up->port);
236	return IRQ_HANDLED;	236	return IRQ_HANDLED;
237	}	237	}
238		238
Lines 604-616 serial_pxa_console_write(struct console co, const char s, unsigned int count) Link Here
604	int locked = 1;	604	int locked = 1;
605		605
606	clk_enable(up->clk);	606	clk_enable(up->clk);
607	local_irq_save(flags);	607	if (oops_in_progress)
608	if (up->port.sysrq)	608	locked = uart_port_trylock_irqsave(&up->port, &flags);
609	locked = 0;
610	else if (oops_in_progress)
611	locked = uart_port_trylock(&up->port);
612	else	609	else
613	uart_port_lock(&up->port);	610	uart_port_lock_irqsave(&up->port, &flags);
614		611
615	/*	612	/*
616	* First save the IER then disable the interrupts	613	* First save the IER then disable the interrupts
Lines 628-637 serial_pxa_console_write(struct console co, const char s, unsigned int count) Link Here
628	serial_out(up, UART_IER, ier);	625	serial_out(up, UART_IER, ier);
629		626
630	if (locked)	627	if (locked)
631	uart_port_unlock(&up->port);	628	uart_port_unlock_irqrestore(&up->port, flags);
632	local_irq_restore(flags);
633	clk_disable(up->clk);	629	clk_disable(up->clk);
634
635	}	630	}
636		631
637	#ifdef CONFIG_CONSOLE_POLL	632	#ifdef CONFIG_CONSOLE_POLL

Lines 394-400 static void rda_uart_receive_chars(struct uart_port *port) Link Here

(-)a/drivers/tty/serial/rda-uart.c (-18 / +10 lines)
394	val &= 0xff;	394	val &= 0xff;
395		395
396	port->icount.rx++;	396	port->icount.rx++;
397	tty_insert_flip_char(&port->state->port, val, flag);	397	if (!uart_prepare_sysrq_char(port, val))
		398	tty_insert_flip_char(&port->state->port, val, flag);
398		399
399	status = rda_uart_read(port, RDA_UART_STATUS);	400	status = rda_uart_read(port, RDA_UART_STATUS);
400	}	401	}
Lines 405-414 static void rda_uart_receive_chars(struct uart_port *port) Link Here
405	static irqreturn_t rda_interrupt(int irq, void *dev_id)	406	static irqreturn_t rda_interrupt(int irq, void *dev_id)
406	{	407	{
407	struct uart_port *port = dev_id;	408	struct uart_port *port = dev_id;
408	unsigned long flags;
409	u32 val, irq_mask;	409	u32 val, irq_mask;
410		410
411	uart_port_lock_irqsave(port, &flags);	411	uart_port_lock(port);
412		412
413	/* Clear IRQ cause */	413	/* Clear IRQ cause */
414	val = rda_uart_read(port, RDA_UART_IRQ_CAUSE);	414	val = rda_uart_read(port, RDA_UART_IRQ_CAUSE);
Lines 425-431 static irqreturn_t rda_interrupt(int irq, void *dev_id) Link Here
425	rda_uart_send_chars(port);	425	rda_uart_send_chars(port);
426	}	426	}
427		427
428	uart_port_unlock_irqrestore(port, flags);	428	uart_unlock_and_check_sysrq(port);
429		429
430	return IRQ_HANDLED;	430	return IRQ_HANDLED;
431	}	431	}
Lines 590-607 static void rda_uart_port_write(struct uart_port port, const char s, Link Here
590	{	590	{
591	u32 old_irq_mask;	591	u32 old_irq_mask;
592	unsigned long flags;	592	unsigned long flags;
593	int locked;	593	int locked = 1;
594		594
595	local_irq_save(flags);	595	if (oops_in_progress)
596		596	locked = uart_port_trylock_irqsave(port, &flags);
597	if (port->sysrq) {	597	else
598	locked = 0;	598	uart_port_lock_irqsave(port, &flags);
599	} else if (oops_in_progress) {
600	locked = uart_port_trylock(port);
601	} else {
602	uart_port_lock(port);
603	locked = 1;
604	}
605		599
606	old_irq_mask = rda_uart_read(port, RDA_UART_IRQ_MASK);	600	old_irq_mask = rda_uart_read(port, RDA_UART_IRQ_MASK);
607	rda_uart_write(port, 0, RDA_UART_IRQ_MASK);	601	rda_uart_write(port, 0, RDA_UART_IRQ_MASK);
Lines 615-623 static void rda_uart_port_write(struct uart_port port, const char s, Link Here
615	rda_uart_write(port, old_irq_mask, RDA_UART_IRQ_MASK);	609	rda_uart_write(port, old_irq_mask, RDA_UART_IRQ_MASK);
616		610
617	if (locked)	611	if (locked)
618	uart_port_unlock(port);	612	uart_port_unlock_irqrestore(port, flags);
619
620	local_irq_restore(flags);
621	}	613	}
622		614
623	static void rda_uart_console_write(struct console co, const char s,	615	static void rda_uart_console_write(struct console co, const char s,

Lines 3145-3152 static int serial_core_add_one_port(struct uart_driver drv, struct uart_port u Link Here

(-)a/drivers/tty/serial/serial_core.c (-8 / +8 lines)
3145	state->uart_port = uport;	3145	state->uart_port = uport;
3146	uport->state = state;	3146	uport->state = state;
3147		3147
		3148	/*
		3149	* If this port is in use as a console then the spinlock is already
		3150	* initialised.
		3151	*/
		3152	if (!uart_console_registered(uport))
		3153	uart_port_spin_lock_init(uport);
		3154
3148	state->pm_state = UART_PM_STATE_UNDEFINED;	3155	state->pm_state = UART_PM_STATE_UNDEFINED;
3149	uport->cons = drv->cons;	3156	uart_port_set_cons(uport, drv->cons);
3150	uport->minor = drv->tty_driver->minor_start + uport->line;	3157	uport->minor = drv->tty_driver->minor_start + uport->line;
3151	uport->name = kasprintf(GFP_KERNEL, "%s%d", drv->dev_name,	3158	uport->name = kasprintf(GFP_KERNEL, "%s%d", drv->dev_name,
3152	drv->tty_driver->name_base + uport->line);	3159	drv->tty_driver->name_base + uport->line);
Lines 3155-3167 static int serial_core_add_one_port(struct uart_driver drv, struct uart_port u Link Here
3155	goto out;	3162	goto out;
3156	}	3163	}
3157		3164
3158	/*
3159	* If this port is in use as a console then the spinlock is already
3160	* initialised.
3161	*/
3162	if (!uart_console_registered(uport))
3163	uart_port_spin_lock_init(uport);
3164
3165	if (uport->cons && uport->dev)	3165	if (uport->cons && uport->dev)
3166	of_console_check(uport->dev->of_node, uport->cons->name, uport->line);	3166	of_console_check(uport->dev->of_node, uport->cons->name, uport->line);
3167		3167

Lines 412-418 static void __ssp_receive_chars(struct sifive_serial_port *ssp) Link Here

(-)a/drivers/tty/serial/sifive.c (-10 / +7 lines)
412	break;	412	break;
413		413
414	ssp->port.icount.rx++;	414	ssp->port.icount.rx++;
415	uart_insert_char(&ssp->port, 0, 0, ch, TTY_NORMAL);	415	if (!uart_prepare_sysrq_char(&ssp->port, ch))
		416	uart_insert_char(&ssp->port, 0, 0, ch, TTY_NORMAL);
416	}	417	}
417		418
418	tty_flip_buffer_push(&ssp->port.state->port);	419	tty_flip_buffer_push(&ssp->port.state->port);
Lines 534-540 static irqreturn_t sifive_serial_irq(int irq, void *dev_id) Link Here
534	if (ip & SIFIVE_SERIAL_IP_TXWM_MASK)	535	if (ip & SIFIVE_SERIAL_IP_TXWM_MASK)
535	__ssp_transmit_chars(ssp);	536	__ssp_transmit_chars(ssp);
536		537
537	uart_port_unlock(&ssp->port);	538	uart_unlock_and_check_sysrq(&ssp->port);
538		539
539	return IRQ_HANDLED;	540	return IRQ_HANDLED;
540	}	541	}
Lines 791-803 static void sifive_serial_console_write(struct console co, const char s, Link Here
791	if (!ssp)	792	if (!ssp)
792	return;	793	return;
793		794
794	local_irq_save(flags);	795	if (oops_in_progress)
795	if (ssp->port.sysrq)	796	locked = uart_port_trylock_irqsave(&ssp->port, &flags);
796	locked = 0;
797	else if (oops_in_progress)
798	locked = uart_port_trylock(&ssp->port);
799	else	797	else
800	uart_port_lock(&ssp->port);	798	uart_port_lock_irqsave(&ssp->port, &flags);
801		799
802	ier = __ssp_readl(ssp, SIFIVE_SERIAL_IE_OFFS);	800	ier = __ssp_readl(ssp, SIFIVE_SERIAL_IE_OFFS);
803	__ssp_writel(0, SIFIVE_SERIAL_IE_OFFS, ssp);	801	__ssp_writel(0, SIFIVE_SERIAL_IE_OFFS, ssp);
Lines 807-814 static void sifive_serial_console_write(struct console co, const char s, Link Here
807	__ssp_writel(ier, SIFIVE_SERIAL_IE_OFFS, ssp);	805	__ssp_writel(ier, SIFIVE_SERIAL_IE_OFFS, ssp);
808		806
809	if (locked)	807	if (locked)
810	uart_port_unlock(&ssp->port);	808	uart_port_unlock_irqrestore(&ssp->port, flags);
811	local_irq_restore(flags);
812	}	809	}
813		810
814	static int sifive_serial_console_setup(struct console co, char options)	811	static int sifive_serial_console_setup(struct console co, char options)

Lines 260-266 static void receive_chars(struct uart_port *port) Link Here

(-)a/drivers/tty/serial/sunplus-uart.c (-12 / +6 lines)
260	if (port->ignore_status_mask & SUP_DUMMY_READ)	260	if (port->ignore_status_mask & SUP_DUMMY_READ)
261	goto ignore_char;	261	goto ignore_char;
262		262
263	if (uart_handle_sysrq_char(port, ch))	263	if (uart_prepare_sysrq_char(port, ch))
264	goto ignore_char;	264	goto ignore_char;
265		265
266	uart_insert_char(port, lsr, SUP_UART_LSR_OE, ch, flag);	266	uart_insert_char(port, lsr, SUP_UART_LSR_OE, ch, flag);
Lines 287-293 static irqreturn_t sunplus_uart_irq(int irq, void *args) Link Here
287	if (isc & SUP_UART_ISC_TX)	287	if (isc & SUP_UART_ISC_TX)
288	transmit_chars(port);	288	transmit_chars(port);
289		289
290	uart_port_unlock(port);	290	uart_unlock_and_check_sysrq(port);
291		291
292	return IRQ_HANDLED;	292	return IRQ_HANDLED;
293	}	293	}
Lines 512-533 static void sunplus_console_write(struct console *co, Link Here
512	unsigned long flags;	512	unsigned long flags;
513	int locked = 1;	513	int locked = 1;
514		514
515	local_irq_save(flags);	515	if (oops_in_progress)
516		516	locked = uart_port_trylock_irqsave(&sunplus_console_ports[co->index]->port, &flags);
517	if (sunplus_console_ports[co->index]->port.sysrq)
518	locked = 0;
519	else if (oops_in_progress)
520	locked = uart_port_trylock(&sunplus_console_ports[co->index]->port);
521	else	517	else
522	uart_port_lock(&sunplus_console_ports[co->index]->port);	518	uart_port_lock_irqsave(&sunplus_console_ports[co->index]->port, &flags);
523		519
524	uart_console_write(&sunplus_console_ports[co->index]->port, s, count,	520	uart_console_write(&sunplus_console_ports[co->index]->port, s, count,
525	sunplus_uart_console_putchar);	521	sunplus_uart_console_putchar);
526		522
527	if (locked)	523	if (locked)
528	uart_port_unlock(&sunplus_console_ports[co->index]->port);	524	uart_port_unlock_irqrestore(&sunplus_console_ports[co->index]->port, flags);
529
530	local_irq_restore(flags);
531	}	525	}
532		526
533	static int __init sunplus_console_setup(struct console co, char options)	527	static int __init sunplus_console_setup(struct console co, char options)

Lines 21-32 static int show_console_dev(struct seq_file m, void v) Link Here

(-)a/fs/proc/consoles.c (-3 / +11 lines)
21	{ CON_ENABLED, 'E' },	21	{ CON_ENABLED, 'E' },
22	{ CON_CONSDEV, 'C' },	22	{ CON_CONSDEV, 'C' },
23	{ CON_BOOT, 'B' },	23	{ CON_BOOT, 'B' },
		24	{ CON_NBCON, 'N' },
24	{ CON_PRINTBUFFER, 'p' },	25	{ CON_PRINTBUFFER, 'p' },
25	{ CON_BRL, 'b' },	26	{ CON_BRL, 'b' },
26	{ CON_ANYTIME, 'a' },	27	{ CON_ANYTIME, 'a' },
27	};	28	};
28	char flags[ARRAY_SIZE(con_flags) + 1];	29	char flags[ARRAY_SIZE(con_flags) + 1];
29	struct console *con = v;	30	struct console *con = v;
		31	char con_write = '-';
30	unsigned int a;	32	unsigned int a;
31	dev_t dev = 0;	33	dev_t dev = 0;
32		34
Lines 57-65 static int show_console_dev(struct seq_file m, void v) Link Here
57	seq_setwidth(m, 21 - 1);	59	seq_setwidth(m, 21 - 1);
58	seq_printf(m, "%s%d", con->name, con->index);	60	seq_printf(m, "%s%d", con->name, con->index);
59	seq_pad(m, ' ');	61	seq_pad(m, ' ');
60	seq_printf(m, "%c%c%c (%s)", con->read ? 'R' : '-',	62	if (con->flags & CON_NBCON) {
61	con->write ? 'W' : '-', con->unblank ? 'U' : '-',	63	if (con->write_atomic \|\| con->write_thread)
62	flags);	64	con_write = 'W';
		65	} else {
		66	if (con->write)
		67	con_write = 'W';
		68	}
		69	seq_printf(m, "%c%c%c (%s)", con->read ? 'R' : '-', con_write,
		70	con->unblank ? 'U' : '-', flags);
63	if (dev)	71	if (dev)
64	seq_printf(m, " %4d:%d", MAJOR(dev), MINOR(dev));	72	seq_printf(m, " %4d:%d", MAJOR(dev), MINOR(dev));
65		73

Lines 16-22 Link Here

(-)a/include/linux/console.h (-17 / +159 lines)
16		16
17	#include <linux/atomic.h>	17	#include <linux/atomic.h>
18	#include <linux/bits.h>	18	#include <linux/bits.h>
		19	#include <linux/irq_work.h>
19	#include <linux/rculist.h>	20	#include <linux/rculist.h>
		21	#include <linux/rcuwait.h>
20	#include <linux/types.h>	22	#include <linux/types.h>
21		23
22	struct vc_data;	24	struct vc_data;
Lines 137-143 static inline int con_debug_leave(void) Link Here
137	*/	139	*/
138		140
139	/**	141	/**
140	* cons_flags - General console flags	142	* enum cons_flags - General console flags
141	* @CON_PRINTBUFFER: Used by newly registered consoles to avoid duplicate	143	* @CON_PRINTBUFFER: Used by newly registered consoles to avoid duplicate
142	* output of messages that were already shown by boot	144	* output of messages that were already shown by boot
143	* consoles or read by userspace via syslog() syscall.	145	* consoles or read by userspace via syslog() syscall.
Lines 218-224 struct nbcon_state { Link Here
218	static_assert(sizeof(struct nbcon_state) <= sizeof(int));	220	static_assert(sizeof(struct nbcon_state) <= sizeof(int));
219		221
220	/**	222	/**
221	* nbcon_prio - console owner priority for nbcon consoles	223	* enum nbcon_prio - console owner priority for nbcon consoles
222	* @NBCON_PRIO_NONE: Unused	224	* @NBCON_PRIO_NONE: Unused
223	* @NBCON_PRIO_NORMAL: Normal (non-emergency) usage	225	* @NBCON_PRIO_NORMAL: Normal (non-emergency) usage
224	* @NBCON_PRIO_EMERGENCY: Emergency output (WARN/OOPS...)	226	* @NBCON_PRIO_EMERGENCY: Emergency output (WARN/OOPS...)
Lines 282-291 struct nbcon_write_context { Link Here
282	bool unsafe_takeover;	284	bool unsafe_takeover;
283	};	285	};
284		286
		287	/**
		288	* struct nbcon_drvdata - Data to allow nbcon acquire in non-print context
		289	* @ctxt: The core console context
		290	* @srcu_cookie: Storage for a console_srcu_lock cookie, if needed
		291	* @owner_index: Storage for the owning console index, if needed
		292	* @locked: Storage for the locked state, if needed
		293	*
		294	* All fields (except for @ctxt) are available exclusively to the driver to
		295	* use as needed. They are not used by the printk subsystem.
		296	*/
		297	struct nbcon_drvdata {
		298	struct nbcon_context __private ctxt;
		299
		300	/* reserved for driver use */
		301	int srcu_cookie;
		302	short owner_index;
		303	bool locked;
		304	};
		305
285	/**	306	/**
286	* struct console - The console descriptor structure	307	* struct console - The console descriptor structure
287	* @name: The name of the console driver	308	* @name: The name of the console driver
288	* @write: Write callback to output messages (Optional)	309	* @write: Legacy write callback to output messages (Optional)
289	* @read: Read callback for console input (Optional)	310	* @read: Read callback for console input (Optional)
290	* @device: The underlying TTY device driver (Optional)	311	* @device: The underlying TTY device driver (Optional)
291	* @unblank: Callback to unblank the console (Optional)	312	* @unblank: Callback to unblank the console (Optional)
Lines 302-311 struct nbcon_write_context { Link Here
302	* @data: Driver private data	323	* @data: Driver private data
303	* @node: hlist node for the console list	324	* @node: hlist node for the console list
304	*	325	*
305	* @write_atomic: Write callback for atomic context
306	* @nbcon_state: State for nbcon consoles	326	* @nbcon_state: State for nbcon consoles
307	* @nbcon_seq: Sequence number of the next record for nbcon to print	327	* @nbcon_seq: Sequence number of the next record for nbcon to print
		328	* @nbcon_prev_seq: Seq num the previous nbcon owner was assigned to print
308	* @pbufs: Pointer to nbcon private buffer	329	* @pbufs: Pointer to nbcon private buffer
		330	* @kthread: Printer kthread for this console
		331	* @rcuwait: RCU-safe wait object for @kthread waking
		332	* @irq_work: Defer @kthread waking to IRQ work context
309	*/	333	*/
310	struct console {	334	struct console {
311	char name[16];	335	char name[16];
Lines 327-337 struct console { Link Here
327	struct hlist_node node;	351	struct hlist_node node;
328		352
329	/* nbcon console specific members */	353	/* nbcon console specific members */
330	bool (write_atomic)(struct console con,	354
331	struct nbcon_write_context *wctxt);	355	/**
		356	* @write_atomic:
		357	*
		358	* NBCON callback to write out text in any context.
		359	*
		360	* This callback is called with the console already acquired. The
		361	* callback can use nbcon_can_proceed() at any time to verify that
		362	* it is still the owner of the console. In the case that it has
		363	* lost ownership, it is no longer allowed to go forward. In this
		364	* case it must back out immediately and carefully. The buffer
		365	* content is also no longer trusted since it no longer belongs to
		366	* the context.
		367	*
		368	* If the callback needs to perform actions where ownership is not
		369	* allowed to be taken over, nbcon_enter_unsafe() and
		370	* nbcon_exit_unsafe() can be used to mark such sections. These
		371	* functions are also points of possible ownership transfer. If
		372	* either function returns false, ownership has been lost.
		373	*
		374	* If the driver must reacquire ownership in order to finalize or
		375	* revert hardware changes, nbcon_reacquire() can be used. However,
		376	* on reacquire the buffer content is no longer available. A
		377	* reacquire cannot be used to resume printing.
		378	*
		379	* This callback can be called from any context (including NMI).
		380	* Therefore it must avoid usage of any locking and instead rely
		381	* on the console ownership for synchronization.
		382	*/
		383	void (write_atomic)(struct console con, struct nbcon_write_context *wctxt);
		384
		385	/**
		386	* @write_thread:
		387	*
		388	* NBCON callback to write out text in task context. (Optional)
		389	*
		390	* This callback is called with the console already acquired. Any
		391	* additional driver synchronization should have been performed by
		392	* device_lock().
		393	*
		394	* This callback is always called from task context but with migration
		395	* disabled.
		396	*
		397	* The same criteria for console ownership verification and unsafe
		398	* sections applies as with write_atomic(). The difference between
		399	* this callback and write_atomic() is that this callback is used
		400	* during normal operation and is always called from task context.
		401	* This provides drivers with a relatively relaxed locking context
		402	* for synchronizing output to the hardware.
		403	*/
		404	void (write_thread)(struct console con, struct nbcon_write_context *wctxt);
		405
		406	/**
		407	* @device_lock:
		408	*
		409	* NBCON callback to begin synchronization with driver code.
		410	*
		411	* Console drivers typically must deal with access to the hardware
		412	* via user input/output (such as an interactive login shell) and
		413	* output of kernel messages via printk() calls. This callback is
		414	* called by the printk-subsystem whenever it needs to synchronize
		415	* with hardware access by the driver. It should be implemented to
		416	* use whatever synchronization mechanism the driver is using for
		417	* itself (for example, the port lock for uart serial consoles).
		418	*
		419	* This callback is always called from task context. It may use any
420	* synchronization method required by the driver. BUT this callback
421	* MUST also disable migration. The console driver may be using a
422	* synchronization mechanism that already takes care of this (such as
423	* spinlocks). Otherwise this function must explicitly call
424	* migrate_disable().
425	*
426	* The flags argument is provided as a convenience to the driver. It
427	* will be passed again to device_unlock(). It can be ignored if the
428	* driver does not need it.
429	*/
430	void (device_lock)(struct console con, unsigned long *flags);
431
432	/**
433	* @device_unlock:
434	*
435	* NBCON callback to finish synchronization with driver code.
436	*
437	* It is the counterpart to device_lock().
438	*
439	* This callback is always called from task context. It must
440	* appropriately re-enable migration (depending on how device_lock()
441	* disabled migration).
442	*
443	* The flags argument is the value of the same variable that was
444	* passed to device_lock().
445	*/
446	void (device_unlock)(struct console con, unsigned long flags);
447
332	atomic_t __private nbcon_state;	448	atomic_t __private nbcon_state;
333	atomic_long_t __private nbcon_seq;	449	atomic_long_t __private nbcon_seq;
		450	atomic_long_t __private nbcon_prev_seq;
		451
		452	/**
		453	* @nbcon_drvdata:
		454	*
		455	* Data for nbcon ownership tracking to allow acquiring nbcon consoles
		456	* in non-printing contexts.
		457	*
		458	* Drivers may need to acquire nbcon consoles in non-printing
		459	* contexts. This is achieved by providing a struct nbcon_drvdata.
		460	* Then the driver can call nbcon_driver_acquire() and
		461	* nbcon_driver_release(). The struct does not require any special
		462	* initialization.
		463	*/
		464	struct nbcon_drvdata *nbcon_drvdata;
		465
334	struct printk_buffers *pbufs;	466	struct printk_buffers *pbufs;
		467	struct task_struct *kthread;
		468	struct rcuwait rcuwait;
		469	struct irq_work irq_work;
335	};	470	};
336		471
337	#ifdef CONFIG_LOCKDEP	472	#ifdef CONFIG_LOCKDEP
Lines 360-387 extern void console_list_unlock(void) __releases(console_mutex); Link Here
360	extern struct hlist_head console_list;	495	extern struct hlist_head console_list;
361		496
362	/**	497	/**
363	* console_srcu_read_flags - Locklessly read the console flags	498	* console_srcu_read_flags - Locklessly read flags of a possibly registered
		499	* console
364	* @con: struct console pointer of console to read flags from	500	* @con: struct console pointer of console to read flags from
365	*	501	*
366	* This function provides the necessary READ_ONCE() and data_race()	502	* Locklessly reading @con->flags provides a consistent read value because
367	* notation for locklessly reading the console flags. The READ_ONCE()	503	* there is at most one CPU modifying @con->flags and that CPU is using only
368	* in this function matches the WRITE_ONCE() when @flags are modified	504	* read-modify-write operations to do so.
369	* for registered consoles with console_srcu_write_flags().
370	*	505	*
371	* Only use this function to read console flags when locklessly	506	* Requires console_srcu_read_lock to be held, which implies that @con might
372	* iterating the console list via srcu.	507	* be a registered console. If the caller is holding the console_list_lock or
		508	* it is certain that the console is not registered, the caller may read
		509	* @con->flags directly instead.
373	*	510	*
374	* Context: Any context.	511	* Context: Any context.
		512	* Return: The current value of the @con->flags field.
375	*/	513	*/
376	static inline short console_srcu_read_flags(const struct console *con)	514	static inline short console_srcu_read_flags(const struct console *con)
377	{	515	{
378	WARN_ON_ONCE(!console_srcu_read_lock_is_held());	516	WARN_ON_ONCE(!console_srcu_read_lock_is_held());
379		517
380	/*	518	/*
381	* Locklessly reading console->flags provides a consistent	519	* The READ_ONCE() matches the WRITE_ONCE() when @flags are modified
382	* read value because there is at most one CPU modifying	520	* for registered consoles with console_srcu_write_flags().
383	* console->flags and that CPU is using only read-modify-write
384	* operations to do so.
385	*/	521	*/
386	return data_race(READ_ONCE(con->flags));	522	return data_race(READ_ONCE(con->flags));
387	}	523	}
Lines 459-471 static inline bool console_is_registered(const struct console *con) Link Here
459	hlist_for_each_entry(con, &console_list, node)	595	hlist_for_each_entry(con, &console_list, node)
460		596
461	#ifdef CONFIG_PRINTK	597	#ifdef CONFIG_PRINTK
		598	extern void nbcon_cpu_emergency_enter(void);
		599	extern void nbcon_cpu_emergency_exit(void);
462	extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt);	600	extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt);
463	extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt);	601	extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt);
464	extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt);	602	extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt);
		603	extern void nbcon_reacquire(struct nbcon_write_context *wctxt);
465	#else	604	#else
		605	static inline void nbcon_cpu_emergency_enter(void) { }
		606	static inline void nbcon_cpu_emergency_exit(void) { }
466	static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; }	607	static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; }
467	static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; }	608	static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; }
468	static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; }	609	static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; }
		610	static inline void nbcon_reacquire(struct nbcon_write_context *wctxt) { }
469	#endif	611	#endif
470		612
471	extern int console_set_on_cmdline;	613	extern int console_set_on_cmdline;

Lines 609-614 extern void __raise_softirq_irqoff(unsigned int nr); Link Here

(-)a/include/linux/interrupt.h (+29 lines)
609	extern void raise_softirq_irqoff(unsigned int nr);	609	extern void raise_softirq_irqoff(unsigned int nr);
610	extern void raise_softirq(unsigned int nr);	610	extern void raise_softirq(unsigned int nr);
611		611
		612	#ifdef CONFIG_PREEMPT_RT
		613	DECLARE_PER_CPU(struct task_struct *, timersd);
		614	DECLARE_PER_CPU(unsigned long, pending_timer_softirq);
		615
		616	extern void raise_timer_softirq(void);
		617	extern void raise_hrtimer_softirq(void);
		618
		619	static inline unsigned int local_pending_timers(void)
		620	{
		621	return __this_cpu_read(pending_timer_softirq);
		622	}
		623
		624	#else
		625	static inline void raise_timer_softirq(void)
		626	{
		627	raise_softirq(TIMER_SOFTIRQ);
		628	}
		629
		630	static inline void raise_hrtimer_softirq(void)
		631	{
		632	raise_softirq_irqoff(HRTIMER_SOFTIRQ);
		633	}
		634
		635	static inline unsigned int local_pending_timers(void)
		636	{
		637	return local_softirq_pending();
		638	}
		639	#endif
		640
612	DECLARE_PER_CPU(struct task_struct *, ksoftirqd);	641	DECLARE_PER_CPU(struct task_struct *, ksoftirqd);
613		642
614	static inline struct task_struct *this_cpu_ksoftirqd(void)	643	static inline struct task_struct *this_cpu_ksoftirqd(void)