Folks,
Attached is a patch adding hangcheck-timer. It is used to detect
when the system goes out to lunch for a period of time, and then
returns. This is interesting for debugging drivers as well as for
clustering environments.
The module sets a timer. When the timer goes off, it then uses
get_cycles() to determine how much real time has passed.
On a normal system, the real elapsed time will be almost
identical to the expected timer duration. However, if a device decided
to udelay for 60 seconds (or some other circumstance), the module takes
notice. If the margin of error passes a threshold, the driver prints a
warning or the machine is rebooted.
There are three parameters.
hangcheck_tick - Timer duration
hangcheck_margin - Allowed margin of between elapsed
jiffies and elapsed wall clock
hangcheck_reboot - If nonzero, reboot if margin
exceeded.
When compiled statically, s/hangcheck/hcheck/ for the kernel
command line.
Marcelo, please apply.
Joel
diff -uNr linux-2.4.20/drivers/char/Config.in linux-2.4.20-hangcheck/drivers/char/Config.in
--- linux-2.4.20/drivers/char/Config.in Thu Nov 28 15:53:12 2002
+++ linux-2.4.20-hangcheck/drivers/char/Config.in Tue Jan 14 17:18:23 2003
@@ -321,4 +321,6 @@
tristate 'ACP Modem (Mwave) support' CONFIG_MWAVE
fi
+tristate 'Hangcheck Timer (kernel pause detection)' CONFIG_HANGCHECK_TIMER
+
endmenu
diff -uNr linux-2.4.20/drivers/char/Makefile linux-2.4.20-hangcheck/drivers/char/Makefile
--- linux-2.4.20/drivers/char/Makefile Thu Nov 28 15:53:12 2002
+++ linux-2.4.20-hangcheck/drivers/char/Makefile Tue Jan 14 17:37:07 2003
@@ -283,6 +283,7 @@
obj-$(CONFIG_WAFER_WDT) += wafer5823wdt.o
obj-$(CONFIG_SOFT_WATCHDOG) += softdog.o
obj-$(CONFIG_AMD7XX_TCO) += amd7xx_tco.o
+obj-$(CONFIG_HANGCHECK_TIMER) += hangcheck-timer.o
subdir-$(CONFIG_MWAVE) += mwave
ifeq ($(CONFIG_MWAVE),y)
diff -uNr linux-2.4.20/drivers/char/hangcheck-timer.c linux-2.4.20-hangcheck/drivers/char/hangcheck-timer.c
--- linux-2.4.20/drivers/char/hangcheck-timer.c Wed Dec 31 16:00:00 1969
+++ linux-2.4.20-hangcheck/drivers/char/hangcheck-timer.c Tue Jan 14 16:57:12 2003
@@ -0,0 +1,173 @@
+/*
+ * hangcheck-timer.c
+ *
+ * Driver for a little io fencing timer.
+ *
+ * Copyright (C) 2002 Oracle Corporation. All rights reserved.
+ *
+ * Author: Joel Becker <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have recieved a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+
+/*
+ * The hangcheck-timer driver uses the TSC to catch delays that
+ * jiffies does not notice. A timer is set. When the timer fires, it
+ * checks whether it was delayed and if that delay exceeds a given
+ * margin of error. The hangcheck_tick module paramter takes the timer
+ * duration in seconds. The hangcheck_margin parameter defines the
+ * margin of error, in seconds. The defaults are 60 seconds for the
+ * timer and 180 seconds for the margin of error. IOW, a timer is set
+ * for 60 seconds. When the timer fires, the callback checks the
+ * actual duration that the timer waited. If the duration exceeds the
+ * alloted time and margin (here 60 + 180, or 240 seconds), the machine
+ * is restarted. A healthy machine will have the duration match the
+ * expected timeout very closely.
+ */
+
+#include <linux/module.h>
+#include <linux/config.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/reboot.h>
+#include <linux/smp_lock.h>
+#include <linux/init.h>
+#include <asm/uaccess.h>
+
+
+#define VERSION_STR "0.5.0"
+
+static void version_hash_print (void)
+{
+ printk(KERN_INFO "I/O fencing modules %s\n", VERSION_STR);
+}
+
+#define DEFAULT_IOFENCE_MARGIN 60 /* Default fudge factor, in seconds */
+#define DEFAULT_IOFENCE_TICK 180 /* Default timer timeout, in seconds */
+
+static int hangcheck_tick = DEFAULT_IOFENCE_TICK;
+static int hangcheck_margin = DEFAULT_IOFENCE_MARGIN;
+static int hangcheck_reboot = 0; /* Do not reboot */
+
+/* options - modular */
+MODULE_PARM(hangcheck_tick,"i");
+MODULE_PARM_DESC(hangcheck_tick, "Timer delay.");
+MODULE_PARM(hangcheck_margin,"i");
+MODULE_PARM_DESC(hangcheck_margin, "If the hangcheck timer has been delayed more than hangcheck_margin seconds, the driver will fire.");
+MODULE_PARM(hangcheck_reboot,"i");
+MODULE_PARM_DESC(hangcheck_reboot, "If nonzero, the machine will reboot when the timer margin is exceeded.");
+MODULE_LICENSE("GPL");
+
+/* options - nonmodular */
+#ifndef MODULE
+
+static int __init hangcheck_parse_tick(char *str) {
+ int par;
+ if (get_option(&str,&par))
+ hangcheck_tick = par;
+ return 1;
+}
+
+static int __init hangcheck_parse_margin(char *str) {
+ int par;
+ if (get_option(&str,&par))
+ hangcheck_margin = par;
+ return 1;
+}
+
+static int __init hangcheck_parse_reboot(char *str) {
+ int par;
+ if (get_option(&str,&par))
+ hangcheck_reboot = par;
+ return 1;
+}
+
+__setup("hcheck_tick", hangcheck_parse_tick);
+__setup("hcheck_margin", hangcheck_parse_margin);
+__setup("hcheck_reboot", hangcheck_parse_reboot);
+#endif /* not MODULE */
+
+
+/* Last time scheduled */
+static unsigned long long hangcheck_tsc, hangcheck_tsc_margin;
+
+static void hangcheck_fire(unsigned long);
+
+static struct timer_list hangcheck_ticktock = {
+ function: hangcheck_fire,
+};
+
+
+static void hangcheck_fire(unsigned long data)
+{
+ unsigned long long cur_tsc, tsc_diff;
+
+ cur_tsc = get_cycles();
+
+ if (cur_tsc > hangcheck_tsc)
+ tsc_diff = cur_tsc - hangcheck_tsc;
+ else
+ tsc_diff = (cur_tsc + (~0ULL - hangcheck_tsc)); /* or something */
+
+#if 0
+ printk(KERN_CRIT "tsc_diff = %lu.%lu, predicted diff is %lu.%lu.\n",
+ (unsigned long) ((tsc_diff >> 32) & 0xFFFFFFFFULL),
+ (unsigned long) (tsc_diff & 0xFFFFFFFFULL),
+ (unsigned long) ((hangcheck_tsc_margin >> 32) & 0xFFFFFFFFULL),
+ (unsigned long) (hangcheck_tsc_margin & 0xFFFFFFFFULL));
+ printk(KERN_CRIT "hangcheck_margin = %lu, HZ = %lu, current_cpu_data.loops_per_jiffy = %lu.\n", hangcheck_margin, HZ, current_cpu_data.loops_per_jiffy);
+#endif
+
+ if (tsc_diff > hangcheck_tsc_margin) {
+ if (hangcheck_reboot) {
+ printk(KERN_CRIT "Hangcheck: hangcheck is restarting the machine.\n");
+ machine_restart(NULL);
+ } else {
+ printk(KERN_CRIT "Hangcheck: hangcheck value past margin!\n");
+ }
+ }
+ mod_timer(&hangcheck_ticktock, jiffies + (hangcheck_tick*HZ));
+ hangcheck_tsc = get_cycles();
+} /* hangcheck_fire() */
+
+
+static int __init hangcheck_init(void)
+{
+ version_hash_print();
+ printk("Hangcheck: starting hangcheck timer (tick is %d seconds, margin is %d seconds).\n",
+ hangcheck_tick, hangcheck_margin);
+
+ hangcheck_tsc_margin = (unsigned long long)(hangcheck_margin + hangcheck_tick) * (unsigned long long)HZ * (unsigned long long)current_cpu_data.loops_per_jiffy;
+
+ hangcheck_tsc = get_cycles();
+ mod_timer(&hangcheck_ticktock, jiffies + (hangcheck_tick*HZ));
+
+ return 0;
+} /* hangcheck_init() */
+
+
+static void __exit hangcheck_exit(void)
+{
+ printk("Stopping hangcheck timer.\n");
+
+ lock_kernel();
+ del_timer(&hangcheck_ticktock);
+ unlock_kernel();
+} /* hangcheck_exit() */
+
+module_init(hangcheck_init);
+module_exit(hangcheck_exit);
--
"Friends may come and go, but enemies accumulate."
- Thomas Jones
Joel Becker
Senior Member of Technical Staff
Oracle Corporation
E-mail: [email protected]
Phone: (650) 506-8127
On Thu, 16 Jan 2003, Joel Becker wrote:
> Folks,
> Attached is a patch adding hangcheck-timer. It is used to detect
> when the system goes out to lunch for a period of time, and then
> returns. This is interesting for debugging drivers as well as for
> clustering environments.
> The module sets a timer. When the timer goes off, it then uses
> get_cycles() to determine how much real time has passed.
> On a normal system, the real elapsed time will be almost
> identical to the expected timer duration. However, if a device decided
> to udelay for 60 seconds (or some other circumstance), the module takes
> notice. If the margin of error passes a threshold, the driver prints a
> warning or the machine is rebooted.
> There are three parameters.
NMI watchdog? Doesn't look like this can catch cases where you have
interrupts disabled and spinning on a lock, if you manage to run this code
then the box isn't really locked up. In which case the software watchdog
should be as good as this.
Zwane
--
function.linuxpower.ca
On Thu, Jan 16, 2003 at 06:43:47PM -0500, Zwane Mwaikambo wrote:
> NMI watchdog? Doesn't look like this can catch cases where you have
> interrupts disabled and spinning on a lock, if you manage to run this code
> then the box isn't really locked up. In which case the software watchdog
> should be as good as this.
Please read again. This isn't about catching a complete lockup.
This is about catching a pause and resume that jiffies does not notice.
Joel
--
Life's Little Instruction Book #15
"Own a great stereo system."
Joel Becker
Senior Member of Technical Staff
Oracle Corporation
E-mail: [email protected]
Phone: (650) 506-8127