#
# Automatically generated make config: don't edit
# Linux kernel version: 2.6.28.7
# Sat Mar 7 20:32:18 2009
#
CONFIG_SCHED_MC=y
CONFIG_MMU=y
CONFIG_LOCKDEP_SUPPORT=y
CONFIG_STACKTRACE_SUPPORT=y
CONFIG_HAVE_LATENCYTOP_SUPPORT=y
CONFIG_RWSEM_XCHGADD_ALGORITHM=y
# CONFIG_ARCH_HAS_ILOG2_U32 is not set
# CONFIG_ARCH_HAS_ILOG2_U64 is not set
CONFIG_GENERIC_HWEIGHT=y
CONFIG_GENERIC_TIME=y
CONFIG_GENERIC_CLOCKEVENTS=y
CONFIG_GENERIC_BUG=y
CONFIG_NO_IOMEM=y
CONFIG_NO_DMA=y
CONFIG_S390=y
CONFIG_DEFCONFIG_LIST="/lib/modules/$UNAME_RELEASE/.config"
#
# General setup
#
CONFIG_EXPERIMENTAL=y
CONFIG_LOCK_KERNEL=y
CONFIG_INIT_ENV_ARG_LIMIT=32
CONFIG_LOCALVERSION=""
# CONFIG_LOCALVERSION_AUTO is not set
CONFIG_SWAP=y
CONFIG_SYSVIPC=y
CONFIG_SYSVIPC_SYSCTL=y
CONFIG_POSIX_MQUEUE=y
CONFIG_BSD_PROCESS_ACCT=y
CONFIG_BSD_PROCESS_ACCT_V3=y
CONFIG_TASKSTATS=y
CONFIG_TASK_DELAY_ACCT=y
CONFIG_TASK_XACCT=y
CONFIG_TASK_IO_ACCOUNTING=y
CONFIG_AUDIT=y
CONFIG_AUDITSYSCALL=y
CONFIG_AUDIT_TREE=y
# CONFIG_IKCONFIG is not set
CONFIG_LOG_BUF_SHIFT=17
CONFIG_CGROUPS=y
# CONFIG_CGROUP_DEBUG is not set
CONFIG_CGROUP_NS=y
# CONFIG_CGROUP_FREEZER is not set
CONFIG_CGROUP_DEVICE=y
CONFIG_CPUSETS=y
CONFIG_GROUP_SCHED=y
CONFIG_FAIR_GROUP_SCHED=y
# CONFIG_RT_GROUP_SCHED is not set
# CONFIG_USER_SCHED is not set
CONFIG_CGROUP_SCHED=y
CONFIG_CGROUP_CPUACCT=y
# CONFIG_RESOURCE_COUNTERS is not set
CONFIG_SYSFS_DEPRECATED=y
CONFIG_SYSFS_DEPRECATED_V2=y
CONFIG_PROC_PID_CPUSET=y
CONFIG_RELAY=y
CONFIG_NAMESPACES=y
CONFIG_UTS_NS=y
CONFIG_IPC_NS=y
CONFIG_USER_NS=y
CONFIG_PID_NS=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_INITRAMFS_SOURCE=""
CONFIG_CC_OPTIMIZE_FOR_SIZE=y
CONFIG_SYSCTL=y
# CONFIG_EMBEDDED is not set
CONFIG_UID16=y
CONFIG_SYSCTL_SYSCALL=y
CONFIG_KALLSYMS=y
# CONFIG_KALLSYMS_ALL is not set
# CONFIG_KALLSYMS_EXTRA_PASS is not set
CONFIG_HOTPLUG=y
CONFIG_PRINTK=y
CONFIG_BUG=y
CONFIG_ELF_CORE=y
# CONFIG_COMPAT_BRK is not set
CONFIG_BASE_FULL=y
CONFIG_FUTEX=y
CONFIG_ANON_INODES=y
CONFIG_EPOLL=y
CONFIG_SIGNALFD=y
CONFIG_TIMERFD=y
CONFIG_EVENTFD=y
CONFIG_SHMEM=y
CONFIG_AIO=y
CONFIG_VM_EVENT_COUNTERS=y
CONFIG_SLAB=y
# CONFIG_SLUB is not set
# CONFIG_SLOB is not set
CONFIG_PROFILING=y
# CONFIG_MARKERS is not set
# CONFIG_OPROFILE is not set
CONFIG_HAVE_OPROFILE=y
# CONFIG_KPROBES is not set
CONFIG_HAVE_SYSCALL_WRAPPERS=y
CONFIG_HAVE_KPROBES=y
CONFIG_HAVE_KRETPROBES=y
CONFIG_HAVE_ARCH_TRACEHOOK=y
# CONFIG_HAVE_GENERIC_DMA_COHERENT is not set
CONFIG_SLABINFO=y
CONFIG_RT_MUTEXES=y
# CONFIG_TINY_SHMEM is not set
CONFIG_BASE_SMALL=0
CONFIG_MODULES=y
CONFIG_MODULE_FORCE_LOAD=y
CONFIG_MODULE_UNLOAD=y
CONFIG_MODULE_FORCE_UNLOAD=y
CONFIG_MODVERSIONS=y
# CONFIG_MODULE_SRCVERSION_ALL is not set
CONFIG_KMOD=y
CONFIG_STOP_MACHINE=y
CONFIG_BLOCK=y
CONFIG_LBD=y
CONFIG_BLK_DEV_IO_TRACE=y
# CONFIG_LSF is not set
CONFIG_BLK_DEV_BSG=y
# CONFIG_BLK_DEV_INTEGRITY is not set
#
# IO Schedulers
#
CONFIG_IOSCHED_NOOP=y
CONFIG_IOSCHED_AS=y
CONFIG_IOSCHED_DEADLINE=y
CONFIG_IOSCHED_CFQ=y
# CONFIG_DEFAULT_AS is not set
# CONFIG_DEFAULT_DEADLINE is not set
CONFIG_DEFAULT_CFQ=y
# CONFIG_DEFAULT_NOOP is not set
CONFIG_DEFAULT_IOSCHED="cfq"
CONFIG_CLASSIC_RCU=y
# CONFIG_FREEZER is not set
#
# Base setup
#
#
# Processor type and features
#
CONFIG_TICK_ONESHOT=y
CONFIG_NO_HZ=y
CONFIG_HIGH_RES_TIMERS=y
CONFIG_GENERIC_CLOCKEVENTS_BUILD=y
# CONFIG_64BIT is not set
CONFIG_32BIT=y
CONFIG_SMP=y
CONFIG_NR_CPUS=32
CONFIG_HOTPLUG_CPU=y
CONFIG_MATHEMU=y
CONFIG_AUDIT_ARCH=y
CONFIG_S390_SWITCH_AMODE=y
CONFIG_S390_EXEC_PROTECT=y
#
# Code generation options
#
CONFIG_MARCH_G5=y
# CONFIG_MARCH_Z900 is not set
# CONFIG_MARCH_Z990 is not set
# CONFIG_MARCH_Z9_109 is not set
CONFIG_PACK_STACK=y
# CONFIG_CHECK_STACK is not set
# CONFIG_WARN_STACK is not set
CONFIG_ARCH_POPULATES_NODE_MAP=y
#
# Kernel preemption
#
CONFIG_PREEMPT_NONE=y
# CONFIG_PREEMPT_VOLUNTARY is not set
# CONFIG_PREEMPT is not set
CONFIG_ARCH_SPARSEMEM_ENABLE=y
CONFIG_ARCH_SPARSEMEM_DEFAULT=y
CONFIG_ARCH_SELECT_MEMORY_MODEL=y
CONFIG_ARCH_ENABLE_MEMORY_HOTPLUG=y
CONFIG_ARCH_ENABLE_MEMORY_HOTREMOVE=y
CONFIG_SELECT_MEMORY_MODEL=y
# CONFIG_FLATMEM_MANUAL is not set
# CONFIG_DISCONTIGMEM_MANUAL is not set
CONFIG_SPARSEMEM_MANUAL=y
CONFIG_SPARSEMEM=y
CONFIG_HAVE_MEMORY_PRESENT=y
CONFIG_SPARSEMEM_STATIC=y
CONFIG_SPARSEMEM_VMEMMAP_ENABLE=y
CONFIG_SPARSEMEM_VMEMMAP=y
# CONFIG_MEMORY_HOTPLUG is not set
CONFIG_PAGEFLAGS_EXTENDED=y
CONFIG_SPLIT_PTLOCK_CPUS=4
CONFIG_MIGRATION=y
# CONFIG_RESOURCES_64BIT is not set
# CONFIG_PHYS_ADDR_T_64BIT is not set
CONFIG_ZONE_DMA_FLAG=0
CONFIG_VIRT_TO_BUS=y
CONFIG_UNEVICTABLE_LRU=y
#
# I/O subsystem configuration
#
CONFIG_MACHCHK_WARNING=y
CONFIG_QDIO=y
# CONFIG_QDIO_DEBUG is not set
CONFIG_CHSC_SCH=m
#
# Misc
#
CONFIG_IPL=y
# CONFIG_IPL_TAPE is not set
CONFIG_IPL_VM=y
CONFIG_BINFMT_ELF=y
# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
# CONFIG_HAVE_AOUT is not set
CONFIG_BINFMT_MISC=m
CONFIG_FORCE_MAX_ZONEORDER=9
# CONFIG_PROCESS_DEBUG is not set
CONFIG_PFAULT=y
# CONFIG_SHARED_KERNEL is not set
CONFIG_CMM=y
CONFIG_CMM_PROC=y
# CONFIG_PAGE_STATES is not set
# CONFIG_VIRT_TIMER is not set
# CONFIG_HZ_100 is not set
CONFIG_HZ_250=y
# CONFIG_HZ_300 is not set
# CONFIG_HZ_1000 is not set
CONFIG_HZ=250
# CONFIG_SCHED_HRTICK is not set
CONFIG_S390_HYPFS_FS=y
CONFIG_KEXEC=y
# CONFIG_ZFCPDUMP is not set
CONFIG_NET=y
#
# Networking options
#
CONFIG_PACKET=y
CONFIG_PACKET_MMAP=y
CONFIG_UNIX=y
CONFIG_XFRM=y
CONFIG_XFRM_USER=m
# CONFIG_XFRM_SUB_POLICY is not set
# CONFIG_XFRM_MIGRATE is not set
# CONFIG_XFRM_STATISTICS is not set
CONFIG_XFRM_IPCOMP=m
CONFIG_NET_KEY=m
# CONFIG_NET_KEY_MIGRATE is not set
CONFIG_IUCV=y
CONFIG_AFIUCV=m
CONFIG_INET=y
CONFIG_IP_MULTICAST=y
CONFIG_IP_ADVANCED_ROUTER=y
CONFIG_ASK_IP_FIB_HASH=y
# CONFIG_IP_FIB_TRIE is not set
CONFIG_IP_FIB_HASH=y
CONFIG_IP_MULTIPLE_TABLES=y
CONFIG_IP_ROUTE_MULTIPATH=y
CONFIG_IP_ROUTE_VERBOSE=y
# CONFIG_IP_PNP is not set
CONFIG_NET_IPIP=m
CONFIG_NET_IPGRE=m
CONFIG_NET_IPGRE_BROADCAST=y
CONFIG_IP_MROUTE=y
CONFIG_IP_PIMSM_V1=y
CONFIG_IP_PIMSM_V2=y
# CONFIG_ARPD is not set
CONFIG_SYN_COOKIES=y
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_IPCOMP=m
CONFIG_INET_XFRM_TUNNEL=m
CONFIG_INET_TUNNEL=m
CONFIG_INET_XFRM_MODE_TRANSPORT=m
CONFIG_INET_XFRM_MODE_TUNNEL=m
CONFIG_INET_XFRM_MODE_BEET=m
CONFIG_INET_LRO=m
CONFIG_INET_DIAG=m
CONFIG_INET_TCP_DIAG=m
CONFIG_TCP_CONG_ADVANCED=y
CONFIG_TCP_CONG_BIC=m
CONFIG_TCP_CONG_CUBIC=y
CONFIG_TCP_CONG_WESTWOOD=m
CONFIG_TCP_CONG_HTCP=m
CONFIG_TCP_CONG_HSTCP=m
CONFIG_TCP_CONG_HYBLA=m
CONFIG_TCP_CONG_VEGAS=m
CONFIG_TCP_CONG_SCALABLE=m
CONFIG_TCP_CONG_LP=m
CONFIG_TCP_CONG_VENO=m
CONFIG_TCP_CONG_YEAH=m
CONFIG_TCP_CONG_ILLINOIS=m
# CONFIG_DEFAULT_BIC is not set
CONFIG_DEFAULT_CUBIC=y
# CONFIG_DEFAULT_HTCP is not set
# CONFIG_DEFAULT_VEGAS is not set
# CONFIG_DEFAULT_WESTWOOD is not set
# CONFIG_DEFAULT_RENO is not set
CONFIG_DEFAULT_TCP_CONG="cubic"
CONFIG_TCP_MD5SIG=y
# CONFIG_IPV6 is not set
# CONFIG_NETLABEL is not set
CONFIG_NETWORK_SECMARK=y
CONFIG_NETFILTER=y
# CONFIG_NETFILTER_DEBUG is not set
CONFIG_NETFILTER_ADVANCED=y
CONFIG_BRIDGE_NETFILTER=y
#
# Core Netfilter Configuration
#
CONFIG_NETFILTER_NETLINK=m
CONFIG_NETFILTER_NETLINK_QUEUE=m
CONFIG_NETFILTER_NETLINK_LOG=m
CONFIG_NF_CONNTRACK=m
CONFIG_NF_CT_ACCT=y
CONFIG_NF_CONNTRACK_MARK=y
CONFIG_NF_CONNTRACK_SECMARK=y
CONFIG_NF_CONNTRACK_EVENTS=y
CONFIG_NF_CT_PROTO_DCCP=m
CONFIG_NF_CT_PROTO_GRE=m
CONFIG_NF_CT_PROTO_SCTP=m
CONFIG_NF_CT_PROTO_UDPLITE=m
CONFIG_NF_CONNTRACK_AMANDA=m
CONFIG_NF_CONNTRACK_FTP=m
CONFIG_NF_CONNTRACK_H323=m
CONFIG_NF_CONNTRACK_IRC=m
CONFIG_NF_CONNTRACK_NETBIOS_NS=m
CONFIG_NF_CONNTRACK_PPTP=m
CONFIG_NF_CONNTRACK_SANE=m
CONFIG_NF_CONNTRACK_SIP=m
CONFIG_NF_CONNTRACK_TFTP=m
CONFIG_NF_CT_NETLINK=m
# CONFIG_NETFILTER_TPROXY is not set
CONFIG_NETFILTER_XTABLES=m
CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=m
CONFIG_NETFILTER_XT_TARGET_DSCP=m
CONFIG_NETFILTER_XT_TARGET_MARK=m
CONFIG_NETFILTER_XT_TARGET_NFLOG=m
CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
CONFIG_NETFILTER_XT_TARGET_NOTRACK=m
CONFIG_NETFILTER_XT_TARGET_RATEEST=m
CONFIG_NETFILTER_XT_TARGET_TRACE=m
CONFIG_NETFILTER_XT_TARGET_SECMARK=m
CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
CONFIG_NETFILTER_XT_MATCH_COMMENT=m
CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
CONFIG_NETFILTER_XT_MATCH_DCCP=m
CONFIG_NETFILTER_XT_MATCH_DSCP=m
CONFIG_NETFILTER_XT_MATCH_ESP=m
CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
CONFIG_NETFILTER_XT_MATCH_HELPER=m
CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
CONFIG_NETFILTER_XT_MATCH_LENGTH=m
CONFIG_NETFILTER_XT_MATCH_LIMIT=m
CONFIG_NETFILTER_XT_MATCH_MAC=m
CONFIG_NETFILTER_XT_MATCH_MARK=m
CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
CONFIG_NETFILTER_XT_MATCH_OWNER=m
CONFIG_NETFILTER_XT_MATCH_POLICY=m
CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
CONFIG_NETFILTER_XT_MATCH_QUOTA=m
CONFIG_NETFILTER_XT_MATCH_RATEEST=m
CONFIG_NETFILTER_XT_MATCH_REALM=m
# CONFIG_NETFILTER_XT_MATCH_RECENT is not set
CONFIG_NETFILTER_XT_MATCH_SCTP=m
CONFIG_NETFILTER_XT_MATCH_STATE=m
CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
CONFIG_NETFILTER_XT_MATCH_STRING=m
CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
CONFIG_NETFILTER_XT_MATCH_TIME=m
CONFIG_NETFILTER_XT_MATCH_U32=m
CONFIG_IP_VS=m
# CONFIG_IP_VS_DEBUG is not set
CONFIG_IP_VS_TAB_BITS=12
#
# IPVS transport protocol load balancing support
#
CONFIG_IP_VS_PROTO_TCP=y
CONFIG_IP_VS_PROTO_UDP=y
CONFIG_IP_VS_PROTO_AH_ESP=y
CONFIG_IP_VS_PROTO_ESP=y
CONFIG_IP_VS_PROTO_AH=y
#
# IPVS scheduler
#
CONFIG_IP_VS_RR=m
CONFIG_IP_VS_WRR=m
CONFIG_IP_VS_LC=m
CONFIG_IP_VS_WLC=m
CONFIG_IP_VS_LBLC=m
CONFIG_IP_VS_LBLCR=m
CONFIG_IP_VS_DH=m
CONFIG_IP_VS_SH=m
CONFIG_IP_VS_SED=m
CONFIG_IP_VS_NQ=m
#
# IPVS application helper
#
CONFIG_IP_VS_FTP=m
#
# IP: Netfilter Configuration
#
CONFIG_NF_DEFRAG_IPV4=m
CONFIG_NF_CONNTRACK_IPV4=m
CONFIG_NF_CONNTRACK_PROC_COMPAT=y
CONFIG_IP_NF_QUEUE=m
CONFIG_IP_NF_IPTABLES=m
CONFIG_IP_NF_MATCH_ADDRTYPE=m
CONFIG_IP_NF_MATCH_AH=m
CONFIG_IP_NF_MATCH_ECN=m
CONFIG_IP_NF_MATCH_TTL=m
CONFIG_IP_NF_FILTER=m
CONFIG_IP_NF_TARGET_REJECT=m
CONFIG_IP_NF_TARGET_LOG=m
CONFIG_IP_NF_TARGET_ULOG=m
CONFIG_NF_NAT=m
CONFIG_NF_NAT_NEEDED=y
CONFIG_IP_NF_TARGET_MASQUERADE=m
CONFIG_IP_NF_TARGET_NETMAP=m
CONFIG_IP_NF_TARGET_REDIRECT=m
CONFIG_NF_NAT_SNMP_BASIC=m
CONFIG_NF_NAT_PROTO_DCCP=m
CONFIG_NF_NAT_PROTO_GRE=m
CONFIG_NF_NAT_PROTO_UDPLITE=m
CONFIG_NF_NAT_PROTO_SCTP=m
CONFIG_NF_NAT_FTP=m
CONFIG_NF_NAT_IRC=m
CONFIG_NF_NAT_TFTP=m
CONFIG_NF_NAT_AMANDA=m
CONFIG_NF_NAT_PPTP=m
CONFIG_NF_NAT_H323=m
CONFIG_NF_NAT_SIP=m
CONFIG_IP_NF_MANGLE=m
CONFIG_IP_NF_TARGET_CLUSTERIP=m
CONFIG_IP_NF_TARGET_ECN=m
CONFIG_IP_NF_TARGET_TTL=m
CONFIG_IP_NF_RAW=m
# CONFIG_IP_NF_SECURITY is not set
CONFIG_IP_NF_ARPTABLES=m
CONFIG_IP_NF_ARPFILTER=m
CONFIG_IP_NF_ARP_MANGLE=m
CONFIG_BRIDGE_NF_EBTABLES=m
CONFIG_BRIDGE_EBT_BROUTE=m
CONFIG_BRIDGE_EBT_T_FILTER=m
CONFIG_BRIDGE_EBT_T_NAT=m
CONFIG_BRIDGE_EBT_802_3=m
CONFIG_BRIDGE_EBT_AMONG=m
CONFIG_BRIDGE_EBT_ARP=m
CONFIG_BRIDGE_EBT_IP=m
CONFIG_BRIDGE_EBT_LIMIT=m
CONFIG_BRIDGE_EBT_MARK=m
CONFIG_BRIDGE_EBT_PKTTYPE=m
CONFIG_BRIDGE_EBT_STP=m
CONFIG_BRIDGE_EBT_VLAN=m
CONFIG_BRIDGE_EBT_ARPREPLY=m
CONFIG_BRIDGE_EBT_DNAT=m
CONFIG_BRIDGE_EBT_MARK_T=m
CONFIG_BRIDGE_EBT_REDIRECT=m
CONFIG_BRIDGE_EBT_SNAT=m
CONFIG_BRIDGE_EBT_LOG=m
CONFIG_BRIDGE_EBT_ULOG=m
CONFIG_BRIDGE_EBT_NFLOG=m
CONFIG_IP_DCCP=m
CONFIG_INET_DCCP_DIAG=m
CONFIG_IP_DCCP_ACKVEC=y
#
# DCCP CCIDs Configuration (EXPERIMENTAL)
#
CONFIG_IP_DCCP_CCID2=m
# CONFIG_IP_DCCP_CCID2_DEBUG is not set
# CONFIG_IP_DCCP_CCID3 is not set
# CONFIG_IP_DCCP_TFRC_LIB is not set
#
# DCCP Kernel Hacking
#
# CONFIG_IP_DCCP_DEBUG is not set
CONFIG_IP_SCTP=m
# CONFIG_SCTP_DBG_MSG is not set
# CONFIG_SCTP_DBG_OBJCNT is not set
# CONFIG_SCTP_HMAC_NONE is not set
# CONFIG_SCTP_HMAC_SHA1 is not set
CONFIG_SCTP_HMAC_MD5=y
CONFIG_TIPC=m
CONFIG_TIPC_ADVANCED=y
CONFIG_TIPC_ZONES=3
CONFIG_TIPC_CLUSTERS=1
CONFIG_TIPC_NODES=255
CONFIG_TIPC_SLAVE_NODES=0
CONFIG_TIPC_PORTS=8191
CONFIG_TIPC_LOG=0
# CONFIG_TIPC_DEBUG is not set
# CONFIG_ATM is not set
CONFIG_STP=m
CONFIG_BRIDGE=m
CONFIG_VLAN_8021Q=m
# CONFIG_VLAN_8021Q_GVRP is not set
# CONFIG_DECNET is not set
CONFIG_LLC=m
# CONFIG_LLC2 is not set
# CONFIG_IPX is not set
# CONFIG_ATALK is not set
# CONFIG_X25 is not set
# CONFIG_LAPB is not set
# CONFIG_ECONET is not set
# CONFIG_WAN_ROUTER is not set
CONFIG_NET_SCHED=y
#
# Queueing/Scheduling
#
CONFIG_NET_SCH_CBQ=m
CONFIG_NET_SCH_HTB=m
CONFIG_NET_SCH_HFSC=m
CONFIG_NET_SCH_PRIO=m
# CONFIG_NET_SCH_MULTIQ is not set
CONFIG_NET_SCH_RED=m
CONFIG_NET_SCH_SFQ=m
CONFIG_NET_SCH_TEQL=m
CONFIG_NET_SCH_TBF=m
CONFIG_NET_SCH_GRED=m
CONFIG_NET_SCH_DSMARK=m
CONFIG_NET_SCH_NETEM=m
CONFIG_NET_SCH_INGRESS=m
#
# Classification
#
CONFIG_NET_CLS=y
CONFIG_NET_CLS_BASIC=m
CONFIG_NET_CLS_TCINDEX=m
CONFIG_NET_CLS_ROUTE4=m
CONFIG_NET_CLS_ROUTE=y
CONFIG_NET_CLS_FW=m
CONFIG_NET_CLS_U32=m
CONFIG_CLS_U32_PERF=y
CONFIG_CLS_U32_MARK=y
CONFIG_NET_CLS_RSVP=m
CONFIG_NET_CLS_RSVP6=m
CONFIG_NET_CLS_FLOW=m
CONFIG_NET_EMATCH=y
CONFIG_NET_EMATCH_STACK=32
CONFIG_NET_EMATCH_CMP=m
CONFIG_NET_EMATCH_NBYTE=m
CONFIG_NET_EMATCH_U32=m
CONFIG_NET_EMATCH_META=m
CONFIG_NET_EMATCH_TEXT=m
CONFIG_NET_CLS_ACT=y
CONFIG_NET_ACT_POLICE=m
CONFIG_NET_ACT_GACT=m
CONFIG_GACT_PROB=y
CONFIG_NET_ACT_MIRRED=m
CONFIG_NET_ACT_IPT=m
CONFIG_NET_ACT_NAT=m
CONFIG_NET_ACT_PEDIT=m
CONFIG_NET_ACT_SIMP=m
# CONFIG_NET_ACT_SKBEDIT is not set
CONFIG_NET_CLS_IND=y
CONFIG_NET_SCH_FIFO=y
#
# Network testing
#
CONFIG_NET_PKTGEN=m
# CONFIG_CAN is not set
CONFIG_AF_RXRPC=m
# CONFIG_AF_RXRPC_DEBUG is not set
CONFIG_RXKAD=m
# CONFIG_PHONET is not set
CONFIG_FIB_RULES=y
# CONFIG_RFKILL is not set
# CONFIG_NET_9P is not set
# CONFIG_PCMCIA is not set
CONFIG_CCW=y
#
# Device Drivers
#
#
# Generic Driver Options
#
CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
CONFIG_STANDALONE=y
CONFIG_PREVENT_FIRMWARE_BUILD=y
CONFIG_FW_LOADER=y
CONFIG_FIRMWARE_IN_KERNEL=y
CONFIG_EXTRA_FIRMWARE=""
# CONFIG_DEBUG_DRIVER is not set
# CONFIG_DEBUG_DEVRES is not set
CONFIG_SYS_HYPERVISOR=y
CONFIG_CONNECTOR=m
CONFIG_BLK_DEV=y
# CONFIG_BLK_DEV_COW_COMMON is not set
CONFIG_BLK_DEV_LOOP=m
CONFIG_BLK_DEV_CRYPTOLOOP=m
CONFIG_BLK_DEV_NBD=m
CONFIG_BLK_DEV_RAM=y
CONFIG_BLK_DEV_RAM_COUNT=16
CONFIG_BLK_DEV_RAM_SIZE=24576
# CONFIG_BLK_DEV_XIP is not set
# CONFIG_CDROM_PKTCDVD is not set
CONFIG_ATA_OVER_ETH=m
#
# S/390 block device drivers
#
CONFIG_BLK_DEV_XPRAM=m
CONFIG_DCSSBLK=m
CONFIG_DASD=m
# CONFIG_DASD_PROFILE is not set
CONFIG_DASD_ECKD=m
CONFIG_DASD_FBA=m
CONFIG_DASD_DIAG=m
# CONFIG_DASD_EER is not set
CONFIG_MISC_DEVICES=y
CONFIG_EEPROM_93CX6=m
CONFIG_ENCLOSURE_SERVICES=m
# CONFIG_C2PORT is not set
#
# SCSI device support
#
CONFIG_RAID_ATTRS=m
CONFIG_SCSI=m
# CONFIG_SCSI_DMA is not set
CONFIG_SCSI_TGT=m
CONFIG_SCSI_NETLINK=y
CONFIG_SCSI_PROC_FS=y
#
# SCSI support type (disk, tape, CD-ROM)
#
CONFIG_BLK_DEV_SD=m
CONFIG_CHR_DEV_ST=m
CONFIG_CHR_DEV_OSST=m
CONFIG_BLK_DEV_SR=m
CONFIG_BLK_DEV_SR_VENDOR=y
CONFIG_CHR_DEV_SG=m
CONFIG_CHR_DEV_SCH=m
CONFIG_SCSI_ENCLOSURE=m
#
# Some SCSI devices (e.g. CD jukebox) support multiple LUNs
#
CONFIG_SCSI_MULTI_LUN=y
CONFIG_SCSI_CONSTANTS=y
CONFIG_SCSI_LOGGING=y
CONFIG_SCSI_SCAN_ASYNC=y
CONFIG_SCSI_WAIT_SCAN=m
#
# SCSI Transports
#
CONFIG_SCSI_SPI_ATTRS=m
CONFIG_SCSI_FC_ATTRS=m
CONFIG_SCSI_FC_TGT_ATTRS=y
CONFIG_SCSI_ISCSI_ATTRS=m
CONFIG_SCSI_SAS_ATTRS=m
CONFIG_SCSI_SAS_LIBSAS=m
CONFIG_SCSI_SAS_HOST_SMP=y
# CONFIG_SCSI_SAS_LIBSAS_DEBUG is not set
# CONFIG_SCSI_SRP_ATTRS is not set
CONFIG_SCSI_LOWLEVEL=y
CONFIG_ISCSI_TCP=m
# CONFIG_SCSI_DEBUG is not set
CONFIG_ZFCP=m
# CONFIG_SCSI_DH is not set
CONFIG_MD=y
# CONFIG_BLK_DEV_MD is not set
CONFIG_BLK_DEV_DM=m
# CONFIG_DM_DEBUG is not set
CONFIG_DM_CRYPT=m
CONFIG_DM_SNAPSHOT=m
CONFIG_DM_MIRROR=m
CONFIG_DM_ZERO=m
CONFIG_DM_MULTIPATH=m
CONFIG_DM_DELAY=m
CONFIG_DM_UEVENT=y
CONFIG_NETDEVICES=y
CONFIG_IFB=m
# CONFIG_DUMMY is not set
CONFIG_BONDING=m
# CONFIG_MACVLAN is not set
CONFIG_EQUALIZER=m
CONFIG_TUN=m
CONFIG_VETH=m
CONFIG_NET_ETHERNET=y
CONFIG_MII=m
# CONFIG_IBM_NEW_EMAC_ZMII is not set
# CONFIG_IBM_NEW_EMAC_RGMII is not set
# CONFIG_IBM_NEW_EMAC_TAH is not set
# CONFIG_IBM_NEW_EMAC_EMAC4 is not set
# CONFIG_IBM_NEW_EMAC_NO_FLOW_CTRL is not set
# CONFIG_IBM_NEW_EMAC_MAL_CLR_ICINTSTAT is not set
# CONFIG_IBM_NEW_EMAC_MAL_COMMON_ERR is not set
# CONFIG_NETDEV_1000 is not set
# CONFIG_NETDEV_10000 is not set
# CONFIG_TR is not set
# CONFIG_WAN is not set
#
# S/390 network device drivers
#
CONFIG_LCS=m
CONFIG_CTCM=m
CONFIG_NETIUCV=m
CONFIG_SMSGIUCV=m
CONFIG_CLAW=m
CONFIG_QETH=m
CONFIG_QETH_L2=m
CONFIG_QETH_L3=m
CONFIG_CCWGROUP=m
# CONFIG_PPP is not set
# CONFIG_SLIP is not set
# CONFIG_NETCONSOLE is not set
# CONFIG_NETPOLL is not set
# CONFIG_NET_POLL_CONTROLLER is not set
#
# Character devices
#
CONFIG_DEVKMEM=y
CONFIG_UNIX98_PTYS=y
# CONFIG_LEGACY_PTYS is not set
CONFIG_HW_RANDOM=m
# CONFIG_R3964 is not set
# CONFIG_RAW_DRIVER is not set
CONFIG_HANGCHECK_TIMER=m
#
# S/390 character device drivers
#
CONFIG_TN3270=y
CONFIG_TN3270_TTY=y
CONFIG_TN3270_FS=m
CONFIG_TN3270_CONSOLE=y
CONFIG_TN3215=y
CONFIG_TN3215_CONSOLE=y
CONFIG_CCW_CONSOLE=y
CONFIG_SCLP_TTY=y
CONFIG_SCLP_CONSOLE=y
CONFIG_SCLP_VT220_TTY=y
CONFIG_SCLP_VT220_CONSOLE=y
CONFIG_SCLP_CPI=m
CONFIG_S390_TAPE=m
#
# S/390 tape interface support
#
CONFIG_S390_TAPE_BLOCK=y
#
# S/390 tape hardware support
#
CONFIG_S390_TAPE_34XX=m
CONFIG_S390_TAPE_3590=m
CONFIG_VMLOGRDR=m
CONFIG_VMCP=m
CONFIG_MONREADER=m
CONFIG_MONWRITER=m
CONFIG_S390_VMUR=m
# CONFIG_POWER_SUPPLY is not set
# CONFIG_THERMAL is not set
# CONFIG_THERMAL_HWMON is not set
# CONFIG_WATCHDOG is not set
# CONFIG_REGULATOR is not set
# CONFIG_MEMSTICK is not set
# CONFIG_NEW_LEDS is not set
# CONFIG_ACCESSIBILITY is not set
# CONFIG_STAGING is not set
#
# File systems
#
CONFIG_EXT2_FS=m
CONFIG_EXT2_FS_XATTR=y
CONFIG_EXT2_FS_POSIX_ACL=y
CONFIG_EXT2_FS_SECURITY=y
# CONFIG_EXT2_FS_XIP is not set
CONFIG_EXT3_FS=m
CONFIG_EXT3_FS_XATTR=y
CONFIG_EXT3_FS_POSIX_ACL=y
CONFIG_EXT3_FS_SECURITY=y
# CONFIG_EXT4_FS is not set
CONFIG_JBD=m
# CONFIG_JBD_DEBUG is not set
CONFIG_FS_MBCACHE=m
# CONFIG_REISERFS_FS is not set
# CONFIG_JFS_FS is not set
CONFIG_FS_POSIX_ACL=y
CONFIG_FILE_LOCKING=y
# CONFIG_XFS_FS is not set
# CONFIG_OCFS2_FS is not set
CONFIG_DNOTIFY=y
CONFIG_INOTIFY=y
CONFIG_INOTIFY_USER=y
CONFIG_QUOTA=y
CONFIG_QUOTA_NETLINK_INTERFACE=y
CONFIG_PRINT_QUOTA_WARNING=y
CONFIG_QFMT_V1=m
CONFIG_QFMT_V2=m
CONFIG_QUOTACTL=y
CONFIG_AUTOFS_FS=m
CONFIG_AUTOFS4_FS=m
CONFIG_FUSE_FS=m
CONFIG_GENERIC_ACL=y
#
# CD-ROM/DVD Filesystems
#
CONFIG_ISO9660_FS=m
CONFIG_JOLIET=y
CONFIG_ZISOFS=y
CONFIG_UDF_FS=m
CONFIG_UDF_NLS=y
#
# DOS/FAT/NT Filesystems
#
CONFIG_FAT_FS=m
CONFIG_MSDOS_FS=m
CONFIG_VFAT_FS=m
CONFIG_FAT_DEFAULT_CODEPAGE=437
CONFIG_FAT_DEFAULT_IOCHARSET="utf8"
CONFIG_NTFS_FS=m
# CONFIG_NTFS_DEBUG is not set
CONFIG_NTFS_RW=y
#
# Pseudo filesystems
#
CONFIG_PROC_FS=y
CONFIG_PROC_KCORE=y
CONFIG_PROC_SYSCTL=y
CONFIG_PROC_PAGE_MONITOR=y
CONFIG_SYSFS=y
CONFIG_TMPFS=y
CONFIG_TMPFS_POSIX_ACL=y
# CONFIG_HUGETLB_PAGE is not set
CONFIG_CONFIGFS_FS=m
#
# Miscellaneous filesystems
#
# CONFIG_ADFS_FS is not set
# CONFIG_AFFS_FS is not set
# CONFIG_ECRYPT_FS is not set
# CONFIG_HFS_FS is not set
# CONFIG_HFSPLUS_FS is not set
# CONFIG_BEFS_FS is not set
# CONFIG_BFS_FS is not set
# CONFIG_EFS_FS is not set
# CONFIG_CRAMFS is not set
# CONFIG_VXFS_FS is not set
# CONFIG_MINIX_FS is not set
# CONFIG_OMFS_FS is not set
# CONFIG_HPFS_FS is not set
# CONFIG_QNX4FS_FS is not set
CONFIG_ROMFS_FS=m
# CONFIG_SYSV_FS is not set
CONFIG_UFS_FS=m
# CONFIG_UFS_FS_WRITE is not set
# CONFIG_UFS_DEBUG is not set
CONFIG_NETWORK_FILESYSTEMS=y
CONFIG_NFS_FS=m
CONFIG_NFS_V3=y
CONFIG_NFS_V3_ACL=y
CONFIG_NFS_V4=y
# CONFIG_NFSD is not set
CONFIG_LOCKD=m
CONFIG_LOCKD_V4=y
CONFIG_NFS_ACL_SUPPORT=m
CONFIG_NFS_COMMON=y
CONFIG_SUNRPC=m
CONFIG_SUNRPC_GSS=m
# CONFIG_SUNRPC_REGISTER_V4 is not set
CONFIG_RPCSEC_GSS_KRB5=m
CONFIG_RPCSEC_GSS_SPKM3=m
# CONFIG_SMB_FS is not set
CONFIG_CIFS=m
# CONFIG_CIFS_STATS is not set
CONFIG_CIFS_WEAK_PW_HASH=y
CONFIG_CIFS_UPCALL=y
CONFIG_CIFS_XATTR=y
CONFIG_CIFS_POSIX=y
# CONFIG_CIFS_DEBUG2 is not set
CONFIG_CIFS_EXPERIMENTAL=y
CONFIG_CIFS_DFS_UPCALL=y
# CONFIG_NCP_FS is not set
# CONFIG_CODA_FS is not set
# CONFIG_AFS_FS is not set
#
# Partition Types
#
CONFIG_PARTITION_ADVANCED=y
# CONFIG_ACORN_PARTITION is not set
# CONFIG_OSF_PARTITION is not set
# CONFIG_AMIGA_PARTITION is not set
# CONFIG_ATARI_PARTITION is not set
CONFIG_IBM_PARTITION=y
# CONFIG_MAC_PARTITION is not set
CONFIG_MSDOS_PARTITION=y
CONFIG_BSD_DISKLABEL=y
# CONFIG_MINIX_SUBPARTITION is not set
# CONFIG_SOLARIS_X86_PARTITION is not set
# CONFIG_UNIXWARE_DISKLABEL is not set
# CONFIG_LDM_PARTITION is not set
# CONFIG_SGI_PARTITION is not set
# CONFIG_ULTRIX_PARTITION is not set
# CONFIG_SUN_PARTITION is not set
CONFIG_KARMA_PARTITION=y
# CONFIG_EFI_PARTITION is not set
# CONFIG_SYSV68_PARTITION is not set
CONFIG_NLS=m
CONFIG_NLS_DEFAULT="utf8"
CONFIG_NLS_CODEPAGE_437=m
# CONFIG_NLS_CODEPAGE_737 is not set
# CONFIG_NLS_CODEPAGE_775 is not set
CONFIG_NLS_CODEPAGE_850=m
# CONFIG_NLS_CODEPAGE_852 is not set
# CONFIG_NLS_CODEPAGE_855 is not set
# CONFIG_NLS_CODEPAGE_857 is not set
# CONFIG_NLS_CODEPAGE_860 is not set
# CONFIG_NLS_CODEPAGE_861 is not set
# CONFIG_NLS_CODEPAGE_862 is not set
# CONFIG_NLS_CODEPAGE_863 is not set
# CONFIG_NLS_CODEPAGE_864 is not set
# CONFIG_NLS_CODEPAGE_865 is not set
# CONFIG_NLS_CODEPAGE_866 is not set
# CONFIG_NLS_CODEPAGE_869 is not set
# CONFIG_NLS_CODEPAGE_936 is not set
# CONFIG_NLS_CODEPAGE_950 is not set
# CONFIG_NLS_CODEPAGE_932 is not set
# CONFIG_NLS_CODEPAGE_949 is not set
# CONFIG_NLS_CODEPAGE_874 is not set
# CONFIG_NLS_ISO8859_8 is not set
# CONFIG_NLS_CODEPAGE_1250 is not set
# CONFIG_NLS_CODEPAGE_1251 is not set
CONFIG_NLS_ASCII=m
CONFIG_NLS_ISO8859_1=m
# CONFIG_NLS_ISO8859_2 is not set
# CONFIG_NLS_ISO8859_3 is not set
# CONFIG_NLS_ISO8859_4 is not set
# CONFIG_NLS_ISO8859_5 is not set
# CONFIG_NLS_ISO8859_6 is not set
# CONFIG_NLS_ISO8859_7 is not set
# CONFIG_NLS_ISO8859_9 is not set
# CONFIG_NLS_ISO8859_13 is not set
# CONFIG_NLS_ISO8859_14 is not set
CONFIG_NLS_ISO8859_15=m
# CONFIG_NLS_KOI8_R is not set
# CONFIG_NLS_KOI8_U is not set
CONFIG_NLS_UTF8=m
CONFIG_DLM=m
CONFIG_DLM_DEBUG=y
#
# Kernel hacking
#
CONFIG_TRACE_IRQFLAGS_SUPPORT=y
CONFIG_PRINTK_TIME=y
CONFIG_ENABLE_WARN_DEPRECATED=y
CONFIG_ENABLE_MUST_CHECK=y
CONFIG_FRAME_WARN=1024
CONFIG_MAGIC_SYSRQ=y
CONFIG_UNUSED_SYMBOLS=y
CONFIG_DEBUG_FS=y
# CONFIG_HEADERS_CHECK is not set
CONFIG_DEBUG_KERNEL=y
CONFIG_SCHED_DEBUG=y
# CONFIG_SCHEDSTATS is not set
CONFIG_TIMER_STATS=y
# CONFIG_DEBUG_OBJECTS is not set
# CONFIG_DEBUG_SLAB is not set
# CONFIG_DEBUG_RT_MUTEXES is not set
# CONFIG_RT_MUTEX_TESTER is not set
# CONFIG_DEBUG_SPINLOCK is not set
# CONFIG_DEBUG_MUTEXES is not set
# CONFIG_DEBUG_LOCK_ALLOC is not set
# CONFIG_PROVE_LOCKING is not set
# CONFIG_LOCK_STAT is not set
# CONFIG_DEBUG_SPINLOCK_SLEEP is not set
# CONFIG_DEBUG_LOCKING_API_SELFTESTS is not set
# CONFIG_DEBUG_KOBJECT is not set
CONFIG_DEBUG_BUGVERBOSE=y
# CONFIG_DEBUG_INFO is not set
# CONFIG_DEBUG_VM is not set
# CONFIG_DEBUG_WRITECOUNT is not set
CONFIG_DEBUG_MEMORY_INIT=y
# CONFIG_DEBUG_LIST is not set
# CONFIG_DEBUG_SG is not set
# CONFIG_FRAME_POINTER is not set
# CONFIG_RCU_TORTURE_TEST is not set
# CONFIG_RCU_CPU_STALL_DETECTOR is not set
# CONFIG_BACKTRACE_SELF_TEST is not set
# CONFIG_DEBUG_BLOCK_EXT_DEVT is not set
# CONFIG_FAULT_INJECTION is not set
# CONFIG_LATENCYTOP is not set
CONFIG_SYSCTL_SYSCALL_CHECK=y
#
# Tracers
#
# CONFIG_IRQSOFF_TRACER is not set
# CONFIG_SCHED_TRACER is not set
# CONFIG_CONTEXT_SWITCH_TRACER is not set
# CONFIG_BOOT_TRACER is not set
# CONFIG_DYNAMIC_PRINTK_DEBUG is not set
# CONFIG_SAMPLES is not set
# CONFIG_DEBUG_PAGEALLOC is not set
#
# Security options
#
CONFIG_KEYS=y
# CONFIG_KEYS_DEBUG_PROC_KEYS is not set
CONFIG_SECURITY=y
# CONFIG_SECURITYFS is not set
CONFIG_SECURITY_NETWORK=y
CONFIG_SECURITY_NETWORK_XFRM=y
CONFIG_SECURITY_FILE_CAPABILITIES=y
CONFIG_SECURITY_DEFAULT_MMAP_MIN_ADDR=0
CONFIG_SECURITY_SELINUX=y
CONFIG_SECURITY_SELINUX_BOOTPARAM=y
CONFIG_SECURITY_SELINUX_BOOTPARAM_VALUE=0
CONFIG_SECURITY_SELINUX_DISABLE=y
CONFIG_SECURITY_SELINUX_DEVELOP=y
CONFIG_SECURITY_SELINUX_AVC_STATS=y
CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE=1
# CONFIG_SECURITY_SELINUX_ENABLE_SECMARK_DEFAULT is not set
# CONFIG_SECURITY_SELINUX_POLICYDB_VERSION_MAX is not set
CONFIG_CRYPTO=y
#
# Crypto core or helper
#
# CONFIG_CRYPTO_FIPS is not set
CONFIG_CRYPTO_ALGAPI=y
CONFIG_CRYPTO_ALGAPI2=y
CONFIG_CRYPTO_AEAD=m
CONFIG_CRYPTO_AEAD2=y
CONFIG_CRYPTO_BLKCIPHER=m
CONFIG_CRYPTO_BLKCIPHER2=y
CONFIG_CRYPTO_HASH=y
CONFIG_CRYPTO_HASH2=y
CONFIG_CRYPTO_RNG=m
CONFIG_CRYPTO_RNG2=y
CONFIG_CRYPTO_MANAGER=y
CONFIG_CRYPTO_MANAGER2=y
CONFIG_CRYPTO_GF128MUL=m
CONFIG_CRYPTO_NULL=m
# CONFIG_CRYPTO_CRYPTD is not set
CONFIG_CRYPTO_AUTHENC=m
CONFIG_CRYPTO_TEST=m
#
# Authenticated Encryption with Associated Data
#
CONFIG_CRYPTO_CCM=m
CONFIG_CRYPTO_GCM=m
CONFIG_CRYPTO_SEQIV=m
#
# Block modes
#
CONFIG_CRYPTO_CBC=m
CONFIG_CRYPTO_CTR=m
CONFIG_CRYPTO_CTS=m
CONFIG_CRYPTO_ECB=m
CONFIG_CRYPTO_LRW=m
CONFIG_CRYPTO_PCBC=m
CONFIG_CRYPTO_XTS=m
#
# Hash modes
#
CONFIG_CRYPTO_HMAC=y
CONFIG_CRYPTO_XCBC=m
#
# Digest
#
CONFIG_CRYPTO_CRC32C=m
CONFIG_CRYPTO_MD4=m
CONFIG_CRYPTO_MD5=y
CONFIG_CRYPTO_MICHAEL_MIC=m
# CONFIG_CRYPTO_RMD128 is not set
# CONFIG_CRYPTO_RMD160 is not set
# CONFIG_CRYPTO_RMD256 is not set
# CONFIG_CRYPTO_RMD320 is not set
CONFIG_CRYPTO_SHA1=m
CONFIG_CRYPTO_SHA256=m
CONFIG_CRYPTO_SHA512=m
CONFIG_CRYPTO_TGR192=m
CONFIG_CRYPTO_WP512=m
#
# Ciphers
#
CONFIG_CRYPTO_AES=m
CONFIG_CRYPTO_ANUBIS=m
CONFIG_CRYPTO_ARC4=m
CONFIG_CRYPTO_BLOWFISH=m
CONFIG_CRYPTO_CAMELLIA=m
CONFIG_CRYPTO_CAST5=m
CONFIG_CRYPTO_CAST6=m
CONFIG_CRYPTO_DES=m
CONFIG_CRYPTO_FCRYPT=m
CONFIG_CRYPTO_KHAZAD=m
CONFIG_CRYPTO_SALSA20=m
CONFIG_CRYPTO_SEED=m
CONFIG_CRYPTO_SERPENT=m
CONFIG_CRYPTO_TEA=m
CONFIG_CRYPTO_TWOFISH=m
CONFIG_CRYPTO_TWOFISH_COMMON=m
#
# Compression
#
CONFIG_CRYPTO_DEFLATE=m
CONFIG_CRYPTO_LZO=m
#
# Random Number Generation
#
# CONFIG_CRYPTO_ANSI_CPRNG is not set
CONFIG_CRYPTO_HW=y
CONFIG_ZCRYPT=m
# CONFIG_ZCRYPT_MONOLITHIC is not set
CONFIG_CRYPTO_SHA1_S390=m
CONFIG_CRYPTO_SHA256_S390=m
CONFIG_CRYPTO_SHA512_S390=m
CONFIG_CRYPTO_DES_S390=m
CONFIG_CRYPTO_AES_S390=m
CONFIG_S390_PRNG=m
#
# Library routines
#
CONFIG_BITREVERSE=y
CONFIG_CRC_CCITT=m
CONFIG_CRC16=m
# CONFIG_CRC_T10DIF is not set
CONFIG_CRC_ITU_T=m
CONFIG_CRC32=y
CONFIG_CRC7=m
CONFIG_LIBCRC32C=m
CONFIG_ZLIB_INFLATE=m
CONFIG_ZLIB_DEFLATE=m
CONFIG_LZO_COMPRESS=m
CONFIG_LZO_DECOMPRESS=m
CONFIG_TEXTSEARCH=y
CONFIG_TEXTSEARCH_KMP=m
CONFIG_TEXTSEARCH_BM=m
CONFIG_TEXTSEARCH_FSM=m
CONFIG_PLIST=y
# CONFIG_VIRTUALIZATION is not set
On Sunday 08 March 2009, Frans Pop wrote:
> Kernels after 2.6.27 fail to boot in the Hercules S/390 emulator, quite
> early in the boot process:
> 0.141419! Initializing cgroup subsys ns
> 0.141605! Initializing cgroup subsys cpuacct
> 0.142009! Initializing cgroup subsys devices
> 0.180403! cpu: 2 configured CPUs, 0 standby CPUs
>
> I've bisected this to the following commit:
> commit 5cd1c9c5cf30d4b33df3d3f74d8142f278d536b7
> Author: Roman Zippel <[email protected]>
> Date: Mon Sep 22 14:42:43 2008 -0700
> timekeeping: fix rounding problem during clock update
>
> After reverting 5cd1c9c5cf30 and the somewhat related 6c9bacb41c10 on
> top of 2.6.28.7, the system boots OK again.
JFYI, with the same reverts 2.6.29-rc7 boots correctly too.
On Sunday 08 March 2009, Frans Pop wrote:
> Kernels after 2.6.27 fail to boot in the Hercules S/390 emulator, quite
> early in the boot process:
[...]
> 0.141419! Initializing cgroup subsys ns
> 0.141605! Initializing cgroup subsys cpuacct
> 0.142009! Initializing cgroup subsys devices
> 0.180403! cpu: 2 configured CPUs, 0 standby CPUs
>
> I've bisected this to the following commit:
> commit 5cd1c9c5cf30d4b33df3d3f74d8142f278d536b7
> Author: Roman Zippel <[email protected]>
> Date: Mon Sep 22 14:42:43 2008 -0700
> timekeeping: fix rounding problem during clock update
After staring at this commit for a while I decided to try the following
patch:
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -547,5 +547,11 @@ void update_wall_time(void)
* add the remainder to the error difference.
*/
xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
+ if (unlikely(clock->xtime_nsec < ((s64)xtime.tv_nsec << clock->shift))) {
+ printk("Negative result: %llu - %lld\n",
+ (unsigned long long)clock->xtime_nsec,
+ (long long)((s64)xtime.tv_nsec << clock->shift));
+ BUG_ON(1);
+ }
clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
And that resulted in:
0.004175! Negative result: 166039808000 - 166039808256
So we're trying to stuff a negative value into an unsigned field, and I
keep seeing on these lists that's a bad idea.
I'll leave it up to you how we get there, but could it possibly be the
same kind of thing as corrected earlier today in Heiko Carsten's patch:
http://lkml.org/lkml/2009/3/9/112 ?
Cheers,
FJP
On Sun, 2009-03-08 at 03:30 +0100, Frans Pop wrote:
> Kernels after 2.6.27 fail to boot in the Hercules S/390 emulator, quite
> early in the boot process:
> 0.000000! Initializing cgroup subsys cpuset
> 0.000000! Initializing cgroup subsys cpu
> 0.000000! Linux version 2.6.29-rc7 (root@aragorn) (gcc version 4.2.4 (Debian 4.2.4-4)) #11 SMP Thu Mar 5 20:02:03 CET 2009
> 0.000000! setup: Linux is running natively in 31-bit mode
> 0.000000! setup: The hardware system has IEEE compatible floating point units
> 0.000000! Zone PFN ranges:
> 0.000000! Normal 0x00000000 -> 0x00010000
> 0.000000! Movable zone start PFN for each node
> 0.000000! early_node_map 1! active PFN ranges
> 0.000000! 0: 0x00000000 -> 0x00010000
> 0.000000! Built 1 zonelists in Zone order, mobility grouping on. Total pages: 65024
> 0.000000! Kernel command line: ro vmpoff="LOGOFF" root=/dev/disk/by-path/ccw-0.0.0120-part1 BOOT_IMAGE=4
> 0.000000! PID hash table entries: 1024 (order: 10, 4096 bytes)
> 0.000503! console ttyS0! enabled
> 0.002333! Dentry cache hash table entries: 32768 (order: 5, 131072 bytes)
> 0.012841! Inode-cache hash table entries: 16384 (order: 4, 65536 bytes)
> 0.129660! Memory: 252032k/262144k available (2271k kernel code, 0k reserved, 922k data, 164k init)
> 0.130086! Write protected kernel read-only data: 0x12000 - 0x2fbfff
> 0.133887! Security Framework initialized
> 0.134064! SELinux: Disabled at boot.
> 0.134996! Mount-cache hash table entries: 512
> 0.141419! Initializing cgroup subsys ns
> 0.141605! Initializing cgroup subsys cpuacct
> 0.142009! Initializing cgroup subsys devices
> 0.180403! cpu: 2 configured CPUs, 0 standby CPUs
> [... here the boot stops, normally followed by: ...]
> 17179568.691707! cpu 0 phys_idx=0 vers=00 ident=002623 machine=3090 unused=0000
> 17179568.699479! cpu 1 phys_idx=1 vers=00 ident=102623 machine=3090 unused=0000
> 17179568.699636! Brought up 2 CPUs
>
> I've bisected this to the following commit:
> commit 5cd1c9c5cf30d4b33df3d3f74d8142f278d536b7
> Author: Roman Zippel <[email protected]>
> Date: Mon Sep 22 14:42:43 2008 -0700
> timekeeping: fix rounding problem during clock update
>
> After reverting 5cd1c9c5cf30 and the somewhat related 6c9bacb41c10 on top
> of 2.6.28.7, the system boots OK again [1].
Huh. No clue right off. What clocksource is in use on this system?
cat /sys/devices/system/clocksource/clocksource0/current_clocksource
Also can you provide the output with the following patch?
thanks
-john
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 900f1b6..e832ffc 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -547,6 +547,7 @@ void update_wall_time(void)
* add the remainder to the error difference.
*/
xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
+ printk("xtime.tv_nsec: %ld clock->xtime_nsec: %lld shift: %ld\n", xtime.tv_nsec, clock->xtime_nsec, clock->shift);
clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
On Tuesday 10 March 2009, John Stultz wrote:
> > After reverting 5cd1c9c5cf30 and the somewhat related 6c9bacb41c10 on
> > top of 2.6.28.7, the system boots OK again [1].
>
> Huh. No clue right off. What clocksource is in use on this system?
$ cat /sys/devices/system/clocksource/clocksource0/current_clocksource
tod
(That's from 2.6.26.)
> Also can you provide the output with the following patch?
I already did something similar. Did you see my follow-up mail?
I added:
+ if (unlikely(clock->xtime_nsec < ((s64)xtime.tv_nsec << clock->shift)))
+ printk("Negative result: %llu - %lld\n",
+ (unsigned long long)clock->xtime_nsec,
+ (long long)((s64)xtime.tv_nsec << clock->shift));
Which resulted in:
Negative result: 166039808000 - 166039808256
Would you still like me to try your patch?
Cheers,
FJP
On Tue, 2009-03-10 at 04:37 +0100, Frans Pop wrote:
> On Tuesday 10 March 2009, John Stultz wrote:
> > > After reverting 5cd1c9c5cf30 and the somewhat related 6c9bacb41c10 on
> > > top of 2.6.28.7, the system boots OK again [1].
> >
> > Huh. No clue right off. What clocksource is in use on this system?
>
> $ cat /sys/devices/system/clocksource/clocksource0/current_clocksource
> tod
> (That's from 2.6.26.)
>
> > Also can you provide the output with the following patch?
>
> I already did something similar. Did you see my follow-up mail?
No, sorry, I must have missed it.
> I added:
> + if (unlikely(clock->xtime_nsec < ((s64)xtime.tv_nsec << clock->shift)))
> + printk("Negative result: %llu - %lld\n",
> + (unsigned long long)clock->xtime_nsec,
> + (long long)((s64)xtime.tv_nsec << clock->shift));
>
> Which resulted in:
> Negative result: 166039808000 - 166039808256
>
> Would you still like me to try your patch?
No no, I'll try to look again at your data in the morning.
thanks
-john
On Mon, 2009-03-09 at 16:04 +0100, Frans Pop wrote:
> On Sunday 08 March 2009, Frans Pop wrote:
> > Kernels after 2.6.27 fail to boot in the Hercules S/390 emulator, quite
> > early in the boot process:
> [...]
> > 0.141419! Initializing cgroup subsys ns
> > 0.141605! Initializing cgroup subsys cpuacct
> > 0.142009! Initializing cgroup subsys devices
> > 0.180403! cpu: 2 configured CPUs, 0 standby CPUs
> >
> > I've bisected this to the following commit:
> > commit 5cd1c9c5cf30d4b33df3d3f74d8142f278d536b7
> > Author: Roman Zippel <[email protected]>
> > Date: Mon Sep 22 14:42:43 2008 -0700
> > timekeeping: fix rounding problem during clock update
>
> After staring at this commit for a while I decided to try the following
> patch:
> --- a/kernel/time/timekeeping.c
> +++ b/kernel/time/timekeeping.c
> @@ -547,5 +547,11 @@ void update_wall_time(void)
> * add the remainder to the error difference.
> */
> xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
> + if (unlikely(clock->xtime_nsec < ((s64)xtime.tv_nsec << clock->shift))) {
> + printk("Negative result: %llu - %lld\n",
> + (unsigned long long)clock->xtime_nsec,
> + (long long)((s64)xtime.tv_nsec << clock->shift));
> + BUG_ON(1);
> + }
> clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
> clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
>
> And that resulted in:
> 0.004175! Negative result: 166039808000 - 166039808256
Hrm. That doesn't make sense.
xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
I'm assuming clock->shift == 12 (from the clocksource_tod definition).
If clock->xtime_nsec == 166039808000, then xtime.tv_nsec should be
40537063, so xtime_tv_nsec << clock->shift would be 166039810048.
So not sure whats going on with the output you got.
Also the negative conditional you add doesn't really make sense either,
as we expect the xtime.tv_nsec << clock->shift to be larger then
clock->xtime_nsec, as we've rounded it up by one. We then accumulate the
negative difference between them into clock->error.
Hmm.. Does the following explicit casting help?
thanks
-john
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 900f1b6..acd130b 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -548,7 +548,7 @@ void update_wall_time(void)
*/
xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
- clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
+ clock->error += (s64)clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
update_xtime_cache(cyc2ns(clock, offset));
On Wednesday 11 March 2009, john stultz wrote:
> On Mon, 2009-03-09 at 16:04 +0100, Frans Pop wrote:
> > And that resulted in:
> > 0.004175! Negative result: 166039808000 - 166039808256
>
> Hrm. That doesn't make sense.
It may not make sense, but that's still what I get ;-)
I've tried again with a bit more elaborate patch:
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -546,9 +546,26 @@ void update_wall_time(void)
/* store full nanoseconds into xtime after rounding it up and
* add the remainder to the error difference.
*/
- xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
- clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
- clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
+ u64 xtns1, xtns2, xtns3;
+
+ xtns1 = clock->xtime_nsec;
+ xtns2 = xtns1;
+ xtime.tv_nsec = ((s64)xtns1 >> clock->shift) + 1;
+ xtns2 -= (s64)xtime.tv_nsec << clock->shift;
+ clock->xtime_nsec = xtns2;
+ xtns3 = clock->xtime_nsec;
+ clock->error += (s64)xtns3 << (NTP_SCALE_SHIFT - clock->shift);
+
+ if (unlikely(xtns1 < ((s64)xtime.tv_nsec << clock->shift)))
+ printk("Wall time error. "
+ "xtns1: %llu, tv_nsec %lu, shifted tv_nsec %lld, "
+ "xtns2: %llu, xtns3: %llu, error: %lld\n",
+ (unsigned long long)xtns1,
+ xtime.tv_nsec,
+ (long long)((s64)xtime.tv_nsec << clock->shift),
+ (unsigned long long)xtns2,
+ (unsigned long long)xtns3,
+ (long long)clock->error);
update_xtime_cache(cyc2ns(clock, offset));
This results in tons of messages passing by and the boot going nowhere.
Sample:
Wall time error. xtns1: 15206684608, tv_nsec 59401112, shifted tv_nsec 15206684672,
xtns2: 18446744063709451552, xtns3: 18446744063709451552, error: -13053227561031008
Wall time error. xtns1: 15206684737, tv_nsec 59401113, shifted tv_nsec 15206684928,
xtns2: 18446744073709551425, xtns3: 18446744073709551425, error: -13053229775623520
Wall time error. xtns1: 15206684862, tv_nsec 59401113, shifted tv_nsec 15206684928,
xtns2: 18446744063709451550, xtns3: 18446744063709451550, error: -13053241856098304
Wall time error. xtns1: 15206684995, tv_nsec 59401114, shifted tv_nsec 15206685184,
xtns2: 18446744073709551427, xtns3: 18446744073709551427, error: -13053244070690816
Wall time error. xtns1: 15206685116, tv_nsec 59401114, shifted tv_nsec 15206685184,
xtns2: 18446744063709451548, xtns3: 18446744063709451548, error: -13053246151065600
So it looks to me as if we're rounding up instead of down (and my
'unlikely' was probably too optimistic).
Code error? Compiler error? Hercules bug possibly?
I've got Hercules set to update the clock every 500 microseconds using
'TIMERINT 500' in the configuration file. From the documentation:
"Specifies the internal timers update interval, in microseconds. This
parameter specifies how frequently Hercules's internal timers-update
thread updates the TOD Clock, CPU Timer, and other architectural
related clock/timer values. The default interval is 50 microseconds,
which strikes a reasonable balance between clock accuracy and overall
host performance. The minimum allowed value is 1 microsecond and the
maximum is 1000000 microseconds (i.e. one second)."
> Hmm.. Does the following explicit casting help?
I included that in the patch, but due to the large number of "errors" I
got I couldn't tell. Can test with just that cast, but it still seems to
me the problem is in the above.
Cheers,
FJP
OK, I think I've gotten a lot further now.
On Wednesday 11 March 2009, john stultz wrote:
> Also the negative conditional you add doesn't really make sense either,
> as we expect the xtime.tv_nsec << clock->shift to be larger then
> clock->xtime_nsec, as we've rounded it up by one. We then accumulate
> the negative difference between them into clock->error.
I'm not at all fluent in casts, bit shifting and stuff, so it took a
while for the quarter to drop. But AFAICT what you're saying here is
exactly the problem.
Indeed you do round xtime.tv_nsec up, so when you do
clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
or
clock->xtime_nsec = clock->xtime_nsec - ((s64)xtime.tv_nsec << clock->shift);
the second argument is always going to be bigger than the first, so you
always end up with a negative value.
> Hmm.. Does the following explicit casting help?
Even with the cast you're just papering over the issue that we're moving a
negative value into a field that is defined as unsigned:
include/linux/clocksource.h: u64 xtime_nsec;
Why does clock->xtime_nsec get set to the _difference_ (-=) at all? It
almost seems to me as if that field is getting abused as a temporary
variable. We're also not doing as the comment says:
/* store full nanoseconds into xtime after rounding it up and
* add the remainder to the error difference.
What we are actually doing is storing the _remainder_ in xtime.
The patch included below gives me saner values, but still leaves a
problem with the calculation of clock->error. Here are the first
wall_update calls after a reboot. This is with the patch and some
debugging code, but *without* actually changing clock->error.
With that the system boots correctly!
0: scale/shift: 32/8, xtime_ns old: 155790080000, new: 155790080256
tv_ns: 608555001, rem: -256, old_err: 0, error: -4294867296
1: scale/shift: 32/8, xtime_ns old: 155790080256, new: 155790080512
tv_ns: 608555002, rem: -256, old_err: 0, error: -4294867296
2: scale/shift: 32/8, xtime_ns old: 155790080512, new: 155790080768
tv_ns: 608555003, rem: -256, old_err: 0, error: -4294867296
3: scale/shift: 32/8, xtime_ns old: 155790080768, new: 155790081024
tv_ns: 608555004, rem: -256, old_err: 0, error: -4294867296
4: scale/shift: 32/8, xtime_ns old: 155790081024, new: 155790081280
tv_ns: 608555005, rem: -256, old_err: 0, error: -4294867296
5: scale/shift: 32/8, xtime_ns old: 155790081280, new: 155790081536
tv_ns: 608555006, rem: -256, old_err: 0, error: -4294867296
First observation is that clock->shift is not 12, but 8! This explains
the "strange" values we got for xtime.tv_nsec. But I agree with you that
from the code in arch/s390/time.c it looks like the value should be 12
for the tod clocksource. No idea what mangles it. It also means that
clock->error gets shifted by 24 (!) as NTP_SCALE_SHIFT is 32.
Second observation is that clock->error (old_err) remains at 0. So
apparently it's not getting set anywhere else if we don't set it here
first. The calculated new error is correct given the shift.
So, lets look next what happens if I allow clock->error to be changed
here. This makes the boot fail and I believe that this is the critical
change in 5cd1c9c5cf30.
0: scale/shift: 32/8, xtime_ns old: 496319488000, new: 496319488256
tv_ns: 1938748001, rem: -256, old_err: 0, error: -4294867296
1: scale/shift: 32/8, xtime_ns old: 496315293952, new: 496315294208
tv_ns: 1938731618, rem: -256, old_err: -4292487689804800, error: -4292501984672096
2: scale/shift: 32/8, xtime_ns old: 496302611296, new: 496302611552
tv_ns: 1938682467, rem: -256, old_err: -12807120030269440, error: -12807124325236736
3: scale/shift: 32/8, xtime_ns old: 496298417248, new: 496298417504
tv_ns: 1938666084, rem: -256, old_err: -14918186650466656, error: -14918180945433952
4: scale/shift: 32/8, xtime_ns old: 496295896064, new: 496295896320
tv_ns: 1938655845, rem: -256, old_err: -15964926015076704, error: -15964920310044000
5: scale/shift: 32/8, xtime_ns old: 496294223456, new: 496294223712
tv_ns: 1938649702, rem: -256, old_err: -16483889798454272, error: -16483904093421568
Note that clock->xtime_nsec is now running backwards and the crazy values
for clock->error.
>From this I conclude that clock->error is getting buggered somewhere
else: we get a completely different value back from what is calculated
here. The calculation here is still correct:
$ echo $(( -4292487689804800 + (-256 << 24) ))
-4292491984772096
I suspect that clock->error running back is what causes my hang.
I hope that I'm at least somewhat on the right track here?
I keep wondering why I'm the only one seeing problems...
Cheers,
FJP
---
Partial fix. This makes clock->xtime_nsec have sane values and gives
correct remainders.
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 900f1b6..fb048cc 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -480,6 +480,7 @@ static void clocksource_adjust(s64 offset)
void update_wall_time(void)
{
cycle_t offset;
+ s64 remainder;
/* Make sure we're fully resumed: */
if (unlikely(timekeeping_suspended))
@@ -547,8 +548,9 @@ void update_wall_time(void)
* add the remainder to the error difference.
*/
xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
- clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
- clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
+ remainder = clock->xtime_nsec - ((s64)xtime.tv_nsec << clock->shift);
+ clock->xtime_nsec = (s64)xtime.tv_nsec << clock->shift;
+ clock->error += remainder << (NTP_SCALE_SHIFT - clock->shift);
update_xtime_cache(cyc2ns(clock, offset));
On Wednesday 11 March 2009, Frans Pop wrote:
> From this I conclude that clock->error is getting buggered somewhere
> else: we get a completely different value back from what is calculated
> here.
Some extra printks at least close the loop:
0: scale/shift: 32/8, xtime_ns old: 168555264000, new: 168555264256
tv_ns: 658419001, rem: -256, old_err: 0, error: -4294867296
bigadjust: adj: 2097152, int: 260046848, offs: 4194304
CS adjust: -4294867296 -> -4292487689804800
1: scale/shift: 32/8, xtime_ns old: 168551069952, new: 168551070208
tv_ns: 658402618, rem: -256, old_err: -4292487689804800, error: -4292501984672096
bigadjust: adj: 4194304, int: 520093696, offs: 12582912
CS adjust: -4292501984672096 -> -12807120030269440
2: scale/shift: 32/8, xtime_ns old: 168538487296, new: 168538487552
tv_ns: 658353467, rem: -256, old_err: -12807120030269440, error: -12807124325236736
bigadjust: adj: 1048576, int: 130023424, offs: 4194304
CS adjust: -12807124325236736 -> -14918186650466656
"bigadjust" is from just before the return in clocksource_bigadjust.
"CS adjust" is from after clock->error gets calculated in clocksource_adjust.
Sorry for the mail flood. This is the last one and then I'm going to wait for some reactions.
On Wednesday 11 March 2009, Frans Pop wrote:
> So, lets look next what happens if I allow clock->error to be changed
> here. This makes the boot fail and I believe that this is the critical
> change in 5cd1c9c5cf30.
[...]
> Note that clock->xtime_nsec is now running backwards and the crazy
> values for clock->error.
>
> From this I conclude that clock->error is getting buggered somewhere
> else: we get a completely different value back from what is calculated
> here. The calculation here is still correct:
> $ echo $(( -4292487689804800 + (-256 << 24) ))
> -4292491984772096
>
> I suspect that clock->error running back is what causes my hang.
s/clock->error/clock->xtime_nsec/ of course.
Looking a bit closer at what Roman's patch 5cd1c9c5cf30 does, I see this:
- clock->xtime_nsec += (s64)xtime.tv_nsec << clock->shift;
+ clock->xtime_nsec = (s64)xtime.tv_nsec << clock->shift;
[...]
clocksource_adjust(offset);
- xtime.tv_nsec = (s64)clock->xtime_nsec >> clock->shift;
+ xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
+ clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
So, in the old situation the code first added xtime.tv_nsec to
clock->xtime_nsec and later subtracted it again, so there's symmetry.
In the new code we no longer do the first, but still do the second. That
seems strange and probably upsets assumptions in the code in between, which
includes the call to clocksource_adjust(). AFAICT this is the root cause of
the overflow visible in my earliest traces.
I've done some tries to correct that, but did not find anything that really
worked.
I also do now know with near certainty where the system hangs with the
vanilla 2.6.28.7: in the 'while (offset >= clock->cycle_interval)' loop in
update_wall_time. That loop should probably have some mechanism to warn if
it's running wild...
This whole code is pretty tricky, but I'm convinced Roman's patch is
structurally broken.
Cheers,
FJP
On Wed, 2009-03-11 at 17:03 +0100, Frans Pop wrote:
> OK, I think I've gotten a lot further now.
>
> On Wednesday 11 March 2009, john stultz wrote:
> > Also the negative conditional you add doesn't really make sense either,
> > as we expect the xtime.tv_nsec << clock->shift to be larger then
> > clock->xtime_nsec, as we've rounded it up by one. We then accumulate
> > the negative difference between them into clock->error.
>
> I'm not at all fluent in casts, bit shifting and stuff, so it took a
> while for the quarter to drop. But AFAICT what you're saying here is
> exactly the problem.
>
> Indeed you do round xtime.tv_nsec up, so when you do
> clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
> or
> clock->xtime_nsec = clock->xtime_nsec - ((s64)xtime.tv_nsec << clock->shift);
> the second argument is always going to be bigger than the first, so you
> always end up with a negative value.
>
> > Hmm.. Does the following explicit casting help?
>
> Even with the cast you're just papering over the issue that we're moving a
> negative value into a field that is defined as unsigned:
> include/linux/clocksource.h: u64 xtime_nsec;
Probably agreed here, xtime_nsec probably should be converted to a s64
as negative values are possible.
However, Its unclear to me if my patch worked or not?
Did you try it alone?
> Why does clock->xtime_nsec get set to the _difference_ (-=) at all? It
> almost seems to me as if that field is getting abused as a temporary
> variable. We're also not doing as the comment says:
> /* store full nanoseconds into xtime after rounding it up and
> * add the remainder to the error difference.
> What we are actually doing is storing the _remainder_ in xtime.
Yes, after Romans patch we do basically store the remainder in
xtime_nsec, however we then use that to add to error and xtime_nsec
isn't used until it is cleared the next time we hit update_wall_clock.
So yea, xtime_nsec scope is now basically a local variable right now,
but there is code in subfunctions that use it so we didn't change it
totally. I agree there is some cleanup needed here.
> The patch included below gives me saner values, but still leaves a
> problem with the calculation of clock->error. Here are the first
> wall_update calls after a reboot. This is with the patch and some
> debugging code, but *without* actually changing clock->error.
>
> With that the system boots correctly!
By this you probably disabled the clock steering. Still not sure how it
affected the issue.
> 0: scale/shift: 32/8, xtime_ns old: 155790080000, new: 155790080256
> tv_ns: 608555001, rem: -256, old_err: 0, error: -4294867296
> 1: scale/shift: 32/8, xtime_ns old: 155790080256, new: 155790080512
> tv_ns: 608555002, rem: -256, old_err: 0, error: -4294867296
> 2: scale/shift: 32/8, xtime_ns old: 155790080512, new: 155790080768
> tv_ns: 608555003, rem: -256, old_err: 0, error: -4294867296
> 3: scale/shift: 32/8, xtime_ns old: 155790080768, new: 155790081024
> tv_ns: 608555004, rem: -256, old_err: 0, error: -4294867296
> 4: scale/shift: 32/8, xtime_ns old: 155790081024, new: 155790081280
> tv_ns: 608555005, rem: -256, old_err: 0, error: -4294867296
> 5: scale/shift: 32/8, xtime_ns old: 155790081280, new: 155790081536
> tv_ns: 608555006, rem: -256, old_err: 0, error: -4294867296
>
> First observation is that clock->shift is not 12, but 8! This explains
> the "strange" values we got for xtime.tv_nsec. But I agree with you that
> from the code in arch/s390/time.c it looks like the value should be 12
> for the tod clocksource. No idea what mangles it. It also means that
> clock->error gets shifted by 24 (!) as NTP_SCALE_SHIFT is 32.
So if the shift value is 8, that likely means the jiffies clock is still
in use here and we haven't switched to the TOD clocksource.
> Second observation is that clock->error (old_err) remains at 0. So
> apparently it's not getting set anywhere else if we don't set it here
> first. The calculated new error is correct given the shift.
>
> So, lets look next what happens if I allow clock->error to be changed
> here. This makes the boot fail and I believe that this is the critical
> change in 5cd1c9c5cf30.
>
> 0: scale/shift: 32/8, xtime_ns old: 496319488000, new: 496319488256
> tv_ns: 1938748001, rem: -256, old_err: 0, error: -4294867296
> 1: scale/shift: 32/8, xtime_ns old: 496315293952, new: 496315294208
> tv_ns: 1938731618, rem: -256, old_err: -4292487689804800, error: -4292501984672096
> 2: scale/shift: 32/8, xtime_ns old: 496302611296, new: 496302611552
> tv_ns: 1938682467, rem: -256, old_err: -12807120030269440, error: -12807124325236736
> 3: scale/shift: 32/8, xtime_ns old: 496298417248, new: 496298417504
> tv_ns: 1938666084, rem: -256, old_err: -14918186650466656, error: -14918180945433952
> 4: scale/shift: 32/8, xtime_ns old: 496295896064, new: 496295896320
> tv_ns: 1938655845, rem: -256, old_err: -15964926015076704, error: -15964920310044000
> 5: scale/shift: 32/8, xtime_ns old: 496294223456, new: 496294223712
> tv_ns: 1938649702, rem: -256, old_err: -16483889798454272, error: -16483904093421568
>
> Note that clock->xtime_nsec is now running backwards and the crazy values
> for clock->error.
That could be acceptable. clocksource_adjust() can do some negative
steering and still have correct output.
> From this I conclude that clock->error is getting buggered somewhere
> else: we get a completely different value back from what is calculated
> here. The calculation here is still correct:
> $ echo $(( -4292487689804800 + (-256 << 24) ))
> -4292491984772096
>
> I suspect that clock->error running back is what causes my hang.
>
> I hope that I'm at least somewhat on the right track here?
> I keep wondering why I'm the only one seeing problems...
Me too. Its a little hard to unpack everything you've gone through here,
as some of your assumptions aren't quite right. Its understandable as
this code is fairly dense, and has evolved over the last couple of years
to handle a number of corner cases found.
But its likely some cleanup here is in order to shake out these signed
shift issues that were not part of the original design.
thanks
-john
On Wed, 2009-03-11 at 20:05 +0100, Frans Pop wrote:
> Sorry for the mail flood. This is the last one and then I'm going to wait for some reactions.
>
> On Wednesday 11 March 2009, Frans Pop wrote:
> > So, lets look next what happens if I allow clock->error to be changed
> > here. This makes the boot fail and I believe that this is the critical
> > change in 5cd1c9c5cf30.
> [...]
> > Note that clock->xtime_nsec is now running backwards and the crazy
> > values for clock->error.
> >
> > From this I conclude that clock->error is getting buggered somewhere
> > else: we get a completely different value back from what is calculated
> > here. The calculation here is still correct:
> > $ echo $(( -4292487689804800 + (-256 << 24) ))
> > -4292491984772096
> >
> > I suspect that clock->error running back is what causes my hang.
>
> s/clock->error/clock->xtime_nsec/ of course.
>
> Looking a bit closer at what Roman's patch 5cd1c9c5cf30 does, I see this:
>
> - clock->xtime_nsec += (s64)xtime.tv_nsec << clock->shift;
> + clock->xtime_nsec = (s64)xtime.tv_nsec << clock->shift;
> [...]
> clocksource_adjust(offset);
> - xtime.tv_nsec = (s64)clock->xtime_nsec >> clock->shift;
> + xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
> clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
> + clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
>
> So, in the old situation the code first added xtime.tv_nsec to
> clock->xtime_nsec and later subtracted it again, so there's symmetry.
>
> In the new code we no longer do the first, but still do the second. That
> seems strange and probably upsets assumptions in the code in between, which
> includes the call to clocksource_adjust(). AFAICT this is the root cause of
> the overflow visible in my earliest traces.
> I've done some tries to correct that, but did not find anything that really
> worked.
No not quite. We use clock->xtime_nsec to store the high precision
xtime.tv_nsec. Its use is as follows:
1) We initialize it to xtime.tv_nsec << clock->shift
2) We accumulate into it
3) We tweak it as needed from clocksource_adjust()
4) We then store its value shifted back down and rounded up into
xtiem.tv_nsec.
5) We calculate the the difference between the rounded up value and
xtime_nsec, and add it to the error.
I'm still a little baffled, but I figure I can try to reproduce this
myself. So I'm working setting up hercules environment here to see if I
can't trigger it. Any help with config or links to your environment
would be great.
thanks
-john
On Wed, 2009-03-11 at 17:30 -0700, john stultz wrote:
> On Wed, 2009-03-11 at 17:03 +0100, Frans Pop wrote:
> > OK, I think I've gotten a lot further now.
> >
> > On Wednesday 11 March 2009, john stultz wrote:
> > > Also the negative conditional you add doesn't really make sense either,
> > > as we expect the xtime.tv_nsec << clock->shift to be larger then
> > > clock->xtime_nsec, as we've rounded it up by one. We then accumulate
> > > the negative difference between them into clock->error.
> >
> > I'm not at all fluent in casts, bit shifting and stuff, so it took a
> > while for the quarter to drop. But AFAICT what you're saying here is
> > exactly the problem.
> >
> > Indeed you do round xtime.tv_nsec up, so when you do
> > clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
> > or
> > clock->xtime_nsec = clock->xtime_nsec - ((s64)xtime.tv_nsec << clock->shift);
> > the second argument is always going to be bigger than the first, so you
> > always end up with a negative value.
> >
> > > Hmm.. Does the following explicit casting help?
> >
> > Even with the cast you're just papering over the issue that we're moving a
> > negative value into a field that is defined as unsigned:
> > include/linux/clocksource.h: u64 xtime_nsec;
>
> Probably agreed here, xtime_nsec probably should be converted to a s64
> as negative values are possible.
>
>
> However, Its unclear to me if my patch worked or not?
> Did you try it alone?
For a cleaner version, could you try the following, against 2.6.29-git
with no other modification?
thanks
-john
xtime_nsec is expected at times to be negative. Instead of trying to
handle all the shifting properly via casts, define it as a s64 instead
of a u64.
NOT FOR INCLUSION
Signed-off-by: John Stultz <[email protected]>
diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h
index f88d32f..e217000 100644
--- a/include/linux/clocksource.h
+++ b/include/linux/clocksource.h
@@ -86,7 +86,7 @@ struct clocksource {
* more than one cache line.
*/
cycle_t cycle_last ____cacheline_aligned_in_smp;
- u64 xtime_nsec;
+ s64 xtime_nsec;
s64 error;
struct timespec raw_time;
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 900f1b6..387be3c 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -546,7 +546,7 @@ void update_wall_time(void)
/* store full nanoseconds into xtime after rounding it up and
* add the remainder to the error difference.
*/
- xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
+ xtime.tv_nsec = (clock->xtime_nsec >> clock->shift) + 1;
clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
On Wed, 11 Mar 2009, john stultz wrote:
> For a cleaner version, could you try the following, against 2.6.29-git
> with no other modification?
cleaner ?
> xtime_nsec is expected at times to be negative. Instead of trying to
> handle all the shifting properly via casts, define it as a s64 instead
> of a u64.
>
> NOT FOR INCLUSION
> Signed-off-by: John Stultz <[email protected]>
>
> diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h
> index f88d32f..e217000 100644
> --- a/include/linux/clocksource.h
> +++ b/include/linux/clocksource.h
> @@ -86,7 +86,7 @@ struct clocksource {
> * more than one cache line.
> */
> cycle_t cycle_last ____cacheline_aligned_in_smp;
> - u64 xtime_nsec;
> + s64 xtime_nsec;
> s64 error;
> struct timespec raw_time;
>
> diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
> index 900f1b6..387be3c 100644
> --- a/kernel/time/timekeeping.c
> +++ b/kernel/time/timekeeping.c
> @@ -546,7 +546,7 @@ void update_wall_time(void)
> /* store full nanoseconds into xtime after rounding it up and
> * add the remainder to the error difference.
> */
> - xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
> + xtime.tv_nsec = (clock->xtime_nsec >> clock->shift) + 1;
> clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
> clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
This code sequence does:
xtime.tv_nsec = (clock->xtime_nsec >> clock->shift) + 1;
clock->xtime_nsec -= xtime.tv_nsec << clock->shift;
Lets substitute a bit:
a = xtime.tv_nsec
b = clock->xtime_nsec
c = clock->shift
r = result (which ends up in b aka clock->xtime_nsec again for the next round)
=> a = (b >> c) + 1
r = b - (a << c)
b >> c = b / 2^c
a << c = a * 2^c
=> a = (b / 2^c) + 1
r = b - (a * 2^c)
=> r = b - ((b / 2^c) + 1) * 2^c
r = b - ((2^c * b / 2^c) + 2^c)
r = b - (2^c * b / 2^c) - 2^c
r = b - b - 2^c
r = -2^c
=> r = -(1 << c)
So the whole business boils down to:
clock->xtime_nsec = -(1 << clock->shift);
Famous last words (see also arch/x86/kernel/tsc.c and
arch/x86/include/asm/timer.h):
[email protected] "math is hard, lets go shopping!"
Thanks,
tglx
On Thu, 2009-03-12 at 02:30 +0100, Thomas Gleixner wrote:
> On Wed, 11 Mar 2009, john stultz wrote:
> > For a cleaner version, could you try the following, against 2.6.29-git
> > with no other modification?
>
> cleaner ?
>
> > xtime_nsec is expected at times to be negative. Instead of trying to
> > handle all the shifting properly via casts, define it as a s64 instead
> > of a u64.
> >
> > NOT FOR INCLUSION
> > Signed-off-by: John Stultz <[email protected]>
> >
> > diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h
> > index f88d32f..e217000 100644
> > --- a/include/linux/clocksource.h
> > +++ b/include/linux/clocksource.h
> > @@ -86,7 +86,7 @@ struct clocksource {
> > * more than one cache line.
> > */
> > cycle_t cycle_last ____cacheline_aligned_in_smp;
> > - u64 xtime_nsec;
> > + s64 xtime_nsec;
> > s64 error;
> > struct timespec raw_time;
> >
> > diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
> > index 900f1b6..387be3c 100644
> > --- a/kernel/time/timekeeping.c
> > +++ b/kernel/time/timekeeping.c
> > @@ -546,7 +546,7 @@ void update_wall_time(void)
> > /* store full nanoseconds into xtime after rounding it up and
> > * add the remainder to the error difference.
> > */
> > - xtime.tv_nsec = ((s64)clock->xtime_nsec >> clock->shift) + 1;
> > + xtime.tv_nsec = (clock->xtime_nsec >> clock->shift) + 1;
> > clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
> > clock->error += clock->xtime_nsec << (NTP_SCALE_SHIFT - clock->shift);
>
> This code sequence does:
>
> xtime.tv_nsec = (clock->xtime_nsec >> clock->shift) + 1;
> clock->xtime_nsec -= xtime.tv_nsec << clock->shift;
>
> Lets substitute a bit:
>
> a = xtime.tv_nsec
> b = clock->xtime_nsec
> c = clock->shift
> r = result (which ends up in b aka clock->xtime_nsec again for the next round)
>
> => a = (b >> c) + 1
> r = b - (a << c)
>
> b >> c = b / 2^c
> a << c = a * 2^c
>
> => a = (b / 2^c) + 1
> r = b - (a * 2^c)
>
> => r = b - ((b / 2^c) + 1) * 2^c
> r = b - ((2^c * b / 2^c) + 2^c)
> r = b - (2^c * b / 2^c) - 2^c
> r = b - b - 2^c
> r = -2^c
>
> => r = -(1 << c)
>
> So the whole business boils down to:
>
> clock->xtime_nsec = -(1 << clock->shift);
Err, not quite. See, we truncate clock->xtime_nsec when we shift it down
and store it into xtime.tv_nsec. Since we don't want to lose truncated
remainder, we simply add one to xtime.tv_nsec, taking the ceiling in
effect.
However, we have to balance this out in order to not rush forward in
time each tick. So we accumulate amount we ended up adding into the
error.
Lets walk through a simple example with actual values.
b = 99
c = 4
=> a = (b >> c) + 1
r = b - (a << c)
=> a = (99 >> 4) + 1
a = 6 + 1
a = 7
r = 99 - (7 << 4)
r = 99 - 112
r = -13
(Your reduction would give -16, since it ignores that shift down
truncates)
In effect, adding 1 to xtime.tv_nsec, is the same as adding 13 to
clock->xtime_nsec.
Maybe a different way of expressing what we're calculating is the
following:
xtime.tv_nsec = (clock->xtime_nsec + (1<<clock->shift)) >> clock->shift
clock->xtime_nsec = (1<<clock->shift)
- (clock->xtime_nsec &((1<<clock->shift)-1)
In other words: The rounded up portion - the masked remainder in
xtime_nsec.
Does that make sense?
thanks
-john
On Wed, 2009-03-11 at 17:34 -0700, john stultz wrote:
> I'm still a little baffled, but I figure I can try to reproduce this
> myself. So I'm working setting up hercules environment here to see if I
> can't trigger it. Any help with config or links to your environment
> would be great.
Ok. I've got this up and reproduced with the hercules emulator.
A couple of interesting hints:
* The issue seems to go away if we disable NO_HZ/dynticks in
the .config
* nohz=off does not affect the issue.
* The issue also seems to go away if HZ is anything except 250
* Commenting out clocksource_adjust(), which does the clock steering,
seems to avoid the issue.
* When the problem triggers we don't seem to transfer to the TOD clock,
and the jiffies clocksource seems to be in use.
* The jiffies clocksource mult value seems to be ever increasing, again
pointing to something wrong in the clock steering.
* Reverting both 6c9bacb41c10 and 5cd1c9c5cf30 against 2.6-git did not
seem to help the issue, as originally reported. I also reverted
49b5cf34727a as well just in case, and it didn't help either.
That's about all I can get for today. I'm out of town until Monday, so
I'll start digging back into it then.
thanks
-john
On Thursday 12 March 2009, John Stultz wrote:
> * Reverting both 6c9bacb41c10 and 5cd1c9c5cf30 against 2.6-git did not
> seem to help the issue, as originally reported. I also reverted
> 49b5cf34727a as well just in case, and it didn't help either.
Hmm, interesting. I think you may have reproduced a similar failure case,
but not exactly the same as mine. I wonder if what you have reproduced is
the same as Debian bug report http://bugs.debian.org/511334. That is with
2.6.26 which would match the fact that the reverts that work reliably for
me did not work for you. I've never had any problems with Debian's 2.6.26
in my configuration.
For me 2.6.28.7 is rock solid with the two reverts. I've attached my
kernel config and my Hercules config. I think the critical thing in my
config may be the "TIMERINT 500" setting for Hercules, which is somewhat
similar to your "goes away if HZ is anything except 250" observation.
> That's about all I can get for today. I'm out of town until Monday, so
> I'll start digging back into it then.
From your other replies I've seen that I have been way off track at some
points. It looks as if I have been looking at events too early in the
boot and been confused by the use of clock->xtime_nsec. My apologies for
any confusion caused by that.
I have a few more tests I think I can usefully do, but will try harder not
to confuse the issue with my inexpert ramblings anymore :-P
My main problem in trying to nail this down has been that any printks that
are added tend to flood the system to such an extend that you cannot
actually observe anything.
Thanks a lot for your replies and work on this.
On Wed, 11 Mar 2009, john stultz wrote:
> Maybe a different way of expressing what we're calculating is the
> following:
>
> xtime.tv_nsec = (clock->xtime_nsec + (1<<clock->shift)) >> clock->shift
> clock->xtime_nsec = (1<<clock->shift)
> - (clock->xtime_nsec &((1<<clock->shift)-1)
>
> In other words: The rounded up portion - the masked remainder in
> xtime_nsec.
>
> Does that make sense?
Yes, now that my brain works again. I knew I was missing something.
You are definitely right: math _is_ hard :)
tglx
0.000000! Initializing cgroup subsys cpuset
0.000000! Initializing cgroup subsys cpu
0.000000! Linux version 2.6.28.7 (root@aragorn) (gcc version 4.2.4 (Debian 4.2.4-4)) #66 SMP Thu Mar 12 16:56:40 CET 2009
0.000000! We are running native (31 bit mode)
0.000000! This machine has an IEEE fpu
0.000000! debug: ignoring loglevel setting.
0.000000! Zone PFN ranges:
0.000000! Normal 0x00000000 -> 0x00010000
0.000000! Movable zone start PFN for each node
0.000000! early_node_map 1! active PFN ranges
0.000000! 0: 0x00000000 -> 0x00010000
0.000000! On node 0 totalpages: 65536
0.000000! Normal zone: 512 pages used for memmap
0.000000! Normal zone: 0 pages reserved
0.000000! Normal zone: 65024 pages, LIFO batch:15
0.000000! Movable zone: 0 pages used for memmap
0.000000! Built 1 zonelists in Zone order, mobility grouping on. Total pages: 65024
0.000000! Kernel command line: ro vmpoff="LOGOFF" root=/dev/disk/by-path/ccw-0.0.0120-part1 ignore_loglevel BOOT_IMAGE=4
0.000000! PID hash table entries: 1024 (order: 10, 4096 bytes)
0.000000! Registering new clock device comparator
0.000000! New clock device comparator registered
0.000633! console ttyS0! enabled
0.002854! Dentry cache hash table entries: 32768 (order: 5, 131072 bytes)
0.004258! timekeeping: clock source changed from none to jiffies (shift: 8)
0.013649! Inode-cache hash table entries: 16384 (order: 4, 65536 bytes)
0.137013! Memory: 251008k/262144k available (2144k kernel code, 0k reserved, 897k data, 152k init)
0.137365! Write protected kernel read-only data: 0x12000 - 0x2d8fff
0.139484! Calibrating delay loop (skipped)... 171.00 BogoMIPS preset
0.140962! Security Framework initialized
0.141155! SELinux: Disabled at boot.
0.142142! Mount-cache hash table entries: 512
0.148444! Initializing cgroup subsys ns
0.148631! Initializing cgroup subsys cpuacct
0.149045! Initializing cgroup subsys devices
0.150337! init: calling smp_prepare_cpus
0.183775! CPUs: 2 configured, 0 standby
0.183947! s390_smp: smp_detect_cpus calling get_online_cpus
0.184162! s390_smp: smp_detect_cpus calling __smp_rescan_cpus
0.184381! s390_smp: smp_rescan_cpus_sigp starting loop
[!!! Hang is here !!!]
[... With 5cd1c9c5 and 6c9bacb4 reverted, the boot continues as follows ...]
[... Next two messages are from Hercules ...]
CPU0000: SIGP Set prefix (0D) CPU0001, PARM 0FEC5000: CC 0
CPU0000: SIGP Restart (06) CPU0001, PARM 00000000: CC 0
[... A correct boot shows similar time lapse since last message ...]
0.525049! s390_smp: smp_rescan_cpus_sigp loop done
0.525310! s390_smp: smp_detect_cpus calling put_online_cpus
0.525555! s390_smp: smp_detect_cpus done
0.526037! cpu 0 phys_idx=0 vers=00 ident=002623 machine=3090 unused=0000
0.526408! s390_smp: start loop smp_create_idle
0.531784! s390_smp: loop smp_create_idle done
0.532009! init: do_pre_smp_initcalls
0.533334! init: start_boot_trace
0.533499! init: smp_init
0.535875! Registering new clock device tod
0.536040! New clock device tod registered
0.536254! cpu 1 phys_idx=1 vers=00 ident=102623 machine=3090 unused=0000
0.545159! Brought up 2 CPUs
0.545323! init: sched_init_smp
0.545724! CPU0 attaching sched-domain:
0.545952! domain 0: span 0-1 level MC
0.546182! groups: 0 1
0.546498! CPU1 attaching sched-domain:
0.546717! domain 0: span 0-1 level MC
0.546943! groups: 1 0
0.548670! init: cpuset_init_smp
0.548847! init: do_basic_setup
0.589620! net_namespace: 524 bytes
0.599256! NET: Registered protocol family 16
0.715112! Switched to high resolution mode on CPU 0
0.715262! Switched to high resolution mode on CPU 1
0.718291! timekeeping: clock source changed from jiffies to tod (shift: 12)
0.731545! NET: Registered protocol family 2
0.787094! IP route cache hash table entries: 2048 (order: 1, 8192 bytes)
0.800004! TCP established hash table entries: 8192 (order: 4, 65536 bytes)
0.804289! TCP bind hash table entries: 8192 (order: 4, 65536 bytes)
0.807714! TCP: Hash tables configured (established 8192 bind 8192)
0.807993! TCP reno registered
0.831294! NET: Registered protocol family 1
0.837270! checking if image is initramfs...
In arch/s390/kernel/smp.c:
static int smp_rescan_cpus_sigp(cpumask_t avail)
{
int cpu_id, logical_cpu;
logical_cpu = first_cpu(avail);
if (logical_cpu == NR_CPUS)
return 0;
printk("s390_smp: smp_rescan_cpus_sigp starting loop\n");
for (cpu_id = 0; cpu_id <= 65535; cpu_id++) {
if (cpu_known(cpu_id))
continue;
__cpu_logical_map[logical_cpu] = cpu_id;
smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
if (!cpu_stopped(logical_cpu))
continue;
cpu_set(logical_cpu, cpu_present_map);
smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
logical_cpu = next_cpu(logical_cpu, avail);
if (logical_cpu == NR_CPUS)
break;
}
printk("s390_smp: smp_rescan_cpus_sigp loop done\n");
return 0;
}
One more.
On Thursday 12 March 2009, Frans Pop wrote:
> I have now been able to trace the hang (full log attached). Where I
> added tracing printks should be fairly obvious, and see attachment.
> No idea what to make of the result.
I added printks that show changes in clock data. I print info for
3 consecutive calls of update_wall_time every 1000 times the function
is called and also after a change of clock source.
With the problem patches reverted this results in (only most relevant boot
messages shown):
0.004316! timekeeping: clock source changed from none to jiffies (shift: 8)
0.004674! timekeeping (jiffies, 8): xtime.tv: 1236941336 -> 927305000
0.004796! clock->xtime: 0 -> 0, error: 0 -> 0
0.009380! timekeeping (jiffies, 8): xtime.tv: 1236941336 -> 927305000
0.009507! clock->xtime: 0 -> 0, error: 0 -> 0
0.014379! timekeeping (jiffies, 8): xtime.tv: 1236941336 -> 927305000
0.014628! clock->xtime: 0 -> 0, error: 0 -> 0
[...]
0.151860! init: calling smp_prepare_cpus
0.184936! CPUs: 2 configured, 0 standby
0.185110! s390_smp: smp_detect_cpus calling get_online_cpus
0.185326! s390_smp: smp_detect_cpus calling __smp_rescan_cpus
0.185544! s390_smp: smp_rescan_cpus_sigp starting loop
CPU0001: SIGP Set prefix (0D) CPU0000, PARM 0FEC5000: CC 0
CPU0001: SIGP Restart (06) CPU0000, PARM 00000000: CC 0
0.515211! s390_smp: smp_rescan_cpus_sigp loop done
0.515426! s390_smp: smp_detect_cpus calling put_online_cpus
0.515648! s390_smp: smp_detect_cpus done
0.515884! cpu 0 phys_idx=1 vers=00 ident=102623 machine=3090 unused=0000
0.516199! s390_smp: start loop smp_create_idle
0.521119! s390_smp: loop smp_create_idle done
0.521304! init: do_pre_smp_initcalls
0.522791! init: start_boot_trace
0.522943! init: smp_init
0.525237! cpu 1 phys_idx=0 vers=00 ident=002623 machine=3090 unused=0000 +
0.535315! Brought up 2 CPUs
0.535475! init: sched_init_smp
0.535878! CPU0 attaching sched-domain:
0.536100! domain 0: span 0-1 level MC
0.536342! groups: 0 1
0.536661! CPU1 attaching sched-domain:
0.536897! domain 0: span 0-1 level MC
0.537126! groups: 1 0
0.538877! init: cpuset_init_smp
0.539072! init: do_basic_setup
0.546056! net_namespace: 524 bytes
0.555855! NET: Registered protocol family 16
0.678223! Switched to high resolution mode on CPU 0
0.678708! Switched to high resolution mode on CPU 1
0.681215! timekeeping: clock source changed from jiffies to tod (shift: 12)
0.681651! timekeeping (tod, 12): xtime.tv: 1236941337 -> 544275946
0.681797! clock->xtime: 0 -> 0, error: 0 -> 0
0.685247! timekeeping (tod, 12): xtime.tv: 1236941337 -> 544275946
0.685393! clock->xtime: 0 -> 0, error: 0 -> 0
0.689345! timekeeping (tod, 12): xtime.tv: 1236941337 -> 544275946
0.689490! clock->xtime: 0 -> 0, error: 0 -> 0
0.698348! NET: Registered protocol family 2
0.754070! IP route cache hash table entries: 2048 (order: 1, 8192 bytes)
0.767402! TCP established hash table entries: 8192 (order: 4, 65536 bytes)
0.771770! TCP bind hash table entries: 8192 (order: 4, 65536 bytes)
0.775119! TCP: Hash tables configured (established 8192 bind 8192)
0.775390! TCP reno registered
0.798979! NET: Registered protocol family 1
0.805108! checking if image is initramfs...
timekeeping (tod, 12): xtime.tv: 1236941341 -> 544275946
4.713613! clock->xtime: 0 -> 0, error: 0 -> 0
4.717402! timekeeping (tod, 12): xtime.tv: 1236941341 -> 544275946
4.717533! clock->xtime: 0 -> 0, error: 0 -> 0
4.721299! timekeeping (tod, 12): xtime.tv: 1236941341 -> 544275946
4.721429! clock->xtime: 0 -> 0, error: 0 -> 0
The values are: "from the very beginning of the function" -> "just after
the calculations". Values are from nsecs fields.
The xtime.tv_nsec which enters the function increases nicely and follows
the timestamps from Hercules, but look rather bogus after the calculations.
With Roman's patch and the later patch this changes to:
0.004593! timekeeping: clock source changed from none to jiffies (shift: 8)
0.005051! timekeeping (jiffies, 8): xtime.tv: 594977000 -> 594977001
0.005097! clock->xtime: 0 -> -256, error: 0 -> -4294867296
0.009608! timekeeping (jiffies, 8): xtime.tv: 594977001 -> 594960618
0.009712! clock->xtime: -256 -> -256, error: -4294867296 -> -4292501984672096
0.014463! Inode-cache hash table entries: 16384 (order: 4, 65536 bytes)
[... Clock has gone back? ...]
0.014859! timekeeping (jiffies, 8): xtime.tv: 594960618 -> 594911467
0.014967! clock->xtime: -256 -> -256, error: -4292501984672096 -> -12807124325236736
[... Followed by a major jump forward? ...]
0.019605! timekeeping (jiffies, 8): xtime.tv: 594911467 -> 594895084
0.019712! clock->xtime: -256 -> -256, error: -12807124325236736 -> -14918180945433952
[...]
0.155939! init: calling smp_prepare_cpus
0.191122! CPUs: 2 configured, 0 standby
0.191308! s390_smp: smp_detect_cpus calling get_online_cpus
0.191614! s390_smp: smp_detect_cpus calling __smp_rescan_cpus
0.191847! s390_smp: smp_rescan_cpus_sigp starting loop
[... Boot hangs here, as in previous traces ...]
[... But strangely enough after some time update_wall_time does get called again ...]
0.348835! timekeeping (jiffies, 8): xtime.tv: 594758872 -> 594758872
0.348933! clock->xtime: -246 -> -11, error: -16046942240899072 -> -16046944321273856
0.349488! timekeeping (jiffies, 8): xtime.tv: 594758872 -> 594758873
0.349606! clock->xtime: -11 -> -244, error: -16046944321273856 -> -16046946535866368
0.350152! timekeeping (jiffies, 8): xtime.tv: 594758873 -> 594758873
0.350251! clock->xtime: -244 -> -13, error: -16046946535866368 -> -16046948616241152
[... Clock has gone back again? ...]
0.394200! timekeeping (jiffies, 8): xtime.tv: 594643725 -> 594643725
0.394308! clock->xtime: -237 -> -20, error: -15554322309840896 -> -15554324390215680
0.394852! timekeeping (jiffies, 8): xtime.tv: 594643725 -> 594643726
0.394956! clock->xtime: -20 -> -235, error: -15554324390215680 -> -15554326604808192
0.395500! timekeeping (jiffies, 8): xtime.tv: 594643726 -> 594643726
0.395602! clock->xtime: -235 -> -22, error: -15554326604808192 -> -15554328685082976
[... And again ...]
0.439589! timekeeping (jiffies, 8): xtime.tv: 594532128 -> 594532129
0.439687! clock->xtime: -28 -> -227, error: -15076881262089568 -> -15076883476682080
0.440219! timekeeping (jiffies, 8): xtime.tv: 594532129 -> 594532129
0.440323! clock->xtime: -227 -> -30, error: -15076883476682080 -> -15076885557056864
0.440864! timekeeping (jiffies, 8): xtime.tv: 594532129 -> 594532130
0.440968! clock->xtime: -30 -> -225, error: -15076885557056864 -> -15076887771649376
[... And again ...]
0.484897! timekeeping (jiffies, 8): xtime.tv: 594423969 -> 594423970
0.485002! clock->xtime: -37 -> -218, error: -14614165981233152 -> -14614168195825664
0.485545! timekeeping (jiffies, 8): xtime.tv: 594423970 -> 594423970
0.485650! clock->xtime: -218 -> -39, error: -14614168195825664 -> -14614160276200448
0.486192! timekeeping (jiffies, 8): xtime.tv: 594423970 -> 594423971
0.486299! clock->xtime: -39 -> -216, error: -14614160276200448 -> -14614162490692960
Here within each set of 3 displays the values look relatively sane, but
the jumps between the bursts cannot be correct.
John: if you'd like the patches I've used for this, please shout.
Cheers,
FJP
On Friday 13 March 2009, Frans Pop wrote:
> I added printks that show changes in clock data. I print info for
> 3 consecutive calls of update_wall_time every 1000 times the function
> is called and also after a change of clock source.
Aaarghh. I'm a moron, but most of you will already have concluded that by
now. I've been looking at nsecs as if they also included secs.
So, here's a trace with Roman's patches that really shows the relevant
data. Basic conclusion remains the same though: time really does look to
be running backwards in the very early part of the boot because the
seconds are not advancing while the nsecs are going backwards.
0.004688! jiffies/8 (1): xtime.tv: 1236962306/99461000 -> 1236962306/99461001
0.004793! clock->xtime: 0 -> -256, error: 0 -> -4294867296
0.009705! jiffies/8 (2): xtime.tv: 1236962306/99461001 -> 1236962306/99444618
0.009821! clock->xtime: -256 -> -256, error: -4294867296 -> -4292501984672096
0.014514! Inode-cache hash table entries: 16384 (order: 4, 65536 bytes)
0.014944! jiffies/8 (3): xtime.tv: 1236962306/99444618 -> 1236962306/99395467
0.015067! clock->xtime: -256 -> -256, error: -4292501984672096 -> -12807124325236736
0.019704! jiffies/8 (4): xtime.tv: 1236962306/99395467 -> 1236962306/99379084
0.019811! clock->xtime: -256 -> -256, error: -12807124325236736 -> -14918180945433952
0.140556! Memory: 251008k/262144k available (2144k kernel code, 0k reserved, 897k data, 152k init)
0.140913! Write protected kernel read-only data: 0x12000 - 0x2d8fff
0.143231! Calibrating delay loop (skipped)... 572.00 BogoMIPS preset
0.144675! Security Framework initialized
0.144870! SELinux: Disabled at boot.
0.146009! Mount-cache hash table entries: 512
0.152474! Initializing cgroup subsys ns
0.152671! Initializing cgroup subsys cpuacct
0.153095! Initializing cgroup subsys devices
0.154303! init: calling smp_prepare_cpus
0.189342! CPUs: 2 configured, 0 standby
0.189516! s390_smp: smp_detect_cpus calling get_online_cpus
0.189742! s390_smp: smp_detect_cpus calling __smp_rescan_cpus
0.189964! s390_smp: smp_rescan_cpus_sigp starting loop
[... System hangs here ...]
[... By this time it looks like the error has stabilized somewhat and nsecs are more stable ...]
[... nsecs are actually larger now than in the display at 0.019704 ...]
0.311447! jiffies/8 (153): xtime.tv: 1236962306/599346640 -> 1236962306/599346640
0.311567! clock->xtime: -228 -> -29, error: -16490881133476192 -> -16490883213850976
0.312174! jiffies/8 (154): xtime.tv: 1236962306/599346640 -> 1236962306/599346641
0.312292! clock->xtime: -29 -> -226, error: -16490883213850976 -> -16490885428443488
0.312884! jiffies/8 (155): xtime.tv: 1236962306/599346641 -> 1236962306/599346641
0.313014! clock->xtime: -226 -> -31, error: -16490885428443488 -> -16490887508818272
[... Only here do seconds start to increase ...]
0.320129! jiffies/8 (304): xtime.tv: 1236962307/99331656 -> 1236962307/99331656
0.320254! clock->xtime: -201 -> -56, error: -16426767068861792 -> -16426769149236576
0.320844! jiffies/8 (305): xtime.tv: 1236962307/99331656 -> 1236962307/99331657
0.320954! clock->xtime: -56 -> -199, error: -16426769149236576 -> -16426781363829088
0.321536! jiffies/8 (306): xtime.tv: 1236962307/99331657 -> 1236962307/99331657
0.321651! clock->xtime: -199 -> -58, error: -16426781363829088 -> -16426783444203872
0.328758! jiffies/8 (455): xtime.tv: 1236962307/599316731 -> 1236962307/599316732
0.328876! clock->xtime: -82 -> -173, error: -16362926407417856 -> -16362928622010368
0.329470! jiffies/8 (456): xtime.tv: 1236962307/599316732 -> 1236962307/599316732
0.329598! clock->xtime: -173 -> -84, error: -16362928622010368 -> -16362920702285152
0.330192! jiffies/8 (457): xtime.tv: 1236962307/599316732 -> 1236962307/599316733
0.330317! clock->xtime: -84 -> -171, error: -16362920702285152 -> -16362922916877664
0.374500! jiffies/8 (1456): xtime.tv: 1236962311/599199308 -> 1236962311/599199308
0.374632! clock->xtime: -165 -> -92, error: -15860566497065312 -> -15860568577440096
On Fri, 2009-03-13 at 12:48 +0100, Frans Pop wrote:
> One more.
>
> On Thursday 12 March 2009, Frans Pop wrote:
> > I have now been able to trace the hang (full log attached). Where I
> > added tracing printks should be fairly obvious, and see attachment.
> > No idea what to make of the result.
>
> I added printks that show changes in clock data. I print info for
> 3 consecutive calls of update_wall_time every 1000 times the function
> is called and also after a change of clock source.
[snip]
> The values are: "from the very beginning of the function" -> "just after
> the calculations". Values are from nsecs fields.
> The xtime.tv_nsec which enters the function increases nicely and follows
> the timestamps from Hercules, but look rather bogus after the calculations.
>
> With Roman's patch and the later patch this changes to:
>
> 0.004593! timekeeping: clock source changed from none to jiffies (shift: 8)
> 0.005051! timekeeping (jiffies, 8): xtime.tv: 594977000 -> 594977001
> 0.005097! clock->xtime: 0 -> -256, error: 0 -> -4294867296
> 0.009608! timekeeping (jiffies, 8): xtime.tv: 594977001 -> 594960618
> 0.009712! clock->xtime: -256 -> -256, error: -4294867296 -> -4292501984672096
> 0.014463! Inode-cache hash table entries: 16384 (order: 4, 65536 bytes)
> [... Clock has gone back? ...]
xtime going backwards is actually allowable, as xtime does not store the
entire state of time.
Time is represented by the equation:
xtime + (offset * mult) >> shift
Since we steer the clock by changing the mult value, imagine if we were
slowing down time, we'd decrese mult by one. However, since offset may
be non-zero, we have to keep the equation balanced, or time might
actually go backwards.
Given:
mult2 == mult1 - 1
xtime1 + (offset * mult1) >> shift == xtime2 + (offset * mult2) >> shift
xtime1 + (offset * mult1) >> shift == xtime2 + (offset * (mult1 - 1)) >> shift
xtime1 + (offset * mult1) >> shift - (offset * (mult1 - 1)) >> shift == xtime2
xtime1 + (offset * mult1 - offset * (mult1 - 1)) >> shift == xtime2
xtime1 + (offset * mult1 - (offset * mult1 - offset*1)) >> shift == xtime2
xtime1 + offset>> shift == xtime2
Now, if we are increasing mult, xtime will be decreased in the same
fashion. So the xtime value going backwards isn't wrong by itself, as
the corresponding offset * newmult will compensate.
xtime_cache actually stores a snapshot of the full state each call to
update_wall_time(), so you might want to use that in your printing
instead.
>From what I've seen so far debugging today it seems something goes off
in clocksource_bigadjust(), and the error continues to grow instead of
being corrected, and we end up constantly increasing mult.
I'm still not quite sure how this links to the hang, but I'm still digging.
thanks
-john
On Thu, 2009-03-12 at 07:51 +0100, Frans Pop wrote:
> On Thursday 12 March 2009, John Stultz wrote:
> > * Reverting both 6c9bacb41c10 and 5cd1c9c5cf30 against 2.6-git did not
> > seem to help the issue, as originally reported. I also reverted
> > 49b5cf34727a as well just in case, and it didn't help either.
>
> Hmm, interesting. I think you may have reproduced a similar failure case,
> but not exactly the same as mine. I wonder if what you have reproduced is
> the same as Debian bug report http://bugs.debian.org/511334. That is with
> 2.6.26 which would match the fact that the reverts that work reliably for
> me did not work for you. I've never had any problems with Debian's 2.6.26
> in my configuration.
Hmmm. I have indeed seen issues w/ the 2.6.26 debian kernel, so that
does seem closer to what I'm seeing.
> For me 2.6.28.7 is rock solid with the two reverts. I've attached my
> kernel config and my Hercules config. I think the critical thing in my
> config may be the "TIMERINT 500" setting for Hercules, which is somewhat
> similar to your "goes away if HZ is anything except 250" observation.
I'll have to checkout that option. Does changing HZ value or disabling
NO_HZ change anything for your situation as well?
> > That's about all I can get for today. I'm out of town until Monday, so
> > I'll start digging back into it then.
>
> From your other replies I've seen that I have been way off track at some
> points. It looks as if I have been looking at events too early in the
> boot and been confused by the use of clock->xtime_nsec. My apologies for
> any confusion caused by that.
>
> I have a few more tests I think I can usefully do, but will try harder not
> to confuse the issue with my inexpert ramblings anymore :-P
No, no, this is subtle and optimized code that keeps very high precision
values. Its not trivial stuff, so your confusion is understandable.
thanks
-john
On Tuesday 17 March 2009, john stultz wrote:
> > For me 2.6.28.7 is rock solid with the two reverts. I've attached my
> > kernel config and my Hercules config. I think the critical thing in
> > my config may be the "TIMERINT 500" setting for Hercules, which is
> > somewhat similar to your "goes away if HZ is anything except 250"
> > observation.
>
> I'll have to checkout that option.
Note that I later established that changing it did not have any effect.
> Does changing HZ value or disabling NO_HZ change anything for your
> situation as well?
If I change that from 250 to 300 the system boots fine. And also if I
switch to NO_HZ.
On Thu, 2009-03-12 at 18:05 +0100, Frans Pop wrote:
> Some other observations:
> * when the hang occurs we're definitely still using jiffies
> * the hang is not in the clock accumulation loop (see below)
> * changing the TIMERINT value does *not* make any difference, I tried a
> few different values (incl. the default 50 and 300)
> * changing the nr of emulated CPUs from 2 to 1 makes no difference
>
>
> I have now been able to trace the hang (full log attached). Where I added
> tracing printks should be fairly obvious, and see attachment.
> No idea what to make of the result.
>
> 0.150337! init: calling smp_prepare_cpus
> 0.183775! CPUs: 2 configured, 0 standby
> 0.183947! s390_smp: smp_detect_cpus calling get_online_cpus
> 0.184162! s390_smp: smp_detect_cpus calling __smp_rescan_cpus
> 0.184381! s390_smp: smp_rescan_cpus_sigp starting loop
> [!!! Hang is here !!!]
>
> [... With 5cd1c9c5 and 6c9bacb4 reverted, the boot continues as follows ...]
> [... Next two messages are from Hercules ...]
> CPU0000: SIGP Set prefix (0D) CPU0001, PARM 0FEC5000: CC 0
> CPU0000: SIGP Restart (06) CPU0001, PARM 00000000: CC 0
> [... Time of previous message is similar for failed and good boot ...]
> 0.525049! s390_smp: smp_rescan_cpus_sigp loop done
> 0.525310! s390_smp: smp_detect_cpus calling put_online_cpus
> 0.525555! s390_smp: smp_detect_cpus done
> 0.526037! cpu 0 phys_idx=0 vers=00 ident=002623 machine=3090 unused=0000
> 0.526408! s390_smp: start loop smp_create_idle
> 0.531784! s390_smp: loop smp_create_idle done
>
> The problem loop is in smp_rescan_cpus_sigp() from arch/s390/kernel/smp.c.
>
> I tried adding printks inside the loop, but that resulted in the boot
> also failing with the two patches reverted!
> So it looks like that loop is somehow very sensitive to timing issues.
> Note the relatively long delay between the start and end of the loop.
Hey Frans,
So I'm looking at the code and I think I've got an idea of what's going
on. Its a little messy and long winded, but stick with me :)
So, right now, I believe the timekeeping or clock steering code is
working properly as designed.
I too saw the code lock up in smp_rescan_cpus, but could find no reason
for it to block there. I do notice that interrupts seem to be working,
so I put a dump_stack() in update_wall_time() every few jiffies and I'd
get the exact same dump every time:
4.642524! ( <0000000000015d68>! show_trace+0x4c/0x88)
4.642885! <0000000000059d4e>! update_wall_time+0x19e/0x778
4.643281! <000000000004541a>! do_timer+0x36/0xe8
4.643627! <000000000005d67e>! tick_periodic+0x7a/0xf8
4.643977! <000000000005d786>! tick_handle_periodic+0x8a/0xe8
4.644350! <0000000000019868>! clock_comparator_work+0x34/0x54
4.644742! <000000000001d368>! do_extint+0x60/0x104
4.645109! <00000000000238ea>! ext_no_vtime+0x12/0x16
4.645480! <0000000000024b52>! cpu_known+0x52/0x180
4.645845! ( <0000000000000000>! 0x0)
4.646139! <000000000032a466>! smp_prepare_cpus+0x172/0x4bc
4.646523! <0000000000320166>! kernel_init+0x52/0x364
4.646903! <000000000001aae2>! kernel_thread_starter+0x6/0xc
4.647288! <000000000001aadc>! kernel_thread_starter+0x0/0xc
So I added a printk in update_wall_time(), and a printk in
tick_handle_periodic(). For a short time, the two messages would
alternate, but then only update_wall_time is printed out.
So we're stuck in interrupt context.
Somehow we're falling behind so that that the loop in
tick_handle_periodic() can't catch up and we just process interrupts
over and over.
In my testing, this isn't really specific to the recent rounding change,
however the rounding change made the issue crop up fast enough that it
could be seen, whereas before the issue wouldn't crop up before the tod
clock was installed. If you boot w/ clocksource=jiffies, you'll probably
see the hang with your working kernels as well, only at a later point
(it would be helpful if you would verify that and let me know).
So why does this happen here, but not w/ HZ != 250, or NOHZ?
Well, from my initial diagnosis, with HZ=250 we some severe error
correction initially. We don't see this with HZ=100 or HZ=1000, and
while we see some initial error correction w/ HZ=300, its not as severe.
So, with NOHZ, the desired accumulation interval is half a second. So w/
HZ=100 that's 50 jiffies, HZ=1000 its 500, HZ=300 is 150 and with HZ=250
it should be 125 jiffies.
However, its being calculated out as 124 jiffies. do_div() seems to be
giving a slightly off answer.
This causes quite a bit of interval error (seen as tick length error in
the code, but w/ NOHZ enabled, tick isn't quite the right word).
Interval error is the difference between how long a the interval should
be and the resulting clocksource interval. So an interval should be
500ms, but since we calculated out 124 jiffies per interval, we see an
4ms error per interval. This causes the jiffies clocksource freq to be
increased to correct for the error, which causes the clock to speed up
by 1/125.
The net effect is that for each jiffy we are increasing time by more
then one jiffy.
Now, back to tick_handle_periodic(). Each interrupt, we call
tick_handle_periodic() to trigger a tick. Each tick we increase jiffies
by one. However, if we're in oneshot mode(NOHZ) we also check the time
to see if we should "catch up", in effect, on ticks that we might have
slept through.
Now, maybe you can see the circularity here? :)
Jiffies is incremented each tick, and time is derived from jiffies.
However, we're using time to decide if we should trigger an extra tick.
This tick will in turn increase jiffies. Since the jiffies clocksource
is being sped up, we increment time by more then a tick. So then when we
check the time again, we're behind! Fire another tick! And on we go...
So two issues that I've found so far:
1) do_div() isn't dividing correctly in this case. Not sure why.
2) We *really* shouldn't be using oneshot mode while the jiffies
clocksource is in use. I believe x86 avoids this specifically, so I'm
not sure why s390 isn't doing the same. Still have to investigate.
So yea, *finally* a bit of progress here. Hopefully I can sort out a
patch shortly.
thanks
-john
On Tue, 2009-03-17 at 19:26 -0700, john stultz wrote:
> So two issues that I've found so far:
> 1) do_div() isn't dividing correctly in this case. Not sure why.
Patch to prove this is part of the issue below. Disables the s390
specific __div64_32() implementation and uses the generic code.
Frans: Mind giving this a try to verify the issue goes away with this?
Martin: I'm not sure if the problem here is the __div64_32
implementation or if the emulator is doing something wrong here. Might
need some help from you in sorting this out.
> 2) We *really* shouldn't be using oneshot mode while the jiffies
> clocksource is in use. I believe x86 avoids this specifically, so I'm
> not sure why s390 isn't doing the same. Still have to investigate.
Still working this.
thanks
-john
diff --git a/arch/s390/lib/div64.c b/arch/s390/lib/div64.c
index a5f8300..6c249fa 100644
--- a/arch/s390/lib/div64.c
+++ b/arch/s390/lib/div64.c
@@ -68,7 +68,7 @@ static uint32_t __div64_31(uint64_t *n, uint32_t base)
words[1] = reg3;
return reg2;
}
-
+#if 0
/*
* Function to divide an unsigned 64 bit integer by an unsigned
* 32 bit integer using the unsigned 64/31 bit division.
@@ -123,9 +123,9 @@ uint32_t __div64_32(uint64_t *n, uint32_t base)
}
return r;
}
-
+#endif
#else /* MARCH_G5 */
-
+#if 0
uint32_t __div64_32(uint64_t *n, uint32_t base)
{
register uint32_t reg2 asm("2");
@@ -145,5 +145,5 @@ uint32_t __div64_32(uint64_t *n, uint32_t base)
words[1] = reg3;
return reg2;
}
-
+#endif
#endif /* MARCH_G5 */
On Tue, 17 Mar 2009 19:54:57 -0700
john stultz <[email protected]> wrote:
> Martin: I'm not sure if the problem here is the __div64_32
> implementation or if the emulator is doing something wrong here. Might
> need some help from you in sorting this out.
__div64_31 is incorrect. Patch and description see below.
How long did you debug this until you finally got down to div64 as the
cause of this? It must have taken you hours!
--
blue skies,
Martin.
"Reality continues to ruin my life." - Calvin.
---
Subject: [PATCH] __div64_31 broken for CONFIG_MARCH_G5
From: Martin Schwidefsky <[email protected]>
The implementation of __div64_31 for G5 machines is broken. The comments
in __div64_31 are correct, only the code does not do what the comments
say. The part "If the remainder has overflown subtract base and increase
the quotient" is only partially realized, the base is subtracted correctly
but the quotient is only increased if the dividend had the last bit set.
Using the correct instruction fixes the problem.
Signed-off-by: Martin Schwidefsky <[email protected]>
---
diff -urpN linux-2.6/arch/s390/lib/div64.c linux-2.6-patched/arch/s390/lib/div64.c
--- linux-2.6/arch/s390/lib/div64.c 2008-12-25 00:26:37.000000000 +0100
+++ linux-2.6-patched/arch/s390/lib/div64.c 2009-03-18 10:06:42.000000000 +0100
@@ -61,7 +61,7 @@ static uint32_t __div64_31(uint64_t *n,
" clr %0,%3\n"
" jl 0f\n"
" slr %0,%3\n"
- " alr %1,%2\n"
+ " ahi %1,1\n"
"0:\n"
: "+d" (reg2), "+d" (reg3), "=d" (tmp)
: "d" (base), "2" (1UL) : "cc" );
On Wednesday 18 March 2009, john stultz wrote:
> In my testing, this isn't really specific to the recent rounding
> change, however the rounding change made the issue crop up fast enough
> that it could be seen, whereas before the issue wouldn't crop up before
> the tod clock was installed. If you boot w/ clocksource=jiffies, you'll
> probably see the hang with your working kernels as well, only at a
> later point (it would be helpful if you would verify that and let me
> know).
Confirmed. It then hangs while checking/loading the initramfs.
On Wednesday 18 March 2009, Martin Schwidefsky wrote:
> From: Martin Schwidefsky <[email protected]>
>
> The implementation of __div64_31 for G5 machines is broken. The
> comments in __div64_31 are correct, only the code does not do what the
> comments say. The part "If the remainder has overflown subtract base
> and increase the quotient" is only partially realized, the base is
> subtracted correctly but the quotient is only increased if the dividend
> had the last bit set. Using the correct instruction fixes the problem.
>
> Signed-off-by: Martin Schwidefsky <[email protected]>
Reported-by: Frans Pop <[email protected]>
Tested-by: Frans Pop <[email protected]>
I've tried this patch with 2.6.28.8 and it fixes the hang! Maybe that
aspect should be mentioned in the commit log?
I've also tested the patch with 2.6.29-rc8 and it also fixes the hang
during login I reported with that [1]. Which means that not only jiffies is
affected, but also tod! And that does not really surprise me because after
the system switches to tod, I also see a continuously increasing error
with clock->xtime_nsec always equal to -4096 (see below).
Am I correct that any kernel starting from 2.6.19 is affected by this, and
that it's the most likely cause of Debian bug report
http://bugs.debian.org/511334? If so, I'll get it pushed into Debian's
stable kernels.
Cheers,
FJP
[1] http://marc.info/?t=123656370500001&r=1&w=2
Ever increasing error with tod on 2.6.28.8 (with Martin's patch applied):
0.672655! timekeeping: clock source changed from jiffies to tod (shift: 12)
0.676889! tod/12 (150): xtime.tv: 1237377507/55524946 -> 1237377507/55524947
0.677020! clock->xtime: 0 -> -4096, error: 0 -> -4294967296
0.680788! tod/12 (151): xtime.tv: 1237377507/55524947 -> 1237377507/55524948
0.680919! clock->xtime: -4096 -> -4096, error: -4294967296 -> -8589934592
0.685280! tod/12 (152): xtime.tv: 1237377507/55524948 -> 1237377507/55524949
0.685411! clock->xtime: -4096 -> -4096, error: -8589934592 -> -12884901888
4.685237! tod/12 (1152): xtime.tv: 1237377511/55525948 -> 1237377511/55525949
4.685356! clock->xtime: -4096 -> -4096, error: -4303557230592 -> -4307852197888
20.700920! tod/12 (5155): xtime.tv: 1237377527/55529951 -> 1237377527/55529952
20.701057! clock->xtime: -4096 -> -4096, error: -21496311316480 -> -21500606283776
32.864888! tod/12 (8160): xtime.tv: 1237377539/55532956 -> 1237377539/55532957
32.865008! clock->xtime: -4096 -> -4096, error: -34402688040960 -> -34406983008256
86.760987! tod/12 (21172): xtime.tv: 1237377593/55545968 -> 1237377593/55545969
86.761120! clock->xtime: -4096 -> -4096, error: -90288802496512 -> -90293097463808
127.100183! tod/12 (29180): xtime.tv: 1237377633/55553976 -> 1237377633/55553977
127.100304! clock->xtime: -4096 -> -4096, error: -124682900602880 -> -124687195570176
491.860765! tod/12 (37189): xtime.tv: 1237377998/55561985 -> 1237377998/55561986
491.860886! clock->xtime: -4096 -> -4096, error: -159081293676544 -> -159085588643840
On Wed, 2009-03-18 at 10:28 +0100, Martin Schwidefsky wrote:
> On Tue, 17 Mar 2009 19:54:57 -0700
> john stultz <[email protected]> wrote:
>
> > Martin: I'm not sure if the problem here is the __div64_32
> > implementation or if the emulator is doing something wrong here. Might
> > need some help from you in sorting this out.
>
> __div64_31 is incorrect. Patch and description see below.
>
> How long did you debug this until you finally got down to div64 as the
> cause of this? It must have taken you hours!
:) Yea, not counting s390 environment ramp up.
Thanks for the quick turnaround on the fix!
-john
On Wed, 2009-03-18 at 13:07 +0100, Frans Pop wrote:
> > Signed-off-by: Martin Schwidefsky <[email protected]>
> Reported-by: Frans Pop <[email protected]>
> Tested-by: Frans Pop <[email protected]>
>
> I've tried this patch with 2.6.28.8 and it fixes the hang! Maybe that
> aspect should be mentioned in the commit log?
>
> I've also tested the patch with 2.6.29-rc8 and it also fixes the hang
> during login I reported with that [1]. Which means that not only jiffies is
> affected, but also tod! And that does not really surprise me because after
> the system switches to tod, I also see a continuously increasing error
> with clock->xtime_nsec always equal to -4096 (see below).
>
> Am I correct that any kernel starting from 2.6.19 is affected by this, and
> that it's the most likely cause of Debian bug report
> http://bugs.debian.org/511334? If so, I'll get it pushed into Debian's
> stable kernels.
Cool!
> Ever increasing error with tod on 2.6.28.8 (with Martin's patch applied):
> 0.672655! timekeeping: clock source changed from jiffies to tod (shift: 12)
> 0.676889! tod/12 (150): xtime.tv: 1237377507/55524946 -> 1237377507/55524947
> 0.677020! clock->xtime: 0 -> -4096, error: 0 -> -4294967296
> 0.680788! tod/12 (151): xtime.tv: 1237377507/55524947 -> 1237377507/55524948
> 0.680919! clock->xtime: -4096 -> -4096, error: -4294967296 -> -8589934592
> 0.685280! tod/12 (152): xtime.tv: 1237377507/55524948 -> 1237377507/55524949
> 0.685411! clock->xtime: -4096 -> -4096, error: -8589934592 -> -12884901888
> 4.685237! tod/12 (1152): xtime.tv: 1237377511/55525948 -> 1237377511/55525949
> 4.685356! clock->xtime: -4096 -> -4096, error: -4303557230592 -> -4307852197888
> 20.700920! tod/12 (5155): xtime.tv: 1237377527/55529951 -> 1237377527/55529952
> 20.701057! clock->xtime: -4096 -> -4096, error: -21496311316480 -> -21500606283776
> 32.864888! tod/12 (8160): xtime.tv: 1237377539/55532956 -> 1237377539/55532957
> 32.865008! clock->xtime: -4096 -> -4096, error: -34402688040960 -> -34406983008256
> 86.760987! tod/12 (21172): xtime.tv: 1237377593/55545968 -> 1237377593/55545969
> 86.761120! clock->xtime: -4096 -> -4096, error: -90288802496512 -> -90293097463808
> 127.100183! tod/12 (29180): xtime.tv: 1237377633/55553976 -> 1237377633/55553977
> 127.100304! clock->xtime: -4096 -> -4096, error: -124682900602880 -> -124687195570176
> 491.860765! tod/12 (37189): xtime.tv: 1237377998/55561985 -> 1237377998/55561986
> 491.860886! clock->xtime: -4096 -> -4096, error: -159081293676544 -> -159085588643840
Hrm. Is the box otherwise working ok? The TOD clock should not be
affected by the second issue (one shot mode) discussed.
thanks
-john
On Wednesday 18 March 2009, John Stultz wrote:
> > Ever increasing error with tod on 2.6.28.8 (with Martin's patch
> > applied):
> > 0.672655! timekeeping: clock source changed from jiffies to tod (shift: 12)
> > 0.676889! tod/12 (150): xtime.tv: 1237377507/55524946 -> 1237377507/55524947
> > 0.677020! clock->xtime: 0 -> -4096, error: 0 -> -4294967296
> > 0.680788! tod/12 (151): xtime.tv: 1237377507/55524947 -> 1237377507/55524948
[...]
> > 491.860765! tod/12 (37189): xtime.tv: 1237377998/55561985 -> 1237377998/55561986
> > 491.860886! clock->xtime: -4096 -> -4096, error: -159081293676544 -> -159085588643840
>
> Hrm. Is the box otherwise working ok? The TOD clock should not be
> affected by the second issue (one shot mode) discussed.
Yes, the box^Wsystem works fine. I've now also seen the eventual correction
of the error in action: after 35 mins of uptime clock->multi changed from
1000 to 999 (with tod).
So the only issue left, though only indirectly related to the hang, is
the initial behavior with clocksource jiffies where clocksource_bigadjust
gets called every time update_wall_time is called (I've confirmed that).
And possibly the cleanup change of clock->xtime_nsec to S64.
I'll happily leave those to you as I readily admit my understanding of the
whole timekeeping thing is still very limited. But if you'd like patches
tested, feel free to CC me.
Thanks again for you excellent work on tracking down the real cause of the
hangs.
Cheers,
FJP
On Mon, 2009-03-23 at 01:11 +0100, Frans Pop wrote:
> On Wednesday 18 March 2009, John Stultz wrote:
> > > Ever increasing error with tod on 2.6.28.8 (with Martin's patch
> > > applied):
> > > 0.672655! timekeeping: clock source changed from jiffies to tod (shift: 12)
> > > 0.676889! tod/12 (150): xtime.tv: 1237377507/55524946 -> 1237377507/55524947
> > > 0.677020! clock->xtime: 0 -> -4096, error: 0 -> -4294967296
> > > 0.680788! tod/12 (151): xtime.tv: 1237377507/55524947 -> 1237377507/55524948
> [...]
> > > 491.860765! tod/12 (37189): xtime.tv: 1237377998/55561985 -> 1237377998/55561986
> > > 491.860886! clock->xtime: -4096 -> -4096, error: -159081293676544 -> -159085588643840
> >
> > Hrm. Is the box otherwise working ok? The TOD clock should not be
> > affected by the second issue (one shot mode) discussed.
>
> Yes, the box^Wsystem works fine. I've now also seen the eventual correction
> of the error in action: after 35 mins of uptime clock->multi changed from
> 1000 to 999 (with tod).
>
> So the only issue left, though only indirectly related to the hang, is
> the initial behavior with clocksource jiffies where clocksource_bigadjust
> gets called every time update_wall_time is called (I've confirmed that).
>
> And possibly the cleanup change of clock->xtime_nsec to S64.
>
> I'll happily leave those to you as I readily admit my understanding of the
> whole timekeeping thing is still very limited. But if you'd like patches
> tested, feel free to CC me.
Here's the fix to the tick_handle_periodic() tripping into an infinite
loop. Again, this was only triggered because the divide error that
caused jiffies to be skewed enough that the clock-steering code
increased the ns per jiffy conversion value enough that any slack we had
in the loop before was lost.
Fixing the divide issue avoids the problem (and is pretty important to
get upstream), but the underlying issue that we allow ONESHOT clockevent
mode to be used while the jiffies clocksource is in use is a concern.
Thomas had pointed out that ppc and other arches that do not have
PERIODIC mode clockevents don't trip over this, but I believe this has
been just luck so far, as we do not enable clocksource switching till
bootup is almost finished (to avoid clocksource churn), so after
interrupts are enabled, but before clocksource switching is allowed,
there is a chance (albeit very very small) that clock steering could
cause a similar problem on other arches.
Thomas, what do you think about this? With it s390 runs fine even
without the do_div() fix.
thanks
-john
The following patch avoids and endless loop issue by requiring that a
highres valid clocksource be installed before we call tick_periodic() in
a loop when using ONESHOT mode. The result is we will only increment
jiffies once per interrupt until a continuous hardware clocksource is
available.
Without this, we can run into a endless loop, where each cycle through
the loop, jiffies is updated which increments time by tick_period or
more (due to clock steering), which can cause the event programming to
think the next event was before the newly incremented time and fail
causing tick_periodic() to be called again and the whole process loops
forever.
Signed-off-by: John Stultz <[email protected]>
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index 21a5ca8..83c4417 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -93,7 +93,17 @@ void tick_handle_periodic(struct clock_event_device *dev)
for (;;) {
if (!clockevents_program_event(dev, next, ktime_get()))
return;
- tick_periodic(cpu);
+ /*
+ * Have to be careful here. If we're in oneshot mode,
+ * before we call tick_periodic() in a loop, we need
+ * to be sure we're using a real hardware clocksource.
+ * Otherwise we could get trapped in an infinite
+ * loop, as the tick_periodic() increments jiffies,
+ * when then will increment time, posibly causing
+ * the loop to trigger again and again.
+ */
+ if (timekeeping_valid_for_hres())
+ tick_periodic(cpu);
next = ktime_add(next, tick_period);
}
}
On Mon, 23 Mar 2009 15:19:00 -0700
John Stultz <[email protected]> wrote:
> Fixing the divide issue avoids the problem (and is pretty important to
> get upstream), but the underlying issue that we allow ONESHOT clockevent
> mode to be used while the jiffies clocksource is in use is a concern.
The bug only appears on 31-bit if the kernel is compiled for a G5. All
current distros use at 64-bit kernels with a cpu >= z900. The severity
is relative .. but the bug fix is upstream and Greg picked it up for
his stable releases.
--
blue skies,
Martin.
"Reality continues to ruin my life." - Calvin.
Hey Thomas,
I think this might have flown past your radar, so I'm resending. Not
super critical, but probably a good thing to have, so its fine for
2.6.31.
Here's the fix to the tick_handle_periodic() tripping into an infinite
loop. This was originally seen on s390 emulator. Again, this was only
triggered because the divide error that caused jiffies to be skewed
enough that the clock-steering code increased the ns per jiffy
conversion value enough that any slack we had in the loop before was
lost.
Fixing the divide issue (already upstream) avoids the problem, but the
underlying issue that we allow ONESHOT clockevent mode to be used while
the jiffies clocksource is in use is still a concern.
Thomas had pointed out that ppc and other arches that do not have
PERIODIC mode clockevents don't trip over this, but I believe this has
been just luck so far, as we do not enable clocksource switching till
bootup is almost finished (to avoid clocksource churn), so after
interrupts are enabled, but before clocksource switching is allowed,
there is a chance (albeit very very small) that clock steering could
cause a similar problem on other arches.
Thomas, what do you think about this? With the s390 emulator that
originally tripped over this issue, this patch makes it runs fine even
without the do_div() fix.
thanks
-john
The following patch avoids and endless loop issue by requiring that a
highres valid clocksource be installed before we call tick_periodic() in
a loop when using ONESHOT mode. The result is we will only increment
jiffies once per interrupt until a continuous hardware clocksource is
available.
Without this, we can run into a endless loop, where each cycle through
the loop, jiffies is updated which increments time by tick_period or
more (due to clock steering), which can cause the event programming to
think the next event was before the newly incremented time and fail
causing tick_periodic() to be called again and the whole process loops
forever.
Signed-off-by: John Stultz <[email protected]>
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index 21a5ca8..83c4417 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -93,7 +93,17 @@ void tick_handle_periodic(struct clock_event_device *dev)
for (;;) {
if (!clockevents_program_event(dev, next, ktime_get()))
return;
- tick_periodic(cpu);
+ /*
+ * Have to be careful here. If we're in oneshot mode,
+ * before we call tick_periodic() in a loop, we need
+ * to be sure we're using a real hardware clocksource.
+ * Otherwise we could get trapped in an infinite
+ * loop, as the tick_periodic() increments jiffies,
+ * when then will increment time, posibly causing
+ * the loop to trigger again and again.
+ */
+ if (timekeeping_valid_for_hres())
+ tick_periodic(cpu);
next = ktime_add(next, tick_period);
}
}