[DragonFly_Stable] panic: vfs_cache: malloc limit exceeded

[YONETANI Tomokazu <qhwt+dfly@xxxxxxxxxx>]

Hello.
Yesterday I upgraded my desktop PC, which had been up for almost a month
without a problem, to DragonFly_Stable. The build and install went fine,
and I kept it running until it panicked in the mid night probably during
or after daily cron job. This is a PentiumIV box running SMP kernel
(but no HyperThreading available). The box has two rootfs'es, and I'm
running older kernel/world now(built on Sept. 9).

GNU gdb 4.18 (FreeBSD)
Copyright 1998 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-unknown-dragonfly"...Deprecated bfd_read called at /home/source/dragonfly/src/gnu/usr.bin/binutils/gdb/../../../../contrib/gdb/gdb/dbxread.c line 2627 in elfstab_build_psymtabs
Deprecated bfd_read called at /home/source/dragonfly/src/gnu/usr.bin/binutils/gdb/../../../../contrib/gdb/gdb/dbxread.c line 933 in fill_symbuf

1 cpu [ff800000,32768]
IdlePTD at phsyical address 0x0047a000
PCB @002b94c0 EIP=c016a743 ESP=ccef3888 EBP=ccef3894
initial pcb at physical address 0x002b94c0
panicstr: vfscache: malloc limit exceeded
panic messages:
---
panic: vfscache: malloc limit exceeded
mp_lock = 00000000; cpuid = 0; lapic.id = 00000000

syncing disks... 63 
done
Uptime: 1d16h59m25s
vn: cdevsw removed

dumping to dev #ad/0x20001, offset 1573960
dump ata0: resetting devices .. done
255 254 [snip, you don't want to see this] 1 0
---
#0  dumpsys () at /home/source/dragonfly/src/sys/kern/kern_shutdown.c:508
508		if (dumping++) {
(kgdb) bt
#0  dumpsys () at /home/source/dragonfly/src/sys/kern/kern_shutdown.c:508
#1  0xc016a53d in boot (howto=256)
    at /home/source/dragonfly/src/sys/kern/kern_shutdown.c:341
#2  0xc016a9ba in poweroff_wait (junk=0xc026da0e, howto=-1071188788)
    at /home/source/dragonfly/src/sys/kern/kern_shutdown.c:620
#3  0xc0168cd2 in malloc (size=68, type=0xc029a4a0, flags=258)
    at /home/source/dragonfly/src/sys/kern/kern_slaballoc.c:391
#4  0xc019b133 in cache_alloc ()
    at /home/source/dragonfly/src/sys/kern/vfs_cache.c:256
#5  0xc019bd36 in cache_enter (dvp=0xcd4841a0, par=0x0, vp=0xcd3e8ae0, 
    cnp=0xccef3bd0) at /home/source/dragonfly/src/sys/kern/vfs_cache.c:998
#6  0xc0203e85 in ufs_lookup (ap=0xccef39d8)
    at /home/source/dragonfly/src/sys/vfs/ufs/ufs_lookup.c:625
#7  0xc02085fe in ufs_vnoperate (ap=0xccef39d8)
    at /home/source/dragonfly/src/sys/vfs/ufs/ufs_vnops.c:2319
#8  0xc01a8892 in vop_cachedlookup (ops=0xc6089000, dvp=0xcd4841a0, 
    vpp=0xccef3bbc, cnp=0xccef3bd0)
    at /home/source/dragonfly/src/sys/kern/vfs_vopops.c:330
#9  0xc019c14a in vfs_cache_lookup (ap=0xccef3a60)
    at /home/source/dragonfly/src/sys/kern/vfs_cache.c:1255
#10 0xc02085fe in ufs_vnoperate (ap=0xccef3a60)
    at /home/source/dragonfly/src/sys/vfs/ufs/ufs_vnops.c:2319
#11 0xc01a8825 in vop_lookup (ops=0xc6089000, dvp=0xcd4841a0, vpp=0xccef3bbc, 
    cnp=0xccef3bd0) at /home/source/dragonfly/src/sys/kern/vfs_vopops.c:313
#12 0xc019ef32 in lookup (ndp=0xccef3ba8)
    at /home/source/dragonfly/src/sys/kern/vfs_lookup.c:463
#13 0xc019ea39 in namei (ndp=0xccef3ba8)
    at /home/source/dragonfly/src/sys/kern/vfs_lookup.c:176
#14 0xc01a5490 in kern_stat (nd=0xccef3ba8, st=0xccef3b48)
    at /home/source/dragonfly/src/sys/kern/vfs_syscalls.c:1630
#15 0xc01a5582 in lstat (uap=0xccef3c24)
    at /home/source/dragonfly/src/sys/kern/vfs_syscalls.c:1678
#16 0xc024ce6d in syscall2 (frame={tf_fs = 672202799, tf_es = 135528495, 
      tf_ds = -1078001617, tf_edi = 134623064, tf_esi = 134622976, 
      tf_ebp = -1077937464, tf_isp = -856736396, tf_ebx = 672131888, 
      tf_edx = 134557696, tf_ecx = 0, tf_eax = 190, tf_trapno = 0, tf_err = 2, 
      tf_eip = 671805240, tf_cs = 31, tf_eflags = 658, tf_esp = -1077937620, 
      tf_ss = 47}) at /home/source/dragonfly/src/sys/i386/i386/trap.c:1350
#17 0xc0237efa in Xint0x80_syscall ()
    at /home/source/dragonfly/src/sys/i386/i386/exception.s:858
#18 0x280ab36e in ?? ()
#19 0x280aac05 in ?? ()
#20 0x8049848 in ?? ()
#21 0x804ba41 in ?? ()
#22 0x804948d in ?? ()
(kgdb) fra 3
#3  0xc0168cd2 in malloc (size=68, type=0xc029a4a0, flags=258)
    at /home/source/dragonfly/src/sys/kern/kern_slaballoc.c:391
391		    panic("%s: malloc limit exceeded", type->ks_shortdesc);
(kgdb) list
386		    ttl += type->ks_memuse[i];
387		type->ks_loosememuse = ttl;
388		if (ttl >= type->ks_limit) {
389		    if (flags & (M_RNOWAIT|M_NULLOK))
390			return(NULL);
391		    panic("%s: malloc limit exceeded", type->ks_shortdesc);
392		}
393	    }
394	
395	    /*
(kgdb) print *type
$1 = {ks_next = 0x203a7325, ks_memuse = {1819042157, 1814061935, 1953066345, 
    1668834592, 1701078373, 1851064420, 1701869669, 1684370531, 1954112032, 
    1868767333, 544501365, 1929405477, 1633837420, 1668246636, 1868767290, 
    1886745202}, ks_loosememuse = 543450484, ks_limit = 1701736314, 
  ks_size = 2037543936, ks_inuse = {543649385, 1713401716, 543516018, 
    1280070990, 1768910880, 1919251566, 1701996032, 1830828645, 1869376609, 
    2037653603, 1814062448, 1936417633, 1734438176, 1291871081, 1701865808, 
    1920090400}, ks_calls = 7309458126808316257, ks_maxused = 1701052465, 
  ks_magic = 1734833506, 
  ks_shortdesc = 0x6f5f7265 <Address 0x6f5f7265 out of bounds>, 
  ks_limblocks = 24430, ks_mapblocks = 24944, ks_reserved = {6515054, 
    544109906, 1969382756, 1919248231}}
(kgdb) print *(struct malloc_type *)0xc029a4a0
$2 = {ks_next = 0xc0299de0, ks_memuse = {25902696, 0 <repeats 15 times>}, 
  ks_loosememuse = 25902696, ks_limit = 25902694, ks_size = 0, ks_inuse = {
    371186, 0 <repeats 15 times>}, ks_calls = 7069901, ks_maxused = 0, 
  ks_magic = 877983977, ks_shortdesc = 0xc026f4cc "vfscache", 
  ks_limblocks = 0, ks_mapblocks = 0, ks_reserved = {0, 0, 0, 0}}

Although this machine has a swap-backed partition as /tmp, the operation
which triggered the panic seems to be an lstat() on a file in /local,
where I keep ports stuff.

(kgdb) fra 15
#15 0xc01a5582 in lstat (uap=0xccef3c24)
    at /home/source/dragonfly/src/sys/kern/vfs_syscalls.c:1678
1678            error = kern_stat(&nd, &st);
(kgdb) print nd
$21 = {ni_dirp = 0x8062f40 "panel-lockscreen.png", ni_segflg = UIO_USERSPACE,
  ni_startdir = 0x0, ni_rootdir = 0xcbe3ede0, ni_topdir = 0x0,
  ni_vp = 0xcd3e8ae0, ni_dvp = 0xcd4841a0, ni_pathlen = 1,
  ni_next = 0xcc936c14 "", ni_loopcnt = 0, ni_cnd = {cn_nameiop = 0,
    cn_flags = 2146436, cn_td = 0xccb15b60, cn_cred = 0xc10c61d8,
    cn_pnbuf = 0xcc936c00 "panel-lockscreen.png",
    cn_nameptr = 0xcc936c00 "panel-lockscreen.png", cn_namelen = 20,
    cn_consume = 0, cn_timeout = -1071339037}}
(kgdb) print nd.ni_vp->v_mount->mnt_stat
$22 = {f_spare2 = 0, f_bsize = 2048, f_iosize = 16384, f_blocks = 4128871,
  f_bfree = 1159702, f_bavail = 829393, f_files = 1034750, f_ffree = 803254,
  f_fsid = {val = {1045662679, 1084546093}}, f_owner = 0, f_type = 1,
  f_flags = 2101248, f_syncwrites = 3, f_asyncwrites = 10813,
  f_fstypename = "ufs", '\000' <repeats 12 times>,
  f_mntonname = "/local", '\000' <repeats 73 times>, f_syncreads = 38456,
  f_asyncreads = 889, f_spares1 = 0,
  f_mntfromname = "/dev/ad0s1f", '\000' <repeats 68 times>, f_spares2 = 0,
  f_spare = {0, 0}}
(kgdb)